mirror of
https://github.com/tree-sitter/tree-sitter.git
synced 2026-09-10 15:36:29 -04:00
7be497a14e
`currentType` and `currentTypeId` return `null` before the first iteration step, after exhaustion, and after a reset. `currentType` also falls back to the language's own name table instead of the literal 'ERROR' when `Language.types` has no entry (auxiliary symbols). Other bindings report `end` where this one reported 'ERROR'.
300 lines
10 KiB
TypeScript
300 lines
10 KiB
TypeScript
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
|
import helper, { type LanguageName } from './helper';
|
|
import { LookaheadIterator, Language } from '../src';
|
|
import { Parser } from '../src';
|
|
import { C } from '../src/constants';
|
|
import { readFile } from 'fs/promises';
|
|
import { pathToFileURL } from 'url';
|
|
|
|
let JavaScript: Language;
|
|
let Rust: Language;
|
|
let languageURL: (name: LanguageName) => string;
|
|
|
|
describe('Language', () => {
|
|
beforeAll(async () => ({ JavaScript, Rust, languageURL } = await helper));
|
|
|
|
describe('.loadSync', () => {
|
|
it('loads a language synchronously from a pre-compiled WebAssembly.Module', async () => {
|
|
const wasmPath = languageURL('javascript');
|
|
const wasmBytes = await readFile(wasmPath);
|
|
const wasmModule = await WebAssembly.compile(wasmBytes);
|
|
|
|
const lang = Language.loadSync(wasmModule);
|
|
expect(lang.name).toBe('javascript');
|
|
expect(lang.abiVersion).toBe(15);
|
|
|
|
// Verify the language actually works by parsing a snippet
|
|
const parser = new Parser();
|
|
parser.setLanguage(lang);
|
|
const tree = parser.parse('const x = 1;');
|
|
expect(tree).not.toBeNull();
|
|
expect(tree!.rootNode.type).toBe('program');
|
|
expect(tree!.rootNode.childCount).toBe(1);
|
|
expect(tree!.rootNode.firstChild!.type).toBe('lexical_declaration');
|
|
parser.delete();
|
|
});
|
|
|
|
it('reports when a module has no language function', () => {
|
|
const loadWebAssemblyModule = C.loadWebAssemblyModule;
|
|
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined);
|
|
C.loadWebAssemblyModule = (() => ({
|
|
not_a_language: () => 0,
|
|
})) as unknown as typeof C.loadWebAssemblyModule;
|
|
|
|
try {
|
|
expect(() => Language.loadSync({})).toThrow(
|
|
'Language.loadSync failed: no language function found in Wasm file',
|
|
);
|
|
expect(log).toHaveBeenCalledWith(expect.stringContaining('not_a_language'));
|
|
} finally {
|
|
C.loadWebAssemblyModule = loadWebAssemblyModule;
|
|
log.mockRestore();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('.load', () => {
|
|
it('loads a language from a file URL', async () => {
|
|
const wasmURL = pathToFileURL(languageURL('javascript'));
|
|
expect(wasmURL).toBeInstanceOf(URL);
|
|
|
|
const lang = await Language.load(wasmURL);
|
|
expect(lang.name).toBe('javascript');
|
|
|
|
// Verify the language actually works by parsing a snippet
|
|
const parser = new Parser();
|
|
parser.setLanguage(lang);
|
|
const tree = parser.parse('const x = 1;');
|
|
expect(tree).not.toBeNull();
|
|
expect(tree!.rootNode.type).toBe('program');
|
|
expect(tree!.rootNode.childCount).toBe(1);
|
|
expect(tree!.rootNode.firstChild!.type).toBe('lexical_declaration');
|
|
parser.delete();
|
|
});
|
|
|
|
it('reports when an async-loaded module has no language function', async () => {
|
|
const loadWebAssemblyModule = C.loadWebAssemblyModule;
|
|
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined);
|
|
C.loadWebAssemblyModule = (() => Promise.resolve({
|
|
not_a_language: () => 0,
|
|
})) as unknown as typeof C.loadWebAssemblyModule;
|
|
|
|
try {
|
|
await expect(Language.load(new Uint8Array([0, 0, 0, 0]))).rejects.toThrow(
|
|
'Language.load failed: no language function found in Wasm file',
|
|
);
|
|
expect(log).toHaveBeenCalledWith(expect.stringContaining('not_a_language'));
|
|
} finally {
|
|
C.loadWebAssemblyModule = loadWebAssemblyModule;
|
|
log.mockRestore();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('.name, .version', () => {
|
|
it('returns the name and version of the language', () => {
|
|
expect(JavaScript.name).toBe('javascript');
|
|
expect(JavaScript.abiVersion).toBe(15);
|
|
});
|
|
});
|
|
|
|
describe('.fieldIdForName, .fieldNameForId', () => {
|
|
it('converts between the string and integer representations of fields', () => {
|
|
const nameId = JavaScript.fieldIdForName('name');
|
|
const bodyId = JavaScript.fieldIdForName('body');
|
|
|
|
expect(nameId).toBeLessThan(JavaScript.fieldCount);
|
|
expect(bodyId).toBeLessThan(JavaScript.fieldCount);
|
|
expect(JavaScript.fieldNameForId(nameId!)).toBe('name');
|
|
expect(JavaScript.fieldNameForId(bodyId!)).toBe('body');
|
|
});
|
|
|
|
it('handles invalid inputs', () => {
|
|
expect(JavaScript.fieldIdForName('namezzz')).toBeNull();
|
|
expect(JavaScript.fieldNameForId(-3)).toBeNull();
|
|
expect(JavaScript.fieldNameForId(10000)).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('.idForNodeType, .nodeTypeForId, .nodeTypeIsNamed', () => {
|
|
it('converts between the string and integer representations of a node type', () => {
|
|
const exportStatementId = JavaScript.idForNodeType('export_statement', true)!;
|
|
const starId = JavaScript.idForNodeType('*', false)!;
|
|
|
|
expect(exportStatementId).toBeLessThan(JavaScript.nodeTypeCount);
|
|
expect(starId).toBeLessThan(JavaScript.nodeTypeCount);
|
|
expect(JavaScript.nodeTypeIsNamed(exportStatementId)).toBe(true);
|
|
expect(JavaScript.nodeTypeForId(exportStatementId)).toBe('export_statement');
|
|
expect(JavaScript.nodeTypeIsNamed(starId)).toBe(false);
|
|
expect(JavaScript.nodeTypeForId(starId)).toBe('*');
|
|
});
|
|
|
|
it('handles invalid inputs', () => {
|
|
expect(JavaScript.nodeTypeForId(-3)).toBeNull();
|
|
expect(JavaScript.nodeTypeForId(10000)).toBeNull();
|
|
expect(JavaScript.idForNodeType('export_statement', false)).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('Supertypes', () => {
|
|
it('gets the supertypes and subtypes of a parser', () => {
|
|
const supertypes = Rust.supertypes;
|
|
const names = supertypes.map((id) => Rust.nodeTypeForId(id));
|
|
expect(names).toEqual([
|
|
'_expression',
|
|
'_literal',
|
|
'_literal_pattern',
|
|
'_pattern',
|
|
'_type'
|
|
]);
|
|
|
|
for (const id of supertypes) {
|
|
const name = Rust.nodeTypeForId(id);
|
|
const subtypes = Rust.subtypes(id);
|
|
let subtypeNames = subtypes.map((id) => Rust.nodeTypeForId(id));
|
|
subtypeNames = [...new Set(subtypeNames)].sort(); // Remove duplicates & sort
|
|
|
|
switch (name) {
|
|
case '_literal':
|
|
expect(subtypeNames).toEqual([
|
|
'boolean_literal',
|
|
'char_literal',
|
|
'float_literal',
|
|
'integer_literal',
|
|
'raw_string_literal',
|
|
'string_literal',
|
|
]);
|
|
break;
|
|
case '_pattern':
|
|
expect(subtypeNames).toEqual([
|
|
'_',
|
|
'_literal_pattern',
|
|
'captured_pattern',
|
|
'const_block',
|
|
'generic_pattern',
|
|
'identifier',
|
|
'macro_invocation',
|
|
'mut_pattern',
|
|
'or_pattern',
|
|
'range_pattern',
|
|
'ref_pattern',
|
|
'reference_pattern',
|
|
'remaining_field_pattern',
|
|
'scoped_identifier',
|
|
'slice_pattern',
|
|
'struct_pattern',
|
|
'tuple_pattern',
|
|
'tuple_struct_pattern',
|
|
]);
|
|
break;
|
|
case '_type':
|
|
expect(subtypeNames).toEqual([
|
|
'abstract_type',
|
|
'array_type',
|
|
'bounded_type',
|
|
'dynamic_type',
|
|
'function_type',
|
|
'generic_type',
|
|
'macro_invocation',
|
|
'metavariable',
|
|
'never_type',
|
|
'pointer_type',
|
|
'primitive_type',
|
|
'reference_type',
|
|
'removed_trait_bound',
|
|
'scoped_type_identifier',
|
|
'tuple_type',
|
|
'type_identifier',
|
|
'unit_type',
|
|
]);
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Lookahead iterator', () => {
|
|
let lookahead: LookaheadIterator;
|
|
let state: number;
|
|
|
|
beforeAll(async () => {
|
|
({ JavaScript } = await helper);
|
|
const parser = new Parser();
|
|
parser.setLanguage(JavaScript);
|
|
const tree = parser.parse('function fn() {}')!;
|
|
parser.delete();
|
|
const cursor = tree.walk();
|
|
expect(cursor.gotoFirstChild()).toBe(true);
|
|
expect(cursor.gotoFirstChild()).toBe(true);
|
|
state = cursor.currentNode.nextParseState;
|
|
lookahead = JavaScript.lookaheadIterator(state)!;
|
|
expect(lookahead).toBeDefined();
|
|
});
|
|
|
|
afterAll(() => { lookahead.delete() });
|
|
|
|
const expected = ['(', 'identifier', '*', 'formal_parameters', 'html_comment', 'comment'];
|
|
|
|
it('should iterate over valid symbols in the state', () => {
|
|
const symbols = Array.from(lookahead);
|
|
expect(symbols).toEqual(expect.arrayContaining(expected));
|
|
expect(symbols).toHaveLength(expected.length);
|
|
});
|
|
|
|
it('should reset to the initial state', () => {
|
|
expect(lookahead.resetState(state)).toBe(true);
|
|
const symbols = Array.from(lookahead);
|
|
expect(symbols).toEqual(expect.arrayContaining(expected));
|
|
expect(symbols).toHaveLength(expected.length);
|
|
});
|
|
|
|
it('should stay exhausted until reset', () => {
|
|
expect(lookahead.resetState(state)).toBe(true);
|
|
expect(Array.from(lookahead)).toHaveLength(expected.length);
|
|
expect(Array.from(lookahead)).toHaveLength(0);
|
|
expect(lookahead.currentType).toBeNull();
|
|
expect(lookahead.currentTypeId).toBeNull();
|
|
});
|
|
|
|
it('should not be positioned before the first step', () => {
|
|
const fresh = JavaScript.lookaheadIterator(state);
|
|
expect(fresh).not.toBeNull();
|
|
expect(fresh?.currentType).toBeNull();
|
|
expect(fresh?.currentTypeId).toBeNull();
|
|
fresh?.delete();
|
|
});
|
|
|
|
it('should reset', () => {
|
|
expect(lookahead.reset(JavaScript, state)).toBe(true);
|
|
const symbols = Array.from(lookahead);
|
|
expect(symbols).toEqual(expect.arrayContaining(expected));
|
|
expect(symbols).toHaveLength(expected.length);
|
|
});
|
|
});
|
|
|
|
describe('Lookahead iterator symbol names', () => {
|
|
let Json: Language;
|
|
let state: number;
|
|
|
|
beforeAll(async () => {
|
|
({ JSON: Json } = await helper);
|
|
const parser = new Parser();
|
|
parser.setLanguage(Json);
|
|
const tree = parser.parse('{"a": 1}')!;
|
|
parser.delete();
|
|
const cursor = tree.walk();
|
|
expect(cursor.gotoFirstChild()).toBe(true); // object
|
|
state = cursor.currentNode.nextParseState;
|
|
});
|
|
|
|
it('hould name symbols missing from Language.types', () => {
|
|
const lookahead = Json.lookaheadIterator(state)!;
|
|
const names = Array.from(lookahead);
|
|
expect(names).toContain('end');
|
|
expect(names).not.toContain('ERROR');
|
|
expect(names).toHaveLength(12);
|
|
lookahead.delete();
|
|
});
|
|
});
|