libs-code/tree-sitter.tree-sitter
1
0
Fork 0
mirror of https://github.com/tree-sitter/tree-sitter.git synced 2026-09-10 07:26:23 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

perf(lib): fast path for ASCII characters in ts_lexer__advance

Browse source 
Yields a 0-5% improvement on parse speed (very grammar dependent).
This commit is contained in:
Will Lillis 2026-08-13 23:29:53 -04:00 committed by Christian Clason
parent 730946cfd5
commit 21e3614b8d
1 changed files with 21 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
lib/src/lexer.c
View file

@ -268,6 +268,27 @@ static void ts_lexer__advance(TSLexer *_self, bool skip) {
LOG("consume", self->data.lookahead)
}
const uint32_t next_position = self->current_position.bytes + 1;
const uint32_t current_range_end =
self->included_ranges[self->current_included_range_index].end_byte;
if (
self->input.encoding == TSInputEncodingUTF8 &&
self->lookahead_size == 1 &&
self->data.lookahead != '\n' &&
next_position < current_range_end &&
next_position < self->chunk_start + self->chunk_size
) {
uint8_t next_byte = (uint8_t)self->chunk[next_position - self->chunk_start];
if (next_byte < 0x80) {
ts_lexer__increment_column_data(self);
self->current_position.bytes++;
self->current_position.extent.column++;
if (skip) self->token_start_position = self->current_position;
self->data.lookahead = next_byte;
return;
}
}
ts_lexer__do_advance(self, skip);
}