From 8e1dbb4617cffbb79e05c446c4bda7eb7922d6cc Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 8 Sep 2024 14:26:34 -0400 Subject: [PATCH] fix: properly handle utf8 code points for highlight and tag assertions (cherry picked from commit 6f050f0da51db6a97a8e59210a35d8d9feb541e6) --- Cargo.lock | Bin 56564 -> 57059 bytes Cargo.toml | 1 + cli/Cargo.toml | 1 + cli/src/query.rs | 10 +++--- cli/src/query_testing.rs | 51 ++++++++++++++++++++++++--- cli/src/test_highlight.rs | 11 +++--- cli/src/test_tags.rs | 11 +++--- cli/src/tests/test_highlight_test.rs | 26 +++++++++----- cli/src/tests/test_tags_test.rs | 16 ++++----- 9 files changed, 92 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d6182c3d3f9536f0843e0feaef45bc1fb5970e6..b1d353409e918a615736b69ef9d9002e98390a75 100644 GIT binary patch delta 887 zcmXAoy^G#O5XZT@oG(xAG(o)!B6tZYtd5~ zu}cvaG$4{-XQ%v%#6l3#*_%MR=s%#~AK={C?|%0)pZV^;zpVZB>)KDt*{eHl7jV~2 z%h}tjZ{7I(`nlPye*MOkYs;MrJLg~*#_1nFod=gaR?>=;xi%yX8W?kP#br$1_W&Z` z(PxL~3Us9sU5ceAX|9daYwwIx-<%rH4&N9LzkGkXy>YNQo-PjdLaa@?gCr?555l2# zpqgS#2t9I5g5Yv;xo6;vTr^^@ftb5T9cS0KrZ3+7;mq+I-f;gu|L*wy@#b{tgQxF} z^ZVN?IDVenJAl1s(H4F0z@dcFdaqe(BSus&sW~DjL5dddwGsj;E2smY0jpcnpZmXU zuY~dP&f+2k1&-u_TWBdlN{&n53=zd=ZbgVA9;tDtkt$knlrFB2%WfRP*7VV5mo~<| z2M2rRQOPxnOD+@})o{3%2$6|>p`r~imn1;fhDpJ<$2*&g%R=EOH2Z&=*o zVo^NWza~(jo@-ByHAfA;s5GY9D|X2#l7VTB4lBe|OiwqtG*$bF5ZU z+iAuYLg9o-Rg$8wLeh_lV^9UKuqu&PI~1^W6KoZYO^*hiHt&6S;zYG?FkQZX_{w;B zcd^$C)R>xt#=_Q^QcWkv6r!dYlv)?Cj1~L2M+276!yN#4SXN z3#QUe{0gziOZWyCyI|=9xc-5uhMC`-bLPj%!=O@$g+4S=Ac(67sFPqqoc6YZu zdGMvEmE^bvP8vaL6%@i!3UYKn7`%EDz)(>zkTv`0Vt|(3Y>fMtj&3@Q-%mNS+sm%d$+12Ek>Nv1^N3#Di|QCpu}X|w5t@zc|vXPVJr=AmS# z8OTZB4OU8olVdWS+Cg4J@s0aq>wrg-?N^sfwF8)7pZG5o*b#;l`;^VOGgF-OL zkoLrD#_mmky7?ZaGg#3JnFOJ@ggK`utle)`;apsMd26%HpI;qMULBop-!Be#T2KdhvPBcs)Ft+)swzoazqi>A-<=yDy??#)8x4BR A6951J diff --git a/Cargo.toml b/Cargo.toml index 1c95c66a6..7594e80b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ strip = false [workspace.dependencies] anstyle = "1.0.8" anyhow = "1.0.89" +bstr = "1.10.0" cc = "1.1.19" clap = { version = "4.5.17", features = [ "cargo", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 493503ce7..34fa31db9 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,6 +27,7 @@ wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"] [dependencies] anstyle.workspace = true anyhow.workspace = true +bstr.workspace = true clap.workspace = true ctor.workspace = true ctrlc.workspace = true diff --git a/cli/src/query.rs b/cli/src/query.rs index bffa05882..f32c5450a 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -9,7 +9,7 @@ use std::{ use anyhow::{Context, Result}; use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; -use crate::query_testing; +use crate::query_testing::{self, to_utf8_point}; #[allow(clippy::too_many_arguments)] pub fn query_files_at_paths( @@ -70,8 +70,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(), - start: capture.node.start_position(), - end: capture.node.end_position(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), }); } } else { @@ -100,8 +100,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(), - start: capture.node.start_position(), - end: capture.node.end_position(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), }); } } diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 3456dbe6e..258770d16 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -1,6 +1,7 @@ use std::fs; use anyhow::{anyhow, Result}; +use bstr::{BStr, ByteSlice}; use lazy_static::lazy_static; use regex::Regex; use tree_sitter::{Language, Parser, Point}; @@ -9,16 +10,56 @@ lazy_static! { static ref CAPTURE_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Utf8Point { + pub row: usize, + pub column: usize, +} + +impl std::fmt::Display for Utf8Point { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "({}, {})", self.row, self.column) + } +} + +impl Utf8Point { + pub const fn new(row: usize, column: usize) -> Self { + Self { row, column } + } +} + +pub fn to_utf8_point(point: Point, source: &[u8]) -> Utf8Point { + if point.column == 0 { + return Utf8Point::new(point.row, 0); + } + + let bstr = BStr::new(source); + let line = bstr.lines_with_terminator().nth(point.row).unwrap(); + let mut utf8_column = 0; + + for (_, grapheme_end, _) in line.grapheme_indices() { + utf8_column += 1; + if grapheme_end >= point.column { + break; + } + } + + Utf8Point { + row: point.row, + column: utf8_column, + } +} + #[derive(Debug, Eq, PartialEq)] pub struct CaptureInfo { pub name: String, - pub start: Point, - pub end: Point, + pub start: Utf8Point, + pub end: Utf8Point, } #[derive(Debug, PartialEq, Eq)] pub struct Assertion { - pub position: Point, + pub position: Utf8Point, pub negative: bool, pub expected_capture_name: String, } @@ -27,7 +68,7 @@ impl Assertion { #[must_use] pub fn new(row: usize, col: usize, negative: bool, expected_capture_name: String) -> Self { Self { - position: Point::new(row, col), + position: Utf8Point::new(row, col), negative, expected_capture_name, } @@ -103,7 +144,7 @@ pub fn parse_position_comments( { assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { - position, + position: to_utf8_point(position, source), negative, expected_capture_name: mat.as_str().to_string(), }); diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 541d98fd3..34be438fb 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -7,7 +7,7 @@ use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, H use tree_sitter_loader::{Config, Loader}; use super::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, test::paint, util, }; @@ -141,7 +141,7 @@ fn test_highlights_indented( } pub fn iterate_assertions( assertions: &[Assertion], - highlights: &[(Point, Point, Highlight)], + highlights: &[(Utf8Point, Utf8Point, Highlight)], highlight_names: &[String], ) -> Result { // Iterate through all of the highlighting assertions, checking each one against the @@ -224,7 +224,7 @@ pub fn get_highlight_positions( highlighter: &mut Highlighter, highlight_config: &HighlightConfiguration, source: &[u8], -) -> Result> { +) -> Result> { let mut row = 0; let mut column = 0; let mut byte_offset = 0; @@ -261,7 +261,10 @@ pub fn get_highlight_positions( } } if let Some(highlight) = highlight_stack.last() { - result.push((start_position, Point::new(row, column), *highlight)); + let utf8_start_position = to_utf8_point(start_position, source.as_bytes()); + let utf8_end_position = + to_utf8_point(Point::new(row, column), source.as_bytes()); + result.push((utf8_start_position, utf8_end_position, *highlight)); } } } diff --git a/cli/src/test_tags.rs b/cli/src/test_tags.rs index c5a1dc021..5b290bda9 100644 --- a/cli/src/test_tags.rs +++ b/cli/src/test_tags.rs @@ -2,12 +2,11 @@ use std::{fs, path::Path}; use anstyle::AnsiColor; use anyhow::{anyhow, Result}; -use tree_sitter::Point; use tree_sitter_loader::{Config, Loader}; use tree_sitter_tags::{TagsConfiguration, TagsContext}; use super::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, test::paint, util, }; @@ -168,7 +167,7 @@ pub fn get_tag_positions( tags_context: &mut TagsContext, tags_config: &TagsConfiguration, source: &[u8], -) -> Result> { +) -> Result> { let (tags_iter, _has_error) = tags_context.generate_tags(tags_config, source, None)?; let tag_positions = tags_iter .filter_map(std::result::Result::ok) @@ -179,7 +178,11 @@ pub fn get_tag_positions( } else { format!("reference.{tag_postfix}") }; - (tag.span.start, tag.span.end, tag_name) + ( + to_utf8_point(tag.span.start, source), + to_utf8_point(tag.span.end, source), + tag_name, + ) }) .collect(); Ok(tag_positions) diff --git a/cli/src/tests/test_highlight_test.rs b/cli/src/tests/test_highlight_test.rs index 8699c2a6f..054e33f82 100644 --- a/cli/src/tests/test_highlight_test.rs +++ b/cli/src/tests/test_highlight_test.rs @@ -1,9 +1,9 @@ -use tree_sitter::{Parser, Point}; +use tree_sitter::Parser; use tree_sitter_highlight::{Highlight, Highlighter}; use super::helpers::fixtures::{get_highlight_config, get_language, test_loader}; use crate::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, Assertion, Utf8Point}, test_highlight::get_highlight_positions, }; @@ -28,6 +28,9 @@ fn test_highlight_test_with_basic_test() { " // ^ variable", " // ^ !variable", "};", + "var y̆y̆y̆y̆ = function() {}", + " // ^ function", + " // ^ keyword", ] .join("\n"); @@ -40,6 +43,8 @@ fn test_highlight_test_with_basic_test() { Assertion::new(1, 11, false, String::from("keyword")), Assertion::new(4, 9, false, String::from("variable")), Assertion::new(4, 11, true, String::from("variable")), + Assertion::new(8, 5, false, String::from("function")), + Assertion::new(8, 11, false, String::from("keyword")), ] ); @@ -50,13 +55,16 @@ fn test_highlight_test_with_basic_test() { assert_eq!( highlight_positions, &[ - (Point::new(1, 0), Point::new(1, 3), Highlight(2)), // "var" - (Point::new(1, 4), Point::new(1, 7), Highlight(0)), // "abc" - (Point::new(1, 10), Point::new(1, 18), Highlight(2)), // "function" - (Point::new(1, 19), Point::new(1, 20), Highlight(1)), // "d" - (Point::new(4, 2), Point::new(4, 8), Highlight(2)), // "return" - (Point::new(4, 9), Point::new(4, 10), Highlight(1)), // "d" - (Point::new(4, 13), Point::new(4, 14), Highlight(1)), // "e" + (Utf8Point::new(1, 0), Utf8Point::new(1, 3), Highlight(2)), // "var" + (Utf8Point::new(1, 4), Utf8Point::new(1, 7), Highlight(0)), // "abc" + (Utf8Point::new(1, 10), Utf8Point::new(1, 18), Highlight(2)), // "function" + (Utf8Point::new(1, 19), Utf8Point::new(1, 20), Highlight(1)), // "d" + (Utf8Point::new(4, 2), Utf8Point::new(4, 8), Highlight(2)), // "return" + (Utf8Point::new(4, 9), Utf8Point::new(4, 10), Highlight(1)), // "d" + (Utf8Point::new(4, 13), Utf8Point::new(4, 14), Highlight(1)), // "e" + (Utf8Point::new(8, 0), Utf8Point::new(8, 3), Highlight(2)), // "var" + (Utf8Point::new(8, 4), Utf8Point::new(8, 8), Highlight(0)), // "y̆y̆y̆y̆" + (Utf8Point::new(8, 11), Utf8Point::new(8, 19), Highlight(2)), // "function" ] ); } diff --git a/cli/src/tests/test_tags_test.rs b/cli/src/tests/test_tags_test.rs index 5e7bf9c93..5f7b88fc2 100644 --- a/cli/src/tests/test_tags_test.rs +++ b/cli/src/tests/test_tags_test.rs @@ -1,9 +1,9 @@ -use tree_sitter::{Parser, Point}; +use tree_sitter::Parser; use tree_sitter_tags::TagsContext; use super::helpers::fixtures::{get_language, get_tags_config}; use crate::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, Assertion, Utf8Point}, test_tags::get_tag_positions, }; @@ -43,18 +43,18 @@ fn test_tags_test_with_basic_test() { tag_positions, &[ ( - Point::new(1, 4), - Point::new(1, 7), + Utf8Point::new(1, 4), + Utf8Point::new(1, 7), "definition.function".to_string() ), ( - Point::new(3, 8), - Point::new(3, 11), + Utf8Point::new(3, 8), + Utf8Point::new(3, 11), "reference.call".to_string() ), ( - Point::new(5, 11), - Point::new(5, 12), + Utf8Point::new(5, 11), + Utf8Point::new(5, 12), "reference.call".to_string() ), ]