From 963248ec01c55730a80ee399a92324e962d8ca66 Mon Sep 17 00:00:00 2001 From: Yudai Takada Date: Thu, 3 Sep 2026 15:21:08 +0900 Subject: [PATCH] fix(lib): decode UTF-16 surrogate pairs with input endianness (#5912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: Parsing UTF-16BE source containing supplementary-plane characters, such as `let emoji = "😀"`, decoded the emoji as two isolated surrogates on little-endian hosts, causing incorrect lexer lookahead and potentially shifted token boundaries. Soluton: Fix the trailing surrogate byte-order conversion in both UTF-16LE and UTF-16BE decoders and adds a regression test for U+1F600. (cherry picked from commit 351bd71e528659938243e324ef91cfa1515827ea) --- crates/cli/src/tests/parser_test.rs | 15 +++++++++++++++ lib/src/unicode.h | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/tests/parser_test.rs b/crates/cli/src/tests/parser_test.rs index 1278d4f47..cb9c3cd75 100644 --- a/crates/cli/src/tests/parser_test.rs +++ b/crates/cli/src/tests/parser_test.rs @@ -253,6 +253,21 @@ fn test_parsing_with_custom_utf16_be_input() { assert_eq!(root.child(0).unwrap().kind(), "function_item"); } +#[test] +fn test_utf16_decodes_surrogate_pairs() { + let mut parser = Parser::new(); + let language = get_test_fixture_language("utf16_surrogate_oob"); + parser.set_language(&language).unwrap(); + + let le = [0xD83D_u16.to_le(), 0xDE00_u16.to_le()]; + let tree = parser.parse_utf16_le(le, None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))"); + + let be = [0xD83D_u16.to_be(), 0xDE00_u16.to_be()]; + let tree = parser.parse_utf16_be(be, None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))"); +} + #[test] fn test_utf16_decode_does_not_read_oob() { // Test for a buffer over-read in ts_decode_utf16_le/be when a lead surrogate diff --git a/lib/src/unicode.h b/lib/src/unicode.h index 59c0ab1e6..196c6bf41 100644 --- a/lib/src/unicode.h +++ b/lib/src/unicode.h @@ -18,7 +18,7 @@ extern "C" { (c)=le16toh((s)[(i)++]); \ if(U16_IS_LEAD(c)) { \ uint16_t __c2; \ - if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ + if((i)!=(length) && U16_IS_TRAIL(__c2=le16toh((s)[(i)]))) { \ ++(i); \ (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ } \ @@ -29,7 +29,7 @@ extern "C" { (c)=be16toh((s)[(i)++]); \ if(U16_IS_LEAD(c)) { \ uint16_t __c2; \ - if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ + if((i)!=(length) && U16_IS_TRAIL(__c2=be16toh((s)[(i)]))) { \ ++(i); \ (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ } \