fix(lib): decode UTF-16 surrogate pairs with input endianness (#5912)

Problem: Parsing UTF-16BE source containing supplementary-plane characters, such as `let emoji = "😀"`, decoded the emoji as two isolated surrogates on little-endian hosts, causing incorrect lexer lookahead and potentially shifted token boundaries.

Soluton: Fix the trailing surrogate byte-order conversion in both UTF-16LE and UTF-16BE decoders and adds a regression test for U+1F600.
(cherry picked from commit 351bd71e52)

crates/cli/src/tests/parser_test.rs

@@ -253,6 +253,21 @@ fn test_parsing_with_custom_utf16_be_input() {
     assert_eq!(root.child(0).unwrap().kind(), "function_item");
 }
 
+#[test]
+fn test_utf16_decodes_surrogate_pairs() {
+    let mut parser = Parser::new();
+    let language = get_test_fixture_language("utf16_surrogate_oob");
+    parser.set_language(&language).unwrap();
+
+    let le = [0xD83D_u16.to_le(), 0xDE00_u16.to_le()];
+    let tree = parser.parse_utf16_le(le, None).unwrap();
+    assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))");
+
+    let be = [0xD83D_u16.to_be(), 0xDE00_u16.to_be()];
+    let tree = parser.parse_utf16_be(be, None).unwrap();
+    assert_eq!(tree.root_node().to_sexp(), "(program (supplementary))");
+}
+
 #[test]
 fn test_utf16_decode_does_not_read_oob() {
     // Test for a buffer over-read in ts_decode_utf16_le/be when a lead surrogate

lib/src/unicode.h

@@ -18,7 +18,7 @@ extern "C" {
     (c)=le16toh((s)[(i)++]); \
     if(U16_IS_LEAD(c)) { \
         uint16_t __c2; \
-        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=le16toh((s)[(i)]))) { \
             ++(i); \
             (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
         } \
@@ -29,7 +29,7 @@ extern "C" {
     (c)=be16toh((s)[(i)++]); \
     if(U16_IS_LEAD(c)) { \
         uint16_t __c2; \
-        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=be16toh((s)[(i)]))) { \
             ++(i); \
             (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
         } \