Merge pull request #2203 from pacocartones/fix/sse-utf8-stream-boundary

fix(web): preserve multi-byte UTF-8 split across SSE chunks in chat stream

cmd/generate_changelog/incoming/2203.txt

@@ -0,0 +1,5 @@
+### PR [#2203](https://github.com/danielmiessler/Fabric/pull/2203) by [pacocartones](https://github.com/pacocartones) and [ksylvan](https://github.com/ksylvan): fix(web): preserve multi-byte UTF-8 split across SSE chunks in chat stream
+
+- Fix: preserve split UTF-8 characters in streaming responses
+- Decode API response chunks with persistent streaming state
+- Reuse streaming decoder when inspecting chat backend events

web/src/lib/api/base.ts

@@ -46,12 +46,14 @@ export const api = {
     const reader = response.body?.getReader();
     if (!reader) throw new Error('Response body is null');
 
+    // Decode in streaming mode: a multi-byte UTF-8 rune split across network
+    // chunks is otherwise decoded as two halves and corrupted into U+FFFD.
     const decoder = new TextDecoder();
     while (true) {
       const { done, value } = await reader.read();
-      yield decoder.decode(value);
-
       if (done) break;
+
+      yield decoder.decode(value, { stream: true });
     }
   }
 };

web/src/lib/services/ChatService.ts

@@ -145,6 +145,9 @@ export class ChatService {
 
 			return response;
 		};
+		// Persistent decoder: a multi-byte UTF-8 rune split across network chunks is
+		// otherwise decoded as two halves and corrupted into U+FFFD before it is buffered.
+		const decoder = new TextDecoder();
 		return new ReadableStream({
 			async start(controller) {
 				try {
@@ -152,7 +155,7 @@ export class ChatService {
 						const { done, value } = await reader.read();
 						if (done) break;
 
-						buffer += new TextDecoder().decode(value);
+						buffer += decoder.decode(value, { stream: true });
 						const segments = buffer.split("\n\n");
 						// Last segment may be incomplete; keep it as buffer
 						buffer = segments.pop() || "";

web/src/routes/chat/+server.ts

@@ -116,10 +116,13 @@ export const POST: RequestHandler = async ({ request }) => {
       throw new Error('No response from fabric backend');
     }
 
-    // Create a TransformStream to inspect the data without modifying it
+    // Create a TransformStream to inspect the data without modifying it.
+    // The decoder is persistent and streaming: a multi-byte UTF-8 rune split
+    // across chunks is otherwise logged as two halves corrupted into U+FFFD.
+    const decoder = new TextDecoder();
     const transformStream = new TransformStream({
       transform(chunk, controller) {
-        const text = new TextDecoder().decode(chunk);
+        const text = decoder.decode(chunk, { stream: true });
         if (text.startsWith('data: ')) {
           try {
             const data = JSON.parse(text.slice(6));