diff --git a/cmd/generate_changelog/incoming/2203.txt b/cmd/generate_changelog/incoming/2203.txt new file mode 100644 index 00000000..2038b745 --- /dev/null +++ b/cmd/generate_changelog/incoming/2203.txt @@ -0,0 +1,5 @@ +### PR [#2203](https://github.com/danielmiessler/Fabric/pull/2203) by [pacocartones](https://github.com/pacocartones) and [ksylvan](https://github.com/ksylvan): fix(web): preserve multi-byte UTF-8 split across SSE chunks in chat stream + +- Fix: preserve split UTF-8 characters in streaming responses +- Decode API response chunks with persistent streaming state +- Reuse streaming decoder when inspecting chat backend events diff --git a/web/src/lib/api/base.ts b/web/src/lib/api/base.ts index 3c421bd9..c34de666 100644 --- a/web/src/lib/api/base.ts +++ b/web/src/lib/api/base.ts @@ -46,12 +46,14 @@ export const api = { const reader = response.body?.getReader(); if (!reader) throw new Error('Response body is null'); + // Decode in streaming mode: a multi-byte UTF-8 rune split across network + // chunks is otherwise decoded as two halves and corrupted into U+FFFD. const decoder = new TextDecoder(); while (true) { const { done, value } = await reader.read(); - yield decoder.decode(value); - if (done) break; + + yield decoder.decode(value, { stream: true }); } } }; diff --git a/web/src/lib/services/ChatService.ts b/web/src/lib/services/ChatService.ts index c84f27e2..d567e596 100644 --- a/web/src/lib/services/ChatService.ts +++ b/web/src/lib/services/ChatService.ts @@ -145,6 +145,9 @@ export class ChatService { return response; }; + // Persistent decoder: a multi-byte UTF-8 rune split across network chunks is + // otherwise decoded as two halves and corrupted into U+FFFD before it is buffered. + const decoder = new TextDecoder(); return new ReadableStream({ async start(controller) { try { @@ -152,7 +155,7 @@ export class ChatService { const { done, value } = await reader.read(); if (done) break; - buffer += new TextDecoder().decode(value); + buffer += decoder.decode(value, { stream: true }); const segments = buffer.split("\n\n"); // Last segment may be incomplete; keep it as buffer buffer = segments.pop() || ""; diff --git a/web/src/routes/chat/+server.ts b/web/src/routes/chat/+server.ts index fb8ced39..7dd31953 100644 --- a/web/src/routes/chat/+server.ts +++ b/web/src/routes/chat/+server.ts @@ -116,10 +116,13 @@ export const POST: RequestHandler = async ({ request }) => { throw new Error('No response from fabric backend'); } - // Create a TransformStream to inspect the data without modifying it + // Create a TransformStream to inspect the data without modifying it. + // The decoder is persistent and streaming: a multi-byte UTF-8 rune split + // across chunks is otherwise logged as two halves corrupted into U+FFFD. + const decoder = new TextDecoder(); const transformStream = new TransformStream({ transform(chunk, controller) { - const text = new TextDecoder().decode(chunk); + const text = decoder.decode(chunk, { stream: true }); if (text.startsWith('data: ')) { try { const data = JSON.parse(text.slice(6));