fix: preserve split UTF-8 characters in streaming responses

- Decode API response chunks with persistent streaming state
- Reuse streaming decoder when inspecting chat backend events
This commit is contained in:
Kayvan Sylvan 2026-08-29 09:26:35 -07:00
parent b185a5c790
commit 2c510971dd
2 changed files with 9 additions and 4 deletions

View file

@ -46,12 +46,14 @@ export const api = {
const reader = response.body?.getReader(); const reader = response.body?.getReader();
if (!reader) throw new Error('Response body is null'); if (!reader) throw new Error('Response body is null');
// Decode in streaming mode: a multi-byte UTF-8 rune split across network
// chunks is otherwise decoded as two halves and corrupted into U+FFFD.
const decoder = new TextDecoder(); const decoder = new TextDecoder();
while (true) { while (true) {
const { done, value } = await reader.read(); const { done, value } = await reader.read();
yield decoder.decode(value);
if (done) break; if (done) break;
yield decoder.decode(value, { stream: true });
} }
} }
}; };

View file

@ -116,10 +116,13 @@ export const POST: RequestHandler = async ({ request }) => {
throw new Error('No response from fabric backend'); throw new Error('No response from fabric backend');
} }
// Create a TransformStream to inspect the data without modifying it // Create a TransformStream to inspect the data without modifying it.
// The decoder is persistent and streaming: a multi-byte UTF-8 rune split
// across chunks is otherwise logged as two halves corrupted into U+FFFD.
const decoder = new TextDecoder();
const transformStream = new TransformStream({ const transformStream = new TransformStream({
transform(chunk, controller) { transform(chunk, controller) {
const text = new TextDecoder().decode(chunk); const text = decoder.decode(chunk, { stream: true });
if (text.startsWith('data: ')) { if (text.startsWith('data: ')) {
try { try {
const data = JSON.parse(text.slice(6)); const data = JSON.parse(text.slice(6));