fix: preserve split UTF-8 characters in streaming responses

- Decode API response chunks with persistent streaming state
- Reuse streaming decoder when inspecting chat backend events
This commit is contained in:
Kayvan Sylvan 2026-08-29 09:26:35 -07:00
parent b185a5c790
commit 2c510971dd
2 changed files with 9 additions and 4 deletions

View file

@ -46,12 +46,14 @@ export const api = {
const reader = response.body?.getReader();
if (!reader) throw new Error('Response body is null');
// Decode in streaming mode: a multi-byte UTF-8 rune split across network
// chunks is otherwise decoded as two halves and corrupted into U+FFFD.
const decoder = new TextDecoder();
while (true) {
const { done, value } = await reader.read();
yield decoder.decode(value);
if (done) break;
yield decoder.decode(value, { stream: true });
}
}
};

View file

@ -116,10 +116,13 @@ export const POST: RequestHandler = async ({ request }) => {
throw new Error('No response from fabric backend');
}
// Create a TransformStream to inspect the data without modifying it
// Create a TransformStream to inspect the data without modifying it.
// The decoder is persistent and streaming: a multi-byte UTF-8 rune split
// across chunks is otherwise logged as two halves corrupted into U+FFFD.
const decoder = new TextDecoder();
const transformStream = new TransformStream({
transform(chunk, controller) {
const text = new TextDecoder().decode(chunk);
const text = decoder.decode(chunk, { stream: true });
if (text.startsWith('data: ')) {
try {
const data = JSON.parse(text.slice(6));