import test from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import { createHash } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; import { fixture, until, assertToolPairs, launchWorkbench } from './helpers.mjs'; import { Provider } from '../src/server/provider.mjs'; import { Runtime } from '../src/server/runtime.mjs'; import { readModelStream } from '../src/core/models/model-stream.mjs'; const sse = event => 'data: ' + (typeof event === 'string' ? event : JSON.stringify(event)) + '\r\n\r\n'; const chat = (delta, finish_reason = null) => sse({ choices: [{ index: 0, delta, finish_reason }] }); const chatEnd = (reason = 'stop') => chat({}, reason) + sse({ choices: [], usage: { prompt_tokens: 20, completion_tokens: 10 } }) + sse('[DONE]'); const native = (type, value, sequence_number) => sse({ type, sequence_number, ...value }); test('HTTP download preserves original malformed SSE bytes and artifact SHA instead of decoding invalid UTF8', async t => { const f = fixture(t), bytes = Buffer.from([100, 97, 116, 97, 58, 32, 0xff, 10, 10]); const artifact = f.store.artifactStore.write(f.session, null, bytes, { kind: 'model-stream-response', mediaType: 'text/event-stream' }); f.store.core.artifact.set(artifact.id, artifact); f.store.save(f.session); f.store.close(); const host = await launchWorkbench(t, f.data); const response = await fetch(host.url + '/api/artifacts/' + artifact.id + '?session=' + f.session.id + '&download=1', { headers: { 'X-Workbench-Token': host.bootstrap.token } }); assert.equal(response.status, 200); const downloaded = Buffer.from(await response.arrayBuffer()); assert.deepEqual(downloaded, bytes); assert.equal(createHash('sha256').update(downloaded).digest('hex'), artifact.sha256); }); async function streamingServer(t, handler) { const requests = [], sockets = new Set(); const server = http.createServer(async (request, response) => { const parts = []; for await (const part of request) parts.push(part); const body = JSON.parse(Buffer.concat(parts)); requests.push({ body, url: request.url }); response.writeHead(200, { 'Content-Type': 'text/event-stream' }); try { await handler(response, body, requests); } catch { response.destroy(); } }); server.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); t.after(() => new Promise(resolve => { for (const socket of sockets) socket.destroy(); server.close(resolve); })); return { requests, baseUrl: 'http://127.0.0.1:' + server.address().port + '/v1' }; } function configure(f, server, protocol = 'chat-completions') { const provider = new Provider(), hub = provider.modelHub(f.store); const connection = hub.upsert({ name: 'Owned stream', baseUrl: server.baseUrl, model: 'owned-stream', protocol, requiresKey: false, streaming: true, capabilityDeclarations: { streaming: 'supported' } }); hub.select({ type: 'connection', id: connection.id }); return { provider, hub, connection }; } test('terminated SSE adapts empty optional tool arguments, retains the exact wire bytes and executes the validated batch once', async t => { const f = fixture(t); const original = chat({ tool_calls: [ { index: 0, id: 'empty-inspection', type: 'function', function: { name: 'inspect_project', arguments: '' } }, { index: 1, id: 'valid-file', type: 'function', function: { name: 'read_file', arguments: '{"path":"README.md"}' } }, ] }) + chatEnd('tool_calls'); const server = await streamingServer(t, (res, body) => res.end(body.messages.some(message => message.role === 'tool') ? chat({ content: '读取已完成。' }) + chatEnd() : original)); const { provider, hub } = configure(f, server), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, '读取项目与 README'); await until(() => !runtime.isActive(f.session.id), 15000); assert.equal(f.session.status, 'completed'); assert.equal(server.requests.length, 2); assert.equal(f.store.core.tool.size, 2); assertToolPairs(f.session.providerMessages); const first = hub.view().attempts.find(attempt => attempt.compatibility?.some(value => value.callId === 'empty-inspection')), artifact = f.store.owned('artifact', first.responseArtifactId, f.session.id); assert.equal(f.store.artifactStore.read(artifact), original); assert.deepEqual(first.compatibility, [{ kind: 'empty_object_arguments', callId: 'empty-inspection', tool: 'inspect_project', originalArguments: '', arguments: '{}' }]); const run = f.store.core.run.get(f.session.runId); assert.equal(run.usage.requests, 2); assert.equal(run.usage.totalTokens, 60); assert.equal(run.consumption.length, 2); }); for (const malformed of ['', '{"path":', 'null']) test('reported SSE usage survives rejected required arguments ' + JSON.stringify(malformed) + ' without dispatching any tool', async t => { const f = fixture(t), original = chat({ tool_calls: [ { index: 0, id: 'valid-inspect', type: 'function', function: { name: 'inspect_project', arguments: '{}' } }, { index: 1, id: 'invalid-read', type: 'function', function: { name: 'read_file', arguments: malformed } }, ] }) + chatEnd('tool_calls'); const server = await streamingServer(t, res => res.end(original)); const { provider, hub } = configure(f, server), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, '读取项目'); await until(() => !runtime.isActive(f.session.id), 15000); assert.equal(f.session.status, 'failed'); assert.equal(server.requests.length, 1); assert.equal(f.store.core.tool.size, 0); assert.match(f.session.failure.message, /read_file.*invalid-read/); const run = f.store.core.run.get(f.session.runId), task = f.store.core.task.get(f.session.taskId); assert.equal(run.usage.totalTokens, 30); assert.equal(task.usage.totalTokens, 30); assert.equal(run.usage.requests, 1); assert.equal(run.usage.unknownRequests, 0); assert.equal(hub.view().attempts[0].usageStatus, 'reported'); assert.equal(hub.view().attempts[0].status, 'failed'); assert.equal(f.store.artifactStore.read(f.store.owned('artifact', hub.view().attempts[0].responseArtifactId, f.session.id)), original); assert.equal(f.store.recordConsumption(f.session, run, run.consumption[0], 999), false); assert.equal(run.usage.requests, 1); }); test('a stream without DONE retains reported usage but cannot adapt or execute even an optional tool', async t => { const f = fixture(t), server = await streamingServer(t, res => res.end(chat({ tool_calls: [ { index: 0, id: 'no-done', type: 'function', function: { name: 'inspect_project', arguments: '' } }, ] }) + chat({}, 'tool_calls') + sse({ choices: [], usage: { prompt_tokens: 31, completion_tokens: 7 } }))); const { provider } = configure(f, server), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, '读取项目'); await until(() => !runtime.isActive(f.session.id), 15000); assert.equal(f.store.core.tool.size, 0); assert.equal(f.session.status, 'failed'); const run = f.store.core.run.get(f.session.runId); assert.equal(run.usage.requests, 1); assert.equal(run.usage.totalTokens, 38); }); test('a timeout after reasoning-only fragments creates one unknown receipt and never invents reported tokens', async t => { const f = fixture(t), server = await streamingServer(t, res => { res.write(chat({ reasoning_content: 'pending reasoning' })); }); const { provider } = configure(f, server); provider.retryPolicy = { timeoutMs: 400, totalTimeoutMs: 1000 }; const runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, '读取项目'); await until(() => !runtime.isActive(f.session.id), 15000); const run = f.store.core.run.get(f.session.runId); assert.equal(server.requests.length, 1); assert.equal(run.usage.requests, 1); assert.equal(run.usage.unknownRequests, 1); assert.equal(run.usage.totalTokens, 0); assert.ok(run.usage.estimatedUnknownTokens > 0); assert.equal(run.consumption.length, 1); assert.equal(f.store.core.tool.size, 0); }); test('real Chat SSE carries UTF8 and split tool JSON; Runtime executes only after DONE, stores partial progress and original usage', async t => { const f = fixture(t); let runtime, observedBeforeEnd = false; const server = await streamingServer(t, async (res, body) => { if (body.messages.some(item => item.role === 'tool')) { res.end(chat({ content: '读取完成。' }) + chatEnd()); return; } const bytes = Buffer.from(chat({ content: '先读取项目。' })); for (let at = 0; at < bytes.length; at += 7) res.write(bytes.subarray(at, at + 7)); res.write(chat({ tool_calls: [{ index: 0, id: 'stream_read', type: 'function', function: { name: 'list_files', arguments: '{"path":' } }] })); await delay(150); observedBeforeEnd = f.session.events.some(item => item.stream?.textPreview === '先读取项目。') && !f.session.events.some(item => item.kind === 'tool'); res.end(chat({ tool_calls: [{ index: 0, function: { arguments: '"","depth":1}' } }] }) + chatEnd('tool_calls')); }); const { provider, hub } = configure(f, server); runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, 'Inspect original project'); await until(() => !runtime.isActive(f.session.id), 15000); assert.equal(f.session.status, 'completed', JSON.stringify(f.session.events.slice(-2))); assert.equal(observedBeforeEnd, true); assert.equal(server.requests.length, 2); assert.equal(f.session.events.filter(item => item.tool === 'list_files' && item.phase === 'completed').length, 1); assertToolPairs(f.session.providerMessages); assert.equal(f.store.core.run.get(f.session.runId).usage.inputTokens, 40); assert.ok(hub.view().attempts.every(item => item.stream.terminal && item.stream.sha256.length === 64)); for (const attempt of hub.view().attempts) { const artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id); assert.equal(artifact.sha256, attempt.stream.sha256); assert.equal(artifact.bytes, attempt.stream.bytes); assert.equal(artifact.kind, 'model-stream-response'); assert.equal(attempt.responseRedacted, false); } }); test('Responses typed SSE validates original item identity, terminal data, raw usage and tool result pair', async t => { const f = fixture(t), call = { type: 'function_call', id: 'fc_stream', call_id: 'call_stream', name: 'list_files', arguments: '', status: 'in_progress' }; const server = await streamingServer(t, async (res, body) => { const done = body.input.some(item => item.type === 'function_call_output'); let seq = 0; res.write(native('response.created', { response: { id: 'resp_stream' } }, seq++)); let output; if (done) { res.write(native('response.output_text.delta', { delta: '完成' }, seq++)); output = [{ type: 'message', id: 'msg_stream', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: '完成' }] }]; } else { res.write(native('response.output_item.added', { output_index: 0, item: call }, seq++)); res.write(native('response.function_call_arguments.delta', { item_id: call.id, delta: '{"path":"",' }, seq++)); res.write(native('response.function_call_arguments.delta', { item_id: call.id, delta: '"depth":1}' }, seq++)); output = [{ ...call, status: 'completed', arguments: '{"path":"","depth":1}' }]; res.write(native('response.function_call_arguments.done', { item_id: call.id, arguments: output[0].arguments }, seq++)); } res.end(native('response.completed', { response: { id: 'resp_stream', status: 'completed', output, usage: { input_tokens: 33, output_tokens: 9, total_tokens: 42 } } }, seq++)); }); const { provider, hub } = configure(f, server, 'responses'), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, 'Inspect'); await until(() => !runtime.isActive(f.session.id), 15000); assert.equal(f.session.status, 'completed', JSON.stringify(f.session.events.slice(-2))); assertToolPairs(f.session.providerMessages); assert.equal(server.requests.length, 2); assert.equal(server.requests[1].body.input.find(item => item.type === 'function_call_output').call_id, 'call_stream'); assert.equal(hub.view().attempts[0].usage.raw.total_tokens, 42); }); test('incomplete network stream retains unknown attempt and never automatically switches pool or dispatches fragment tools', async t => { const f = fixture(t), server = await streamingServer(t, async res => res.end(chat({ content: 'partial' }) + chat({ tool_calls: [{ index: 0, id: 'partial_tool', function: { name: 'list_files', arguments: '{"path":' } }] }))); const { provider, hub, connection } = configure(f, server), other = hub.upsert({ ...connection, id: undefined, name: 'Other', revision: undefined }); const pool = hub.pool({ name: 'No replay', members: [{ connectionId: connection.id }, { connectionId: other.id }] }); hub.select({ type: 'pool', id: pool.id }); const runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, 'Inspect'); await until(() => !runtime.isActive(f.session.id)); assert.equal(server.requests.length, 1); assert.equal(f.session.events.filter(item => item.kind === 'tool').length, 0); const attempt = hub.view().attempts[0]; assert.equal(attempt.status, 'unknown'); assert.equal(attempt.partial.textPreview, 'partial'); assert.equal(attempt.usageStatus, 'unknown'); const artifact = f.store.owned('artifact', attempt.responseArtifactId, f.session.id); assert.equal(artifact.sha256, attempt.stream.sha256); assert.equal(artifact.bytes, attempt.stream.capturedBytes); assert.equal(attempt.stream.terminal, false); assert.equal(f.session.events.find(item => item.stream)?.stream.status, 'incomplete'); }); test('stopping an active HTTP stream closes only original request and keeps partial text without executing tools', async t => { const f = fixture(t); let closed = false; const server = await streamingServer(t, async res => { res.on('close', () => { closed = true; }); res.write(chat({ content: '尚未完成' })); }); const { provider, hub } = configure(f, server), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, 'Inspect'); await until(() => f.session.events.some(item => item.stream?.textCharacters > 0)); runtime.stop(f.session.id); await until(() => !runtime.isActive(f.session.id) && closed); assert.equal(server.requests.length, 1); assert.equal(f.session.events.filter(item => item.kind === 'tool').length, 0); assert.equal(hub.view().attempts[0].status, 'interrupted'); assert.equal(f.session.events.find(item => item.stream)?.stream.status, 'interrupted'); const attempt = hub.view().attempts[0]; assert.equal(f.store.owned('artifact', attempt.responseArtifactId, f.session.id).sha256, attempt.stream.sha256); assert.equal([...hub.leases.values()].reduce((a, b) => a + b, 0), 0); }); test('explicit streaming probe uses actual SSE despite unknown declaration and binds its own original attempt', async t => { const f = fixture(t), server = await streamingServer(t, async res => res.end(chat({ content: 'OK' }) + chatEnd())); const provider = new Provider(), hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned probe', model: 'm', baseUrl: server.baseUrl, requiresKey: false }); await hub.probe(connection.id, 'streaming'); const view = hub.view().connections.find(item => item.id === connection.id); assert.equal(view.capabilities.streaming, 'verified'); assert.equal(view.streaming, false); assert.equal(server.requests[0].body.stream, true); assert.equal(view.capabilityEvidence.attemptId, hub.view().attempts[0].id); assert.equal(f.session.events.length, 0); }); test('coalesced initial SSE events publish the latest partial state even when the upstream stays idle', async t => { const f = fixture(t), server = await streamingServer(t, async res => { res.write(native('response.created', { response: { id: 'resp_idle' } }, 0) + native('response.output_text.delta', { delta: '等待中的实际片段' }, 1)); }); const { provider } = configure(f, server, 'responses'), runtime = new Runtime(f.store, provider); t.after(() => runtime.stopAll()); runtime.start(f.session.id, 'Inspect'); await until(() => f.session.events.some(item => item.stream?.textPreview === '等待中的实际片段')); assert.equal(runtime.isActive(f.session.id), true); runtime.stop(f.session.id); await until(() => !runtime.isActive(f.session.id)); }); test('stream timeout closes the original HTTP response, preserves redacted capture and never retries', async t => { const f = fixture(t); let closed = false; const server = await streamingServer(t, async res => { res.on('close', () => { closed = true; }); res.write(chat({ content: 'owned-secret partial' })); }); const { provider, hub } = configure(f, server); provider.setKey('owned-secret'); await assert.rejects(provider.complete(provider.prepareConfig(f.store.config, f.store), [{ role: 'user', content: 'Inspect' }], f.project, undefined, undefined, null, [], { timeoutMs: 100, totalTimeoutMs: 300, attempts: 3 }), error => error.diagnostic.kind === 'timeout' && !error.diagnostic.retryable); await until(() => closed); assert.equal(server.requests.length, 1); assert.equal(hub.view().attempts[0].status, 'unknown'); assert.equal(hub.view().attempts[0].partial.textPreview.includes('owned-secret'), false); }); test('stream idle timeout preserves incomplete tool arguments, records unknown use and never fails over', async t => { const f = fixture(t); let closed = false; const original = chat({ tool_calls: [{ index: 0, id: 'idle-original', type: 'function', function: { name: 'propose_file', arguments: '{"path":"incomplete.html",' } }] }); const server = await streamingServer(t, res => { res.on('close', () => { closed = true; }); res.write(original); }); const { provider, hub, connection } = configure(f, server), other = hub.upsert({ ...connection, id: undefined, name: 'No failover', revision: undefined }); const pool = hub.pool({ name: 'Idle capture', members: [{ connectionId: connection.id }, { connectionId: other.id }] }); hub.select({ type: 'pool', id: pool.id }); const progress = []; await assert.rejects(provider.complete(provider.prepareConfig(f.store.config, f.store), [{ role: 'user', content: 'owned timeout' }], f.project, undefined, event => progress.push(event), null, [], { timeoutMs: 5000, totalTimeoutMs: 5000, idleTimeoutMs: 150 }), error => { assert.equal(error.diagnostic.timeout.kind, 'idle'); assert.equal(error.diagnostic.timeout.receivedBytes, Buffer.byteLength(original)); assert.equal(error.diagnostic.retryable, false); return true; }); await until(() => closed); assert.equal(server.requests.length, 1); assert.equal(f.session.changes.length, 0); const attempt = hub.view().attempts[0]; assert.equal(attempt.status, 'unknown'); assert.equal(attempt.usageStatus, 'unknown'); assert.equal(attempt.stream.terminal, false); assert.equal(attempt.stream.bytes, Buffer.byteLength(original)); const capture = progress.find(event => event.consumption)?.consumption; assert.equal(capture.usageStatus, 'unknown'); }); test('SSE accepts CR-only final event boundaries and retains exact invalid UTF8 bytes without dispatch', async () => { const valid = (chat({ content: 'OK' }) + chatEnd()).replaceAll('\r\n', '\r'); const result = await readModelStream(new Response(valid, { headers: { 'Content-Type': 'text/event-stream' } }), {}, undefined); assert.equal(result.value.choices[0].message.content, 'OK'); const invalid = Buffer.from([100, 97, 116, 97, 58, 32, 0xff, 10, 10]); let capture; await assert.rejects(readModelStream(new Response(invalid, { headers: { 'Content-Type': 'text/event-stream' } }), {}, undefined, event => { if (event.streamCapture) capture = event; })); assert.deepEqual(capture.streamCapture, invalid); assert.equal(capture.stream.sha256, createHash('sha256').update(invalid).digest('hex')); assert.equal(capture.stream.terminal, false); }); test('stream parser rejects truncation, finish length, malformed JSON, duplicate sequence, changed final item and oversized data', async t => { const cases = [ ['chat-completions', chat({ content: 'partial' })], ['chat-completions', chat({ content: 'partial' }) + chatEnd('length')], ['chat-completions', 'data: {broken}\n\n'], ['chat-completions', 'data: ' + 'x'.repeat(4 * 1024 * 1024) + '\n\n'], ['responses', native('response.created', { response: { id: 'r' } }, 0) + native('response.output_text.delta', { delta: 'x' }, 0)], ['responses', native('response.output_text.delta', { delta: 'x' }, 0) + native('response.completed', { response: { status: 'completed', output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'changed' }] }] } }, 1)], ]; for (const [index, [protocol, body]] of cases.entries()) await t.test('reject ' + index, async () => { await assert.rejects(readModelStream(new Response(body, { headers: { 'content-type': 'text/event-stream' } }), { protocol }, undefined)); }); });