Files
Agent-Workbench/scripts/verify-live-session.mjs
2026-10-29 21:09:50 +08:00

83 lines
6.6 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
const sessionId = process.env.AGENT_WORKBENCH_SESSION || '569c9457-d399-46e6-b53e-4b1fdbb02109';
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
assert.ok(['1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0'].includes(bootstrap.version));
assert.equal(bootstrap.config.mode, 'compatible');
assert.ok(bootstrap.config.hasKey, 'The existing in-memory connection must be present.');
const api = async (route, input) => {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token },
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
const value = await response.json(); assert.ok(response.ok, value.error); return value;
};
const before = await api('/sessions/' + sessionId);
assert.equal(before.status, 'completed', 'Finish the original session before a read-only validation.');
const project = bootstrap.projects.find(item => item.id === before.projectId);
assert.ok(project);
const approved = before.changes.filter(change => change.status === 'applied');
assert.equal(approved.length, 2, 'This validation targets the original two applied changes.');
const hash = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
const hashes = approved.map(change => ({ path: change.path, sha256: hash(path.join(project.root, change.path)) }));
const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
const route = '/sessions/' + sessionId;
const prompt = '刚才工作台里的 fetch failed 到底是怎么回事,跟项目的修改有关吗?src/catalog.js 和 tests/catalog.test.mjs 现在到底是什么状态?'
+ '这次请只读诊断,不修改文件、不提出新改动、不运行终端命令。';
const started = Date.now(); let done = false;
fs.mkdirSync('.qa', { recursive: true });
fs.writeFileSync('.qa/live-session-baseline.json', JSON.stringify({ sessionId, capturedAt: new Date().toISOString(),
messageCount: before.messages.length, messagesSha256: digest(before.messages), eventCount: before.events.length,
changesSha256: digest(before.changes), files: hashes }, null, 2));
try {
await api(route + '/messages', { content: prompt });
let current, lastProgress = '';
while (Date.now() - started < 180000) {
current = await api(route);
const tools = current.events.slice(before.events.length).filter(event => event.kind === 'tool');
const progress = current.status + '|' + current.events.length;
if (progress !== lastProgress) {
console.log(JSON.stringify({ status: current.status, tools: tools.map(event => ({ tool: event.tool, phase: event.phase })), elapsedSeconds: Math.round((Date.now() - started) / 1000) }));
lastProgress = progress;
}
assert.equal(current.approvals.filter(approval => approval.status === 'pending').length, 0, 'Read-only diagnosis must not request a terminal command.');
assert.ok(tools.length <= 4, 'Diagnosis exceeded its bounded read-only scope.');
if (!['running', 'awaiting_approval', 'stopping'].includes(current.status)) break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
assert.ok(current && !['running', 'awaiting_approval', 'stopping'].includes(current.status), 'Actual model validation timed out.');
assert.equal(current.status, 'completed', current.failure?.message || 'Read-only diagnosis did not complete.');
const tools = current.events.slice(before.events.length).filter(event => event.kind === 'tool');
assert.ok(tools.some(event => event.tool === 'inspect_session' && event.phase === 'completed'), 'The model did not inspect the actual session.');
assert.ok(tools.every(event => event.tool === 'inspect_session'
|| (event.tool === 'read_file' && approved.some(change => change.path === event.input?.path))),
'Diagnosis may inspect the runtime and the named applied files; unrelated searches and mutations are outside scope.');
assert.deepEqual(current.changes, before.changes, 'Diagnosis must preserve file proposals and their current states.');
assert.deepEqual(current.messages.slice(0, before.messages.length), before.messages, 'Existing conversation history must be preserved.');
for (const file of hashes) assert.equal(hash(path.join(project.root, file.path)), file.sha256);
const answer = current.messages.slice(before.messages.length).filter(message => message.role === 'assistant').at(-1)?.content || '';
assert.match(answer, /模型/); assert.match(answer, /请求|连接/); assert.match(answer, /已应用|applied/);
assert.ok(!/请.{0,12}(提供|上传).{0,15}(工作台|会话|工具).{0,6}(日志|记录)/.test(answer), 'The model must use workbench records already available.');
const health = await (await fetch(url + '/api/health')).json();
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), url, pid: health.pid,
sessionId, projectRoot: project.root, model: bootstrap.config.model, endpoint: bootstrap.config.baseUrl,
elapsedSeconds: Math.round((Date.now() - started) / 1000), status: current.status,
tools: tools.map(event => ({ tool: event.tool, phase: event.phase })), fileStates: current.changes.map(change => ({ path: change.path, status: change.status })),
answer, preservedHistory: { messageCount: before.messages.length, sha256: digest(current.messages.slice(0, before.messages.length)) }, files: hashes,
evidence: ['Actual configured model inspected the original session', 'Historical fetch failure attributed to the model request',
'Current applied states used; optional reads limited to the two named applied files', 'No project searches, new proposals or terminal commands', 'Original messages and file hashes preserved'],
credentialsHandling: 'The script used the existing in-memory connection through the local service; no API Key was accessed or recorded.' };
fs.writeFileSync('.qa/live-session-validation.json', JSON.stringify(result, null, 2)); done = true;
console.log(JSON.stringify(result, null, 2));
} catch (error) {
fs.writeFileSync('.qa/live-session-validation.json', JSON.stringify({ success: false, version: bootstrap.version,
testedAt: new Date().toISOString(), sessionId, model: bootstrap.config.model, error: error.message,
elapsedSeconds: Math.round((Date.now() - started) / 1000) }, null, 2));
throw error;
} finally {
if (!done) await api(route + '/stop', {}).catch(() => {});
}