83 lines
6.5 KiB
JavaScript
83 lines
6.5 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
|
|
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
|
|
const sessionId = process.env.AGENT_WORKBENCH_SESSION || '569c9457-d399-46e6-b53e-4b1fdbb02109';
|
|
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
|
assert.equal(bootstrap.version, '1.0.0');
|
|
assert.equal(bootstrap.config.mode, 'compatible');
|
|
assert.ok(bootstrap.config.hasKey, 'The existing in-memory connection must be present.');
|
|
const api = async (route, input) => {
|
|
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
|
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token },
|
|
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
|
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
|
};
|
|
const before = await api('/sessions/' + sessionId);
|
|
assert.equal(before.status, 'completed', 'Finish the original session before a read-only validation.');
|
|
const project = bootstrap.projects.find(item => item.id === before.projectId);
|
|
assert.ok(project);
|
|
const approved = before.changes.filter(change => change.status === 'applied');
|
|
assert.equal(approved.length, 2, 'This validation targets the original two applied changes.');
|
|
const hash = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
const hashes = approved.map(change => ({ path: change.path, sha256: hash(path.join(project.root, change.path)) }));
|
|
const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
const route = '/sessions/' + sessionId;
|
|
const prompt = '刚才工作台里的 fetch failed 到底是怎么回事,跟项目的修改有关吗?src/catalog.js 和 tests/catalog.test.mjs 现在到底是什么状态?'
|
|
+ '这次请只读诊断,不修改文件、不提出新改动、不运行终端命令。';
|
|
const started = Date.now(); let done = false;
|
|
fs.mkdirSync('.qa', { recursive: true });
|
|
fs.writeFileSync('.qa/live-session-baseline.json', JSON.stringify({ sessionId, capturedAt: new Date().toISOString(),
|
|
messageCount: before.messages.length, messagesSha256: digest(before.messages), eventCount: before.events.length,
|
|
changesSha256: digest(before.changes), files: hashes }, null, 2));
|
|
try {
|
|
await api(route + '/messages', { content: prompt });
|
|
let current, lastProgress = '';
|
|
while (Date.now() - started < 180000) {
|
|
current = await api(route);
|
|
const tools = current.events.slice(before.events.length).filter(event => event.kind === 'tool');
|
|
const progress = current.status + '|' + current.events.length;
|
|
if (progress !== lastProgress) {
|
|
console.log(JSON.stringify({ status: current.status, tools: tools.map(event => ({ tool: event.tool, phase: event.phase })), elapsedSeconds: Math.round((Date.now() - started) / 1000) }));
|
|
lastProgress = progress;
|
|
}
|
|
assert.equal(current.approvals.filter(approval => approval.status === 'pending').length, 0, 'Read-only diagnosis must not request a terminal command.');
|
|
assert.ok(tools.length <= 4, 'Diagnosis exceeded its bounded read-only scope.');
|
|
if (!['running', 'awaiting_approval', 'stopping'].includes(current.status)) break;
|
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
}
|
|
assert.ok(current && !['running', 'awaiting_approval', 'stopping'].includes(current.status), 'Actual model validation timed out.');
|
|
assert.equal(current.status, 'completed', current.failure?.message || 'Read-only diagnosis did not complete.');
|
|
const tools = current.events.slice(before.events.length).filter(event => event.kind === 'tool');
|
|
assert.ok(tools.some(event => event.tool === 'inspect_session' && event.phase === 'completed'), 'The model did not inspect the actual session.');
|
|
assert.ok(tools.every(event => event.tool === 'inspect_session'
|
|
|| (event.tool === 'read_file' && approved.some(change => change.path === event.input?.path))),
|
|
'Diagnosis may inspect the runtime and the named applied files; unrelated searches and mutations are outside scope.');
|
|
assert.deepEqual(current.changes, before.changes, 'Diagnosis must preserve file proposals and their current states.');
|
|
assert.deepEqual(current.messages.slice(0, before.messages.length), before.messages, 'Existing conversation history must be preserved.');
|
|
for (const file of hashes) assert.equal(hash(path.join(project.root, file.path)), file.sha256);
|
|
const answer = current.messages.slice(before.messages.length).filter(message => message.role === 'assistant').at(-1)?.content || '';
|
|
assert.match(answer, /模型/); assert.match(answer, /请求|连接/); assert.match(answer, /已应用|applied/);
|
|
assert.ok(!/请.{0,12}(提供|上传).{0,15}(工作台|会话|工具).{0,6}(日志|记录)/.test(answer), 'The model must use workbench records already available.');
|
|
const health = await (await fetch(url + '/api/health')).json();
|
|
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), url, pid: health.pid,
|
|
sessionId, projectRoot: project.root, model: bootstrap.config.model, endpoint: bootstrap.config.baseUrl,
|
|
elapsedSeconds: Math.round((Date.now() - started) / 1000), status: current.status,
|
|
tools: tools.map(event => ({ tool: event.tool, phase: event.phase })), fileStates: current.changes.map(change => ({ path: change.path, status: change.status })),
|
|
answer, preservedHistory: { messageCount: before.messages.length, sha256: digest(current.messages.slice(0, before.messages.length)) }, files: hashes,
|
|
evidence: ['Actual configured model inspected the original session', 'Historical fetch failure attributed to the model request',
|
|
'Current applied states used; optional reads limited to the two named applied files', 'No project searches, new proposals or terminal commands', 'Original messages and file hashes preserved'],
|
|
credentialsHandling: 'The script used the existing in-memory connection through the local service; no API Key was accessed or recorded.' };
|
|
fs.writeFileSync('.qa/live-session-validation.json', JSON.stringify(result, null, 2)); done = true;
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} catch (error) {
|
|
fs.writeFileSync('.qa/live-session-validation.json', JSON.stringify({ success: false, version: bootstrap.version,
|
|
testedAt: new Date().toISOString(), sessionId, model: bootstrap.config.model, error: error.message,
|
|
elapsedSeconds: Math.round((Date.now() - started) / 1000) }, null, 2));
|
|
throw error;
|
|
} finally {
|
|
if (!done) await api(route + '/stop', {}).catch(() => {});
|
|
}
|