77 lines
5.5 KiB
JavaScript
77 lines
5.5 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { randomUUID } from 'node:crypto';
|
|
|
|
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
|
|
const bootstrapResponse = await fetch(url + '/api/bootstrap');
|
|
assert.ok(bootstrapResponse.ok, '工作台未连接');
|
|
const bootstrap = await bootstrapResponse.json();
|
|
assert.equal(bootstrap.config.mode, 'compatible', '请先保存兼容模型配置');
|
|
assert.ok(bootstrap.config.model, '没有选择模型');
|
|
const root = path.resolve('.qa', 'live-model-' + randomUUID(), '模型连接验证');
|
|
const proof = randomUUID();
|
|
fs.mkdirSync(root, { recursive: true });
|
|
fs.writeFileSync(path.join(root, 'facts.txt'), 'PROOF_TOKEN=' + proof + '\nINPUT_NUMBER=21\n', 'utf8');
|
|
fs.writeFileSync(path.join(root, 'README.md'), '# 模型连接验证\n\n此目录只有自动化验证的合成数据。\n', 'utf8');
|
|
const api = async (route, input) => {
|
|
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
|
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token },
|
|
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
|
const body = await response.json();
|
|
if (!response.ok) throw new Error(body.error || ('请求失败 ' + response.status));
|
|
return body;
|
|
};
|
|
const project = await api('/projects', { root });
|
|
const created = await api('/sessions', { projectId: project.id });
|
|
let completed = false;
|
|
const started = Date.now();
|
|
try {
|
|
await api('/sessions/' + created.id + '/messages', { content:
|
|
'这是工作台的真实模型连接验证。请先调用 read_file 读取 facts.txt,取得 PROOF_TOKEN 和 INPUT_NUMBER;不能从猜测得出内容。' +
|
|
'随后调用 propose_file,提出新文件 MODEL-RESULT.md,内容包含读取到的完整 PROOF_TOKEN,以及 INPUT_NUMBER 乘以 2 的值,格式写成 PROOF_TOKEN=... 和 DOUBLED_NUMBER=...。' +
|
|
'只产生提案,不能声称文件已经写入。最后用一句中文说明结果。不要运行终端命令,不要读取其他项目,不要重复提出相同提案。' });
|
|
let state, lastProgress = '';
|
|
while (Date.now() - started < 120000) {
|
|
state = await api('/sessions/' + created.id);
|
|
const progress = state.status + '|' + state.events.length;
|
|
if (progress !== lastProgress) {
|
|
console.log(JSON.stringify({ status: state.status, tools: state.events.filter(event => event.kind === 'tool').map(event => ({ tool: event.tool, phase: event.phase })), elapsedSeconds: Math.round((Date.now() - started) / 1000) }));
|
|
lastProgress = progress;
|
|
}
|
|
for (const approval of state.approvals.filter(approval => approval.status === 'pending')) {
|
|
await api('/sessions/' + created.id + '/approvals/' + approval.id, { allow: false });
|
|
}
|
|
if (!['running', 'awaiting_approval', 'stopping'].includes(state.status)) break;
|
|
if (state.events.filter(event => event.kind === 'tool').length > 6) throw new Error('验证任务超出预期工具次数,已停止');
|
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
}
|
|
if (!state || ['running', 'awaiting_approval', 'stopping'].includes(state.status)) throw new Error('模型验证超时');
|
|
if (state.status === 'failed') throw new Error(state.messages.filter(message => message.role === 'system').at(-1)?.content || '模型请求失败');
|
|
assert.equal(state.executionMode, 'compatible');
|
|
assert.ok(state.events.some(event => event.tool === 'read_file' && event.phase === 'completed' && event.input?.path === 'facts.txt'), '未取得真实文件读取结果');
|
|
const proposal = state.changes.find(change => change.path === 'MODEL-RESULT.md' && change.status === 'pending');
|
|
assert.ok(proposal, '模型没有生成预期文件提案');
|
|
assert.ok(proposal.content.includes(proof), '提案没有包含实际读取的 token');
|
|
assert.match(proposal.content, /DOUBLED_NUMBER\s*=\s*42/);
|
|
assert.equal(fs.existsSync(path.join(root, 'MODEL-RESULT.md')), false, '审核前文件不应写入');
|
|
await api('/sessions/' + created.id + '/changes/' + proposal.id + '/apply', {});
|
|
assert.equal(fs.readFileSync(path.join(root, 'MODEL-RESULT.md'), 'utf8'), proposal.content);
|
|
state = await api('/sessions/' + created.id);
|
|
completed = true;
|
|
const result = { success: true, testedAt: new Date().toISOString(), endpoint: bootstrap.config.baseUrl,
|
|
model: bootstrap.config.model, projectRoot: root, projectId: project.id, sessionId: created.id,
|
|
status: state.status, elapsedSeconds: Math.round((Date.now() - started) / 1000), usage: state.usage,
|
|
tools: state.events.filter(event => event.kind === 'tool').map(event => ({ tool: event.tool, phase: event.phase })),
|
|
evidence: ['真实配置的模型服务响应', 'read_file 读取随机合成数据', 'propose_file 使用读取结果', '审核前不写盘', '核对提案后应用成功'],
|
|
credentialsHandling: '验证脚本使用服务内存中的连接配置,没有读取或记录 API Key' };
|
|
fs.writeFileSync('.qa/live-model-validation.json', JSON.stringify(result, null, 2));
|
|
console.log(JSON.stringify(result, null, 2));
|
|
} catch (error) {
|
|
fs.writeFileSync('.qa/live-model-validation.json', JSON.stringify({ success: false, testedAt: new Date().toISOString(), model: bootstrap.config.model,
|
|
sessionId: created.id, projectRoot: root, error: error.message, elapsedSeconds: Math.round((Date.now() - started) / 1000) }, null, 2));
|
|
throw error;
|
|
} finally {
|
|
if (!completed) { try { await api('/sessions/' + created.id + '/stop', {}); } catch {} }
|
|
}
|