test: consume preserved diagnostic feedback after context projection
This commit is contained in:
@@ -18,7 +18,7 @@ const api = async (route, input) => {
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
||||
};
|
||||
let mode = 'failure', requestCount = 0, sessionId = '', page;
|
||||
let mode = 'failure', requestCount = 0, sessionId = '', page, diagnosticReads = 0, diagnosticProjectionUsed = false;
|
||||
const passed = [], pageErrors = [], networkErrors = [];
|
||||
const model = http.createServer(async (request, response) => {
|
||||
try {
|
||||
@@ -36,8 +36,17 @@ const model = http.createServer(async (request, response) => {
|
||||
if (mode === 'recover') { assert.equal(input.messages.at(-1).role, 'tool'); return answer('**模型请求已恢复**,已完成的读取结果保留。'); }
|
||||
if (mode === 'question') {
|
||||
assert.match(input.messages[0].content, /model_connection/); assert.match(input.messages[0].content, /503/);
|
||||
if (!results.length) return tool('inspect_session', {});
|
||||
const snapshot = JSON.parse(results.at(-1).content); assert.equal(snapshot.lastFailure.httpStatus, 503);
|
||||
// Large owned snapshots may be projected into mandatory latest-tool
|
||||
// feedback. The fixture must consume that saved result like the model,
|
||||
// rather than replaying an inspection whose raw group was archived.
|
||||
const feedback = input.messages.filter(message => message.role === 'system').map(message => {
|
||||
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
|
||||
}).find(value => value.latest_tools)?.data.entries.find(entry => entry.tool === 'inspect_session');
|
||||
const snapshot = results.length ? JSON.parse(results.at(-1).content) : feedback?.result;
|
||||
if (!snapshot) { diagnosticReads++; return tool('inspect_session', {}); }
|
||||
assert.equal(snapshot.session.id, sessionId); assert.equal(snapshot.lastFailure.source, 'model_connection');
|
||||
if (feedback) { diagnosticProjectionUsed = true; assert.equal(feedback.recorded_state, 'completed'); assert.match(snapshot.lastFailure.message, /HTTP 503/); }
|
||||
else assert.equal(snapshot.lastFailure.httpStatus, 503);
|
||||
return answer('已读取工作台执行记录。上次错误发生在请求模型服务时:**HTTP 503**。项目读取已经成功,文件未改动。');
|
||||
}
|
||||
if (mode === 'proposal') {
|
||||
@@ -116,6 +125,8 @@ try {
|
||||
mode = 'question'; await page.getByLabel('任务内容').fill('刚才工作台报错怎么回事?请查执行记录。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await page.getByText(/已读取工作台执行记录/).waitFor(); await waitState('completed');
|
||||
const answered = await state(); assert.ok(answered.events.some(event => event.tool === 'inspect_session' && event.phase === 'completed'));
|
||||
assert.equal(diagnosticReads, 1, 'Read the actual saved diagnosis once, including when its raw group is projected.');
|
||||
assert.equal(answered.events.filter(event => event.tool === 'inspect_session').length, 1);
|
||||
assert.equal(answered.events.filter(event => event.tool === 'search_files').length, 0);
|
||||
assert.ok(!await page.locator('.message-assistant .message-text').last().innerText().then(text => text.includes('**')));
|
||||
await screenshot('formal-diagnostic-answer'); mark('an error question receives the actual session diagnosis and formatted answer');
|
||||
@@ -158,7 +169,7 @@ try {
|
||||
mode = 'transient'; requestCount = 0; await task('验证自动恢复'); await waitState('completed'); assert.equal(requestCount, 2);
|
||||
await screenshot('formal-desktop'); mark('a transient service failure recovers within the bounded retry policy');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(networkErrors, []);
|
||||
const result = { success: true, version: bootstrap.version, url, date: new Date().toISOString(), passed, pageErrors, requestFailures: networkErrors, modelValidation: 'controlled local HTTP fixture; actual configured model is verified separately' };
|
||||
const result = { success: true, version: bootstrap.version, url, date: new Date().toISOString(), passed, pageErrors, requestFailures: networkErrors, diagnosticReads, diagnosticProjectionUsed, modelValidation: 'controlled local HTTP fixture; actual configured model is verified separately' };
|
||||
fs.writeFileSync('.qa/diagnostics-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
if (page) await page.screenshot({ path: '.qa/screenshots/formal-ui-failure.png', scale: 'css' }).catch(() => {});
|
||||
|
||||
Reference in New Issue
Block a user