fix: retain verifiable task observations within context budgets

This commit is contained in:
Agent Workbench Dev
2026-10-30 01:28:23 +08:00
parent 04a74babc6
commit f12a12e12e
43 changed files with 933 additions and 46 deletions
+35 -4
View File
@@ -1,7 +1,7 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { randomUUID, createHash } from 'node:crypto';
import { spawn } from 'node:child_process';
import { chromium, expect } from 'playwright/test';
import { Store } from '../src/server/store.mjs';
@@ -12,7 +12,7 @@ import { call } from '../tests/helpers.mjs';
const root = path.resolve('.qa', 'context-ui-' + randomUUID()), data = path.join(root, 'data');
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
const store = new Store(data), cases = {};
for (const name of ['long', 'other', 'paused', 'paused-repeat']) {
for (const name of ['long', 'other', 'paused', 'paused-repeat', 'working']) {
const projectRoot = path.join(root, 'context-' + name); fs.mkdirSync(projectRoot);
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned context UI fixture\n');
const project = store.addProject(projectRoot), session = store.createSession(project.id, '上下文验收 ' + name);
@@ -36,6 +36,22 @@ for (const name of ['long', 'other', 'paused', 'paused-repeat']) {
session.providerMessages.push({ role: 'assistant', content: 'Saved fixture read.', tool_calls: [tool] },
{ role: 'tool', tool_call_id: tool.id, content: JSON.stringify(result) });
}
if (name === 'working') {
const content = 'export const actual = "当前任务真实代码🙂";\n';
fs.writeFileSync(path.join(projectRoot, 'actual.mjs'), content);
const values = [{ tool: 'read_file', args: { path: 'actual.mjs' }, result: { path: 'actual.mjs', content,
sha256: createHash('sha256').update(content).digest('hex'), start_line: 1, end_line: 2, total_lines: 2, truncated: false } },
{ tool: 'query_code', args: { query: 'large' }, result: { results: 'Large owned mixed group '.repeat(16000) } }];
const calls = values.map(value => call(randomUUID(), value.tool, value.args)), messages = [];
for (const [index, invocation] of calls.entries()) {
const event = store.event(session, { kind: 'tool', tool: invocation.function.name, toolCallId: invocation.id,
runId: run.id, taskId: run.taskId, phase: 'queued', startedAt: new Date().toISOString(), label: '工作依据样本' });
store.registerInvocation(session, invocation, run, event); store.beginInvocation(session, invocation, run, event);
store.finishInvocation(session, event, values[index].result);
messages.push({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(values[index].result) });
}
session.providerMessages.push({ role: 'assistant', content: '', tool_calls: calls }, ...messages); store.save(session);
}
const step = store.beginStep(session, run), first = planner.prepare(session, run, store.config, {
system: goal + '\n' + JSON.stringify({ project_id: project.id, session_id: session.id, task_id: run.taskId }), tools: toolDefinitions, stepId: step.id });
const response = { message: { role: 'assistant', content: '独立界面样本已保存;没有请求真实模型。' }, usage: { prompt_tokens: 2400, completion_tokens: 300 } };
@@ -60,9 +76,9 @@ child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
await until(() => url || child.exitCode !== null); assert.ok(url, errors);
const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
assert.ok(['1.5.0', '1.5.1', '1.6.0'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
assert.ok(['1.5.0', '1.5.1', '1.6.0', '1.6.1'].includes(bootstrap.version)); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
const proofPrefix = bootstrap.version === '1.5.0' ? 'context' : bootstrap.version === '1.5.1' ? 'context-fix' : 'workspaces-context';
const proofPrefix = bootstrap.version === '1.5.0' ? 'context' : bootstrap.version === '1.5.1' ? 'context-fix' : bootstrap.version === '1.6.0' ? 'workspaces-context' : 'working-context';
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
page.on('pageerror', error => pageErrors.push(error.message));
@@ -147,6 +163,21 @@ try {
const resumed = await api('/sessions/' + cases['paused-repeat'].session.id);
assert.equal(resumed.execution.task.id, pausedTaskId); assert.equal(resumed.execution.run.resumeOf, cases['paused-repeat'].runs[0]);
mark('390px repeated-read pause explains the cause and explicit continuation retains the same Task');
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('working');
await expect(panel()).toContainText('当前任务依据'); await expect(panel()).toContainText('1 项代码与执行结果');
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
await expect(artifact.locator('pre')).toContainText('result_artifact_id'); await screenshot('desktop-working-evidence');
await page.keyboard.press('Escape'); mark('desktop working evidence shows actual code, original result provenance and the current Task');
await page.setViewportSize({ width: 390, height: 844 }); await choose('working');
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
const workingBounds = await artifact.boundingBox(); assert.ok(workingBounds.x >= 0 && workingBounds.x + workingBounds.width <= 390);
await screenshot('mobile-working-evidence'); await page.keyboard.press('Escape'); await expect(panel()).toBeVisible();
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click();
await expect(artifact.locator('pre')).toContainText('working_set'); await page.keyboard.press('Escape');
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
mark('390px working evidence and sent request stay readable and closing the artifact retains task navigation');
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), passed, pageErrors, requestFailures, screenshots, root, data,
mode: 'saved synthetic context evidence; no real model or project commands executed by this UI verifier', browser: browser.version() };