docs(release): seal and deliver task agreement 1.25 with retained production state
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
param([string]$AppRoot = (Split-Path -Parent $PSScriptRoot), [string]$DataRoot = '', [int]$Port = 4318)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$AppRoot = (Resolve-Path -LiteralPath $AppRoot).Path
|
||||
if (-not $DataRoot) { $DataRoot = Join-Path $env:LOCALAPPDATA 'AgentWorkbench' }
|
||||
$DesktopExe = Join-Path $AppRoot 'desktop\AgentWorkbench.exe'
|
||||
$NodeExe = Join-Path $AppRoot 'runtime\node.exe'
|
||||
$MainFile = Join-Path $AppRoot 'dist-desktop\main.cjs'
|
||||
$DesktopArgs = @('--app-root', $AppRoot, '--data-root', $DataRoot, '--port', [string]$Port)
|
||||
if (-not (Test-Path -LiteralPath $DesktopExe)) {
|
||||
$DesktopExe = Join-Path $AppRoot 'runtime\electron\electron.exe'
|
||||
if (-not (Test-Path -LiteralPath $DesktopExe)) { $DesktopExe = Join-Path $AppRoot 'node_modules\electron\dist\electron.exe' }
|
||||
if (-not (Test-Path -LiteralPath $NodeExe)) { $NodeExe = Join-Path (Split-Path -Parent $AppRoot) 'tools\node-v24.22.1-win-x64\node.exe' }
|
||||
$DesktopArgs = @($MainFile) + $DesktopArgs + @('--node', $NodeExe)
|
||||
}
|
||||
foreach ($Required in @($DesktopExe, $NodeExe, $MainFile)) {
|
||||
if (-not (Test-Path -LiteralPath $Required -PathType Leaf)) { throw "Desktop runtime file is missing: $Required" }
|
||||
}
|
||||
$QuotedArgs = ($DesktopArgs | ForEach-Object { if ($_ -match '"') { throw 'Invalid launch path.' }; '"' + $_ + '"' }) -join ' '
|
||||
Start-Process -FilePath $DesktopExe -ArgumentList $QuotedArgs -WorkingDirectory $AppRoot -WindowStyle Hidden
|
||||
@@ -0,0 +1,19 @@
|
||||
[CmdletBinding()]
|
||||
param([int]$Port = 0, [switch]$NoBrowser, [string]$DataRoot = '', [string]$NodePath = '')
|
||||
$ErrorActionPreference = 'Stop'
|
||||
[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false)
|
||||
$appRoot = [IO.Path]::GetFullPath((Split-Path -Parent $PSScriptRoot))
|
||||
if ($Port -eq 0) { $Port = if ($env:AGENT_WORKBENCH_PORT) { [int]$env:AGENT_WORKBENCH_PORT } else { 4318 } }
|
||||
if (-not $DataRoot) { $DataRoot = if ($env:AGENT_WORKBENCH_DATA_DIR) { $env:AGENT_WORKBENCH_DATA_DIR } else { Join-Path $env:LOCALAPPDATA 'AgentWorkbench' } }
|
||||
$DataRoot = [IO.Path]::GetFullPath($DataRoot)
|
||||
if (-not $NodePath) { $NodePath = Join-Path $appRoot 'runtime\node.exe' }
|
||||
if (-not (Test-Path -LiteralPath $NodePath)) { $NodePath = (Get-Command node.exe -ErrorAction Stop).Source }
|
||||
$launcher = Join-Path $appRoot 'dist-server\host-launcher.mjs'
|
||||
if (-not (Test-Path -LiteralPath $launcher)) { throw 'Build the source first, or use the complete portable package.' }
|
||||
$output = & $NodePath $launcher start --app-root $appRoot --data-root $DataRoot --node $NodePath --port ([string]$Port)
|
||||
if ($LASTEXITCODE -ne 0) { throw 'The host could not be verified. See the original diagnostic above.' }
|
||||
$result = $output | ConvertFrom-Json
|
||||
Write-Output ('Agent Workbench is ready: ' + $result.url)
|
||||
Write-Output ('Data: ' + $DataRoot)
|
||||
Write-Output ('Verified host PID: ' + $result.identity.pid + ' · Instance: ' + $result.identity.instanceId)
|
||||
if (-not $NoBrowser) { Start-Process -FilePath $result.url }
|
||||
@@ -0,0 +1,14 @@
|
||||
[CmdletBinding()]
|
||||
param([int]$Port = 0, [string]$DataRoot = '', [string]$NodePath = '')
|
||||
$ErrorActionPreference = 'Stop'
|
||||
[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false)
|
||||
$appRoot = [IO.Path]::GetFullPath((Split-Path -Parent $PSScriptRoot))
|
||||
if ($Port -eq 0) { $Port = if ($env:AGENT_WORKBENCH_PORT) { [int]$env:AGENT_WORKBENCH_PORT } else { 4318 } }
|
||||
if (-not $DataRoot) { $DataRoot = if ($env:AGENT_WORKBENCH_DATA_DIR) { $env:AGENT_WORKBENCH_DATA_DIR } else { Join-Path $env:LOCALAPPDATA 'AgentWorkbench' } }
|
||||
$DataRoot = [IO.Path]::GetFullPath($DataRoot)
|
||||
if (-not $NodePath) { $NodePath = Join-Path $appRoot 'runtime\node.exe' }
|
||||
if (-not (Test-Path -LiteralPath $NodePath)) { $NodePath = (Get-Command node.exe -ErrorAction Stop).Source }
|
||||
$output = & $NodePath (Join-Path $appRoot 'dist-server\host-launcher.mjs') stop --app-root $appRoot --data-root $DataRoot --node $NodePath --port ([string]$Port)
|
||||
if ($LASTEXITCODE -ne 0) { throw 'The original host stop remains unverified. See the diagnostic above.' }
|
||||
$result = $output | ConvertFrom-Json
|
||||
Write-Output ('Agent Workbench host: ' + $result.state)
|
||||
@@ -0,0 +1,76 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import net from 'node:net';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { publicSnapshot, compareOriginalState, hash } from './retained-host-state.mjs';
|
||||
import { inspectHostInstallation } from '../src/core/desktop/app-installation.mjs';
|
||||
|
||||
// Explicit local release stages. Never sends a model request, stops an existing
|
||||
// task/window, reads a private credential map, or retries an unknown installer.
|
||||
const root = path.resolve('.qa/adoption-ab-release/formal'), url = 'http://127.0.0.1:4318', mode = process.argv[2];
|
||||
assert.ok(['before', 'before-recheck', 'upgrade', 'final'].includes(mode)); fs.mkdirSync(root, { recursive: true });
|
||||
const baseline = mode === 'before' || mode === 'before-recheck';
|
||||
const read = file => JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
|
||||
const save = (name, value) => fs.writeFileSync(path.join(root, name), JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
|
||||
assert.equal(fs.existsSync(path.join(root, mode + '.json')), false, 'Preserve the original attempt; no automatic replay.');
|
||||
async function api(route) {
|
||||
const boot = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' }, signal: AbortSignal.timeout(20000) })).json();
|
||||
const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': boot.token, Connection: 'close' }, signal: AbortSignal.timeout(20000) });
|
||||
assert.equal(response.status, 200, route); return response.json();
|
||||
}
|
||||
async function inspectorClosed() {
|
||||
await new Promise((resolve, reject) => { const socket = net.createConnection({ host: '127.0.0.1', port: 9229 }); socket.setTimeout(1500);
|
||||
socket.once('connect', () => { socket.destroy(); reject(Error('Inspector is already open.')); });
|
||||
socket.once('error', error => { socket.destroy(); error.code === 'ECONNREFUSED' ? resolve() : reject(error); });
|
||||
socket.once('timeout', () => { socket.destroy(); reject(Error('Inspector state unknown.')); }); });
|
||||
}
|
||||
const startedAt = new Date().toISOString(); let result;
|
||||
try {
|
||||
await inspectorClosed(); const host = await api('/host/status'), dataRoot = host.identity.dataRoot;
|
||||
assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
|
||||
// An open idle window can be recorded by a read-only baseline. Installation
|
||||
// and the final retention check still require every registered window closed.
|
||||
if (!baseline) assert.equal(host.desktopWindows.count, 0);
|
||||
const installRoot = path.dirname(path.dirname(host.identity.runtimeRoot)); assert.equal(path.basename(path.dirname(host.identity.runtimeRoot)), 'versions');
|
||||
const current = await publicSnapshot(url, dataRoot); assert.equal(current.pid, 36552); assert.equal(current.hasKey, true); assert.deepEqual(current.active, []);
|
||||
if (baseline) {
|
||||
assert.equal(current.version, '1.24.8'); save('state-before.json', current); save('host-before.json', host); save('pointer-before.json', read(path.join(installRoot, 'current.json')));
|
||||
result = { success: true, version: current.version, pid: current.pid, dataRoot, installRoot, projects: current.projects,
|
||||
sessions: current.database.sessionIds.length, desktopWindows: host.desktopWindows.count,
|
||||
tables: Object.fromEntries(Object.entries(current.database.tables).map(([name, table]) => [name, table.count])) };
|
||||
} else {
|
||||
const before = read(path.join(root, 'state-before.json')); compareOriginalState(before, current);
|
||||
const candidate = read('.qa/adoption-ab-release/candidate.json'), freeze = read('.qa/adoption-ab-release/freeze.json');
|
||||
assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
|
||||
for (const input of freeze.inputs) assert.equal(hash(fs.readFileSync(input.path)), input.sha256, 'Frozen input drift.');
|
||||
for (const stage of ['rebuild', 'archives', 'portable', 'install', 'source-a', 'source-a-desktop', 'source-b', 'source-b-desktop', 'portable-a', 'portable-a-desktop', 'portable-b', 'portable-b-desktop', 'portable-files']) {
|
||||
const receipt = read('.qa/adoption-ab-release/receipts/' + stage + '.json'); assert.equal(receipt.success, true, stage);
|
||||
assert.equal(receipt.sourceCommit, freeze.sourceCommit); assert.equal(receipt.candidate, candidate.portableRoot); assert.equal(hash(fs.readFileSync(receipt.log)), receipt.logSha256);
|
||||
}
|
||||
if (mode === 'upgrade') {
|
||||
assert.equal(current.version, before.version); assert.deepEqual(read(path.join(installRoot, 'current.json')), read(path.join(root, 'pointer-before.json')));
|
||||
assert.equal(hash(fs.readFileSync(candidate.installer.path)), candidate.installer.sha256);
|
||||
save('candidate.json', candidate); save('preinstall-state.json', current);
|
||||
const fd = fs.openSync(path.join(root, 'installer-process.log'), 'wx'); let code;
|
||||
try { code = await new Promise((resolve, reject) => {
|
||||
const child = spawn(candidate.installer.path, ['/VERYSILENT', '/SUPPRESSMSGBOXES', '/NORESTART', '/LOG=' + path.join(root, 'installer.log'), '/DIR=' + installRoot, '/DATA_ROOT=' + dataRoot, '/PORT=4318'], { windowsHide: true, stdio: ['ignore', fd, fd] });
|
||||
child.once('error', reject); child.once('close', resolve);
|
||||
}); } finally { fs.closeSync(fd); }
|
||||
save('installer-exit.json', { code, finishedAt: new Date().toISOString() }); assert.equal(code, 0, 'Do not replay a failed/unknown formal installation.');
|
||||
} else { assert.equal(read(path.join(root, 'upgrade.json')).success, true); assert.deepEqual(candidate, read(path.join(root, 'candidate.json'))); }
|
||||
const after = await publicSnapshot(url, dataRoot), retention = compareOriginalState(before, after), afterHost = await api('/host/status');
|
||||
assert.equal(after.version, '1.24.9'); assert.equal(after.pid, before.pid);
|
||||
assert.equal(afterHost.identity.instanceId, read(path.join(root, 'host-before.json')).identity.instanceId);
|
||||
const installed = path.join(installRoot, 'versions', path.basename(candidate.portableRoot)); assert.equal(fs.realpathSync.native(after.runtimeRoot), fs.realpathSync.native(installed));
|
||||
const identity = inspectHostInstallation(installed), pointer = read(path.join(installRoot, 'current.json'));
|
||||
assert.equal(pointer.active.directory, path.basename(installed)); assert.equal(pointer.active.manifestSha256, identity.manifestSha256); assert.equal(pointer.active.runtimeIdentity, identity.identitySha256);
|
||||
const operation = fs.readdirSync(path.join(installRoot, '.installation')).filter(name => /^progress-.*\.json$/.test(name)).map(name => read(path.join(installRoot, '.installation', name))).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0];
|
||||
assert.equal(operation.state, 'completed'); assert.equal(operation.candidate.manifestSha256, identity.manifestSha256); await inspectorClosed();
|
||||
save(mode + '-state-after.json', after); result = { success: true, version: after.version, pid: after.pid, runtimeRoot: after.runtimeRoot,
|
||||
projects: after.projects, sessions: after.database.sessionIds.length, hasKey: after.hasKey, retention, sourceCommit: freeze.sourceCommit,
|
||||
pointer, operation, inspectorClosed: true, desktopWindows: afterHost.desktopWindows.count };
|
||||
}
|
||||
} catch (error) { result = { success: false, error: error.stack }; process.exitCode = 1; }
|
||||
result = { ...result, mode, startedAt, finishedAt: new Date().toISOString(), realModelRequests: 0, keyReadOrResaved: false };
|
||||
save(mode + '.json', result); console.log(JSON.stringify({ success: result.success, mode, version: result.version, pid: result.pid, error: result.error }));
|
||||
@@ -0,0 +1,77 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync, spawn } from 'node:child_process';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
|
||||
const root = path.resolve('.qa/adoption-ab-release'), mode = process.argv[2];
|
||||
const read = file => JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
|
||||
const save = (name, value) => { const file = path.join(root, name); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', { flag: 'wx' }); };
|
||||
const hash = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
const record = file => ({ path: file.replaceAll('\\', '/'), bytes: fs.statSync(file).size, sha256: hash(file) });
|
||||
const walk = directory => fs.readdirSync(directory, { withFileTypes: true }).flatMap(row => row.isDirectory() ? walk(path.join(directory, row.name)) : [path.join(directory, row.name)]);
|
||||
const python = 'C:/Users/wangy/AppData/Local/Programs/Python/Python312/python.exe';
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
if (mode === 'freeze') {
|
||||
assert.equal(execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8', windowsHide: true }).trim(), '');
|
||||
assert.equal(read('package.json').version, '1.24.9');
|
||||
const inputs = execFileSync('git', ['ls-files', '-z', '--', 'src', 'public', 'scripts', 'tests', 'examples', 'benchmarks', 'research', 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', 'mobile.html'], { encoding: 'utf8', windowsHide: true }).split('\0').filter(Boolean).sort();
|
||||
const freeze = { sourceCommit: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(), version: '1.24.9',
|
||||
inputs: inputs.map(record), compiled: ['dist-client', 'dist-server', 'dist-desktop'].flatMap(walk).sort().map(record), realModelRequests: 0 };
|
||||
save('freeze.json', freeze); console.log(JSON.stringify({ success: true, sourceCommit: freeze.sourceCommit, inputs: freeze.inputs.length, compiled: freeze.compiled.length })); process.exit(0);
|
||||
}
|
||||
const freeze = read(path.join(root, 'freeze.json'));
|
||||
for (const row of freeze.inputs) assert.equal(hash(row.path), row.sha256, 'Frozen input drift: ' + row.path);
|
||||
if (mode === 'register') {
|
||||
const candidate = read('releases/latest.json'); assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
|
||||
assert.equal(read(path.join(candidate.portableRoot, 'MANIFEST.json')).version, freeze.version);
|
||||
for (const row of freeze.compiled) assert.equal(hash(path.join(candidate.portableRoot, row.path)), row.sha256);
|
||||
save('candidate.json', candidate); console.log(JSON.stringify({ success: true, candidate: candidate.portableRoot })); process.exit(0);
|
||||
}
|
||||
const candidate = read(path.join(root, 'candidate.json')); assert.deepEqual(read('releases/latest.json'), candidate);
|
||||
const commands = {
|
||||
rebuild: [process.execPath, ['scripts/verify-packaged-build.mjs']],
|
||||
archives: [python, ['-I', 'scripts/verify-archives.py']],
|
||||
portable: [process.execPath, ['scripts/verify-release.mjs']],
|
||||
install: [process.execPath, ['scripts/verify-desktop-install.mjs']],
|
||||
'source-a': [process.execPath, ['scripts/verify-requirements-ui.mjs']],
|
||||
'source-a-desktop': [process.execPath, ['scripts/verify-requirements-desktop.mjs']],
|
||||
'source-b': [process.execPath, ['scripts/verify-turn-progress-ui.mjs']],
|
||||
'source-b-desktop': [process.execPath, ['scripts/verify-turn-progress-ui.mjs', '--desktop']],
|
||||
'portable-a': [process.execPath, ['scripts/verify-requirements-ui.mjs']],
|
||||
'portable-a-desktop': [process.execPath, ['scripts/verify-requirements-desktop.mjs']],
|
||||
'portable-b': [process.execPath, ['scripts/verify-turn-progress-ui.mjs']],
|
||||
'portable-b-desktop': [process.execPath, ['scripts/verify-turn-progress-ui.mjs', '--desktop']],
|
||||
'portable-files': [process.execPath, ['scripts/verify-local-files.mjs']],
|
||||
};
|
||||
assert.ok(commands[mode], 'Unknown release stage.');
|
||||
const output = path.join(root, 'receipts', mode + '.json'), log = path.join(root, 'logs', mode + '.log');
|
||||
assert.equal(fs.existsSync(output), false, 'Preserve previous stage receipt; do not replay unknown work.'); fs.mkdirSync(path.dirname(log), { recursive: true });
|
||||
const fd = fs.openSync(log, 'wx'), startedAt = new Date().toISOString();
|
||||
const env = projectEnvironment({ AGENT_WORKBENCH_TEST_PYTHON: python,
|
||||
...((mode.startsWith('portable-') || mode === 'install') ? { AGENT_WORKBENCH_QA_APP_ROOT: candidate.portableRoot } : {}) });
|
||||
let result, code;
|
||||
try {
|
||||
console.log('START ' + mode);
|
||||
code = await new Promise((resolve, reject) => { const child = spawn(...commands[mode], { env, windowsHide: true, stdio: ['ignore', fd, fd] }); child.once('error', reject); child.once('close', resolve); });
|
||||
fs.closeSync(fd); assert.equal(code, 0, 'Inspect preserved log: ' + log);
|
||||
assert.deepEqual(read('releases/latest.json'), candidate);
|
||||
let file;
|
||||
if (mode === 'archives') file = '.qa/archives-validation.json';
|
||||
else if (mode === 'portable') file = '.qa/release-validation.json';
|
||||
else if (mode === 'install') file = read('.qa/desktop-install-location.json').result;
|
||||
else {
|
||||
const summaries = fs.readFileSync(log, 'utf8').trim().split(/\r?\n/).filter(line => line.startsWith('{')).map(line => { try { return JSON.parse(line); } catch { return null; } });
|
||||
const summary = summaries.findLast(row => row?.success !== undefined); assert.ok(summary);
|
||||
file = summary.output ?? path.join(summary.directory ?? summary.root, 'result.json');
|
||||
}
|
||||
const value = read(file); assert.equal(value.success, true);
|
||||
if (mode !== 'archives') assert.equal(value.version, freeze.version);
|
||||
if (mode.startsWith('portable-')) assert.equal(fs.realpathSync.native(value.appRoot), fs.realpathSync.native(candidate.portableRoot));
|
||||
if (mode === 'install') assert.equal(fs.realpathSync.native(value.packagedSource), fs.realpathSync.native(candidate.portableRoot));
|
||||
if (mode === 'rebuild') assert.equal(value.sourceCommit, freeze.sourceCommit);
|
||||
result = { success: true, resultPath: path.resolve(file), result: value };
|
||||
} catch (error) { result = { success: false, code, error: error.stack }; process.exitCode = 1; }
|
||||
result = { ...result, mode, log, logSha256: hash(log), startedAt, finishedAt: new Date().toISOString(), sourceCommit: freeze.sourceCommit, candidate: candidate.portableRoot, realModelRequests: 0 };
|
||||
save('receipts/' + mode + '.json', result); console.log(JSON.stringify({ success: result.success, mode, output, resultPath: result.resultPath, error: result.error }));
|
||||
@@ -0,0 +1,71 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
const root = path.resolve('docs/live-original-purge-h-evidence');
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const read = relative => JSON.parse(fs.readFileSync(path.join(root, relative)));
|
||||
const manifest = read('manifest.json');
|
||||
for (const entry of manifest.entries) {
|
||||
const raw = fs.readFileSync(path.join(root, entry.path)); assert.equal(raw.length, entry.bytes); assert.equal(sha(raw), entry.sha256);
|
||||
}
|
||||
const core = {};
|
||||
for (const row of read('receipts/owned-records.json')) (core[row.kind] ??= new Map()).set(row.id, JSON.parse(row.payload));
|
||||
const artifacts = new Map(manifest.artifacts.map(a => [a.id, a]));
|
||||
function artifact(id) { const a = artifacts.get(id); assert.ok(a, id); return JSON.parse(fs.readFileSync(path.join(root, a.file))); }
|
||||
const attempts = read('receipts/all-model-attempts.json');
|
||||
function strings(value, output = [], depth = 0) {
|
||||
if (depth > 12) return output;
|
||||
if (typeof value === 'string') {
|
||||
output.push(value);
|
||||
for (const text of new Set([value, ...value.split('\n').filter(line => /^\s*[\[{]/.test(line))])) {
|
||||
if (/^\s*[\[{]/.test(text)) { try { strings(JSON.parse(text), output, depth + 1); } catch {} }
|
||||
}
|
||||
} else if (value && typeof value === 'object') for (const v of Object.values(value)) strings(v, output, depth + 1);
|
||||
return output;
|
||||
}
|
||||
function covered(ranges,start,end){if(end<=start)return false;let cursor=start;for(const r of ranges.slice().sort((a,b)=>a.start-b.start)){if(r.end<=cursor)continue;if(r.start>cursor)return false;cursor=Math.max(cursor,r.end);if(cursor>=end)return true;}return false;}
|
||||
assert.equal(covered([{start:0,end:4},{start:4,end:8}],2,6),true);
|
||||
assert.equal(covered([{start:0,end:2},{start:4,end:8}],1,5),false);
|
||||
assert.equal(covered([{start:0,end:8}],3,3),false);
|
||||
const reports = [];
|
||||
for (const sample of manifest.samples.slice(0, 2)) {
|
||||
const exported=read('owned-exports/'+sample.sessionId+'.json');
|
||||
const applied=exported.session.changes.filter(c=>c.status==='applied'&&c.appliedAt);
|
||||
const currentSource=(r,at)=>{const c=applied.filter(c=>c.path===r.path&&c.appliedAt<at).sort((a,b)=>a.appliedAt.localeCompare(b.appliedAt)).at(-1);return !c||c.sha256===r.sourceSha256;};
|
||||
const tools = [...core.tool.values()].filter(t => t.sessionId === sample.sessionId)
|
||||
.map(t => ({ ...t, result: t.result ?? (t.resultArtifactId ? artifact(t.resultArtifactId) : null) }))
|
||||
.sort((a, b) => a.startedAt.localeCompare(b.startedAt));
|
||||
const reads = tools.filter(t => t.tool === 'read_file' && t.result?.content).map(t => ({ path: t.result.path, start: t.result.start_line, end: t.result.end_line,
|
||||
complete: !t.result.truncated && t.result.start_line === 1 && t.result.end_line === t.result.total_lines,
|
||||
content: t.result.content, sourceSha256:t.result.sha256, toolId: t.id, at: t.finishedAt }));
|
||||
const ranges = new Map();
|
||||
const pages = tools.filter(t => t.tool === 'read_artifact').map(t => {
|
||||
const r = t.result, id = r?.artifact_id, a = artifacts.get(id);
|
||||
if (!id) return { toolId: t.id, state: t.state, resultKeys: Object.keys(r ?? {}) };
|
||||
const prior = ranges.get(id) ?? [], start = r.offset, end = r.next_offset;
|
||||
const alreadyCovered = covered(prior,start,end);
|
||||
const overlap = prior.some(p => p.start < end && start < p.end);
|
||||
prior.push({ start, end }); ranges.set(id, prior);
|
||||
let source;
|
||||
try { const value = JSON.parse(r.content); source = { path: value.path, tool: value.tool, keys: Object.keys(value).slice(0, 10) }; } catch {}
|
||||
return { toolId: t.id, at: t.finishedAt, artifactId: id, kind: a?.kind, sha256: a?.sha256, offset: start, nextOffset: end,
|
||||
total: r.total_characters, truncated: r.truncated, alreadyCovered, overlap, source };
|
||||
});
|
||||
const requestReports = attempts.filter(a => a.sessionId === sample.sessionId).sort((a, b) => a.startedAt.localeCompare(b.startedAt)).map((a, i) => {
|
||||
const request = artifact(a.requestArtifactId), bodyStrings = strings(request.messages ?? request.input ?? request);
|
||||
const before = reads.filter(r => r.at < a.startedAt);
|
||||
return { request: i + 1, attemptId: a.id, reportedTokens: a.usage?.total_tokens, startedAt: a.startedAt,
|
||||
completeSource: before.filter(r => r.complete).map(r => ({ path: r.path, toolId: r.toolId, sourceSha256:r.sourceSha256, currentAtDispatch:currentSource(r,a.startedAt), retained: bodyStrings.some(s => s.includes(r.content)) })),
|
||||
toolsBefore: tools.filter(t => t.finishedAt < a.startedAt).length };
|
||||
});
|
||||
const prepared = tools.findIndex(t => t.tool === 'prepare_change_set' && t.state === 'completed');
|
||||
reports.push({ name: sample.name, sessionId: sample.sessionId, tools: tools.map(t => ({ toolId: t.id, name: t.tool, state: t.state, at: t.finishedAt,
|
||||
resultKeys: Object.keys(t.result ?? {}), ...(t.tool === 'prepare_change_set' ? { result: t.result } : {}) })),
|
||||
reads: reads.map(({ content, ...r }) => ({ ...r, characters: content.length })), pages, requests: requestReports,
|
||||
afterPrepared: prepared < 0 ? [] : tools.slice(prepared + 1).map(t => ({ name: t.tool, at: t.finishedAt, state: t.state })) });
|
||||
}
|
||||
const result = { success: true, manifestSha256: sha(fs.readFileSync(path.join(root, 'manifest.json'))), realModelRequests: 0, auditVersion:2, originalAuditSha256:sha(fs.readFileSync('docs/live-retest-h-efficiency-audit.json')), corrections:['Repeated artifact page coverage uses the union of all prior returned character ranges.','Source reads invalidated by an applied change are explicitly marked stale; their absence is not a lost-current-source finding.'],
|
||||
scope: 'Offline observations of sealed new H samples only. Exact source retention checks inspect nested JSON tool content; repeated artifact pages are measured by original byte-identity and returned character interval. No source changes or inferred model cognition.', reports };
|
||||
fs.writeFileSync('docs/live-retest-h-efficiency-audit-v2.json', JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
||||
for(const r of reports)console.log(JSON.stringify({name:r.name,toolCalls:r.tools.length,completeReads:r.reads.filter(x=>x.complete).length,repeatedPages:r.pages.filter(p=>p.alreadyCovered&&p.nextOffset>p.offset).length,requests:r.requests.map(q=>({request:q.request,completeRead:q.completeSource.length,currentSource:q.completeSource.filter(f=>f.currentAtDispatch).length,currentRetained:q.completeSource.filter(f=>f.currentAtDispatch&&f.retained).length})),afterPrepared:r.afterPrepared}));
|
||||
@@ -0,0 +1,44 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { validateInstalledVersion } from '../src/core/desktop/install-manager.mjs';
|
||||
|
||||
export function buildWindowsInstaller({ root, portableRoot, outputRoot, name = null, qaFixture = null }) {
|
||||
const packageRoot = path.join(root, 'node_modules/innosetup-compiler'), compiler = path.join(packageRoot, 'bin/ISCC.exe');
|
||||
const meta = JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf8'));
|
||||
if (meta.version !== '6.3.1' || !fs.existsSync(compiler)) throw new Error('The pinned, script-free Inno Setup compiler 6.3.1 is required.');
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(portableRoot, 'MANIFEST.json'), 'utf8'));
|
||||
if (manifest.app !== 'agent-workbench' || manifest.kind !== 'portable') throw new Error('Installer requires a full portable candidate.');
|
||||
validateInstalledVersion(portableRoot);
|
||||
for (const value of [root, portableRoot, outputRoot]) if (!path.isAbsolute(value) || /["\r\n]/.test(value)) throw new Error('Installer build requires absolute local paths.');
|
||||
const directory = path.join(root, '.qa', 'installer-build-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
fs.mkdirSync(outputRoot, { recursive: true });
|
||||
const outputName = name ?? path.basename(portableRoot) + '-Setup';
|
||||
if (!/^[A-Za-z0-9._-]+$/.test(outputName) || fs.existsSync(path.join(outputRoot, outputName + '.exe'))) throw new Error('Installer output must be a new simple name.');
|
||||
let template = fs.readFileSync(path.join(root, 'src/desktop/installer.iss'), 'utf8');
|
||||
if (qaFixture) {
|
||||
if (!/^[a-f0-9-]{36}$/.test(qaFixture.appId ?? '') || !['none', 'corrupt-installed-candidate'].includes(qaFixture.fault ?? 'none')) throw new Error('Invalid isolated installer QA fixture.');
|
||||
template = template.replace('E914E883-46F9-4B82-B27F-BDB2B40DCC90', qaFixture.appId)
|
||||
.replace('DefaultGroupName=Agent Workbench', 'DefaultGroupName=Agent Workbench QA ' + qaFixture.appId);
|
||||
if (qaFixture.fault === 'corrupt-installed-candidate') template = template.replace("if CurStep = ssPostInstall then begin", "if CurStep = ssPostInstall then begin\n { Explicit owned QA fault, never included by the product packaging CLI. }\n if not SaveStringToFile(ExpandConstant('{app}\\versions\\{#VersionDirectory}\\dist-client\\index.html'), 'owned-candidate-corruption', True) then RaiseException('QA fault injection failed');");
|
||||
}
|
||||
const literal = value => String(value).replaceAll('"', '""');
|
||||
const manifestSha256 = createHash('sha256').update(fs.readFileSync(path.join(portableRoot, 'MANIFEST.json'))).digest('hex');
|
||||
const prefix = [['PortableRoot', portableRoot], ['OutputRoot', outputRoot], ['OutputName', outputName], ['Version', manifest.version], ['VersionDirectory', path.basename(portableRoot)], ['ManifestSha256', manifestSha256]]
|
||||
.map(([key, value]) => '#define ' + key + ' "' + literal(value) + '"').join('\n') + '\n';
|
||||
const source = path.join(directory, 'installer.iss'); fs.writeFileSync(source, '\uFEFF' + prefix + template, 'utf8');
|
||||
let output;
|
||||
try { output = execFileSync(compiler, ['/Q', source], { cwd: directory, windowsHide: true, encoding: 'utf8', timeout: 600000, maxBuffer: 4 * 1024 * 1024 }); }
|
||||
catch (error) { const bytes = value => Buffer.isBuffer(value) ? value : Buffer.from(value ?? ''); fs.writeFileSync(path.join(directory, 'compile-failure.txt'), Buffer.concat([bytes(error.stdout), bytes(error.stderr)])); throw error; }
|
||||
fs.writeFileSync(path.join(directory, 'compile.txt'), output);
|
||||
const file = path.join(outputRoot, outputName + '.exe');
|
||||
return { path: file, bytes: fs.statSync(file).size, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex'), compiler: { version: meta.version, sha256: createHash('sha256').update(fs.readFileSync(compiler)).digest('hex') }, source, version: manifest.version, candidateDirectory: path.basename(portableRoot), qaFixture };
|
||||
}
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const args = process.argv.slice(2), values = {};
|
||||
if (args.length === 1 && args[0] === '--help') { console.log('node scripts/build-installer.mjs --portable-root DIRECTORY [--output-root DIRECTORY] — compile the pinned user-level installer from an immutable manifest.'); process.exit(0); }
|
||||
while (args.length) { const key = args.shift(); if (!['--portable-root', '--output-root'].includes(key) || values[key] || !args.length) throw new Error('Use --help.'); values[key] = args.shift(); }
|
||||
console.log(JSON.stringify(buildWindowsInstaller({ root: process.cwd(), portableRoot: values['--portable-root'], outputRoot: values['--output-root'] ?? path.resolve('releases') })));
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
// Vite's dependency graph defines the complete mobile allowlist. Desktop
|
||||
// entries, arbitrary files and future chunks are not implicitly exposed.
|
||||
const root = path.resolve('dist-client'), manifest = JSON.parse(fs.readFileSync(path.join(root, '.vite/manifest.json'), 'utf8'));
|
||||
const seen = new Set(), assets = new Set();
|
||||
function visit(name) {
|
||||
if (seen.has(name)) return; seen.add(name);
|
||||
const entry = manifest[name]; if (!entry) throw new Error('Mobile build dependency is missing: ' + name);
|
||||
for (const file of [entry.file, ...(entry.css ?? []), ...(entry.assets ?? [])]) {
|
||||
if (!/^assets\/[A-Za-z0-9_.-]+\.(js|css)$/.test(file)) throw new Error('Unexpected mobile asset: ' + file);
|
||||
if (!fs.statSync(path.join(root, file)).isFile()) throw new Error('Mobile asset is not a file: ' + file);
|
||||
assets.add('/' + file);
|
||||
}
|
||||
for (const imported of [...(entry.imports ?? []), ...(entry.dynamicImports ?? [])]) visit(imported);
|
||||
}
|
||||
visit('mobile.html');
|
||||
const list = [...assets].sort();
|
||||
fs.writeFileSync(path.join(root, 'mobile-assets.json'), JSON.stringify(list, null, 2) + '\n');
|
||||
// Reuse our existing raster mark for iPhone's home-screen icon; no generated
|
||||
// media or network dependency is needed for this product asset.
|
||||
const icon = fs.readFileSync(path.join(root, 'workbench-icon.png'));
|
||||
fs.writeFileSync(path.join(root, 'mobile-touch-icon.png'), icon);
|
||||
const version = createHash('sha256').update(fs.readFileSync(path.join(root, 'mobile.html'))).update(JSON.stringify(list)).digest('hex').slice(0, 20);
|
||||
const files = ['/', '/mobile.webmanifest', '/mobile-icon.svg', '/mobile-touch-icon.png', ...list];
|
||||
const worker = `const CACHE = 'workbench-mobile-${version}';
|
||||
const FILES = ${JSON.stringify(files)};
|
||||
self.addEventListener('install', event => event.waitUntil(caches.open(CACHE).then(cache => cache.addAll(FILES)).then(() => self.skipWaiting())));
|
||||
self.addEventListener('activate', event => event.waitUntil(caches.keys().then(names => Promise.all(names.filter(name => name.startsWith('workbench-mobile-') && name !== CACHE).map(name => caches.delete(name)))).then(() => self.clients.claim())));
|
||||
self.addEventListener('fetch', event => {
|
||||
const url = new URL(event.request.url);
|
||||
if (event.request.method !== 'GET' || url.origin !== self.location.origin || url.search || !FILES.includes(url.pathname)) return;
|
||||
event.respondWith(fetch(event.request).then(response => {
|
||||
if (response.ok) { const copy = response.clone(); event.waitUntil(caches.open(CACHE).then(cache => cache.put(event.request, copy))); }
|
||||
return response;
|
||||
}).catch(async () => (await caches.match(event.request)) || Response.error()));
|
||||
});
|
||||
`;
|
||||
fs.writeFileSync(path.join(root, 'mobile-sw.js'), worker);
|
||||
console.log('Mobile shell built: ' + list.length + ' allowed assets; API and project data are never cached.');
|
||||
@@ -0,0 +1,46 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
// Development-only provenance recipe. Runtime code imports the resulting JSON;
|
||||
// it neither trusts the QA directory nor learns new executable inputs on launch.
|
||||
const evidence='docs/original-purge-load-evidence',raw=fs.readFileSync(path.join(evidence,'manifest.json'));
|
||||
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
assert.equal(sha(raw),'acabdce397139dcd59cbe7f66229a1353d4f95a5ceb7a555977e544b17ea1066');
|
||||
const manifest=JSON.parse(raw),root=process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT;
|
||||
assert.ok(root&&path.isAbsolute(root));
|
||||
const prefix=path.resolve(root)+path.sep,files=new Map();
|
||||
const identify=file=>{const raw=fs.readFileSync(file);return {relative:path.relative(root,file).replaceAll('\\','/'),bytes:raw.length,sha256:sha(raw)};};
|
||||
for(const input of manifest.loadedModuleInputs.filter(input=>input.path.startsWith(prefix))){
|
||||
assert.equal(sha(fs.readFileSync(path.join(evidence,input.snapshot))),input.sha256);
|
||||
const identity=identify(input.path);assert.equal(identity.sha256,input.sha256);files.set(identity.relative,identity);
|
||||
// Node package resolution reads these metadata before source loading.
|
||||
for(let parent=path.dirname(input.path);parent.startsWith(prefix)||parent===root;parent=path.dirname(parent)){
|
||||
const file=path.join(parent,'package.json');if(fs.existsSync(file)){const item=identify(file);files.set(item.relative,item);}
|
||||
if(parent===root)break;
|
||||
}
|
||||
}
|
||||
for(const relative of ['packages/bundle/sdk-minimal/cordis.patch.yml','pnpm-lock.yaml']){
|
||||
const input=identify(path.join(root,relative));files.set(relative,input);
|
||||
}
|
||||
const nativePackage='node_modules/.pnpm/node-addon-require-builtin-win32-x64-msvc@0.1.9/node_modules/node-addon-require-builtin-win32-x64-msvc';
|
||||
const native=identify(path.join(root,nativePackage,'prebuilt/win32-x64-msvc-napi-v9.node'));
|
||||
assert.equal(native.sha256,'ead441d11202126827a789a7eb88e79abe8f7f374b16e7273c4244225c8d6c2b');
|
||||
for(const relative of ['prebuilds.json','prebuilt/win32-x64-msvc-napi-v9.node']){
|
||||
const input=identify(path.join(root,nativePackage,relative));files.set(input.relative,input);
|
||||
}
|
||||
const koffi=identify(path.join(root,'node_modules/.pnpm/@koromix+koffi-win32-x64@3.1.1/node_modules/@koromix/koffi-win32-x64/win32_x64/koffi.node'));
|
||||
assert.equal(koffi.sha256,'eb0587c804bb76262968bf83028528da4dc2d4322f558845cce205c709e2ba20');files.set(koffi.relative,koffi);
|
||||
const upstream=manifest.upstreamInputs.map(({relative,bytes,sha256,snapshot})=>{
|
||||
assert.equal(sha(fs.readFileSync(path.join(evidence,snapshot))),sha256);return {relative,bytes,sha256};
|
||||
}).sort((a,b)=>a.relative.localeCompare(b.relative,'en'));
|
||||
const lock={schemaVersion:1,purgeVersion:'1.1.78',harnessVersion:'0.2.1-rc.2',
|
||||
provenance:{manifestSha256:sha(raw),sourceCommit:manifest.sourceCommit,scope:'Observed JS/JSON plus resolution metadata and explicit native require-builtin input.',
|
||||
nativePeEvidenceIds:['ev_4a114ba373a5039c34e7fcdad6b9202b42d00f16cd4ae6d80304feb4d93cc19c','ev_b9670d77723678e6c17415368494a06583872fe1e41bd739d795c0c6a20ddca2']},
|
||||
original:upstream,harness:[...files.values()].sort((a,b)=>a.relative.localeCompare(b.relative,'en')),
|
||||
nativeAddons:[{...native,staticImports:['KERNEL32.dll'],staticDelayImports:['node.exe']},
|
||||
{...koffi,staticImports:['KERNEL32.dll','ADVAPI32.dll'],staticDelayImports:['NODE.EXE']}]};
|
||||
const output=process.argv[2]??'src/core/engines/original-purge-lock.json';assert.equal(fs.existsSync(output),false);
|
||||
fs.writeFileSync(output,JSON.stringify(lock,null,2)+'\n',{flag:'wx'});
|
||||
console.log(JSON.stringify({output,original:lock.original.length,harness:lock.harness.length,sha256:sha(fs.readFileSync(output))}));
|
||||
@@ -0,0 +1,19 @@
|
||||
import { build } from 'esbuild';
|
||||
import fs from 'node:fs';
|
||||
// Bundled CommonJS dependencies (QR PNG encoder) still import Node builtins.
|
||||
const nodeRequire = { js: "import { createRequire as __awbCreateRequire } from 'node:module'; const require = __awbCreateRequire(import.meta.url);" };
|
||||
fs.mkdirSync('dist-server', { recursive: true });
|
||||
await build({ entryPoints: ['src/server/index.mjs'], outfile: 'dist-server/server.mjs', bundle: true,
|
||||
platform: 'node', target: 'node24', format: 'esm', sourcemap: false, legalComments: 'eof', banner: nodeRequire });
|
||||
await build({ entryPoints: ['src/server/hot-upgrade.mjs'], outfile: 'dist-server/hot-upgrade.mjs', bundle: true,
|
||||
platform: 'node', target: 'node24', format: 'esm', sourcemap: false, legalComments: 'eof', banner: nodeRequire });
|
||||
await build({ entryPoints: ['src/desktop/host-cli.mjs'], outfile: 'dist-server/host-launcher.mjs', bundle: true,
|
||||
platform: 'node', target: 'node24', format: 'esm', sourcemap: false, legalComments: 'eof' });
|
||||
await build({ entryPoints: ['src/desktop/install-cli.mjs'], outfile: 'dist-server/install-manager.mjs', bundle: true,
|
||||
platform: 'node', target: 'node24', format: 'esm', sourcemap: false, legalComments: 'eof' });
|
||||
fs.mkdirSync('dist-desktop', { recursive: true });
|
||||
await build({ entryPoints: ['src/desktop/main.mjs'], outfile: 'dist-desktop/main.cjs', bundle: true, external: ['electron'],
|
||||
platform: 'node', target: 'node24', format: 'cjs', sourcemap: false, legalComments: 'eof' });
|
||||
await build({ entryPoints: ['src/desktop/preload.cjs'], outfile: 'dist-desktop/preload.cjs', bundle: true, external: ['electron'],
|
||||
platform: 'node', target: 'node24', format: 'cjs', sourcemap: false, legalComments: 'eof' });
|
||||
console.log('Server bundle built.');
|
||||
@@ -0,0 +1,35 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export function planBrowserBundle(appRoot) {
|
||||
const lock = JSON.parse(fs.readFileSync(path.join(appRoot, 'package-lock.json'), 'utf8'));
|
||||
const packages = ['playwright', 'playwright-core'].map(name => {
|
||||
const root = path.join(appRoot, 'node_modules', name), meta = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
||||
const declaration = lock.packages['node_modules/' + name];
|
||||
if (meta.version !== '1.64.1' || declaration.version !== meta.version || !declaration.integrity) throw new Error('Portable browser packages must match locked Playwright 1.64.1.');
|
||||
if (!fs.existsSync(path.join(root, 'LICENSE'))) throw new Error('The browser package requires its original LICENSE.');
|
||||
return { name, version: meta.version, root, integrity: declaration.integrity };
|
||||
});
|
||||
return { version: '1.64.1', packages };
|
||||
}
|
||||
export function copyBrowserBundle(runtimeDirectory, plan) {
|
||||
const runtime = path.resolve(runtimeDirectory), output = path.join(runtime, 'node_modules'), packages = [];
|
||||
const copy = (root, destination, relative = '') => {
|
||||
const records = [];
|
||||
for (const entry of fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (entry.isSymbolicLink()) throw new Error('Browser runtime input cannot be a link.');
|
||||
const file = path.join(relative, entry.name), target = path.resolve(destination, file);
|
||||
if (!target.startsWith(output + path.sep)) throw new Error('Browser bundle escaped the owned runtime.');
|
||||
if (entry.isDirectory()) { fs.mkdirSync(target, { recursive: true }); records.push(...copy(root, destination, file)); continue; }
|
||||
const bytes = fs.readFileSync(path.join(root, file)); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, bytes);
|
||||
records.push({ path: file.replaceAll('\\', '/'), bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') });
|
||||
}
|
||||
return records;
|
||||
};
|
||||
for (const pkg of plan.packages) packages.push({ name: pkg.name, version: pkg.version, integrity: pkg.integrity,
|
||||
files: copy(pkg.root, path.join(output, pkg.name)) });
|
||||
const result = { schemaVersion: 1, kind: 'workbench-browser-runtime', playwrightVersion: plan.version, browser: 'local Microsoft Edge',
|
||||
scope: 'win32-x64 static build checks; browser executable remains the locally installed Edge', packages };
|
||||
fs.writeFileSync(path.join(runtime, 'BROWSER-RUNTIME.json'), JSON.stringify(result, null, 2) + '\n'); return result;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
export function planDesktopBundle(root) {
|
||||
const meta = JSON.parse(fs.readFileSync(path.join(root, 'node_modules/electron/package.json'), 'utf8'));
|
||||
if (meta.version !== '45.2.0') throw new Error('Desktop packaging requires the fixed Electron 45.2.0 runtime.');
|
||||
const installed = path.join(root, 'node_modules/electron/dist');
|
||||
const directory = fs.existsSync(path.join(installed, 'electron.exe')) ? installed : path.join(root, 'runtime/electron');
|
||||
const executable = path.join(directory, 'electron.exe');
|
||||
if (!fs.existsSync(executable) || digest(executable) !== '010ea51bfb528c8143513439aa93fb40754403daa2023f9decd4517da71706ab') throw new Error('The actual Windows Electron runtime differs from the fixed binary.');
|
||||
for (const file of ['LICENSE', 'LICENSES.chromium.html']) if (!fs.existsSync(path.join(directory, file))) throw new Error('Electron runtime license missing.');
|
||||
return { directory, version: meta.version, sha256: digest(executable), bytes: fs.statSync(executable).size };
|
||||
}
|
||||
export function buildNativeLauncher(root, output) {
|
||||
const compiler = path.join(process.env.SystemRoot || 'C:\\Windows', 'Microsoft.NET/Framework64/v4.0.30319/csc.exe');
|
||||
if (!fs.existsSync(compiler)) throw new Error('The Windows .NET Framework compiler is required to build the fixed desktop launcher.');
|
||||
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||
const inputs = { compiler: digest(compiler), source: digest(path.join(root, 'src/desktop/Launcher.cs')), icon: digest(path.join(root, 'public/workbench-icon.ico')) };
|
||||
const recordFile = output + '.build.json';
|
||||
if (fs.existsSync(output) && fs.existsSync(recordFile)) {
|
||||
const cached = JSON.parse(fs.readFileSync(recordFile, 'utf8'));
|
||||
if (JSON.stringify(cached.inputs) === JSON.stringify(inputs) && cached.sha256 === digest(output)) return cached;
|
||||
}
|
||||
try { execFileSync(compiler, ['/nologo', '/target:winexe', '/platform:x64', '/optimize+', '/utf8output',
|
||||
'/reference:System.Web.Extensions.dll', '/reference:System.Windows.Forms.dll', '/win32icon:' + path.join(root, 'public/workbench-icon.ico'),
|
||||
'/out:' + output, path.join(root, 'src/desktop/Launcher.cs')], { cwd: root, windowsHide: true, timeout: 30000, stdio: 'pipe' }); }
|
||||
catch (error) { fs.writeFileSync(output + '.compile-failure.txt', Buffer.concat([error.stdout ?? Buffer.alloc(0), error.stderr ?? Buffer.alloc(0)])); throw error; }
|
||||
const record = { bytes: fs.statSync(output).size, sha256: digest(output), inputs, compiler: 'Windows .NET Framework C# compiler; source-bound cached original output, no deterministic-build claim.' };
|
||||
fs.writeFileSync(recordFile, JSON.stringify(record, null, 2)); return record;
|
||||
}
|
||||
export function copyDesktopBundle(root, output, plan, { source = false, launcher = null } = {}) {
|
||||
if (source && digest(path.join(plan.directory, 'resources/default_app.asar')) !== 'eac55d6cfcf4eab7e31a45356567f549ef7eb669e3699226b48a8492ad34f510') {
|
||||
throw new Error('The source Electron runtime requires its fixed default application loader.');
|
||||
}
|
||||
const destination = path.join(output, source ? 'runtime/electron' : 'desktop');
|
||||
fs.cpSync(plan.directory, destination, { recursive: true, filter: file => source || path.basename(file) !== 'default_app.asar' });
|
||||
if (!source) {
|
||||
fs.renameSync(path.join(destination, 'electron.exe'), path.join(destination, 'AgentWorkbench.exe'));
|
||||
const app = path.join(destination, 'resources/app'); fs.mkdirSync(app, { recursive: true });
|
||||
fs.writeFileSync(path.join(app, 'package.json'), JSON.stringify({ name: 'agent-workbench-desktop', version: JSON.parse(fs.readFileSync(path.join(root, 'package.json'))).version, main: 'main.cjs' }) + '\n');
|
||||
fs.writeFileSync(path.join(app, 'main.cjs'), "require('../../../dist-desktop/main.cjs');\n");
|
||||
if (launcher) fs.copyFileSync(launcher, path.join(output, 'AgentWorkbench.exe'));
|
||||
}
|
||||
const identity = { version: plan.version, sha256: plan.sha256, bytes: plan.bytes, executable: source ? 'runtime/electron/electron.exe' : 'desktop/AgentWorkbench.exe',
|
||||
scope: 'Fixed original Windows Electron runtime, full license texts, isolated application loader and native bootstrap; original executable hash unchanged.' };
|
||||
fs.writeFileSync(path.join(output, 'DESKTOP-RUNTIME.json'), JSON.stringify(identity, null, 2) + '\n');
|
||||
return identity;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { REA_VERSION, REA_PROTOCOL, resolveReaInstallation } from '../src/core/tools/rea-adapter.mjs';
|
||||
|
||||
const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
const inside = (root, filename) => path.resolve(filename).startsWith(path.resolve(root) + path.sep);
|
||||
const packageMetadata = root => JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
||||
|
||||
export function planReaBundle(appRoot) {
|
||||
const installation = resolveReaInstallation(appRoot);
|
||||
if (installation.status !== 'available') throw new Error('REA release dependency is unavailable: ' + installation.message);
|
||||
const modules = path.dirname(installation.packageRoot), sourceRoot = path.dirname(modules);
|
||||
const originalLock = path.join(sourceRoot, 'package-lock.json'), lockPath = fs.existsSync(originalLock) ? originalLock : path.join(sourceRoot, 'REA-PACKAGE-LOCK.json');
|
||||
if (path.basename(modules) !== 'node_modules' || !fs.existsSync(lockPath)) throw new Error('Use the isolated, locked REA production installation when packaging.');
|
||||
const lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
|
||||
if (lock.packages?.['node_modules/rea-agents']?.version !== REA_VERSION || lock.packages['']?.dependencies?.['rea-agents'] !== REA_VERSION) throw new Error('REA installation lock does not match the fixed release contract.');
|
||||
const packages = [], absentOptional = [];
|
||||
for (const [relative, expected] of Object.entries(lock.packages)) {
|
||||
if (!relative.startsWith('node_modules/')) continue;
|
||||
const absolute = path.resolve(sourceRoot, relative);
|
||||
if (!inside(modules, absolute)) throw new Error('REA dependency escapes its installation: ' + relative);
|
||||
if (!fs.existsSync(path.join(absolute, 'package.json'))) {
|
||||
if (expected.optional || expected.dev) { if (expected.optional) absentOptional.push(relative); continue; }
|
||||
throw new Error('REA required installed dependency is missing: ' + relative);
|
||||
}
|
||||
if (expected.dev) throw new Error('REA release input must contain only production dependencies: ' + relative);
|
||||
if (fs.lstatSync(absolute).isSymbolicLink() || !inside(modules, fs.realpathSync.native(absolute))) throw new Error('REA release dependencies must be regular contained directories.');
|
||||
const metadata = packageMetadata(absolute);
|
||||
if (metadata.version !== expected.version) throw new Error('REA installed dependency declaration differs from its lock: ' + relative);
|
||||
const licenseFiles = fs.readdirSync(absolute).filter(name => /^(licen[cs]e|copying)([.-].*)?$/i.test(name) && fs.statSync(path.join(absolute, name)).isFile());
|
||||
packages.push({ path: relative, name: metadata.name, version: metadata.version, license: metadata.license ?? expected.license ?? null,
|
||||
integrity: expected.integrity ?? null, licenseFiles });
|
||||
}
|
||||
const topLevel = packages.filter(item => !packages.some(parent => item !== parent && item.path.startsWith(parent.path + '/')));
|
||||
if (!packages.some(item => item.name === 'rea-agents' && item.version === REA_VERSION)) throw new Error('REA package is absent from the installed production graph.');
|
||||
return { installation, sourceRoot, lockPath, lockSha256: digest(lockPath), packages, topLevel, absentOptional, builtAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
export function copyReaBundle(destinationRuntime, plan) {
|
||||
fs.mkdirSync(destinationRuntime, { recursive: true });
|
||||
for (const item of plan.topLevel) {
|
||||
const destination = path.join(destinationRuntime, item.path);
|
||||
if (!inside(destinationRuntime, destination)) throw new Error('REA destination is outside the new release runtime.');
|
||||
if (fs.existsSync(destination)) throw new Error('Refusing to overwrite a runtime dependency: ' + item.path);
|
||||
fs.cpSync(path.join(plan.sourceRoot, item.path), destination, { recursive: true, filter: file => {
|
||||
if (fs.lstatSync(file).isSymbolicLink()) throw new Error('REA runtime contains an unsupported link: ' + file);
|
||||
return true;
|
||||
} });
|
||||
}
|
||||
fs.copyFileSync(plan.lockPath, path.join(destinationRuntime, 'REA-PACKAGE-LOCK.json'));
|
||||
const files = [];
|
||||
const walk = relative => {
|
||||
for (const entry of fs.readdirSync(path.join(destinationRuntime, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const filename = path.join(relative, entry.name);
|
||||
if (entry.isDirectory()) walk(filename);
|
||||
else if (entry.isFile()) files.push({ path: filename.replaceAll('\\', '/'), bytes: fs.statSync(path.join(destinationRuntime, filename)).size,
|
||||
sha256: digest(path.join(destinationRuntime, filename)) });
|
||||
else throw new Error('REA runtime contains a non-regular entry: ' + filename);
|
||||
}
|
||||
};
|
||||
for (const item of plan.topLevel) walk(item.path);
|
||||
const manifest = { schemaVersion: 1, name: 'rea-agents', version: REA_VERSION, protocolVersion: REA_PROTOCOL, builtAt: plan.builtAt,
|
||||
installationLock: { path: 'REA-PACKAGE-LOCK.json', sha256: plan.lockSha256 }, packages: plan.packages,
|
||||
absentOptional: plan.absentOptional, files,
|
||||
identityScope: 'Copied installed production package declarations and file snapshots; MCP handshake and static operations are independently verified.',
|
||||
capabilities: ['javascript-asar-static', 'pe-imports-static', 'pe-resources-static'] };
|
||||
fs.writeFileSync(path.join(destinationRuntime, 'REA-RUNTIME.json'), JSON.stringify(manifest, null, 2) + '\n');
|
||||
return { version: manifest.version, protocolVersion: manifest.protocolVersion, packages: manifest.packages.length,
|
||||
files: files.length, bytes: files.reduce((sum, item) => sum + item.bytes, 0), lockSha256: plan.lockSha256 };
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { _electron, expect } from 'playwright/test';
|
||||
import { createRequestHandler, createSessionPublisher } from '../src/server/app.mjs';
|
||||
import { hostAuthority } from '../src/core/desktop/host-authority.mjs';
|
||||
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
export async function desktopProjectQa({ store, provider, runtime, sample, kind }) {
|
||||
const appRoot = path.resolve(process.env.AGENT_WORKBENCH_DESKTOP_QA_APP_ROOT || process.cwd()), server = http.createServer(), clients = new Set(), token = randomUUID();
|
||||
runtime.appDirectory = appRoot; store.claimRuntime(); store.on('update', createSessionPublisher(clients));
|
||||
server.on('request', createRequestHandler({ appDirectory: appRoot, server, store, provider, runtime, token, clients, started: Date.now(), getPort: () => server.address().port, shutdown: () => {} }));
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
const authority = hostAuthority({ appDirectory: appRoot, server, store, provider, runtime }); authority.initialize();
|
||||
let application, page, selected; const events = [], errors = [], screenshots = [];
|
||||
async function launch() {
|
||||
const executable = desktopExecutableIdentity(appRoot).path, packaged = executable.toLowerCase().endsWith('agentworkbench.exe');
|
||||
application = await _electron.launch({ executablePath: executable,
|
||||
args: [...(packaged ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', store.directory, '--node', process.execPath, '--port', String(server.address().port), '--hidden-window'],
|
||||
cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
||||
page = await application.firstWindow(); page.on('pageerror', error => errors.push(error.message));
|
||||
await page.locator('.connection.connected').waitFor({ timeout: 30000 });
|
||||
if (selected) await select(selected);
|
||||
}
|
||||
async function select(session) {
|
||||
selected = session;
|
||||
const project = page.getByRole('combobox', { name: '当前项目', exact: true });
|
||||
if (!await project.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await project.selectOption(session.projectId);
|
||||
await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session')), { timeout: 20000 }).toBe(session.id);
|
||||
}
|
||||
async function closeInspector() { const close = page.getByRole('button', { name: '关闭检视区', exact: true }); if (await close.isVisible().catch(() => false)) await close.click(); }
|
||||
async function review(session, ids) {
|
||||
await select(session); await closeInspector();
|
||||
const prepared = [...store.core.change_set.values()].find(value => value.sessionId === session.id && value.status === 'prepared' && JSON.stringify(value.files.map(file => file.changeId).sort()) === JSON.stringify(ids.slice().sort()));
|
||||
if (prepared) {
|
||||
await page.getByRole('button', { name: '隔离协作', exact: true }).click();
|
||||
await page.getByRole('button', { name: '打开父工程变更集审查', exact: true }).click();
|
||||
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
||||
} else {
|
||||
if (await page.getByRole('button', { name: '查看改动', exact: true }).isVisible()) await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
||||
else await page.locator('.inspector-tabs').getByRole('button', { name: /^改动/ }).click();
|
||||
await page.getByRole('button', { name: '组合审查', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '选择要一起审查的提案', exact: true }).getByRole('button', { name: '封存审查范围', exact: true }).click();
|
||||
}
|
||||
const dialog = page.getByRole('dialog', { name: '多文件变更集审查', exact: true });
|
||||
const confirm = dialog.getByRole('button', { name: '确认应用全部 ' + ids.length + ' 项', exact: true }); await confirm.waitFor();
|
||||
const matching = [...store.core.change_set.values()].filter(value => value.sessionId === session.id && value.status !== 'completed' && JSON.stringify(value.files.map(file => file.changeId).sort()) === JSON.stringify(ids.slice().sort()));
|
||||
assert.equal(matching.length, 1); const set = matching[0];
|
||||
const response = page.waitForResponse(value => new URL(value.url()).pathname === '/api/sessions/' + session.id + '/change-sets/' + set.id + '/apply'
|
||||
&& value.request().method() === 'POST', { timeout: 120000 });
|
||||
await confirm.click(); const applied = await response;
|
||||
assert.equal(applied.status(), 202, 'Original desktop apply request was rejected');
|
||||
const operation = runtime.changeSetManager().active.get(set.id);
|
||||
if (operation) await operation.promise;
|
||||
assert.equal(set.status, 'completed', JSON.stringify({ status: set.status, summary: set.summary, requiresVerification: set.requiresVerification }));
|
||||
await expect(dialog).toContainText('全部应用完成', { timeout: 20000 }); await page.keyboard.press('Escape');
|
||||
assert.equal(set.status, 'completed'); events.push({ action: 'actual-desktop-file-review', taskId: session.taskId, changeSetId: set.id, fingerprint: set.fingerprint, paths: set.files.map(value => value.path) }); return set;
|
||||
}
|
||||
async function approve(session, approval) {
|
||||
await select(session); await closeInspector();
|
||||
const button = page.getByRole('button', { name: '确认执行', exact: true }); await button.waitFor({ timeout: 20000 });
|
||||
assert.equal(session.approvals.filter(value => value.status === 'pending').length, 1); assert.equal(session.approvals.find(value => value.status === 'pending').id, approval.id);
|
||||
await button.click(); await expect.poll(() => approval.status).toBe('approved'); events.push({ action: 'actual-desktop-original-approval', approvalId: approval.id, taskId: session.taskId });
|
||||
}
|
||||
async function cycle(session, label) {
|
||||
await select(session); const identity = authority.identity(), eventSeq = session.eventSeq, taskId = session.taskId, runId = session.runId;
|
||||
const originalEvents = () => {
|
||||
const events = []; let cursor = 0;
|
||||
while (cursor < eventSeq) {
|
||||
const chunk = store.eventsAfter(cursor, { sessionId: session.id, limit: 500 });
|
||||
events.push(...chunk.events.filter(value => value.seq <= eventSeq));
|
||||
if (!chunk.hasMore || chunk.nextCursor >= eventSeq) break;
|
||||
assert.ok(chunk.nextCursor > cursor); cursor = chunk.nextCursor;
|
||||
}
|
||||
return events;
|
||||
};
|
||||
const retainedEvents = originalEvents(); assert.equal(retainedEvents.at(-1).seq, eventSeq);
|
||||
const completed = () => [...store.core.tool.values()].filter(value => value.sessionId === session.id && value.state === 'completed');
|
||||
const prior = hash(Buffer.from(JSON.stringify(completed()))), approvals = session.approvals.map(value => ({ id: value.id, status: value.status }));
|
||||
await application.close(); application = null;
|
||||
assert.equal(runtime.isActive(session.id), true); assert.equal(authority.identity().instanceId, identity.instanceId); assert.equal(process.pid, identity.pid);
|
||||
await launch(); assert.equal(session.taskId, taskId); assert.equal(session.runId, runId); assert.ok(session.eventSeq >= eventSeq); assert.deepEqual(originalEvents(), retainedEvents);
|
||||
const replay = await page.evaluate(async ({ sessionId, cursor }) => {
|
||||
const bootstrap = await (await fetch('/api/bootstrap')).json();
|
||||
const response = await fetch('/api/events/history?session=' + encodeURIComponent(sessionId) + '&after=' + cursor + '&limit=500', { headers: { 'x-workbench-token': bootstrap.token } });
|
||||
if (!response.ok) throw Error('Original desktop event replay failed (' + response.status + ')'); return response.json();
|
||||
}, { sessionId: session.id, cursor: eventSeq - 1 });
|
||||
assert.deepEqual(replay.events[0], retainedEvents.at(-1)); assert.ok(replay.events.every((value, index) => !index || value.seq > replay.events[index - 1].seq));
|
||||
assert.equal(hash(Buffer.from(JSON.stringify(completed()))), prior); assert.deepEqual(session.approvals.map(value => ({ id: value.id, status: value.status })), approvals);
|
||||
events.push({ action: label, hostPid: process.pid, taskId, runId, eventSeq, reopenedEventSeq: session.eventSeq, replayCursor: replay.nextCursor,
|
||||
retainedEvents: retainedEvents.length, retainedEventsSha256: hash(Buffer.from(JSON.stringify(retainedEvents))), instanceId: identity.instanceId, completedToolSha256: prior, approvals });
|
||||
}
|
||||
async function capture(session, label) {
|
||||
await select(session); await closeInspector(); await page.getByRole('button', { name: '隔离协作', exact: true }).click();
|
||||
for (const width of [1440, 390]) {
|
||||
const native = await application.browserWindow(page); await native.evaluate((value, width) => { value.showInactive(); value.setContentSize(width, 900); }, width);
|
||||
await expect.poll(() => page.evaluate(() => innerWidth)).toBe(width);
|
||||
const filename = path.join(sample, kind + '-' + label + '-' + width + '.png'); await page.screenshot({ path: filename }); fs.writeFileSync(filename + '.html', await page.content());
|
||||
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false); screenshots.push({ file: filename, width, sha256: hash(fs.readFileSync(filename)) });
|
||||
}
|
||||
await (await application.browserWindow(page)).evaluate(value => value.setContentSize(1440, 1000));
|
||||
// Electron resize and React's responsive state settle independently.
|
||||
// Await both before querying a control whose accessible name changes.
|
||||
await expect.poll(() => page.evaluate(() => innerWidth)).toBe(1440);
|
||||
await expect(page.getByRole('button', { name: '关闭检视区', exact: true })).toBeHidden();
|
||||
await page.keyboard.press('Escape'); await expect(page.getByRole('dialog', { name: '隔离协作', exact: true })).toBeHidden();
|
||||
await closeInspector();
|
||||
}
|
||||
await launch();
|
||||
return { select, review, approve, cycle, capture, events, screenshots, errors, get page() { return page; },
|
||||
async resize(width, height) { await (await application.browserWindow(page)).evaluate((value, size) => { value.showInactive(); value.setContentSize(size.width, size.height); }, { width, height }); await expect.poll(() => page.evaluate(() => innerWidth)).toBe(width); },
|
||||
async download(filename, trigger) {
|
||||
const id = randomUUID();
|
||||
await application.evaluate(({ session }, { id, filename }) => {
|
||||
globalThis.workbenchQaDownloads ??= {};
|
||||
globalThis.workbenchQaDownloads[id] = { state: 'waiting' };
|
||||
session.defaultSession.once('will-download', (_event, item) => {
|
||||
const value = globalThis.workbenchQaDownloads[id]; value.suggestedFilename = item.getFilename(); value.state = 'started';
|
||||
item.setSavePath(filename); // Actual Electron DownloadItem; only the owned destination selection is a QA fixture.
|
||||
item.once('done', (_event, state) => { value.state = state; value.bytes = item.getReceivedBytes(); value.totalBytes = item.getTotalBytes(); });
|
||||
});
|
||||
}, { id, filename });
|
||||
await trigger(); await expect.poll(() => application.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id].state, id), { timeout: 30000 }).toBe('completed');
|
||||
const value = await application.evaluate((_electron, id) => globalThis.workbenchQaDownloads[id], id);
|
||||
assert.equal(value.bytes, value.totalBytes); assert.equal(fs.statSync(filename).size, value.bytes);
|
||||
events.push({ action: 'actual-electron-native-download', ...value, file: filename, sha256: hash(fs.readFileSync(filename)), scope: 'Actual DownloadItem and transferred bytes; own output path chosen by QA without exercising the Windows save dialog.' });
|
||||
return value;
|
||||
},
|
||||
scope: 'Real Electron renderer and fixed native main/preload; own in-process Node fixture host, actual native Harness/HTTP/check/Git. The whole fixture host is never exited from its desktop UI.',
|
||||
async failure(error) { if (!page || page.isClosed()) return; const filename = path.join(sample, kind + '-desktop-failure'); fs.writeFileSync(filename + '.html', await page.content()); fs.writeFileSync(filename + '.json', JSON.stringify({ error: error.stack, dialogs: await page.getByRole('dialog').allTextContents(), errors, events }, null, 2)); await (await application.browserWindow(page)).evaluate(value => value.showInactive()); await page.screenshot({ path: filename + '.png' }); },
|
||||
async close() { await application?.close(); for (const response of clients) response.destroy(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); assert.deepEqual(errors, []); } };
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const REQUIRED_REPORTS = Object.freeze(['docs/RESEARCH.md', 'docs/research/CURSOR-MECHANISMS.md', 'docs/research/CONTEXT-RECOVERY.md', 'docs/research/TOOL-LOOP.md']);
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const simple = value => value.replaceAll('\\', '/');
|
||||
|
||||
export function createDocumentationBundle({ root, portableRoot, evidenceRoot, archiveName, sourceCommit }) {
|
||||
if (!/^[a-f0-9]{40}$/.test(sourceCommit) || !/^[A-Za-z0-9._-]+\.zip$/.test(archiveName)) throw new Error('Documentation needs the fixed source commit and separate archive name.');
|
||||
if (fs.existsSync(evidenceRoot)) throw new Error('Historical evidence output must be a new directory.');
|
||||
const entries = [];
|
||||
const walk = relative => {
|
||||
for (const entry of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) {
|
||||
const next = path.posix.join(relative, entry.name);
|
||||
if (entry.isSymbolicLink()) throw new Error('Documentation source cannot contain links: ' + next);
|
||||
if (entry.isDirectory()) walk(next);
|
||||
else if (entry.isFile()) {
|
||||
const bytes = fs.readFileSync(path.join(root, next));
|
||||
entries.push({ path: next, bytes: bytes.length, sha256: hash(bytes) });
|
||||
} else throw new Error('Documentation source must contain regular files: ' + next);
|
||||
}
|
||||
};
|
||||
walk('docs'); entries.sort((a, b) => a.path.localeCompare(b.path));
|
||||
const byPath = new Map(entries.map(row => [row.path, row]));
|
||||
for (const file of REQUIRED_REPORTS) if (!byPath.has(file)) throw new Error('Required original research report missing: ' + file);
|
||||
// Text guides and original research remain offline. Large historical raw
|
||||
// receipts/screenshots are independently shipped with their original paths.
|
||||
const selected = new Set(entries.filter(row => /^docs\/[^/]+\.md$/.test(row.path) || /^docs\/research\/[^/]+\.(?:md|json)$/.test(row.path)).map(row => row.path));
|
||||
const references = [], shipped = [];
|
||||
fs.mkdirSync(evidenceRoot, { recursive: true });
|
||||
for (const row of entries) {
|
||||
const target = path.join(evidenceRoot, row.path); fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.copyFileSync(path.join(root, row.path), target);
|
||||
if (hash(fs.readFileSync(target)) !== row.sha256) throw new Error('Original evidence copy differs: ' + row.path);
|
||||
}
|
||||
const evidenceManifest = { schema: 1, app: 'agent-workbench-evidence', sourceCommit, scope: 'Original docs tree, including all successful/failed historical evidence. No runtime execution dependency.', files: entries };
|
||||
const evidenceBytes = Buffer.from(JSON.stringify(evidenceManifest, null, 2) + '\n');
|
||||
fs.writeFileSync(path.join(evidenceRoot, 'MANIFEST.json'), evidenceBytes, { flag: 'wx' });
|
||||
const rewrite = (file, original) => original.replace(/\]\(([^)]+)\)/g, (whole, raw) => {
|
||||
const [target] = raw.split('#');
|
||||
if (!target || /^[a-z]+:/i.test(target) || target.startsWith('/')) return whole;
|
||||
let decoded;
|
||||
try { decoded = decodeURI(target); } catch { references.push({ from: file, target, disposition: 'malformed-historical-reference' }); return whole; }
|
||||
const resolved = simple(path.posix.normalize(path.posix.join(path.posix.dirname(file), decoded)));
|
||||
if (!resolved.startsWith('docs/')) return whole;
|
||||
const source = byPath.get(resolved);
|
||||
if (!source) {
|
||||
references.push({ from: file, target, disposition: 'historical-source-not-in-repository' });
|
||||
return whole;
|
||||
}
|
||||
if (selected.has(resolved)) return whole;
|
||||
references.push({ from: file, target: resolved, sha256: source.sha256, disposition: REQUIRED_REPORTS.includes(file) ? 'original-report-reference-in-evidence-archive' : 'evidence-catalog-link' });
|
||||
if (REQUIRED_REPORTS.includes(file)) return whole; // Preserve all four original report bytes exactly.
|
||||
const catalogue = path.posix.relative(path.posix.dirname(file), 'docs/EVIDENCE-CATALOG.md');
|
||||
return '](' + catalogue + '#e-' + source.sha256.slice(0, 16) + ')';
|
||||
});
|
||||
for (const file of [...selected, 'README.md']) {
|
||||
const original = fs.readFileSync(path.join(root, file));
|
||||
const content = Buffer.from(rewrite(file, original.toString('utf8')));
|
||||
const destination = path.join(portableRoot, file); fs.mkdirSync(path.dirname(destination), { recursive: true });
|
||||
fs.writeFileSync(destination, content);
|
||||
shipped.push({ path: file, bytes: content.length, sha256: hash(content), sourceSha256: hash(original), originalBytesPreserved: content.equals(original) });
|
||||
}
|
||||
const linked = new Map(references.filter(row => row.sha256).map(row => [row.target, byPath.get(row.target)]));
|
||||
const catalogue = ['# 历史验收资料', '', '运行包保留离线使用说明和四份原研究报告。完整历史原件另存于配套证据包,包含原路径、SHA-256、成功与失败记录;源码包也保留完整资料。', '',
|
||||
`配套归档:\`${archiveName}\``, '', `原件清单 SHA-256:\`${hash(evidenceBytes)}\``, '',
|
||||
'请将证据包解压到单独目录,按其中的原 docs/ 路径查看。不要解压到受完整性核对的程序目录。缺少证据包不影响运行工作台。', ''];
|
||||
for (const [file, row] of [...linked].sort(([a], [b]) => a.localeCompare(b))) catalogue.push(`<a id="e-${row.sha256.slice(0, 16)}"></a>`, `## ${file}`, '', `原件:\`${file}\` · ${row.bytes} 字节`, '', `SHA-256:\`${row.sha256}\``, '');
|
||||
fs.writeFileSync(path.join(portableRoot, 'docs/EVIDENCE-CATALOG.md'), catalogue.join('\n'));
|
||||
const result = { schema: 1, sourceCommit, archiveName, evidenceManifestSha256: hash(evidenceBytes), originalFiles: entries.length, originalBytes: entries.reduce((sum, row) => sum + row.bytes, 0),
|
||||
shipped, references, originalReports: REQUIRED_REPORTS.map(file => byPath.get(file)), evidenceEntries: entries };
|
||||
fs.writeFileSync(path.join(portableRoot, 'DOCUMENTATION.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
return { ...result, evidenceRoot };
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// A declared wire fixture. It does not load the third-party SDK or call a model.
|
||||
export function createEngineFixture(directory, options = {}) {
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const version = '0.2.1-rc.2';
|
||||
const manifests = [
|
||||
['apps/cli/package.json', '@deepseek-ai/dsh'],
|
||||
['packages/bundle/sdk-minimal/package.json', '@deepseek-ai/dsh-sdk-minimal'],
|
||||
['packages/sdk/server/package.json', '@deepseek-ai/dsh-sdk-jsonrpc-server'],
|
||||
['packages/sdk/protocol/package.json', '@deepseek-ai/dsh-sdk-protocol'],
|
||||
['packages/sdk/client/package.json', '@deepseek-ai/dsh-sdk-client'],
|
||||
];
|
||||
const write = (relative, content) => { const filename = path.join(directory, relative); fs.mkdirSync(path.dirname(filename), { recursive: true }); fs.writeFileSync(filename, content); };
|
||||
for (const [relative, name] of manifests) write(relative, JSON.stringify({ name, version, type: 'module', fixture: true }));
|
||||
for (const relative of ['packages/boot/cmdline/lib/index.js', 'packages/bundle/sdk-app/lib/index.js', 'packages/sdk/server/lib/index.js', 'packages/sdk/protocol/lib/index.js']) write(relative, '// Owned protocol fixture; not the actual SDK.\n');
|
||||
write('packages/bundle/sdk-minimal/cordis.patch.yml', '# Owned wire-fixture profile\n');
|
||||
write('pnpm-lock.yaml', '# Owned deterministic fixture identity\n');
|
||||
write('apps/cli/node_modules/@deepseek-ai/cordis/package.json', JSON.stringify({ name: '@deepseek-ai/cordis', type: 'module', exports: './index.mjs', fixture: true }));
|
||||
write('apps/cli/node_modules/@deepseek-ai/cordis/index.mjs', '// Owned Cordis identity fixture; never imported.\n');
|
||||
write('fixture-options.json', JSON.stringify(options));
|
||||
write('apps/cli/lib/bin.js', `import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createInterface } from 'node:readline';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
|
||||
const options = JSON.parse(fs.readFileSync(path.join(root, 'fixture-options.json'), 'utf8'));
|
||||
const patch = JSON.parse(fs.readFileSync(process.argv[process.argv.indexOf('--patch') + 1], 'utf8'));
|
||||
const config = patch[0].insert[0].config;
|
||||
const audit = value => fs.appendFileSync(path.join(root, 'wire.jsonl'), JSON.stringify(value) + '\\n');
|
||||
const send = value => process.stdout.write(JSON.stringify(value) + '\\n');
|
||||
audit({ type: 'launch', cwd: process.cwd(), nodeOptionsInherited: !!process.env.NODE_OPTIONS,
|
||||
secretInherited: !!process.env.OWNED_ENGINE_TEST_SECRET, profile: process.argv[process.argv.indexOf('--profile') + 1] });
|
||||
if (options.mode !== 'no-ready') setTimeout(() => {
|
||||
fs.writeFileSync(config.marker, JSON.stringify({ schemaVersion: 1, nonce: options.mode === 'bad-ready' ? 'wrong' : config.nonce,
|
||||
pid: process.pid, signal: 'appReady', readyAt: new Date().toISOString() }), { flag: 'wx' });
|
||||
}, options.readyDelayMs ?? 0);
|
||||
const reader = createInterface({ input: process.stdin });
|
||||
reader.on('line', async line => {
|
||||
const value = JSON.parse(line); audit({ type: 'wire', value });
|
||||
if (!value.method) return;
|
||||
const params = value.params ?? {};
|
||||
let result = {};
|
||||
if (value.method === 'initialize') {
|
||||
if (!fs.existsSync(config.marker)) { send({ jsonrpc: '2.0', id: value.id, error: { code: -32603, message: 'initialize before appReady' } }); return; }
|
||||
if (options.mode === 'hang-init') return;
|
||||
if (options.mode === 'malformed') { process.stdout.write('{broken-json}\\n'); return; }
|
||||
if (options.mode === 'reverse-request') send({ jsonrpc: '2.0', id: 'reverse-control', method: 'approve-command', params: { command: 'fixture-never-executed' } });
|
||||
if (options.mode === 'unowned-event') send({ jsonrpc: '2.0', method: 'session.status', params: { sessionId: randomUUID(), status: 'idle' } });
|
||||
if (options.mode === 'model-request') await fetch(process.env.DEEPSEEK_BASE_URL + '/chat/completions', { method: 'POST', body: '{}' });
|
||||
result = { serverInfo: { name: 'deepseek-harness-sdk-runtime', version: options.mode === 'wrong-version' ? '9.9' : '0.0.1' } };
|
||||
} else if (value.method === 'session/working-directory/get') {
|
||||
if (options.mode === 'change-input') fs.appendFileSync(path.join(root, 'packages/boot/cmdline/lib/index.js'), '// changed\\n');
|
||||
if (options.mode === 'change-workspace') fs.writeFileSync(path.join(process.cwd(), 'README.md'), 'fixture mutated');
|
||||
result = { cwd: options.mode === 'wrong-cwd' ? root : process.cwd() };
|
||||
} else if (['session/wait', 'session/resume', 'session/cancel'].includes(value.method)) {
|
||||
send({ jsonrpc: '2.0', id: value.id, error: { code: -32603, message: value.method === 'session/cancel'
|
||||
? 'unknown DeepSeek Harness SDK runtime method: session/cancel' : 'unknown SDK session: ' + params.sessionId } }); return;
|
||||
} else if (value.method !== 'shutdown') {
|
||||
send({ jsonrpc: '2.0', id: value.id, error: { code: -32601, message: 'unknown fixture method' } }); return;
|
||||
}
|
||||
send({ jsonrpc: '2.0', id: value.id, result });
|
||||
if (value.method === 'shutdown') setImmediate(() => process.exit(0));
|
||||
});
|
||||
process.stdin.on('end', () => process.exit(0));
|
||||
`);
|
||||
return directory;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Generate the product's deterministic original bracket/workbench icon (PNG and ICO)."""
|
||||
import pathlib
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
size = 256
|
||||
rows = []
|
||||
for y in range(size):
|
||||
row = bytearray([0])
|
||||
for x in range(size):
|
||||
color = (28, 64, 115, 255)
|
||||
if (46 <= x < 64 and 64 <= y < 192 or 46 <= x < 98 and (64 <= y < 82 or 174 <= y < 192)
|
||||
or 192 <= x < 210 and 64 <= y < 192 or 158 <= x < 210 and (64 <= y < 82 or 174 <= y < 192)):
|
||||
color = (238, 246, 255, 255)
|
||||
if 100 <= x < 156 and 119 <= y < 137:
|
||||
color = (105, 182, 247, 255)
|
||||
row.extend(color)
|
||||
rows.append(bytes(row))
|
||||
|
||||
def chunk(tag, value):
|
||||
return struct.pack('>I', len(value)) + tag + value + struct.pack('>I', zlib.crc32(tag + value))
|
||||
|
||||
png = (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', size, size, 8, 6, 0, 0, 0))
|
||||
+ chunk(b'IDAT', zlib.compress(b''.join(rows), 9)) + chunk(b'IEND', b''))
|
||||
root = pathlib.Path(__file__).resolve().parent.parent / 'public'
|
||||
(root / 'workbench-icon.png').write_bytes(png)
|
||||
(root / 'workbench-icon.ico').write_bytes(struct.pack('<HHH', 0, 1, 1) + struct.pack('<BBBBHHII', 0, 0, 0, 0, 1, 32, len(png), 22) + png)
|
||||
print('Original desktop icon generated: PNG and ICO, 256 × 256.')
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Read a pinned, clean public source checkout; never import or run its code."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
PIN = "2d0b22e0b3554686e1e4131f8bdc568ef768e060"
|
||||
PRODUCT_PIN = "676fc554"
|
||||
URL = "https://github.com/huiliyi37/Tianshu-harness"
|
||||
|
||||
# Identifiers are used by the research report. Empty needles anchor at line one.
|
||||
SOURCES = [
|
||||
("S01", "package.json", '"version"'),
|
||||
("S02", "LICENSE", ""),
|
||||
("S03", "README.md", "**提交、验证、审查分离的交付**"),
|
||||
("S04", "src/agent/runtime-hooks.ts", "export type RuntimeHookPhase"),
|
||||
("S05", "src/agent/create-runtime-hooks.ts", "export function createDefaultRuntimeHooks"),
|
||||
("S06", "src/agent/loop-factory.ts", "const hooks = createDefaultRuntimeHooks"),
|
||||
("S07", "src/agent/turn-perception.ts", ""),
|
||||
("S08", "src/agent/tool-execution.ts", "export class ToolExecutionController"),
|
||||
("S09", "src/agent/turn-orchestrator.ts", ""),
|
||||
("S10", "src/context/task-contract.ts", "export interface TaskContract"),
|
||||
("S11", "src/agent/evidence-obligation.ts", "export type ObligationState"),
|
||||
("S12", "src/agent/obligation-tracker.ts", "export class ObligationTracker"),
|
||||
("S13", "src/agent/loop.ts", "captureAcceptance(items:"),
|
||||
("S14", "src/agent/advisory-bus.ts", "export class AdvisoryBus"),
|
||||
("S15", "src/agent/runtime-advice-facts.ts", ""),
|
||||
("S16", "src/agent/convergence-detector.ts", "const effectiveScore ="),
|
||||
("S17", "src/agent/convergence-message.ts", ""),
|
||||
("S18", "src/agent/score-history.ts", ""),
|
||||
("S19", "src/tools/write-tool-helpers.ts", ""),
|
||||
("S20", "src/agent/exploration-stall.ts", ""),
|
||||
("S21", "src/agent/__tests__/convergence-detector.test.ts", ""),
|
||||
("S22", "src/agent/delivery-gate-v2.ts", ""),
|
||||
("S23", "src/agent/deliver-task.ts", ""),
|
||||
("S24", "src/agent/evidence.ts", ""),
|
||||
("S25", "src/agent/verification-snapshot.ts", ""),
|
||||
("S26", "src/agent/verification-scope-current.ts", ""),
|
||||
("S27", "src/agent/tool-pipeline.ts", "export function executeToolUse("),
|
||||
("S28", "src/agent/approval-input-boundary.ts", ""),
|
||||
("S29", "src/prompt/engine.ts", "export class PromptEngine"),
|
||||
("S30", "src/agent/request-context-controller.ts", "export class RequestContextController"),
|
||||
("S31", "src/agent/context-budget-preparation.ts", "export async function prepareContextRequest"),
|
||||
("S32", "src/agent/compact-boundary-coordinator.ts", ""),
|
||||
("S33", "src/context/request-budget.ts", ""),
|
||||
("S34", "src/tools/tool-preset.ts", "minimal 36 / 26"),
|
||||
("S35", "src/tools/default-registry.ts", ""),
|
||||
("S36", "src/agent/tool-surface.ts", "export function describeToolSurface"),
|
||||
("S37", "src/agent/work-order.ts", "const workerBudgetSchema"),
|
||||
("S38", "src/agent/worker-runtime.ts", "export function createWorkerRuntimeFactory"),
|
||||
("S39", "src/agent/worker-session.ts", "async function runWorkerSessionImpl"),
|
||||
("S40", "src/agent/hands-session.ts", "const gateOn ="),
|
||||
("S41", "src/agent/worker-write-gate.ts", "export async function evaluateWorkerWriteGate"),
|
||||
("S42", "src/agent/worker-objective-gate.ts", "export function reconcileWithObjective"),
|
||||
("S43", "src/agent/coordinator.ts", "result: reconcileWithObjective(order"),
|
||||
("S44", "src/model/model-binding.ts", "export function resolveModelBinding"),
|
||||
("S45", "src/config/provider-keys.ts", "export function findModelOwner"),
|
||||
("S46", "src/api/fallback-client.ts", "getMainPrefixProof?.() ||"),
|
||||
("S47", "src/memory/formation-policy.ts", ""),
|
||||
("S48", "src/memory/essence-gate.ts", "export async function runEssenceGate"),
|
||||
("S49", "src/memory/unified-memory.ts", "export function recallMemoryEntries"),
|
||||
("S50", "src/agent/hooks/essence-gate-hook.ts", ""),
|
||||
("S51", "src/agent/skill-distill.ts", "export function renderSkillDraftMarkdown"),
|
||||
("S52", "src/agent/hooks/skill-distill-hook.ts", ""),
|
||||
("S53", "src/skills/skill-loader.ts", "Two tiers:"),
|
||||
("S54", "src/mcp/manager.ts", "export class McpManager"),
|
||||
("S55", "src/plugins/plugin-loader.ts", "export async function initializePlugins"),
|
||||
("S56", "src/tools/computer-use/bridge.ts", ""),
|
||||
("S57", "src/tools/computer-use/stub.ts", ""),
|
||||
("S58", "docs/architecture-overview.md", ""),
|
||||
("S59", "docs/reference/observability-harness.md", ""),
|
||||
("S60", "docs/CVM运行时对Agent模型的实证影响.md", "版权与许可"),
|
||||
("S61", "docs/reference/cvm-cognitive-runtime.md", "版权与许可"),
|
||||
("S62", "src/cli/entry.ts", ""),
|
||||
("S63", "src/main.ts", ""),
|
||||
]
|
||||
PRODUCT = [
|
||||
"src/server/runtime.mjs", "src/core/context/working-set.mjs",
|
||||
"src/core/context/request-plan.mjs", "src/core/context/delivery-progress.mjs",
|
||||
"src/core/tasks/task-plan-manager.mjs", "src/core/tasks/worker-manager.mjs",
|
||||
"src/core/git-delivery.mjs", "src/core/models/model-hub.mjs",
|
||||
"src/core/models/protocols.mjs", "src/core/memory/project-memory.mjs",
|
||||
"src/core/evaluation/evaluation-manager.mjs", "src/core/browser/browser-manager.mjs",
|
||||
"src/shared/contracts/task-plans.mjs", "docs/PLAN-1.25.md",
|
||||
]
|
||||
|
||||
|
||||
def git(root, *args):
|
||||
return subprocess.check_output(["git", "-C", str(root), *args])
|
||||
|
||||
|
||||
def digest(root, rel):
|
||||
data = (root / rel).read_bytes()
|
||||
return {"path": rel, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(),
|
||||
"lines": len(data.decode("utf-8").splitlines())}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
source = args.source.resolve()
|
||||
product = Path(__file__).resolve().parents[1]
|
||||
assert git(source, "rev-parse", "HEAD").decode().strip() == PIN, "Unexpected source revision"
|
||||
assert git(source, "status", "--porcelain") == b"", "Source checkout is not clean"
|
||||
tracked = git(source, "ls-files", "-z").decode("utf-8").split("\0")[:-1]
|
||||
ts = [p for p in tracked if p.startswith("src/") and p.endswith((".ts", ".tsx"))]
|
||||
tests = [p for p in ts if "/__tests__/" in p or ".test." in p or ".spec." in p]
|
||||
code = [p for p in ts if p not in set(tests)]
|
||||
hooks = [p for p in code if p.startswith("src/agent/hooks/")]
|
||||
entries = []
|
||||
for identifier, rel, needle in SOURCES:
|
||||
entry = {"id": identifier, **digest(source, rel)}
|
||||
lines = (source / rel).read_text(encoding="utf-8").splitlines()
|
||||
matches = [i + 1 for i, line in enumerate(lines) if needle in line] if needle else [1]
|
||||
assert matches, f"Anchor not found: {identifier} {rel} {needle}"
|
||||
entry.update(anchorLine=matches[0], anchorNeedle=needle,
|
||||
url=f"{URL}/blob/{PIN}/{rel}#L{matches[0]}",
|
||||
gitBlob=git(source, "rev-parse", f"HEAD:{rel}").decode().strip())
|
||||
entries.append(entry)
|
||||
result = {
|
||||
"schemaVersion": 1, "repository": URL + ".git", "commit": PIN,
|
||||
"commitInfo": git(source, "show", "-s", "--format=%cI%n%s", "HEAD").decode().splitlines(),
|
||||
"packageVersion": json.loads((source / "package.json").read_text(encoding="utf-8"))["version"],
|
||||
"codeLicense": "Apache-2.0",
|
||||
"documentExceptions": ["docs/CVM运行时对Agent模型的实证影响.md", "docs/reference/cvm-cognitive-runtime.md"],
|
||||
"documentExceptionLicense": "CC-BY-NC-ND-4.0; linked and license-recorded only; not reproduced or adapted",
|
||||
"publicTree": {name: any(p.startswith(name + "/") for p in tracked) for name in ["desktop", "src/pro", "vscode-extension", "src/tools/computer-use"]},
|
||||
"inventory": {"trackedFiles": len(tracked), "srcTsTsxFiles": len(ts), "testFilesByPathOrName": len(tests),
|
||||
"sourceFilesByExclusion": len(code), "sourceLinesSplitlines": sum(digest(source, p)["lines"] for p in code),
|
||||
"hookSourceFiles": len(hooks)},
|
||||
"inventoryMethod": "git ls-files; src/ .ts/.tsx; tests are /__tests__/ or .test. or .spec.; UTF-8 splitlines counts; no tests executed",
|
||||
"sourceFiles": entries,
|
||||
"productBaselineCommit": git(product, "rev-parse", PRODUCT_PIN).decode().strip(),
|
||||
"productFiles": [digest(product, p) for p in PRODUCT],
|
||||
"executionScope": {"fullApplicationStarted": False, "thirdPartyDependenciesInstalled": False, "modelRequests": 0,
|
||||
"pureProbeOnly": "Five reviewed modules; separate original output and driver are sealed alongside this index"},
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(result, stream, ensure_ascii=False, indent=2)
|
||||
stream.write("\n")
|
||||
print(json.dumps({"commit": PIN, "inventory": result["inventory"], "indexedSources": len(entries), "productFiles": len(PRODUCT)}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
import fs from 'node:fs';
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4319';
|
||||
fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
try {
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 960 }, deviceScaleFactor: 1 });
|
||||
const errors = [];
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForLoadState('networkidle', { timeout: 6000 }).catch(() => {});
|
||||
await page.locator('.connection.connected').waitFor({ timeout: 8000 });
|
||||
await page.screenshot({ path: '.qa/screenshots/initial-desktop.png', fullPage: true });
|
||||
fs.writeFileSync('.qa/initial-dom.html', await page.content());
|
||||
console.log(JSON.stringify({ url, errors, buttons: await page.getByRole('button').allTextContents(),
|
||||
labeledButtons: await page.locator('button[aria-label]').evaluateAll(nodes => nodes.map(node => node.getAttribute('aria-label'))),
|
||||
text: (await page.locator('body').innerText()).slice(0, 3500) }, null, 2));
|
||||
} finally { await browser.close(); }
|
||||
@@ -0,0 +1,49 @@
|
||||
// Legacy model fields describe the dispatched route, not authenticated model weights.
|
||||
// Public execution views may contain only the most recent Runs; gaps stay explicit.
|
||||
export function observedTaskModels(state,taskId=state?.execution?.task?.id){
|
||||
const execution=state?.execution??{},runs=new Map(),attempts=new Map(),issues=[];
|
||||
for(const run of [...(execution.runs??[]),execution.run].filter(Boolean)){
|
||||
if(taskId&&run.taskId===taskId&&typeof run.id==='string')runs.set(run.id,run);
|
||||
}
|
||||
const task=execution.task?.id===taskId?execution.task:null;
|
||||
const validCount=value=>Number.isSafeInteger(value)&&value>=0;
|
||||
let declared=task&&validCount(task.usage?.requests)?task.usage.requests:null;
|
||||
if(declared===null&&runs.size&&[...runs.values()].every(run=>validCount(run.usage?.requests)))
|
||||
declared=[...runs.values()].reduce((total,run)=>total+run.usage.requests,0);
|
||||
for(const run of runs.values())for(const receipt of run.consumption??[]){
|
||||
if(typeof receipt.attemptId!=='string'||!receipt.attemptId){issues.push({kind:'missing_attempt_id',runId:run.id});continue;}
|
||||
const attempt={runId:run.id,attemptId:receipt.attemptId,requestId:receipt.requestId??null,
|
||||
model:typeof receipt.model==='string'&&receipt.model.trim()?receipt.model:null,
|
||||
protocol:typeof receipt.protocol==='string'&&receipt.protocol.trim()?receipt.protocol:null,
|
||||
connectionId:receipt.connectionId??null,outcome:receipt.outcome??null,
|
||||
requestedModel:receipt.modelIdentity?.requestedModel??receipt.model??null,
|
||||
modelIdentity:receipt.modelIdentity?structuredClone(receipt.modelIdentity):null,
|
||||
usageStatus:receipt.usageStatus??'unknown',usage:receipt.usage??null};
|
||||
const previous=attempts.get(attempt.attemptId);
|
||||
if(previous){
|
||||
if(JSON.stringify(previous)!==JSON.stringify(attempt))issues.push({kind:'conflicting_attempt',attemptId:attempt.attemptId,runIds:[previous.runId,run.id]});
|
||||
continue;
|
||||
}
|
||||
attempts.set(attempt.attemptId,attempt);
|
||||
if(!attempt.model||!attempt.protocol)issues.push({kind:'missing_route_identity',attemptId:attempt.attemptId});
|
||||
if(attempt.usageStatus==='reported'&&!attempt.usage)issues.push({kind:'missing_reported_usage',attemptId:attempt.attemptId});
|
||||
}
|
||||
const observed=[...attempts.values()],models=[...new Set(observed.map(value=>value.model).filter(Boolean))],routes=new Map();
|
||||
for(const attempt of observed){
|
||||
const identity={connectionId:attempt.connectionId,model:attempt.model,protocol:attempt.protocol},key=JSON.stringify(identity);
|
||||
if(!routes.has(key))routes.set(key,{...identity,requests:0,reportedUsageRequests:0,unknownUsageRequests:0});
|
||||
const route=routes.get(key);route.requests++;
|
||||
if(attempt.usageStatus==='reported'&&attempt.usage)route.reportedUsageRequests++;else route.unknownUsageRequests++;
|
||||
}
|
||||
if(!taskId)issues.push({kind:'missing_task_identity'});
|
||||
if(declared!==null&&declared!==observed.length)issues.push({kind:'request_coverage_gap',recordedRequests:declared,observedAttempts:observed.length});
|
||||
const status=issues.length?'incomplete':!observed.length?declared===0?'not_observed':'unavailable':models.length>1?'mixed':'observed';
|
||||
const identities=observed.map(attempt=>attempt.modelIdentity),identityCounts={};
|
||||
for(const identity of identities){const key=identity?.status??'historical_unchecked';identityCounts[key]=(identityCounts[key]??0)+1;}
|
||||
return {schemaVersion:1,source:'task_run_consumption',taskId:taskId??null,status,
|
||||
identityScope:'request_routes_and_upstream_reported_identifiers_not_model_weights',
|
||||
responseIdentity:{counts:identityCounts,reportedModels:[...new Set(identities.flatMap(value=>value?.reportedModels??[]))],
|
||||
allAccepted:observed.length>0&&issues.length===0&&identities.every(value=>value?.complete===true&&['matched','mapped'].includes(value.status))},
|
||||
model:['observed','mixed'].includes(status)&&models.length===1?models[0]:null,models,
|
||||
observedAttempts:observed.length,recordedRequests:declared,routes:[...routes.values()],attempts:observed,issues};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import { chromium } from 'playwright';
|
||||
import { until } from '../tests/helpers.mjs';
|
||||
|
||||
// Independent acceptance of the built app; this helper never supplies source fixes.
|
||||
export async function verifyLiveReactClient(projectRoot, outputDirectory) {
|
||||
const dist = fs.realpathSync.native(path.join(projectRoot, 'dist')), pageErrors = [], requestFailures = [], screenshots = [];
|
||||
const server = http.createServer((request, response) => {
|
||||
try {
|
||||
const pathname = decodeURIComponent(new URL(request.url, 'http://127.0.0.1').pathname);
|
||||
const candidate = path.resolve(dist, '.' + (pathname === '/' ? '/index.html' : pathname));
|
||||
if (!candidate.startsWith(dist + path.sep) || !fs.existsSync(candidate) || !fs.statSync(candidate).isFile()
|
||||
|| !fs.realpathSync.native(candidate).startsWith(dist + path.sep)) { response.writeHead(404); response.end(); return; }
|
||||
const mime = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8' }[path.extname(candidate)] ?? 'application/octet-stream';
|
||||
response.writeHead(200, { 'Content-Type': mime, 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' }); response.end(fs.readFileSync(candidate));
|
||||
} catch { response.writeHead(400); response.end(); }
|
||||
});
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1280, height: 920 } });
|
||||
page.on('pageerror', error => pageErrors.push(error.message)); page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const count = async expected => { await until(async () => await page.locator('article').count() === expected); };
|
||||
const text = async (locator, expected) => { await until(async () => (await locator.innerText()) === expected); };
|
||||
fs.mkdirSync(outputDirectory, { recursive: true });
|
||||
try {
|
||||
await page.goto('http://127.0.0.1:' + server.address().port); await page.waitForLoadState('networkidle'); await count(3);
|
||||
await page.getByLabel('任务状态').selectOption('pending'); await count(2);
|
||||
await page.getByLabel('任务状态').selectOption('done'); await count(1);
|
||||
await page.getByLabel('任务搜索').fill(' 不存在 '); await count(0); await page.getByText('没有匹配的任务', { exact: true }).waitFor({ state: 'visible' });
|
||||
await page.getByLabel('任务搜索').fill(' 验证 '); await count(1);
|
||||
await page.getByLabel('任务搜索').fill(''); await page.getByLabel('任务状态').selectOption('all'); await count(3);
|
||||
await page.locator('article[data-task="api"]').getByRole('button', { name: '标为完成', exact: true }).click();
|
||||
await text(page.locator('article[data-task="api"] .tag'), '已完成'); await text(page.locator('article[data-task="tests"] .tag'), '待完成');
|
||||
await text(page.getByLabel('项目统计').locator('div').filter({ hasText: '完成任务' }).locator('strong'), '2');
|
||||
await text(page.getByLabel('项目统计').locator('div').filter({ hasText: '待办工时' }).locator('strong'), '5小时');
|
||||
const desktop = path.join(outputDirectory, 'live-react-desktop.png'); await page.screenshot({ path: desktop, scale: 'css' }); screenshots.push(desktop);
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await page.getByLabel('任务状态').selectOption('pending'); await count(1);
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
const mobile = path.join(outputDirectory, 'live-react-mobile.png'); await page.screenshot({ path: mobile, scale: 'css' }); screenshots.push(mobile);
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
return { success: true, browser: browser.version(), pageErrors, requestFailures, screenshots,
|
||||
verified: ['pending/done filters', 'trimmed search', 'empty state', 'one-task toggle', 'unfiltered statistics', '390px layout'] };
|
||||
} finally { await browser.close(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); }
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
// Own deterministic inputs; no model connection, user project or tool execution.
|
||||
const phase = process.argv[2] ?? 'baseline';
|
||||
assert.ok(['baseline', 'comparison'].includes(phase));
|
||||
const output = path.resolve('.qa', 'project-cache-' + phase + '-' + randomUUID());
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
let baselineIdentity = null, modulePath = path.resolve('src/core/context/project-index.mjs');
|
||||
if (phase === 'baseline') {
|
||||
const commit = '606b5ec72edb2f86864b6cb2dfa1fab3f9e1af1a', gitPath = 'src/core/context/project-index.mjs';
|
||||
const original = execFileSync('git', ['show', commit + ':' + gitPath], { windowsHide: true, encoding: 'utf8' });
|
||||
const inspector = pathToFileURL(path.resolve('src/core/project-inspector.mjs')).href, files = pathToFileURL(path.resolve('src/server/files.mjs')).href;
|
||||
const adjusted = original.replace("'../project-inspector.mjs'", JSON.stringify(inspector)).replace("'../../server/files.mjs'", JSON.stringify(files));
|
||||
assert.notEqual(adjusted, original); modulePath = path.join(output, 'original-index.mjs'); fs.writeFileSync(modulePath, adjusted);
|
||||
baselineIdentity = { commit, gitPath, originalSha256: hash(original), adjustedSha256: hash(adjusted), changes: 'Only two import paths relocated to current unchanged default inspection/file interfaces. Original parsing/cache algorithm preserved.' };
|
||||
}
|
||||
const { ProjectIndex } = await import(pathToFileURL(modulePath).href);
|
||||
const rows = [];
|
||||
const originalRead = fs.readFileSync;
|
||||
const originalOpen = fs.openSync, originalClose = fs.closeSync, handles = new Map();
|
||||
fs.openSync = function (filename, ...args) { const handle = originalOpen.call(this, filename, ...args); handles.set(handle, filename); return handle; };
|
||||
fs.closeSync = function (handle) { handles.delete(handle); return originalClose.call(this, handle); };
|
||||
let currentRoot = null, reads = 0, bytes = 0;
|
||||
fs.readFileSync = function (filename, ...args) {
|
||||
const result = originalRead.call(this, filename, ...args);
|
||||
const source = typeof filename === 'number' ? handles.get(filename) : filename;
|
||||
if (typeof source === 'string' && currentRoot && path.resolve(source).startsWith(currentRoot + path.sep)) {
|
||||
reads++; bytes += Buffer.byteLength(result);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
try {
|
||||
for (const language of ['node', 'react', 'python']) {
|
||||
const root = path.join(output, language); fs.mkdirSync(root);
|
||||
const inputs = [];
|
||||
const write = (name, content) => { fs.writeFileSync(path.join(root, name), content); inputs.push({ path: name, sha256: hash(content), bytes: Buffer.byteLength(content) }); };
|
||||
if (language !== 'python') write('package.json', JSON.stringify({ name: 'cache-measure-' + language, type: 'module' }) + '\n');
|
||||
else write('pyproject.toml', '[project]\nname = "cache-measure-python"\nversion = "0.1.0"\n');
|
||||
const extension = language === 'node' ? 'mjs' : language === 'react' ? 'tsx' : 'py';
|
||||
for (let file = 0; file < 72; file++) {
|
||||
const declarations = Array.from({ length: 36 }, (_, item) => language === 'python'
|
||||
? `def calculate_${file}_${item}(value):\n return value + ${item}\n`
|
||||
: language === 'react' ? `export const Component${file}_${item} = ({value}: {value: number}) => <button aria-label="查看 ${file}-${item}">{value + ${item}}</button>;\n`
|
||||
: `export function calculate${file}_${item}(value) { const result = value + ${item}; return result; }\n`);
|
||||
write('module-' + file + '.' + extension, declarations.join(''));
|
||||
}
|
||||
const project = { id: 'measure-' + language, root: fs.realpathSync.native(root), name: '固定' + language + '解析样本' };
|
||||
const cacheDirectory = path.join(output, 'derived-' + language);
|
||||
const create = () => new ProjectIndex(phase === 'comparison' ? { cacheDirectory } : {});
|
||||
let indexer = create(), first;
|
||||
const measure = async (name, indexer) => {
|
||||
reads = 0; bytes = 0; currentRoot = project.root;
|
||||
const started = performance.now(), index = await indexer.refresh(project), durationMs = performance.now() - started;
|
||||
currentRoot = null;
|
||||
const result = { language, case: name, durationMs, sourceReads: reads, sourceBytesRead: bytes, refresh: index.refresh,
|
||||
cache: index.cache ?? null, sourceFingerprint: index.sourceFingerprint, indexFingerprint: index.indexFingerprint,
|
||||
coverage: index.coverage };
|
||||
rows.push(result);
|
||||
assert.equal(index.coverage.snapshotComplete, true); assert.equal(index.coverage.unindexedFiles, 0);
|
||||
console.log(language + '/' + name + ': ' + durationMs.toFixed(1) + 'ms, reads=' + reads + ', updated=' + index.refresh.updatedFiles + ', reused=' + index.refresh.reusedFiles);
|
||||
return index;
|
||||
};
|
||||
first = await measure('cold', indexer);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const warm = await measure('warm-' + (i + 1), indexer);
|
||||
assert.equal(warm.indexFingerprint, first.indexFingerprint);
|
||||
}
|
||||
const restarted = await measure('new-indexer', create());
|
||||
assert.equal(restarted.indexFingerprint, first.indexFingerprint);
|
||||
const target = path.join(root, 'module-0.' + extension), old = fs.statSync(target);
|
||||
const content = originalRead(target, 'utf8'), changed = content.replace('value + 0', 'value + 9');
|
||||
assert.notEqual(content, changed); assert.equal(Buffer.byteLength(content), Buffer.byteLength(changed));
|
||||
fs.writeFileSync(target, changed); fs.utimesSync(target, old.atime, old.mtime);
|
||||
const modified = await measure('same-size-mtime-change', indexer);
|
||||
assert.notEqual(modified.sourceFingerprint, first.sourceFingerprint); assert.equal(modified.refresh.updatedFiles, 1);
|
||||
fs.writeFileSync(path.join(output, language + '-inputs.json'), JSON.stringify({ project, inputs,
|
||||
mutation: { path: path.basename(target), before: hash(content), after: hash(changed), preservedSize: true, restoredMtime: true } }, null, 2) + '\n');
|
||||
}
|
||||
} finally { fs.readFileSync = originalRead; fs.openSync = originalOpen; fs.closeSync = originalClose; }
|
||||
const report = { success: true, phase, at: new Date().toISOString(), node: process.version,
|
||||
baselineIdentity,
|
||||
interpretation: 'Local derived parsing only. Source reads include declarations and all current hashes. No model calls or token/cost inference.', rows };
|
||||
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(report, null, 2) + '\n');
|
||||
console.log('EVIDENCE ' + output);
|
||||
@@ -0,0 +1,140 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { planReaBundle, copyReaBundle } from './bundle-rea.mjs';
|
||||
import { planBrowserBundle, copyBrowserBundle } from './bundle-browser.mjs';
|
||||
import { planDesktopBundle, buildNativeLauncher, copyDesktopBundle } from './bundle-desktop.mjs';
|
||||
import { buildWindowsInstaller } from './build-installer.mjs';
|
||||
import { createDocumentationBundle } from './documentation-bundle.mjs';
|
||||
import { archiveWindowsDirectory } from './release-archive.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.some(value => !['--help', '--immutable'].includes(value))) throw new Error('Unsupported argument; use --help.');
|
||||
if (args.includes('--help')) { console.log('node scripts/package.mjs [--immutable] - create new portable/source releases and archives; existing release folders are never refreshed.'); process.exit(0); }
|
||||
const root = process.cwd();
|
||||
if (execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8', windowsHide: true }).trim()) throw new Error('Commit the reviewed release inputs before packaging; sourceCommit must identify the actual shipped sources.');
|
||||
const meta = JSON.parse(fs.readFileSync('package.json', 'utf8'));
|
||||
const lock = JSON.parse(fs.readFileSync('package-lock.json', 'utf8'));
|
||||
const reaPlan = planReaBundle(root);
|
||||
const browserPlan = planBrowserBundle(root);
|
||||
const desktopPlan = planDesktopBundle(root);
|
||||
const releases = path.resolve(root, 'releases');
|
||||
fs.mkdirSync(releases, { recursive: true });
|
||||
const baseName = 'Agent-Workbench-' + meta.version;
|
||||
const buildSuffix = args.includes('--immutable') || fs.existsSync(path.join(releases, baseName + '-win-x64')) ? '-' + new Date().toISOString().replace(/[-:TZ.]/g, '') : '';
|
||||
const portableRoot = path.join(releases, baseName + '-win-x64' + buildSuffix);
|
||||
const sourceRoot = path.join(releases, baseName + '-source' + buildSuffix);
|
||||
const evidenceRoot = path.join(releases, baseName + '-evidence' + buildSuffix);
|
||||
for (const target of [portableRoot, sourceRoot]) {
|
||||
if (!path.resolve(target).startsWith(releases + path.sep) || fs.existsSync(target)) throw new Error('Release output must be a new folder under releases.');
|
||||
fs.mkdirSync(target, { recursive: true });
|
||||
}
|
||||
|
||||
const portableFiles = ['dist-client', 'dist-server', 'dist-desktop', 'public', 'examples', 'benchmarks', 'research', 'CHANGELOG.md', 'DESIGN.md', 'AGENTS.md', 'Start-Workbench.cmd', 'Stop-Workbench.cmd', 'Start-Desktop.cmd'];
|
||||
const sourceFiles = ['src', 'public', 'scripts', 'tests', 'docs', 'examples', 'benchmarks', 'research', 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', 'mobile.html', '.gitignore', '.gitattributes', 'README.md', 'CHANGELOG.md', 'DESIGN.md', 'AGENTS.md', 'task_plan.md', 'findings.md', 'progress.md', 'Start-Workbench.cmd', 'Stop-Workbench.cmd', 'Start-Desktop.cmd'];
|
||||
const copyTree = (source, destination) => fs.cpSync(source, destination, { recursive: true, filter: (from, to) => {
|
||||
if (!fs.statSync(from).isFile() || !fs.existsSync(to)) return true;
|
||||
return !fs.readFileSync(from).equals(fs.readFileSync(to));
|
||||
} });
|
||||
for (const name of portableFiles) copyTree(path.join(root, name), path.join(portableRoot, name));
|
||||
fs.mkdirSync(path.join(portableRoot, 'scripts'), { recursive: true });
|
||||
for (const name of ['Start-Workbench.ps1', 'Stop-Workbench.ps1', 'Start-Desktop.ps1', 'upgrade-running.mjs', 'python-check-runner.py']) fs.copyFileSync(path.join(root, 'scripts', name), path.join(portableRoot, 'scripts', name));
|
||||
for (const name of sourceFiles) copyTree(path.join(root, name), path.join(sourceRoot, name));
|
||||
const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8', windowsHide: true }).trim();
|
||||
const documentation = createDocumentationBundle({ root, portableRoot, evidenceRoot, archiveName: path.basename(evidenceRoot) + '.zip', sourceCommit });
|
||||
|
||||
const preferredNode = path.resolve(root, '..', 'tools', 'node-v24.22.1-win-x64', 'node.exe');
|
||||
const nodeBinary = process.env.AGENT_WORKBENCH_NODE_PATH || (fs.existsSync(preferredNode) ? preferredNode : process.execPath);
|
||||
const nodeLicense = path.join(path.dirname(nodeBinary), 'LICENSE');
|
||||
if (!fs.existsSync(nodeLicense)) throw new Error('The bundled Node runtime requires its complete LICENSE next to node.exe.');
|
||||
const nodeVersion = execFileSync(nodeBinary, ['--version'], { encoding: 'utf8', windowsHide: true }).trim();
|
||||
if (!/^v24\./.test(nodeVersion)) throw new Error('This package was validated with Node 24.');
|
||||
fs.mkdirSync(path.join(portableRoot, 'runtime'), { recursive: true });
|
||||
const runtimeDestination = path.join(portableRoot, 'runtime', 'node.exe');
|
||||
const digest = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
if (!fs.existsSync(runtimeDestination) || digest(runtimeDestination) !== digest(nodeBinary)) fs.copyFileSync(nodeBinary, runtimeDestination);
|
||||
const npmRoot = path.join(path.dirname(nodeBinary), 'node_modules', 'npm');
|
||||
if (!fs.existsSync(path.join(npmRoot, 'bin', 'npm-cli.js'))) throw new Error('The portable engineering runner requires the npm shipped with this Node distribution.');
|
||||
copyTree(npmRoot, path.join(portableRoot, 'runtime', 'node_modules', 'npm'));
|
||||
fs.copyFileSync(nodeLicense, path.join(portableRoot, 'runtime', 'LICENSE'));
|
||||
fs.mkdirSync(path.join(sourceRoot, 'runtime'), { recursive: true });
|
||||
fs.copyFileSync(nodeBinary, path.join(sourceRoot, 'runtime', 'node.exe'));
|
||||
fs.copyFileSync(nodeLicense, path.join(sourceRoot, 'runtime', 'LICENSE'));
|
||||
copyTree(npmRoot, path.join(sourceRoot, 'runtime', 'node_modules', 'npm'));
|
||||
const launcherPath = path.join(root, '.qa', 'native-launcher', 'AgentWorkbench.exe');
|
||||
const nativeLauncher = buildNativeLauncher(root, launcherPath);
|
||||
const desktopRuntime = copyDesktopBundle(root, portableRoot, desktopPlan, { launcher: launcherPath });
|
||||
copyDesktopBundle(root, sourceRoot, desktopPlan, { source: true });
|
||||
const reaRuntime = copyReaBundle(path.join(portableRoot, 'runtime'), reaPlan);
|
||||
copyReaBundle(path.join(sourceRoot, 'runtime'), reaPlan);
|
||||
const browserRuntime = copyBrowserBundle(path.join(portableRoot, 'runtime'), browserPlan);
|
||||
copyBrowserBundle(path.join(sourceRoot, 'runtime'), browserPlan);
|
||||
|
||||
const notices = ['Agent Workbench third-party notices', '',
|
||||
'Original product code is supplied for the project owner. Its redistribution terms are determined by the project owner.',
|
||||
'Third-party components remain governed by their own licenses. Full available license texts are reproduced below and in licenses/.', '',
|
||||
'The release does not contain Claude Code or Cursor client binaries or recovered application source.', ''];
|
||||
const licenseRoot = path.join(root, 'licenses'); fs.mkdirSync(licenseRoot, { recursive: true });
|
||||
let licenseCount = 0;
|
||||
for (const [location, entry] of Object.entries(lock.packages).sort(([a], [b]) => a.localeCompare(b))) {
|
||||
if (!location.startsWith('node_modules/')) continue;
|
||||
const packageRoot = path.resolve(root, location);
|
||||
if (!packageRoot.startsWith(path.join(root, 'node_modules') + path.sep) || !fs.existsSync(packageRoot)) continue;
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf8'));
|
||||
const files = fs.readdirSync(packageRoot).filter(name => /^(licen[cs]e|copying)([.-].*)?$/i.test(name) && fs.statSync(path.join(packageRoot, name)).isFile());
|
||||
notices.push('='.repeat(72), (pkg.name || location) + ' ' + (pkg.version || entry.version),
|
||||
'License: ' + (typeof pkg.license === 'object' ? JSON.stringify(pkg.license) : pkg.license || entry.license || 'see upstream package'),
|
||||
'Source package: ' + location, '');
|
||||
if (!files.length) notices.push('License metadata is recorded above; associated parent package license text is included where supplied.', '');
|
||||
for (const file of files) {
|
||||
const text = fs.readFileSync(path.join(packageRoot, file), 'utf8');
|
||||
notices.push(file, '', text, '');
|
||||
fs.writeFileSync(path.join(licenseRoot, (pkg.name + '-' + pkg.version + '-' + file).replace(/[^a-z0-9_.-]/gi, '_') + '.txt'), text);
|
||||
licenseCount++;
|
||||
}
|
||||
}
|
||||
const nodeLicenseText = fs.readFileSync(nodeLicense, 'utf8');
|
||||
const reaLicenseText = fs.readFileSync(path.join(reaPlan.installation.packageRoot, 'LICENSE'), 'utf8');
|
||||
fs.writeFileSync(path.join(licenseRoot, 'REA-' + reaRuntime.version + '-LICENSE.txt'), reaLicenseText);
|
||||
notices.push('='.repeat(72), 'REA (rea-agents) ' + reaRuntime.version, 'License: MIT', 'Source: https://github.com/morluto/rea', '', reaLicenseText,
|
||||
'The fixed production dependency declarations, integrity metadata and file hashes are recorded in runtime/REA-RUNTIME.json.',
|
||||
'Original package and dependency license texts remain under runtime/node_modules/. REA additional notices are retained at runtime/node_modules/rea-agents/THIRD_PARTY_NOTICES.md.', '');
|
||||
fs.writeFileSync(path.join(licenseRoot, 'Node-' + nodeVersion + '-LICENSE.txt'), nodeLicenseText);
|
||||
notices.push('='.repeat(72), 'Node.js ' + nodeVersion, '', nodeLicenseText);
|
||||
notices.push('='.repeat(72), 'npm bundled with Node.js', 'Complete package and dependency licenses are retained under runtime/node_modules/npm/.', '', fs.readFileSync(path.join(npmRoot, 'LICENSE'), 'utf8'));
|
||||
fs.writeFileSync(path.join(root, 'THIRD_PARTY_NOTICES.txt'), notices.join('\n'));
|
||||
for (const output of [portableRoot, sourceRoot]) {
|
||||
fs.cpSync(licenseRoot, path.join(output, 'licenses'), { recursive: true });
|
||||
fs.copyFileSync(path.join(root, 'THIRD_PARTY_NOTICES.txt'), path.join(output, 'THIRD_PARTY_NOTICES.txt'));
|
||||
}
|
||||
|
||||
function filesIn(directory, relative = '') {
|
||||
return fs.readdirSync(path.join(directory, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
||||
const file = path.join(relative, entry.name);
|
||||
return entry.isDirectory() ? filesIn(directory, file) : [file];
|
||||
});
|
||||
}
|
||||
function manifest(directory, kind) {
|
||||
const files = filesIn(directory).filter(file => file !== 'MANIFEST.json').map(file => {
|
||||
const bytes = fs.readFileSync(path.join(directory, file));
|
||||
return { path: file.replaceAll('\\', '/'), bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
|
||||
});
|
||||
const result = { app: 'agent-workbench', version: meta.version, kind, platform: 'win32-x64', nodeVersion,
|
||||
createdAt: new Date().toISOString(), files };
|
||||
fs.writeFileSync(path.join(directory, 'MANIFEST.json'), JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
}
|
||||
const portableManifest = manifest(portableRoot, 'portable');
|
||||
manifest(sourceRoot, 'source');
|
||||
const installer = buildWindowsInstaller({ root, portableRoot, outputRoot: releases });
|
||||
function archive(directory) {
|
||||
const destination = directory + '.zip';
|
||||
if (!destination.startsWith(releases + path.sep)) throw new Error('Archive path is outside releases.');
|
||||
archiveWindowsDirectory(directory, destination);
|
||||
return { path: destination, bytes: fs.statSync(destination).size, sha256: createHash('sha256').update(fs.readFileSync(destination)).digest('hex') };
|
||||
}
|
||||
const result = { portableRoot, sourceRoot, evidenceRoot, portable: archive(portableRoot), source: archive(sourceRoot), evidence: archive(evidenceRoot), documentation: { sourceCommit, originalFiles: documentation.originalFiles, originalBytes: documentation.originalBytes, shippedFiles: documentation.shipped.length, shippedBytes: documentation.shipped.reduce((sum, row) => sum + row.bytes, 0), evidenceManifestSha256: documentation.evidenceManifestSha256 }, installer, desktopRuntime, nativeLauncher, nodeVersion, reaRuntime, portableFiles: portableManifest.files.length, licenseTexts: licenseCount + 2 };
|
||||
result.browserRuntime = { playwrightVersion: browserRuntime.playwrightVersion, packages: browserRuntime.packages.map(pkg => ({ name: pkg.name, version: pkg.version, files: pkg.files.length })) };
|
||||
fs.writeFileSync(path.join(releases, 'latest.json'), JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Collect actual Workbench Python checks after their execution is approved."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def checked_path(root, relative):
|
||||
if not isinstance(relative, str) or not relative or "\0" in relative:
|
||||
raise ValueError("Invalid planned Python path")
|
||||
candidate = (root / relative).resolve(strict=True)
|
||||
if not candidate.is_relative_to(root):
|
||||
raise ValueError("Python input is outside the sealed project")
|
||||
return candidate
|
||||
|
||||
|
||||
def verify_sources(root, files):
|
||||
for item in files:
|
||||
filename = checked_path(root, item["path"])
|
||||
if not filename.is_file() or sha(filename.read_bytes()) != item["sha256"]:
|
||||
raise ValueError("Planned Python source changed: " + item["path"])
|
||||
|
||||
|
||||
def compile_sources(root, files):
|
||||
failures = []
|
||||
for item in files:
|
||||
filename = checked_path(root, item["path"])
|
||||
data = filename.read_bytes()
|
||||
try:
|
||||
compile(data, str(filename), "exec", dont_inherit=True)
|
||||
except (SyntaxError, ValueError) as error:
|
||||
failures.append({"path": item["path"], "message": str(error), "line": getattr(error, "lineno", None)})
|
||||
traceback.print_exception(type(error), error, error.__traceback__)
|
||||
print("Python source compilation: %d / %d" % (len(files) - len(failures), len(files)))
|
||||
return {"compilation": {"compiled": len(files) - len(failures), "total": len(files), "failures": failures}}, 1 if failures else 0
|
||||
|
||||
|
||||
def count_records(records):
|
||||
states = [item["status"] for item in records]
|
||||
return {"total": len(states), "passed": states.count("passed"), "failed": states.count("failed"),
|
||||
"skipped": states.count("skipped"), "cancelled": states.count("pending")}
|
||||
|
||||
|
||||
def unittest_sources(root, settings):
|
||||
import unittest
|
||||
|
||||
class CollectedResult(unittest.TextTestResult):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.records = []
|
||||
self.active = {}
|
||||
self.collection_errors = 0
|
||||
self.collection_skipped = 0
|
||||
|
||||
def startTest(self, test):
|
||||
super().startTest(test)
|
||||
# Test.id() is a display name. Repeated cases and subtests belong
|
||||
# to their actual startTest execution ordinal, not that name.
|
||||
self.active[id(test)] = len(self.records)
|
||||
self.records.append({"status": "pending"})
|
||||
|
||||
def stopTest(self, test):
|
||||
self.active.pop(id(test), None)
|
||||
super().stopTest(test)
|
||||
|
||||
def mark(self, test, status):
|
||||
index = self.active.get(id(test))
|
||||
if index is None:
|
||||
index = self.active.get(id(getattr(test, "test_case", None)))
|
||||
if index is None:
|
||||
if status == "failed":
|
||||
self.collection_errors += 1
|
||||
elif status == "skipped":
|
||||
self.collection_skipped += 1
|
||||
return
|
||||
record = self.records[index]
|
||||
if record["status"] != "failed" and (record["status"] != "skipped" or status == "failed"):
|
||||
record["status"] = status
|
||||
|
||||
def addSuccess(self, test):
|
||||
self.mark(test, "passed")
|
||||
super().addSuccess(test)
|
||||
|
||||
def addFailure(self, test, error):
|
||||
self.mark(test, "failed")
|
||||
super().addFailure(test, error)
|
||||
|
||||
def addError(self, test, error):
|
||||
self.mark(test, "failed")
|
||||
super().addError(test, error)
|
||||
|
||||
def addSkip(self, test, reason):
|
||||
self.mark(test, "skipped")
|
||||
super().addSkip(test, reason)
|
||||
|
||||
def addExpectedFailure(self, test, error):
|
||||
self.mark(test, "skipped")
|
||||
super().addExpectedFailure(test, error)
|
||||
|
||||
def addUnexpectedSuccess(self, test):
|
||||
self.mark(test, "failed")
|
||||
super().addUnexpectedSuccess(test)
|
||||
|
||||
def addSubTest(self, test, subtest, error):
|
||||
if error is not None:
|
||||
self.mark(test, "failed")
|
||||
super().addSubTest(test, subtest, error)
|
||||
|
||||
start = checked_path(root, settings["startDirectory"])
|
||||
if not start.is_dir():
|
||||
raise ValueError("Unittest discovery needs a project directory")
|
||||
sys.path[:0] = [str(root), str(root / "src")]
|
||||
loader = unittest.TestLoader()
|
||||
suite = loader.discover(str(start), pattern=settings["pattern"])
|
||||
result = unittest.TextTestRunner(verbosity=2, resultclass=CollectedResult).run(suite)
|
||||
if result.testsRun != len(result.records):
|
||||
raise ValueError("Unittest execution count was not collected consistently")
|
||||
return {"tests": count_records(result.records), "framework": "unittest", "frameworkVersion": sys.version.split()[0],
|
||||
"collectionErrors": result.collection_errors, "collectionSkipped": result.collection_skipped}, 0 if result.wasSuccessful() else 1
|
||||
|
||||
|
||||
def pytest_sources(root, settings):
|
||||
# Import the host framework before adding project imports. Autoloaded
|
||||
# environment plugins and addopts are excluded by the host check contract.
|
||||
import pytest
|
||||
sys.path[:0] = [str(root), str(root / "src")]
|
||||
|
||||
class Collector:
|
||||
def __init__(self):
|
||||
self.records = []
|
||||
self.ordinals = {}
|
||||
self.active = None
|
||||
self.collection_errors = 0
|
||||
self.collection_skipped = 0
|
||||
|
||||
def pytest_collection_finish(self, session):
|
||||
self.records = [{"status": "pending", "nodeid": item.nodeid} for item in session.items]
|
||||
self.ordinals = {}
|
||||
for index, item in enumerate(session.items):
|
||||
self.ordinals.setdefault(id(item), []).append(index)
|
||||
|
||||
def pytest_collectreport(self, report):
|
||||
if report.failed:
|
||||
self.collection_errors += 1
|
||||
elif report.skipped:
|
||||
self.collection_skipped += 1
|
||||
|
||||
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
|
||||
def pytest_runtest_protocol(self, item, nextitem):
|
||||
pending = self.ordinals.get(id(item), [])
|
||||
self.active = pending.pop(0) if pending else None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.active = None
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if self.active is None or self.records[self.active]["nodeid"] != report.nodeid:
|
||||
self.collection_errors += 1
|
||||
return
|
||||
record = self.records[self.active]
|
||||
if report.failed:
|
||||
record["status"] = "failed"
|
||||
elif record["status"] != "failed" and (report.skipped or hasattr(report, "wasxfail")):
|
||||
record["status"] = "skipped"
|
||||
elif record["status"] == "pending" and report.when == "call" and report.passed:
|
||||
record["status"] = "passed"
|
||||
|
||||
collector = Collector()
|
||||
args = [str(checked_path(root, relative)) for relative in settings["testPaths"]]
|
||||
args += ["-p", "no:cacheprovider", "--import-mode=importlib", "-o", "addopts="]
|
||||
if settings.get("configPath"):
|
||||
args += ["-c", str(checked_path(root, settings["configPath"]))]
|
||||
code = int(pytest.main(args, plugins=[collector]))
|
||||
return {"tests": count_records(collector.records), "framework": "pytest", "frameworkVersion": pytest.__version__,
|
||||
"collectionErrors": collector.collection_errors, "collectionSkipped": collector.collection_skipped}, code
|
||||
|
||||
|
||||
def main():
|
||||
payload_bytes = sys.stdin.buffer.read()
|
||||
payload = json.loads(payload_bytes)
|
||||
result_path = Path(sys.argv[1]).resolve()
|
||||
root = Path(payload["root"]).resolve(strict=True)
|
||||
if not root.is_dir() or result_path.is_relative_to(root):
|
||||
raise ValueError("Result collection must use the owned host directory")
|
||||
executable = Path(os.path.realpath(sys.executable))
|
||||
base_executable = Path(os.path.realpath(getattr(sys, "_base_executable", sys.executable)))
|
||||
identity = {"schemaVersion": 1, "kind": "python-check-result", "invocationId": payload["invocationId"],
|
||||
"inputSha256": sha(payload_bytes), "mode": payload["mode"], "runnerSha256": sha(Path(__file__).read_bytes()),
|
||||
"runtime": {"executable": str(executable), "version": sys.version.split()[0], "sha256": sha(executable.read_bytes()),
|
||||
"baseExecutable": str(base_executable), "baseSha256": sha(base_executable.read_bytes())}}
|
||||
try:
|
||||
verify_sources(root, payload["files"])
|
||||
if payload["mode"] == "compile":
|
||||
evidence, code = compile_sources(root, payload["files"])
|
||||
elif payload["mode"] == "unittest":
|
||||
evidence, code = unittest_sources(root, payload["settings"])
|
||||
elif payload["mode"] == "pytest":
|
||||
evidence, code = pytest_sources(root, payload["settings"])
|
||||
else:
|
||||
raise ValueError("Unknown built-in Python check")
|
||||
output = {**identity, **evidence, "exitCode": code}
|
||||
except BaseException as error:
|
||||
traceback.print_exception(type(error), error, error.__traceback__)
|
||||
output = {**identity, "exitCode": 1, "collectorError": type(error).__name__ + ": " + str(error)}
|
||||
code = 1
|
||||
temporary = result_path.with_suffix(".tmp")
|
||||
with temporary.open("x", encoding="utf-8") as stream:
|
||||
json.dump(output, stream, ensure_ascii=False)
|
||||
temporary.replace(result_path)
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
// Keep every release's acceptance files separate from historical evidence.
|
||||
const prefixes = Object.freeze({
|
||||
'1.4.0': 'm2', '1.5.0': 'context', '1.5.1': 'context-fix', '1.6.0': 'workspaces',
|
||||
'1.6.1': 'working-context', '1.6.2': 'working-retention', '1.7.0': 'python-engine',
|
||||
'1.8.0': 'rea-engine', '1.9.0': 'external-engines', '1.10.0': 'browser-engine',
|
||||
'1.10.1': 'browser-hotfix', '1.11.0': 'task-plans', '1.12.0': 'model-center',
|
||||
'1.12.1': 'milestone-audit',
|
||||
'1.13.0': 'workers',
|
||||
'1.14.0': 'controlled-engine',
|
||||
'1.15.0': 'desktop-host',
|
||||
'1.16.0': 'mobile-companion',
|
||||
'1.17.0': 'model-protocols',
|
||||
'1.18.0': 'vision-inputs',
|
||||
'1.19.0': 'project-cache',
|
||||
'1.20.0': 'engineering-evals',
|
||||
'1.20.1': 'task-control',
|
||||
'1.21.0': 'reproducible-env',
|
||||
'1.22.0': 'embedded-browser',
|
||||
'1.22.1': 'local-files',
|
||||
'1.23.0': 'release-efficiency',
|
||||
'1.24.0': 'existing-results',
|
||||
'1.24.1': 'delivery-reliability',
|
||||
'1.24.2': 'delivery-integrity',
|
||||
'1.24.3': 'model-selection',
|
||||
'1.24.5': 'live-context-o',
|
||||
'1.24.6': 'network-q',
|
||||
'1.24.7': 'reading-s',
|
||||
'1.24.9': 'adoption-ab',
|
||||
'1.25.0': 'task-agreement',
|
||||
});
|
||||
export function releaseEvidencePrefix(version) {
|
||||
const prefix = prefixes[version];
|
||||
if (!prefix) throw new Error('Review evidence paths when introducing release ' + version + '.');
|
||||
return prefix;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pytest==8.4.2
|
||||
@@ -0,0 +1,14 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
export function archiveWindowsDirectory(directory, destination) {
|
||||
if (fs.existsSync(destination)) throw new Error('Archive output already exists; preserve the original candidate.');
|
||||
const quote = value => "'" + value.replaceAll("'", "''") + "'";
|
||||
// Windows PowerShell runs on .NET Framework. Namespace both paths so deeply
|
||||
// nested source/evidence files do not fall back to the legacy MAX_PATH limit.
|
||||
const command = "$ErrorActionPreference='Stop'; Add-Type -AssemblyName System.IO.Compression.FileSystem; [IO.Compression.ZipFile]::CreateFromDirectory("
|
||||
+ quote(path.toNamespacedPath(path.resolve(directory))) + ', ' + quote(path.toNamespacedPath(path.resolve(destination)))
|
||||
+ ', [IO.Compression.CompressionLevel]::Optimal, $true)';
|
||||
execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', command], { windowsHide: true, stdio: 'pipe' });
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// Self-owned inert artifact fixtures. These bytes are only parsed by REA;
|
||||
// neither the JavaScript bootstrap nor the synthetic PE image is executed.
|
||||
export function ownedAsar(files) {
|
||||
let offset = 0; const directory = { files: {} }, bodies = [];
|
||||
for (const [name, value] of Object.entries(files)) {
|
||||
const bytes = Buffer.from(value, 'utf8'); directory.files[name] = { size: bytes.length, offset: String(offset) };
|
||||
offset += bytes.length; bodies.push(bytes);
|
||||
}
|
||||
const json = Buffer.from(JSON.stringify(directory)), header = Buffer.alloc(8 + Math.ceil(json.length / 4) * 4);
|
||||
header.writeUInt32LE(header.length - 4, 0); header.writeUInt32LE(json.length, 4); json.copy(header, 8);
|
||||
const size = Buffer.alloc(8); size.writeUInt32LE(4, 0); size.writeUInt32LE(header.length, 4);
|
||||
return Buffer.concat([size, header, ...bodies]);
|
||||
}
|
||||
|
||||
export function ownedPe() {
|
||||
const bytes = Buffer.alloc(1024), pe = 0x80, optional = pe + 24, section = optional + 0xf0;
|
||||
bytes.write('MZ'); bytes.writeUInt32LE(pe, 0x3c); bytes.write('PE\0\0', pe, 'ascii');
|
||||
bytes.writeUInt16LE(0x8664, pe + 4); bytes.writeUInt16LE(1, pe + 6); bytes.writeUInt16LE(0xf0, pe + 20); bytes.writeUInt16LE(0x2022, pe + 22);
|
||||
bytes.writeUInt16LE(0x20b, optional); bytes.writeBigUInt64LE(0x140000000n, optional + 24);
|
||||
bytes.writeUInt32LE(0x1000, optional + 32); bytes.writeUInt32LE(0x200, optional + 36);
|
||||
bytes.writeUInt16LE(6, optional + 40); bytes.writeUInt16LE(6, optional + 48);
|
||||
bytes.writeUInt32LE(0x2000, optional + 56); bytes.writeUInt32LE(0x200, optional + 60); bytes.writeUInt16LE(3, optional + 68);
|
||||
bytes.writeUInt32LE(16, optional + 108);
|
||||
bytes.writeUInt32LE(0x1000, optional + 120); bytes.writeUInt32LE(40, optional + 124);
|
||||
bytes.writeUInt32LE(0x1100, optional + 128); bytes.writeUInt32LE(0x80, optional + 132);
|
||||
bytes.write('.rdata', section, 'ascii'); bytes.writeUInt32LE(512, section + 8); bytes.writeUInt32LE(0x1000, section + 12);
|
||||
bytes.writeUInt32LE(512, section + 16); bytes.writeUInt32LE(0x200, section + 20); bytes.writeUInt32LE(0x40000040, section + 36);
|
||||
const raw = rva => rva - 0x1000 + 0x200;
|
||||
bytes.writeUInt32LE(0x1040, raw(0x1000)); bytes.writeUInt32LE(0x1080, raw(0x1000) + 12); bytes.writeUInt32LE(0x1060, raw(0x1000) + 16);
|
||||
bytes.writeBigUInt64LE(0x1090n, raw(0x1040)); bytes.writeBigUInt64LE(0x1090n, raw(0x1060));
|
||||
bytes.write('KERNEL32.dll\0', raw(0x1080), 'ascii'); bytes.write('ExitProcess\0', raw(0x1090) + 2, 'ascii');
|
||||
for (const [rva, name, child] of [[0x1100, 10, 0x80000020], [0x1120, 1, 0x80000040], [0x1140, 0x409, 0x60]]) {
|
||||
bytes.writeUInt16LE(1, raw(rva) + 14); bytes.writeUInt32LE(name, raw(rva) + 16); bytes.writeUInt32LE(child, raw(rva) + 20);
|
||||
}
|
||||
bytes.writeUInt32LE(0x1170, raw(0x1160)); bytes.writeUInt32LE(8, raw(0x1160) + 4); bytes.write('REA-DATA', raw(0x1170), 'ascii');
|
||||
return bytes;
|
||||
}
|
||||
|
||||
export function createResearchSamples(directory) {
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const marker = path.join(directory, 'bootstrap-executed.txt');
|
||||
const asar = path.join(directory, 'owned-agent.asar'), pe = path.join(directory, 'owned-imports-resources.dll');
|
||||
fs.writeFileSync(asar, ownedAsar({
|
||||
'package.json': JSON.stringify({ name: 'owned-agent-fixture', version: '1.0.0', main: 'main.js' }),
|
||||
'main.js': "const fs = require('node:fs');\nfs.writeFileSync(" + JSON.stringify(marker) + ", 'must not execute');\nexports.inspect = async file => fs.promises.readFile(file, 'utf8');\n",
|
||||
'preload.js': "export const contract = { approval: 'per-call', replay: false };\n",
|
||||
}), { flag: 'wx' });
|
||||
fs.writeFileSync(pe, ownedPe(), { flag: 'wx' });
|
||||
return { asar, pe, marker };
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { snapshotPersistentData } from '../src/core/desktop/install-manager.mjs';
|
||||
export const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
export const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object' ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
export const digest = value => hash(JSON.stringify(canonical(value)));
|
||||
export function databaseSnapshot(dataRoot) {
|
||||
const database = new DatabaseSync(path.join(dataRoot, 'workbench.sqlite'), { readOnly: true }); database.exec('BEGIN');
|
||||
try {
|
||||
assert.equal(database.prepare('PRAGMA quick_check').get().quick_check, 'ok');
|
||||
const tables = {};
|
||||
for (const { name, sql } of database.prepare("SELECT name,sql FROM sqlite_schema WHERE type='table' ORDER BY name").all()) {
|
||||
assert.match(name, /^[a-z_]+$/);
|
||||
const rows = database.prepare('SELECT * FROM "' + name + '" ORDER BY rowid').all().filter(row => !(name === 'metadata' && row.key === 'runtimeOwner'));
|
||||
tables[name] = { count: rows.length, schemaSha256: hash(sql), sha256: digest(rows), rows: rows.map((row, index) => ({ id: row.id ?? row.key ?? row.seq ?? index, ...(row.kind ? { kind: row.kind } : {}), sha256: digest(row) })) };
|
||||
}
|
||||
const sessionIds = database.prepare('SELECT id FROM sessions ORDER BY id').all().map(row => row.id);
|
||||
const owner = database.prepare("SELECT value FROM metadata WHERE key='runtimeOwner'").get();
|
||||
return { tables, sessionIds, ownerSha256: owner ? digest(owner) : null, scope: 'Every original table schema and row field hashed, excluding only runtimeOwner lifecycle metadata. No row payload or credential is returned.' };
|
||||
} finally { database.exec('ROLLBACK'); database.close(); }
|
||||
}
|
||||
export async function publicSnapshot(url, dataRoot) {
|
||||
const healthResponse = await fetch(url + '/api/health', { headers: { Connection: 'close' }, signal: AbortSignal.timeout(20000) }); assert.equal(healthResponse.status, 200);
|
||||
const health = await healthResponse.json(), bootResponse = await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' }, signal: AbortSignal.timeout(20000) }); assert.equal(bootResponse.status, 200);
|
||||
const bootstrap = await bootResponse.json(); const token = bootstrap.token;
|
||||
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': token, Connection: 'close' }, signal: AbortSignal.timeout(20000) }); assert.ok(response.ok, route + ': HTTP ' + response.status); return response.json(); };
|
||||
const database = databaseSnapshot(dataRoot), exports = [];
|
||||
for (const id of database.sessionIds) {
|
||||
const report = await api('/sessions/' + id + '/export');
|
||||
exports.push({ id, project: digest(report.project), session: digest(report.session), execution: digest(report.execution) });
|
||||
}
|
||||
const center = await api('/model-center');
|
||||
// This is the documented public model-center view; Provider's private map and
|
||||
// API key are never accessed. Only this public object is retained for schema-aware comparison.
|
||||
const modelCenter = structuredClone(center);
|
||||
return { observedAt: new Date().toISOString(), version: health.version, pid: health.pid, runtimeRoot: bootstrap.runtimeRoot,
|
||||
configSha256: digest(bootstrap.config), projectsSha256: digest(bootstrap.projects), sessionsSha256: digest(bootstrap.sessions), modelCenterSha256: digest(modelCenter), modelCenter,
|
||||
projects: bootstrap.projects.length, visibleSessions: bootstrap.sessions.length, hasKey: !!bootstrap.config.hasKey,
|
||||
active: bootstrap.sessions.filter(value => ['running', 'awaiting_approval', 'stopping'].includes(value.status)).map(value => ({ id: value.id, status: value.status })),
|
||||
storage: bootstrap.storage, eventCursor: bootstrap.eventCursor, database, exports, persistent: snapshotPersistentData(dataRoot),
|
||||
scope: 'Read-only public state, every original session export digest, original table/row/schema hashes and every original artifact byte hash. No credential extraction, model probe, execution or mutation.' };
|
||||
}
|
||||
export function compareOriginalState(before, after, { modelCenterSchemaChange = false } = {}) {
|
||||
for (const field of ['pid', 'hasKey', 'configSha256', 'projectsSha256', 'sessionsSha256', 'eventCursor']) assert.deepEqual(after[field], before[field], field);
|
||||
assert.deepEqual(canonical(after.database.tables), canonical(before.database.tables), 'All original table schemas and every persistent row');
|
||||
assert.deepEqual(after.database.sessionIds, before.database.sessionIds); assert.deepEqual(after.exports, before.exports);
|
||||
assert.deepEqual(after.persistent, before.persistent, 'Every original artifact byte, persistent row and public disk config');
|
||||
if (modelCenterSchemaChange) {
|
||||
const original = structuredClone(before.modelCenter), next = structuredClone(after.modelCenter);
|
||||
// 1.15 changes only the public explanatory text about optional DPAPI storage.
|
||||
// Existing IDs, routes, revisions, credentials-present bits and all original attempts remain exact.
|
||||
assert.match(next.storage.restart, /临时凭据.*已明确本机加密保存/);
|
||||
next.storage.restart = original.storage.restart;
|
||||
assert.deepEqual(next, original, 'Original public model center, except versioned storage explanation');
|
||||
return { modelCenterChange: { path: '/storage/restart', original: before.modelCenter.storage.restart, actual: after.modelCenter.storage.restart }, originalRows: 'exact', originalExports: 'exact', originalArtifactBytes: 'exact' };
|
||||
}
|
||||
assert.equal(after.modelCenterSha256, before.modelCenterSha256);
|
||||
return { originalRows: 'exact', originalExports: 'exact', originalArtifactBytes: 'exact', originalModelCenter: 'exact' };
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
const root = path.resolve('docs/turn-progress-b-evidence'), qa = path.resolve('.qa/requirements-progress');
|
||||
assert.equal(fs.existsSync(root), false, 'Do not overwrite an existing stage seal.'); fs.mkdirSync(root);
|
||||
const digest = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const copy = (source, relative) => {
|
||||
assert.equal(fs.lstatSync(source).isSymbolicLink(), false); const target = path.join(root, relative);
|
||||
if (fs.statSync(source).isDirectory()) { fs.mkdirSync(target, { recursive: true }); for (const name of fs.readdirSync(source)) copy(path.join(source, name), relative + '/' + name); }
|
||||
else { fs.mkdirSync(path.dirname(target), { recursive: true }); fs.copyFileSync(source, target, fs.constants.COPYFILE_EXCL); }
|
||||
};
|
||||
const sourceFiles = [
|
||||
'src/core/context/turn-progress.mjs', 'src/core/context/turn-progress-recorder.mjs', 'src/core/delivery-report.mjs', 'src/server/runtime.mjs',
|
||||
'src/client/TurnProgress.tsx', 'src/client/TaskPanel.tsx', 'src/client/TaskPanel.css', 'src/client/types.ts', 'tests/turn-progress.test.mjs',
|
||||
'scripts/verify-turn-progress-ui.mjs', 'scripts/verify-turn-progress-evidence.mjs', 'scripts/verify-adoption-evidence.mjs', 'scripts/seal-turn-progress-evidence.mjs',
|
||||
].map(file => { const snapshot = 'source/' + file; copy(path.resolve(file), snapshot); return { path: file, snapshot }; });
|
||||
const testRuns = [
|
||||
['shadow-first.tap', 11, 9, 2], ['shadow-second.tap', 12, 12, 0], ['shadow-related.tap', 68, 68, 0],
|
||||
['shadow-final.tap', 31, 31, 0], ['shadow-source-final.tap', 31, 31, 0],
|
||||
].map(([name, tests, passed, failed]) => { copy(path.join(qa, name), 'logs/' + name); return { path: 'logs/' + name, tests, passed, failed }; });
|
||||
for (const name of ['build-shadow-first.log', 'build-shadow-second.log', 'build-shadow-final.log', 'build-shadow-check-wait.log', 'build-shadow-source-final.log', 'official-readonly-boundary.json']) copy(path.join(qa, name), 'logs/' + name);
|
||||
const uiRuns = [], observedRuns = [];
|
||||
for (const [name, folder, final] of [
|
||||
['shadow-ui-first', 'web-first', false], ['shadow-electron-first', 'electron-first', false],
|
||||
['shadow-ui-final', 'web-touch', false], ['shadow-electron-final', 'electron-touch', false],
|
||||
['shadow-ui-source-final', 'web-final', true], ['shadow-electron-source-final', 'electron-final', true],
|
||||
]) {
|
||||
const log = fs.readFileSync(path.join(qa, name + '.log'), 'utf8'); copy(path.join(qa, name + '.log'), 'logs/' + name + '.log');
|
||||
const result = JSON.parse(log.trim().split(/\r?\n/).at(-1)), source = path.resolve(result.directory), relative = 'ui/' + folder;
|
||||
assert.ok(source.startsWith(qa + path.sep)); assert.equal(result.success, true); copy(source, relative);
|
||||
uiRuns.push({ path: relative + '/result.json', success: true, passed: 9 }); if (final) observedRuns.push(relative);
|
||||
}
|
||||
const caseReview = {
|
||||
scope: 'Fixed public-fact cases manually reviewed against original assertions; no population accuracy or model benefit claim.',
|
||||
cases: [
|
||||
{ case: 'normal_exploration', expected: 'new observed content is evidence growth', evidence: 'logs/shadow-source-final.tap', assertion: 'fixed normal exploration, pagination, SHA changes and repeated ranges' },
|
||||
{ case: 'pagination_and_repetition', expected: '10 new + 10 new + repeated candidate, original step_limit', evidence: 'ui/web-final/pages-original.json' },
|
||||
{ case: 'read_only_research', expected: 'reading is useful; summary wording is not additional engineering evidence; no shadow interruption', evidence: 'logs/shadow-source-final.tap', assertion: 'read-only research is valid work' },
|
||||
{ case: 'long_check', expected: 'approved actual running job stays waiting, then counted result', evidence: 'logs/shadow-source-final.tap', assertion: 'actual approved long check' },
|
||||
{ case: 'failure_repair_evidence', expected: 'applied file, actual counted check and current requirement proof grow after baseline', evidence: 'logs/shadow-source-final.tap', assertion: 'real applied files, counted checks and current requirement proofs' },
|
||||
{ case: 'permission_denial', expected: 'waiting then original permission failure; earlier same-turn read consumed once', evidence: 'ui/electron-final/approval-denied.json' },
|
||||
{ case: 'broken_tool_or_network', expected: 'new failure is evidence; original failure categories remain distinct', evidence: 'logs/shadow-source-final.tap', assertion: 'new permissions, network and broken tools' },
|
||||
{ case: 'invalid_input_repetition', expected: 'repeat candidate, no new early stop; original four-request step_limit', evidence: 'logs/shadow-source-final.tap', assertion: 'actual invalid tool calls repeat under the original step budget' },
|
||||
{ case: 'bounded_unknown', expected: 'unretained facts stay unknown; old receipts retained and scope cannot mix', evidence: 'logs/shadow-source-final.tap', assertion: 'receipt retention stays aligned at the cap' },
|
||||
],
|
||||
limitations: ['No real model delivery sample.', 'No intervention thresholds or global false-positive/false-negative rate.', 'No Purge comparison or network recovery claim.'],
|
||||
};
|
||||
fs.writeFileSync(path.join(root, 'case-review.json'), JSON.stringify(caseReview, null, 2));
|
||||
const walk = directory => fs.readdirSync(directory, { withFileTypes: true }).flatMap(row => row.isDirectory() ? walk(path.join(directory, row.name)) : [path.join(directory, row.name)]);
|
||||
const files = walk(root).map(file => { const bytes = fs.readFileSync(file); return { path: path.relative(root, file).replaceAll('\\', '/'), bytes: bytes.length, sha256: digest(bytes) }; }).sort((a, b) => a.path.localeCompare(b.path));
|
||||
const manifest = { kind: 'adoption-stage-evidence', stage: 'turn-progress-b', status: 'development_slice_validated_not_released',
|
||||
realModelRequests: 0, officialServiceChanged: false, sourceFiles, testRuns, uiRuns, observedRuns, files, bytes: files.reduce((sum, row) => sum + row.bytes, 0) };
|
||||
fs.writeFileSync(path.join(root, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
||||
console.log(JSON.stringify({ root, files: files.length, bytes: manifest.bytes, sourceFiles: sourceFiles.length, uiRuns: uiRuns.length }));
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import net from 'node:net';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { publicSnapshot, compareOriginalState, hash } from './retained-host-state.mjs';
|
||||
import { inspectHostInstallation } from '../src/core/desktop/app-installation.mjs';
|
||||
|
||||
// Explicit local release stages. Never sends a model request, stops an existing
|
||||
// task/window, reads a private credential map, or retries an unknown installer.
|
||||
const root = path.resolve('.qa/task-agreement-release/formal'), url = 'http://127.0.0.1:4318', mode = process.argv[2];
|
||||
assert.ok(['before', 'before-recheck', 'upgrade', 'final'].includes(mode)); fs.mkdirSync(root, { recursive: true });
|
||||
const baseline = mode === 'before' || mode === 'before-recheck';
|
||||
const read = file => JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
|
||||
const save = (name, value) => fs.writeFileSync(path.join(root, name), JSON.stringify(value, null, 2) + '\n', { flag: 'wx' });
|
||||
assert.equal(fs.existsSync(path.join(root, mode + '.json')), false, 'Preserve the original attempt; no automatic replay.');
|
||||
async function api(route) {
|
||||
const boot = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' }, signal: AbortSignal.timeout(20000) })).json();
|
||||
const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': boot.token, Connection: 'close' }, signal: AbortSignal.timeout(20000) });
|
||||
assert.equal(response.status, 200, route); return response.json();
|
||||
}
|
||||
async function inspectorClosed() {
|
||||
await new Promise((resolve, reject) => { const socket = net.createConnection({ host: '127.0.0.1', port: 9229 }); socket.setTimeout(1500);
|
||||
socket.once('connect', () => { socket.destroy(); reject(Error('Inspector is already open.')); });
|
||||
socket.once('error', error => { socket.destroy(); error.code === 'ECONNREFUSED' ? resolve() : reject(error); });
|
||||
socket.once('timeout', () => { socket.destroy(); reject(Error('Inspector state unknown.')); }); });
|
||||
}
|
||||
const startedAt = new Date().toISOString(); let result;
|
||||
try {
|
||||
await inspectorClosed(); const host = await api('/host/status'), dataRoot = host.identity.dataRoot;
|
||||
assert.equal(host.identity.pid, 36552); assert.equal(host.activity.active, false);
|
||||
// An open idle window can be recorded by a read-only baseline. Installation
|
||||
// and the final retention check still require every registered window closed.
|
||||
if (!baseline) assert.equal(host.desktopWindows.count, 0);
|
||||
const installRoot = path.dirname(path.dirname(host.identity.runtimeRoot)); assert.equal(path.basename(path.dirname(host.identity.runtimeRoot)), 'versions');
|
||||
const current = await publicSnapshot(url, dataRoot); assert.equal(current.pid, 36552); assert.equal(current.hasKey, true); assert.deepEqual(current.active, []);
|
||||
if (baseline) {
|
||||
assert.equal(current.version, '1.24.9'); save('state-before.json', current); save('host-before.json', host); save('pointer-before.json', read(path.join(installRoot, 'current.json')));
|
||||
result = { success: true, version: current.version, pid: current.pid, dataRoot, installRoot, projects: current.projects,
|
||||
sessions: current.database.sessionIds.length, desktopWindows: host.desktopWindows.count,
|
||||
tables: Object.fromEntries(Object.entries(current.database.tables).map(([name, table]) => [name, table.count])) };
|
||||
} else {
|
||||
const before = read(path.join(root, 'state-before.json')); compareOriginalState(before, current);
|
||||
const candidate = read('.qa/task-agreement-release/candidate.json'), freeze = read('.qa/task-agreement-release/freeze.json');
|
||||
assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
|
||||
for (const input of freeze.inputs) assert.equal(hash(fs.readFileSync(input.path)), input.sha256, 'Frozen input drift.');
|
||||
for (const stage of ['rebuild', 'archives', 'portable', 'install', 'source-a', 'source-a-desktop', 'source-b', 'source-b-desktop', 'portable-a', 'portable-a-desktop', 'portable-b', 'portable-b-desktop', 'portable-files', 'source-c', 'source-c-desktop', 'source-d', 'source-d-desktop', 'portable-c', 'portable-c-desktop', 'portable-d', 'portable-d-desktop']) {
|
||||
const receipt = read('.qa/task-agreement-release/receipts/' + stage + '.json'); assert.equal(receipt.success, true, stage);
|
||||
assert.equal(receipt.sourceCommit, freeze.sourceCommit); assert.equal(receipt.candidate, candidate.portableRoot); assert.equal(hash(fs.readFileSync(receipt.log)), receipt.logSha256);
|
||||
}
|
||||
if (mode === 'upgrade') {
|
||||
assert.equal(current.version, before.version); assert.deepEqual(read(path.join(installRoot, 'current.json')), read(path.join(root, 'pointer-before.json')));
|
||||
assert.equal(hash(fs.readFileSync(candidate.installer.path)), candidate.installer.sha256);
|
||||
save('candidate.json', candidate); save('preinstall-state.json', current);
|
||||
const fd = fs.openSync(path.join(root, 'installer-process.log'), 'wx'); let code;
|
||||
try { code = await new Promise((resolve, reject) => {
|
||||
const child = spawn(candidate.installer.path, ['/VERYSILENT', '/SUPPRESSMSGBOXES', '/NORESTART', '/LOG=' + path.join(root, 'installer.log'), '/DIR=' + installRoot, '/DATA_ROOT=' + dataRoot, '/PORT=4318'], { windowsHide: true, stdio: ['ignore', fd, fd] });
|
||||
child.once('error', reject); child.once('close', resolve);
|
||||
}); } finally { fs.closeSync(fd); }
|
||||
save('installer-exit.json', { code, finishedAt: new Date().toISOString() }); assert.equal(code, 0, 'Do not replay a failed/unknown formal installation.');
|
||||
} else { assert.equal(read(path.join(root, 'upgrade.json')).success, true); assert.deepEqual(candidate, read(path.join(root, 'candidate.json'))); }
|
||||
const after = await publicSnapshot(url, dataRoot), retention = compareOriginalState(before, after), afterHost = await api('/host/status');
|
||||
assert.equal(after.version, '1.25.0'); assert.equal(after.pid, before.pid);
|
||||
assert.equal(afterHost.identity.instanceId, read(path.join(root, 'host-before.json')).identity.instanceId);
|
||||
const installed = path.join(installRoot, 'versions', path.basename(candidate.portableRoot)); assert.equal(fs.realpathSync.native(after.runtimeRoot), fs.realpathSync.native(installed));
|
||||
const identity = inspectHostInstallation(installed), pointer = read(path.join(installRoot, 'current.json'));
|
||||
assert.equal(pointer.active.directory, path.basename(installed)); assert.equal(pointer.active.manifestSha256, identity.manifestSha256); assert.equal(pointer.active.runtimeIdentity, identity.identitySha256);
|
||||
const operation = fs.readdirSync(path.join(installRoot, '.installation')).filter(name => /^progress-.*\.json$/.test(name)).map(name => read(path.join(installRoot, '.installation', name))).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0];
|
||||
assert.equal(operation.state, 'completed'); assert.equal(operation.candidate.manifestSha256, identity.manifestSha256); await inspectorClosed();
|
||||
save(mode + '-state-after.json', after); result = { success: true, version: after.version, pid: after.pid, runtimeRoot: after.runtimeRoot,
|
||||
projects: after.projects, sessions: after.database.sessionIds.length, hasKey: after.hasKey, retention, sourceCommit: freeze.sourceCommit,
|
||||
pointer, operation, inspectorClosed: true, desktopWindows: afterHost.desktopWindows.count };
|
||||
}
|
||||
} catch (error) { result = { success: false, error: error.stack }; process.exitCode = 1; }
|
||||
result = { ...result, mode, startedAt, finishedAt: new Date().toISOString(), realModelRequests: 0, keyReadOrResaved: false };
|
||||
save(mode + '.json', result); console.log(JSON.stringify({ success: result.success, mode, version: result.version, pid: result.pid, error: result.error }));
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync, spawn } from 'node:child_process';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
|
||||
const root = path.resolve('.qa/task-agreement-release'), mode = process.argv[2];
|
||||
const read = file => JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
|
||||
const save = (name, value) => { const file = path.join(root, name); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', { flag: 'wx' }); };
|
||||
const hash = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
const record = file => ({ path: file.replaceAll('\\', '/'), bytes: fs.statSync(file).size, sha256: hash(file) });
|
||||
const walk = directory => fs.readdirSync(directory, { withFileTypes: true }).flatMap(row => row.isDirectory() ? walk(path.join(directory, row.name)) : [path.join(directory, row.name)]);
|
||||
const python = 'C:/Users/wangy/AppData/Local/Programs/Python/Python312/python.exe';
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
if (mode === 'freeze') {
|
||||
assert.equal(execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8', windowsHide: true }).trim(), '');
|
||||
assert.equal(read('package.json').version, '1.25.0');
|
||||
const inputs = execFileSync('git', ['ls-files', '-z', '--', 'src', 'public', 'scripts', 'tests', 'examples', 'benchmarks', 'research', 'package.json', 'package-lock.json', 'vite.config.ts', 'tsconfig.json', 'index.html', 'mobile.html'], { encoding: 'utf8', windowsHide: true }).split('\0').filter(Boolean).sort();
|
||||
const freeze = { sourceCommit: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }).trim(), version: '1.25.0',
|
||||
inputs: inputs.map(record), compiled: ['dist-client', 'dist-server', 'dist-desktop'].flatMap(walk).sort().map(record), realModelRequests: 0 };
|
||||
save('freeze.json', freeze); console.log(JSON.stringify({ success: true, sourceCommit: freeze.sourceCommit, inputs: freeze.inputs.length, compiled: freeze.compiled.length })); process.exit(0);
|
||||
}
|
||||
const freeze = read(path.join(root, 'freeze.json'));
|
||||
for (const row of freeze.inputs) assert.equal(hash(row.path), row.sha256, 'Frozen input drift: ' + row.path);
|
||||
if (mode === 'register') {
|
||||
const candidate = read('releases/latest.json'); assert.equal(candidate.documentation.sourceCommit, freeze.sourceCommit);
|
||||
assert.equal(read(path.join(candidate.portableRoot, 'MANIFEST.json')).version, freeze.version);
|
||||
for (const row of freeze.compiled) assert.equal(hash(path.join(candidate.portableRoot, row.path)), row.sha256);
|
||||
save('candidate.json', candidate); console.log(JSON.stringify({ success: true, candidate: candidate.portableRoot })); process.exit(0);
|
||||
}
|
||||
const candidate = read(path.join(root, 'candidate.json')); assert.deepEqual(read('releases/latest.json'), candidate);
|
||||
const commands = {
|
||||
rebuild: [process.execPath, ['scripts/verify-packaged-build.mjs']],
|
||||
archives: [python, ['-I', 'scripts/verify-archives.py']],
|
||||
portable: [process.execPath, ['scripts/verify-release.mjs']],
|
||||
install: [process.execPath, ['scripts/verify-desktop-install.mjs']],
|
||||
'source-a': [process.execPath, ['scripts/verify-requirements-ui.mjs']],
|
||||
'source-a-desktop': [process.execPath, ['scripts/verify-requirements-desktop.mjs']],
|
||||
'source-b': [process.execPath, ['scripts/verify-turn-progress-ui.mjs']],
|
||||
'source-b-desktop': [process.execPath, ['scripts/verify-turn-progress-ui.mjs', '--desktop']],
|
||||
'portable-a': [process.execPath, ['scripts/verify-requirements-ui.mjs']],
|
||||
'portable-a-desktop': [process.execPath, ['scripts/verify-requirements-desktop.mjs']],
|
||||
'portable-b': [process.execPath, ['scripts/verify-turn-progress-ui.mjs']],
|
||||
'portable-b-desktop': [process.execPath, ['scripts/verify-turn-progress-ui.mjs', '--desktop']],
|
||||
'portable-files': [process.execPath, ['scripts/verify-local-files.mjs']],
|
||||
'source-c': [process.execPath, ['scripts/verify-task-agreement-ui.mjs']],
|
||||
'source-c-desktop': [process.execPath, ['scripts/verify-task-agreement-ui.mjs', '--desktop']],
|
||||
'source-d': [process.execPath, ['scripts/verify-task-follow-up-ui.mjs']],
|
||||
'source-d-desktop': [process.execPath, ['scripts/verify-task-follow-up-ui.mjs', '--desktop']],
|
||||
'portable-c': [process.execPath, ['scripts/verify-task-agreement-ui.mjs']],
|
||||
'portable-c-desktop': [process.execPath, ['scripts/verify-task-agreement-ui.mjs', '--desktop']],
|
||||
'portable-d': [process.execPath, ['scripts/verify-task-follow-up-ui.mjs']],
|
||||
'portable-d-desktop': [process.execPath, ['scripts/verify-task-follow-up-ui.mjs', '--desktop']],
|
||||
};
|
||||
assert.ok(commands[mode], 'Unknown release stage.');
|
||||
const output = path.join(root, 'receipts', mode + '.json'), log = path.join(root, 'logs', mode + '.log');
|
||||
assert.equal(fs.existsSync(output), false, 'Preserve previous stage receipt; do not replay unknown work.'); fs.mkdirSync(path.dirname(log), { recursive: true });
|
||||
const fd = fs.openSync(log, 'wx'), startedAt = new Date().toISOString();
|
||||
const env = projectEnvironment({ AGENT_WORKBENCH_TEST_PYTHON: python,
|
||||
...((mode.startsWith('portable-') || mode === 'install') ? { AGENT_WORKBENCH_QA_APP_ROOT: candidate.portableRoot } : {}) });
|
||||
let result, code;
|
||||
try {
|
||||
console.log('START ' + mode);
|
||||
code = await new Promise((resolve, reject) => { const child = spawn(...commands[mode], { env, windowsHide: true, stdio: ['ignore', fd, fd] }); child.once('error', reject); child.once('close', resolve); });
|
||||
fs.closeSync(fd); assert.equal(code, 0, 'Inspect preserved log: ' + log);
|
||||
assert.deepEqual(read('releases/latest.json'), candidate);
|
||||
let file;
|
||||
if (mode === 'archives') file = '.qa/archives-validation.json';
|
||||
else if (mode === 'portable') file = '.qa/release-validation.json';
|
||||
else if (mode === 'install') file = read('.qa/desktop-install-location.json').result;
|
||||
else {
|
||||
const summaries = fs.readFileSync(log, 'utf8').trim().split(/\r?\n/).filter(line => line.startsWith('{')).map(line => { try { return JSON.parse(line); } catch { return null; } });
|
||||
const summary = summaries.findLast(row => row?.success !== undefined); assert.ok(summary);
|
||||
file = summary.output ?? path.join(summary.directory ?? summary.root, 'result.json');
|
||||
}
|
||||
const value = read(file); assert.equal(value.success, true);
|
||||
const count = { a: 7, 'a-desktop': 3, b: 9, 'b-desktop': 9, c: 6, 'c-desktop': 6, d: 5, 'd-desktop': 5 }[mode.replace(/^(source|portable)-/, '')];
|
||||
if (count) { assert.equal(value.passed.length, count); assert.deepEqual(value.errors, []); assert.equal(value.realModelRequests, 0); }
|
||||
if (mode !== 'archives') assert.equal(value.version, freeze.version);
|
||||
if (mode.startsWith('portable-')) assert.equal(fs.realpathSync.native(value.appRoot), fs.realpathSync.native(candidate.portableRoot));
|
||||
if (mode === 'install') assert.equal(fs.realpathSync.native(value.packagedSource), fs.realpathSync.native(candidate.portableRoot));
|
||||
if (mode === 'rebuild') assert.equal(value.sourceCommit, freeze.sourceCommit);
|
||||
result = { success: true, resultPath: path.resolve(file), result: value };
|
||||
} catch (error) { result = { success: false, code, error: error.stack }; process.exitCode = 1; }
|
||||
result = { ...result, mode, log, logSha256: hash(log), startedAt, finishedAt: new Date().toISOString(), sourceCommit: freeze.sourceCommit, candidate: candidate.portableRoot, realModelRequests: 0 };
|
||||
save('receipts/' + mode + '.json', result); console.log(JSON.stringify({ success: result.success, mode, output, resultPath: result.resultPath, error: result.error }));
|
||||
@@ -0,0 +1,23 @@
|
||||
import fs from 'node:fs';
|
||||
import postcss from 'postcss';
|
||||
|
||||
const file = 'src/client/styles.css';
|
||||
const root = postcss.parse(fs.readFileSync(file, 'utf8'));
|
||||
const rgb = hex => hex.match(/../g).map(value => parseInt(value, 16));
|
||||
const luminance = color => color.map(value => value / 255).map(value => value <= 0.04045 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4)
|
||||
.reduce((sum, value, index) => sum + value * [0.2126, 0.7152, 0.0722][index], 0);
|
||||
const ink = rgb('1c2e32');
|
||||
const background = luminance(rgb('edf3f1'));
|
||||
root.walkDecls(declaration => {
|
||||
if (declaration.prop === 'font-size' && /^\d+px$/.test(declaration.value) && parseInt(declaration.value) < 11) declaration.value = '11px';
|
||||
if (declaration.prop !== 'color' || !/^#[0-9a-f]{6}$/i.test(declaration.value)) return;
|
||||
if (/\.toast|\.brand-mark|\.send-button/.test(declaration.parent.selector ?? '')) return;
|
||||
const original = rgb(declaration.value.slice(1));
|
||||
let color = original;
|
||||
for (let mix = 0; (background + 0.05) / (luminance(color) + 0.05) < 4.7 && mix <= 1; mix += 0.03) {
|
||||
color = original.map((value, index) => Math.round(value * (1 - mix) + ink[index] * mix));
|
||||
}
|
||||
declaration.value = '#' + color.map(value => value.toString(16).padStart(2, '0')).join('');
|
||||
});
|
||||
fs.writeFileSync(file, root.toString(), 'utf8');
|
||||
console.log('Text contrast and minimum caption sizes calibrated.');
|
||||
@@ -0,0 +1,168 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
|
||||
function localUrl(value) {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== 'http:' || url.hostname !== '127.0.0.1' || url.username || url.password || url.search || url.hash) throw new Error('Upgrade only accepts the local Workbench address.');
|
||||
return url.origin;
|
||||
}
|
||||
|
||||
async function within(promise, milliseconds, label) {
|
||||
let timer;
|
||||
try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(label + ' timed out.')), milliseconds); })]); }
|
||||
finally { clearTimeout(timer); }
|
||||
}
|
||||
|
||||
async function readUpgradeJson(address, milliseconds, stage) {
|
||||
try {
|
||||
// Inspector pauses can outlast a server's keep-alive timeout. Use a fresh
|
||||
// connection for each control request rather than reuse a stale pooled one.
|
||||
const response = await fetch(address, { headers: { Connection: 'close' }, signal: AbortSignal.timeout(milliseconds) });
|
||||
if (!response.ok) throw new Error('HTTP ' + response.status);
|
||||
return await response.json();
|
||||
} catch (error) {
|
||||
throw new Error(stage + ': ' + error.message + (error.cause?.code ? ' (' + error.cause.code + ')' : ''), { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
class Protocol {
|
||||
constructor(socket) {
|
||||
this.socket = socket; this.nextId = 1; this.pending = new Map(); this.listeners = new Map();
|
||||
socket.addEventListener('message', event => {
|
||||
const message = JSON.parse(String(event.data));
|
||||
if (message.id) {
|
||||
const pending = this.pending.get(message.id); if (!pending) return;
|
||||
this.pending.delete(message.id);
|
||||
if (message.error) pending.reject(new Error(message.error.message)); else pending.resolve(message.result);
|
||||
} else for (const listener of this.listeners.get(message.method) ?? []) listener(message.params);
|
||||
});
|
||||
socket.addEventListener('close', () => { for (const pending of this.pending.values()) pending.reject(new Error('Inspector connection closed.')); this.pending.clear(); });
|
||||
}
|
||||
on(method, listener) { this.listeners.set(method, [...(this.listeners.get(method) ?? []), listener]); }
|
||||
send(method, params = {}) {
|
||||
return within(new Promise((resolve, reject) => {
|
||||
const id = this.nextId++; this.pending.set(id, { resolve, reject }); this.socket.send(JSON.stringify({ id, method, params }));
|
||||
}), 10000, method);
|
||||
}
|
||||
}
|
||||
|
||||
function sameFile(url, file) {
|
||||
try { return path.resolve(fileURLToPath(url)).toLowerCase() === path.resolve(file).toLowerCase(); } catch { return false; }
|
||||
}
|
||||
function sameTarget(target, file) {
|
||||
if (sameFile(target.url, file)) return true;
|
||||
const displayName = path.resolve(file).replaceAll('\\', '_').replaceAll('/', '_');
|
||||
return target.title === displayName && target.url === 'file://' + displayName;
|
||||
}
|
||||
|
||||
export async function upgradeRunning({ url, pid, currentRoot, newRoot, handlerRoot }) {
|
||||
const origin = localUrl(url), original = fs.realpathSync.native(currentRoot), next = fs.realpathSync.native(newRoot);
|
||||
const originalScript = path.join(original, 'dist-server', 'server.mjs');
|
||||
const upgradeScript = path.join(next, 'dist-server', 'hot-upgrade.mjs');
|
||||
if (!fs.existsSync(originalScript) || !fs.existsSync(upgradeScript) || !fs.existsSync(path.join(next, 'dist-client', 'index.html'))) throw new Error('Upgrade bundles are incomplete.');
|
||||
if (!Number.isInteger(pid) || pid < 1 || pid === process.pid) throw new Error('Invalid target process.');
|
||||
const before = await readUpgradeJson(origin + '/api/bootstrap', 5000, 'Read original Workbench');
|
||||
if (!before.token || !['0.1.0', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0', '1.5.1', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.20.2', '1.20.3', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.8', '1.24.9', '1.25.0'].includes(before.version)) throw new Error('This migration accepts a supported existing Workbench.');
|
||||
if (before.sessions.some(session => ['running', 'awaiting_approval', 'stopping'].includes(session.status))) throw new Error('Wait for existing tasks and approvals before upgrading.');
|
||||
let protocol, socket, enabledHere = false, paused = false;
|
||||
const breakpoints = [];
|
||||
try {
|
||||
let targets;
|
||||
try { targets = await readUpgradeJson('http://127.0.0.1:9229/json/list', 500, 'Discover inspector'); } catch {}
|
||||
if (targets && !targets.some(target => sameTarget(target, originalScript))) throw new Error('The local inspector port belongs to another process.');
|
||||
if (!targets) {
|
||||
process._debugProcess(pid); enabledHere = true;
|
||||
for (let attempt = 0; attempt < 40 && !targets; attempt++) {
|
||||
await delay(100);
|
||||
try { targets = await readUpgradeJson('http://127.0.0.1:9229/json/list', 500, 'Discover opened inspector'); } catch {}
|
||||
}
|
||||
}
|
||||
const target = targets?.find(item => sameTarget(item, originalScript));
|
||||
if (!target || !target.webSocketDebuggerUrl?.startsWith('ws://127.0.0.1:9229/')) throw new Error('The expected Workbench inspector was not found.');
|
||||
socket = new WebSocket(target.webSocketDebuggerUrl);
|
||||
await within(new Promise((resolve, reject) => { socket.addEventListener('open', resolve, { once: true }); socket.addEventListener('error', reject, { once: true }); }), 10000, 'Inspector connection');
|
||||
protocol = new Protocol(socket);
|
||||
const scripts = [];
|
||||
protocol.on('Debugger.scriptParsed', script => scripts.push(script));
|
||||
let pausedResolve;
|
||||
const pausedEvent = new Promise(resolve => { pausedResolve = resolve; });
|
||||
protocol.on('Debugger.paused', event => { paused = true; pausedResolve(event); });
|
||||
await protocol.send('Runtime.enable');
|
||||
const identity = await protocol.send('Runtime.evaluate', { expression: 'process.pid', returnByValue: true });
|
||||
if (identity.result.value !== pid) throw new Error('The inspector belongs to a different process.');
|
||||
await protocol.send('Debugger.enable');
|
||||
const previousRoot = handlerRoot || before.runtimeRoot;
|
||||
const expectedScripts = [originalScript, ...(previousRoot ? [path.join(fs.realpathSync.native(previousRoot), 'dist-server', 'hot-upgrade.mjs')] : [])];
|
||||
const candidates = scripts.filter(item => expectedScripts.some(file => sameFile(item.url, file)));
|
||||
if (!candidates.length) throw new Error('The existing server module is missing from inspector metadata.');
|
||||
for (const script of candidates) {
|
||||
const { scriptSource } = await protocol.send('Debugger.getScriptSource', { scriptId: script.scriptId });
|
||||
// Bundled transport libraries can parse request.url too. Select the
|
||||
// actual loopback handler with its local base and observed port binding.
|
||||
const locations = scriptSource.split('\n').flatMap((line, index) => /const url = new URL\(request\.url,\s*["']http:\/\/127\.0\.0\.1:["']\s*\+\s*actualPort\d*\)/.test(line) ? [index] : []);
|
||||
if (locations.length !== 1) throw new Error('The existing request handler could not be identified uniquely.');
|
||||
breakpoints.push((await protocol.send('Debugger.setBreakpoint', { location: { scriptId: script.scriptId, lineNumber: locations[0] } })).breakpointId);
|
||||
}
|
||||
const probe = readUpgradeJson(origin + '/api/health', 15000, 'Observe owned request handler');
|
||||
probe.catch(() => {});
|
||||
const event = await within(pausedEvent, 10000, 'Workbench request observation');
|
||||
const captureExpression = "(() => {"
|
||||
+ " const currentServer = typeof server !== 'undefined' ? server : typeof server2 !== 'undefined' ? server2 : undefined;"
|
||||
+ " const currentStore = typeof store !== 'undefined' ? store : typeof store2 !== 'undefined' ? store2 : undefined;"
|
||||
+ " const currentProvider = typeof provider !== 'undefined' ? provider : typeof provider2 !== 'undefined' ? provider2 : undefined;"
|
||||
+ " const currentRuntime = typeof runtime !== 'undefined' ? runtime : typeof runtime2 !== 'undefined' ? runtime2 : undefined;"
|
||||
+ " const currentToken = typeof token !== 'undefined' ? token : typeof token2 !== 'undefined' ? token2 : undefined;"
|
||||
+ " const currentClients = typeof clients !== 'undefined' ? clients : typeof clients2 !== 'undefined' ? clients2 : undefined;"
|
||||
+ " const currentStarted = typeof started !== 'undefined' ? started : typeof started2 !== 'undefined' ? started2 : undefined;"
|
||||
+ " const currentExampleProject = typeof exampleProject !== 'undefined' ? exampleProject : typeof exampleProject2 !== 'undefined' ? exampleProject2 : undefined;"
|
||||
+ " const currentShutdown = typeof shutdown !== 'undefined' ? shutdown : typeof shutdown2 !== 'undefined' ? shutdown2 : undefined;"
|
||||
+ " const servers = currentServer ? [currentServer] : process._getActiveHandles().filter(handle => handle?.listening && handle.address?.()?.address === '127.0.0.1' && handle.address?.()?.port === " + new URL(origin).port + ");"
|
||||
+ " if (servers.length !== 1 || !currentStore || !currentProvider || !currentRuntime || !currentToken || !currentClients || !currentShutdown) throw new Error('The existing Workbench context is incomplete.');"
|
||||
+ " globalThis.__awUpgradeContext = { server: servers[0], store: currentStore, provider: currentProvider, runtime: currentRuntime, token: currentToken, clients: currentClients, started: currentStarted, exampleProject: currentExampleProject, shutdown: currentShutdown }; return 'context-ready'; })()";
|
||||
const captured = await protocol.send('Debugger.evaluateOnCallFrame', { callFrameId: event.callFrames[0].callFrameId,
|
||||
expression: captureExpression, returnByValue: true });
|
||||
if (captured.exceptionDetails || captured.result.value !== 'context-ready') throw new Error('Could not retain the existing runtime objects.');
|
||||
for (const breakpointId of breakpoints.splice(0)) await protocol.send('Debugger.removeBreakpoint', { breakpointId });
|
||||
await protocol.send('Debugger.resume'); paused = false; await probe;
|
||||
const expression = '(() => { try { const module = process.getBuiltinModule("module").createRequire(' + JSON.stringify(upgradeScript) + ')(' + JSON.stringify(upgradeScript) + '); return module.installUpgrade(globalThis.__awUpgradeContext, { appDirectory: ' + JSON.stringify(next) + ' }); } finally { delete globalThis.__awUpgradeContext; } })()';
|
||||
const upgraded = await protocol.send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true });
|
||||
if (upgraded.exceptionDetails) throw new Error('Runtime upgrade failed: ' + (upgraded.exceptionDetails.exception?.description ?? upgraded.exceptionDetails.text));
|
||||
const result = upgraded.result.value;
|
||||
const after = await readUpgradeJson(origin + '/api/bootstrap', 5000, 'Read upgraded Workbench');
|
||||
const health = await readUpgradeJson(origin + '/api/health', 5000, 'Verify upgraded PID');
|
||||
if (health.pid !== pid || result.pid !== pid || after.version !== result.version || after.config.hasKey !== before.config.hasKey
|
||||
|| after.config.model !== before.config.model || after.config.baseUrl !== before.config.baseUrl
|
||||
|| JSON.stringify(after.projects) !== JSON.stringify(before.projects)
|
||||
|| JSON.stringify(after.sessions) !== JSON.stringify(before.sessions)) throw new Error('The upgraded runtime did not preserve the existing state.');
|
||||
return { ...result, verified: true, connectionPreserved: true, url: origin };
|
||||
} finally {
|
||||
if (protocol) {
|
||||
for (const breakpointId of breakpoints) { try { await protocol.send('Debugger.removeBreakpoint', { breakpointId }); } catch {} }
|
||||
try { if (paused) await protocol.send('Debugger.resume'); } catch {}
|
||||
try { await protocol.send('Runtime.evaluate', { expression: 'delete globalThis.__awUpgradeContext; true', returnByValue: true }); } catch {}
|
||||
if (enabledHere) {
|
||||
try { await protocol.send('Runtime.evaluate', { expression: "setTimeout(() => process.getBuiltinModule('inspector').close(), 25); true", returnByValue: true }); } catch {}
|
||||
}
|
||||
}
|
||||
socket?.close();
|
||||
if (enabledHere) await delay(150);
|
||||
}
|
||||
}
|
||||
|
||||
// This file is also bundled as an installation library; its CLI must not run
|
||||
// merely because esbuild gives it the install-manager entry's import.meta.url.
|
||||
if (path.basename(fileURLToPath(import.meta.url)) === 'upgrade-running.mjs' && process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const raw = process.argv.slice(2), args = {}, options = new Set(['url', 'pid', 'current-root', 'new-root', 'handler-root']);
|
||||
if (raw.length === 1 && raw[0] === '--help') { console.log('node scripts/upgrade-running.mjs --pid PID --current-root DIR --new-root DIR [--url http://127.0.0.1:4318] [--handler-root DIR]'); process.exit(0); }
|
||||
for (let index = 0; index < raw.length; index += 2) {
|
||||
const key = raw[index].startsWith('--') ? raw[index].slice(2) : '';
|
||||
if (!options.has(key) || Object.hasOwn(args, key) || !raw[index + 1] || raw[index + 1].startsWith('--')) throw new Error('Unknown or incomplete argument; use --help.');
|
||||
args[key] = raw[index + 1];
|
||||
}
|
||||
if (!/^\d+$/.test(args.pid ?? '') || !path.isAbsolute(args['current-root'] ?? '') || !path.isAbsolute(args['new-root'] ?? '')
|
||||
|| args['handler-root'] !== undefined && !path.isAbsolute(args['handler-root'])) throw new Error('PID and absolute current/new roots are required arguments; use --help.');
|
||||
const result = await upgradeRunning({ url: args.url || 'http://127.0.0.1:4318', pid: Number(args.pid), currentRoot: args['current-root'], newRoot: args['new-root'], handlerRoot: args['handler-root'] });
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const [directory, ...options] = process.argv.slice(2);
|
||||
if (!directory || options.some((value, index) => value !== '--index' && value !== '--source-ref' && options[index - 1] !== '--source-ref'))
|
||||
throw new Error('Usage: node scripts/verify-adoption-evidence.mjs <sealed-directory> [--index] [--source-ref <commit|:>]');
|
||||
const root = path.resolve(directory), gitRoot = process.cwd(), useIndex = options.includes('--index');
|
||||
const sourceRef = options.includes('--source-ref') ? options[options.indexOf('--source-ref') + 1] : null;
|
||||
if (sourceRef) assert.ok(sourceRef === ':' || /^[a-f0-9]{7,40}$/.test(sourceRef));
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const git = object => execFileSync('git', ['show', object], { cwd: gitRoot, windowsHide: true, maxBuffer: 16 * 1024 * 1024 });
|
||||
const contained = rel => {
|
||||
assert.ok(typeof rel === 'string' && rel && !rel.includes('\\') && !path.isAbsolute(rel) && !rel.split('/').some(part => !part || part === '..'));
|
||||
const file = path.resolve(root, rel); assert.ok(file.startsWith(root + path.sep)); assert.ok(fs.lstatSync(file).isFile()); return file;
|
||||
};
|
||||
const manifestBytes = fs.readFileSync(path.join(root, 'manifest.json')), manifest = JSON.parse(manifestBytes);
|
||||
assert.equal(manifest.kind, 'adoption-stage-evidence'); assert.equal(manifest.realModelRequests, 0); assert.equal(manifest.officialServiceChanged, false);
|
||||
const names = new Set(); let bytes = 0;
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(!names.has(row.path)); names.add(row.path); const data = fs.readFileSync(contained(row.path));
|
||||
assert.equal(data.length, row.bytes, row.path); assert.equal(sha(data), row.sha256, row.path); bytes += data.length;
|
||||
if (row.path.endsWith('.png')) assert.equal(data.subarray(0, 8).toString('hex'), '89504e470d0a1a0a');
|
||||
if (useIndex) assert.equal(sha(git(':' + path.relative(gitRoot, path.join(root, row.path)).replaceAll('\\', '/'))), row.sha256);
|
||||
}
|
||||
assert.equal(bytes, manifest.bytes);
|
||||
const walk = dir => fs.readdirSync(dir, { withFileTypes: true }).flatMap(row => {
|
||||
const file = path.join(dir, row.name); assert.equal(fs.lstatSync(file).isSymbolicLink(), false);
|
||||
return row.isDirectory() ? walk(file) : [path.relative(root, file).replaceAll('\\', '/')];
|
||||
});
|
||||
assert.deepEqual(walk(root).filter(name => name !== 'manifest.json').sort(), [...names].sort());
|
||||
if (useIndex) assert.equal(sha(git(':' + path.relative(gitRoot, path.join(root, 'manifest.json')).replaceAll('\\', '/'))), sha(manifestBytes));
|
||||
for (const check of manifest.testRuns ?? []) {
|
||||
const log = fs.readFileSync(contained(check.path), 'utf8');
|
||||
for (const [label, count] of Object.entries({ tests: check.tests, pass: check.passed, fail: check.failed }))
|
||||
assert.match(log, new RegExp('(?:ℹ |# )' + label + ' ' + count + '(?:\\r?\\n|$)'));
|
||||
}
|
||||
for (const check of manifest.uiRuns ?? []) {
|
||||
const result = JSON.parse(fs.readFileSync(contained(check.path), 'utf8'));
|
||||
assert.equal(result.success, check.success); assert.equal(result.passed.length, check.passed); assert.equal(result.realModelRequests, 0);
|
||||
}
|
||||
if (sourceRef) for (const row of manifest.sourceFiles) {
|
||||
assert.ok(names.has(row.snapshot)); assert.ok(row.path && !row.path.includes('..') && !path.isAbsolute(row.path));
|
||||
const original = fs.readFileSync(contained(row.snapshot)); const recorded = git(sourceRef === ':' ? ':' + row.path : sourceRef + ':' + row.path);
|
||||
// Source snapshots preserve original checkout bytes. The repository's text
|
||||
// attributes store LF; accept only that declared text representation change.
|
||||
const gitText = Buffer.from(original.toString('utf8').replaceAll('\r\n', '\n'));
|
||||
assert.equal(sha(recorded), sha(original) === sha(recorded) ? sha(original) : sha(gitText), 'Source ref differs: ' + row.path);
|
||||
}
|
||||
console.log(JSON.stringify({ ok: true, stage: manifest.stage, files: names.size, bytes, testRuns: manifest.testRuns?.length ?? 0,
|
||||
uiRuns: manifest.uiRuns?.length ?? 0, sourceFiles: manifest.sourceFiles.length, indexChecked: useIndex, sourceRef, realModelRequests: 0 }));
|
||||
@@ -0,0 +1,119 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
if (process.argv.slice(2).some(value => value !== '--help')) throw new Error('Unsupported argument; use --help.');
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-all-ui.mjs - run serial desktop/390px checks in independent local hosts; use explicit QA Python/Harness/Purge source paths for 1.15.'); process.exit(0); }
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
assert.ok(['1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version), 'Review evidence paths when introducing the next release.');
|
||||
const prefix = releaseEvidencePrefix(version), resultFile = `.qa/${prefix}-all-ui-validation.json`, results = [];
|
||||
const qaEnvironment = Object.fromEntries(['AGENT_WORKBENCH_TEST_PYTHON', 'AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_178_ROOT', 'AGENT_WORKBENCH_PURGE_181_ROOT']
|
||||
.filter(name => process.env[name]).map(name => [name, process.env[name]]));
|
||||
for (const [name, value] of Object.entries(qaEnvironment)) assert.ok(path.isAbsolute(value) && fs.existsSync(value), 'Use an existing absolute QA runtime/source path: ' + name);
|
||||
const cases = [
|
||||
...(version === '1.25.0' ? [['verify-task-agreement-ui.mjs', null], ['verify-task-follow-up-ui.mjs', null]] : []),
|
||||
...(['1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-file-results-ui.mjs', null]] : []),
|
||||
...(['1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-release-efficiency-ui.mjs', null]] : []),
|
||||
...(['1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-local-files.mjs', null]] : []),
|
||||
...(['1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version)?[['verify-embedded-browser.mjs',null]]:[]),
|
||||
...(['1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version)?[['verify-environment-ui.mjs',null],['verify-model-fidelity-ui.mjs',null],['verify-model-wait-ui.mjs',null],['verify-task-control-ui.mjs',null]]:[]),
|
||||
...(['1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-evaluations-ui.mjs', null]] : []),
|
||||
...(['1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-project-cache-ui.mjs', null]] : []),
|
||||
...(['1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-vision-ui.mjs', null]] : []),
|
||||
...(['1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-model-protocols-ui.mjs', null]] : []),
|
||||
...(['1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-companion-ui.mjs', null]] : []),
|
||||
...(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-controlled-ui.mjs', null], ['verify-purge-ui.mjs', null]] : []),
|
||||
...(['1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-workers-ui.mjs', null]] : []),
|
||||
...(['1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-task-plans-ui.mjs', `docs/${prefix}-task-ui-validation.json`]] : []),
|
||||
...(['1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-browser-ui.mjs', `docs/${prefix}-browser-ui-validation.json`]] : []),
|
||||
['verify-ui.mjs', '.qa/ui-validation.json'], ['verify-diagnostics-ui.mjs', '.qa/diagnostics-ui-validation.json'],
|
||||
['verify-execution-ui.mjs', '.qa/execution-ui-validation.json'], ['verify-delivery-ui.mjs', '.qa/delivery-ui-validation.json'],
|
||||
['verify-index-ui.mjs', '.qa/index-ui-result.json'], ['verify-change-sets-ui.mjs', `docs/${prefix}-batch-ui-validation.json`],
|
||||
['verify-context-ui.mjs', `.qa/${version === '1.6.0' ? 'workspaces-context' : prefix}-ui-validation.json`],
|
||||
['verify-workspaces-ui.mjs', `docs/${prefix === 'workspaces' ? prefix : prefix + '-workspaces'}-ui-validation.json`],
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-python-ui.mjs', `docs/${prefix}-python-ui-validation.json`]] : []),
|
||||
...(['1.7.0', '1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-react-delivery.mjs', version === '1.7.0' ? 'docs/react-engine-delivery-validation.json' : `docs/${prefix}-react-delivery-validation.json`]] : []),
|
||||
...(['1.8.0', '1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-research-ui.mjs', `docs/${prefix}-research-ui-validation.json`]] : []),
|
||||
...(['1.9.0', '1.10.0', '1.10.1', '1.11.0', '1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? [['verify-engines-ui.mjs', version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${prefix}-engines-ui-validation.json`]] : []),
|
||||
...(['1.12.0', '1.12.1', '1.13.0', '1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.9', '1.25.0'].includes(version) ? ['model-center', 'project-memory', 'task-checkpoints', 'skills', 'mcp', 'mcp-recovery'].map(name => [`verify-${name}-ui.mjs`, null]) : []),
|
||||
];
|
||||
fs.mkdirSync('.qa', { recursive: true });
|
||||
const externalHost = new Set(['verify-ui.mjs', 'verify-diagnostics-ui.mjs', 'verify-execution-ui.mjs', 'verify-index-ui.mjs']);
|
||||
async function startHost(script) {
|
||||
const data = path.resolve('.qa', 'ui-host-' + script.replace('.mjs', '') + '-' + randomUUID());
|
||||
fs.mkdirSync(data, { recursive: true });
|
||||
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env: { ...gitEnvironment(),
|
||||
AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let url = '', output = '', errors = '';
|
||||
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
|
||||
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
||||
const deadline = Date.now() + 10000;
|
||||
while (!url && child.exitCode === null && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 30));
|
||||
if (!url) { child.kill(); throw new Error('Independent UI host failed: ' + errors); }
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.equal(bootstrap.version, version); assert.equal(bootstrap.config.hasKey, false); assert.equal(bootstrap.config.mode, 'demo');
|
||||
return { url, data, async close() {
|
||||
try {
|
||||
await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' });
|
||||
const stopDeadline = Date.now() + 10000;
|
||||
while (child.exitCode === null && child.signalCode === null && Date.now() < stopDeadline) await new Promise(resolve => setTimeout(resolve, 30));
|
||||
} finally { if (child.exitCode === null && child.signalCode === null) child.kill(); }
|
||||
} };
|
||||
}
|
||||
for (const [script, fixedProof] of cases) {
|
||||
console.log('UI regression: ' + script); const started = Date.now(), log = `.qa/${prefix}-${script.replace('.mjs', '')}.log`;
|
||||
const stream = fs.createWriteStream(log);
|
||||
let host;
|
||||
try {
|
||||
if (externalHost.has(script)) host = await startHost(script);
|
||||
const output = await new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, [path.join('scripts', script)], { cwd: process.cwd(), env: { ...gitEnvironment(), ...qaEnvironment, ...(host ? { AGENT_WORKBENCH_URL: host.url } : {}) }, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let tail = ''; const capture = bytes => { stream.write(bytes); tail = (tail + bytes.toString('utf8')).slice(-10000); };
|
||||
child.stdout.on('data', capture); child.stderr.on('data', capture);
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
if (process.platform === 'win32' && child.pid) {
|
||||
try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000, stdio: 'ignore' }); } catch { child.kill(); }
|
||||
} else child.kill();
|
||||
}, 240000);
|
||||
child.once('error', error => { clearTimeout(timer); reject(error); });
|
||||
child.once('close', code => { clearTimeout(timer); code === 0 && !timedOut ? resolve(tail) : reject(new Error(script + (timedOut ? ' timed out' : ' failed (' + code + ')') + ': ' + tail)); });
|
||||
});
|
||||
stream.end();
|
||||
const summary = output.trim().split(/\r?\n/).findLast(line => line.startsWith('{"success":'));
|
||||
const detail = summary ? JSON.parse(summary) : null;
|
||||
const directory = [...output.matchAll(/^EVIDENCE (.+)\r?$/gm)].at(-1)?.[1].trim() || detail?.directory || detail?.root;
|
||||
const proof = fixedProof || (directory && path.join(directory, 'result.json'));
|
||||
assert.ok(proof, 'Verifier must identify its current evidence.');
|
||||
if (!fixedProof) {
|
||||
const relative = path.relative(path.resolve('.qa'), path.resolve(proof));
|
||||
assert.ok(relative && !relative.startsWith('..') && !path.isAbsolute(relative), 'Evidence must belong to this workspace QA directory.');
|
||||
}
|
||||
const evidence = JSON.parse(fs.readFileSync(proof, 'utf8'));
|
||||
assert.notEqual(evidence.success, false); assert.notEqual(evidence.passed, false); if (evidence.version) assert.equal(evidence.version, version);
|
||||
const passed = [evidence.passed, evidence.cases, evidence.assertions, evidence.checks, evidence.results].find(Array.isArray);
|
||||
assert.ok(Array.isArray(passed) && passed.length > 0);
|
||||
const errors = evidence.pageErrors ?? [], failures = evidence.requestFailures ?? [];
|
||||
assert.equal(errors.length, 0); assert.equal(failures.length, 0);
|
||||
assert.deepEqual(evidence.errors ?? [], []);
|
||||
const saved = ['verify-controlled-ui.mjs', 'verify-purge-ui.mjs'].includes(script) ? `docs/${prefix}-${script.replace('verify-', '').replace('-ui.mjs', '')}-final-ui-validation.json`
|
||||
: script === 'verify-ui.mjs' ? `docs/${prefix}-basic-ui-validation.json`
|
||||
: ['verify-workspaces-ui.mjs', 'verify-task-plans-ui.mjs'].includes(script) ? proof
|
||||
: `docs/${prefix}-${script.replace('verify-', '').replace('.mjs', '')}-validation.json`;
|
||||
if (path.resolve(saved) !== path.resolve(proof)) fs.copyFileSync(proof, saved);
|
||||
results.push({ script, success: true, groups: passed.length, durationMs: Date.now() - started, evidence: saved, log });
|
||||
console.log('PASS ' + script + ': ' + passed.length + ' groups');
|
||||
} catch (error) {
|
||||
stream.end(); fs.writeFileSync(resultFile, JSON.stringify({ success: false, version, at: new Date().toISOString(), results, error: error.message, script, log }, null, 2) + '\n');
|
||||
throw error;
|
||||
} finally { await host?.close(); }
|
||||
}
|
||||
const result = { success: true, version, at: new Date().toISOString(), groups: results.reduce((sum, item) => sum + item.groups, 0), results,
|
||||
scope: 'Every verifier owns an independent port and synthetic data directory. Synthetic provider fixtures verify protocols; configured-model acceptance is separate. No production service or real key read/copied.' };
|
||||
fs.writeFileSync(resultFile, JSON.stringify(result, null, 2) + '\n');
|
||||
fs.copyFileSync(resultFile, 'docs/' + prefix + '-all-ui-validation.json'); console.log(JSON.stringify(result, null, 2));
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-anthropic-messages-ui.mjs [--app-root ABS] — isolated Messages editor, actual JSON/SSE, tool roundtrip, identity mismatch, URLs, raw errors, reload and 390px; synthetic credentials only.'); process.exit(0); }
|
||||
const args = process.argv.slice(2); assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
||||
const appRoot = fs.realpathSync.native(args[1] ?? process.cwd());
|
||||
const directory = path.resolve('.qa', 'messages-ui-m-' + randomUUID()); fs.mkdirSync(directory, { recursive: true }); console.log('EVIDENCE ' + directory);
|
||||
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t); f.store.close();
|
||||
const requests = [], sockets = new Set(), checks = [], screenshots = [], errors = [];
|
||||
let mode = 'ok';
|
||||
const msg = (content, stop_reason) => ({ id: 'msg_owned_ui', type: 'message', role: 'assistant', model: mode === 'mismatch' ? 'gpt-owned-other' : 'claude-owned-ui', content, stop_reason,
|
||||
usage: { input_tokens: 40, cache_creation_input_tokens: 10, cache_read_input_tokens: 20, output_tokens: 8 } });
|
||||
const modelServer = http.createServer(async (req, res) => {
|
||||
try {
|
||||
const parts = []; for await (const chunk of req) parts.push(chunk);
|
||||
const body = JSON.parse(Buffer.concat(parts)), raw = Buffer.concat(parts);
|
||||
requests.push({ route: req.url, body, auth: req.headers.authorization === 'Bearer owned-ui-key', keyHeaderPresent: req.headers['x-api-key'] !== undefined, version: req.headers['anthropic-version'], sha256: createHash('sha256').update(raw).digest('hex') });
|
||||
assert.equal(req.url, '/v1/messages'); assert.equal(req.headers.authorization, 'Bearer owned-ui-key'); assert.equal(req.headers['x-api-key'], undefined); assert.equal(req.headers['anthropic-version'], '2023-06-01');
|
||||
if (mode === 'error') { res.writeHead(429, { 'Content-Type': 'application/json', 'request-id': 'req_owned_limit', 'Retry-After': '1' }); res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'owned Messages limit' } })); return; }
|
||||
const prompt = body.messages.filter(m => m.role === 'user').flatMap(m => m.content).filter(b => b.type === 'text').map(b => b.text).join('\n');
|
||||
const nonce = prompt.match(/nonce ([a-f0-9-]+)/)?.[1], probe = prompt === 'Reply OK.' || !!nonce;
|
||||
const completedTool = body.messages.some(m => m.content.some(b => b.type === 'tool_result'));
|
||||
const tool = { type: 'tool_use', id: 'call_owned_ui', name: nonce ? 'workbench_capability_echo' : 'list_files', input: nonce ? { nonce } : { path: '', depth: 1 } };
|
||||
const content = mode === 'mismatch' || probe && !nonce || completedTool ? [{ type: 'text', text: '已核对。' }] : [tool];
|
||||
const final = msg(content, content[0].type === 'tool_use' ? 'tool_use' : 'end_turn');
|
||||
if (!body.stream) { res.writeHead(200, { 'Content-Type': 'application/json', 'request-id': 'req_owned_native' }); res.end(JSON.stringify(final)); return; }
|
||||
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'request-id': 'req_owned_native' });
|
||||
const send = (type, fields = {}) => res.write('event: ' + type + '\ndata: ' + JSON.stringify({ type, ...fields }) + '\n\n');
|
||||
send('message_start', { message: { ...final, content: [], stop_reason: null, usage: { ...final.usage, output_tokens: 1 } } });
|
||||
content.forEach((block, index) => {
|
||||
send('content_block_start', { index, content_block: block.type === 'text' ? { ...block, text: '' } : { ...block, input: {} } });
|
||||
send('content_block_delta', { index, delta: block.type === 'text' ? { type: 'text_delta', text: block.text } : { type: 'input_json_delta', partial_json: JSON.stringify(block.input) } });
|
||||
send('content_block_stop', { index });
|
||||
});
|
||||
send('message_delta', { delta: { stop_reason: final.stop_reason, stop_sequence: null }, usage: { output_tokens: final.usage.output_tokens } }); send('message_stop'); res.end();
|
||||
} catch (cause) { errors.push('owned upstream: ' + cause.message); res.destroy(); }
|
||||
});
|
||||
modelServer.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
|
||||
await new Promise(resolve => modelServer.listen(0, '127.0.0.1', resolve));
|
||||
const baseUrl = 'http://127.0.0.1:' + modelServer.address().port;
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath: args.length ? path.join(appRoot, 'runtime/node.exe') : process.execPath });
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
page.on('pageerror', cause => errors.push(cause.message));
|
||||
const mark = message => { checks.push(message); console.log('PASS ' + message); };
|
||||
const dialog = () => page.getByRole('dialog', { name: '模型中心', exact: true });
|
||||
async function open() { if (!await dialog().isVisible()) await page.getByRole('button', { name: '打开模型中心', exact: true }).click(); return dialog(); }
|
||||
async function shot(name) {
|
||||
const file = path.join(directory, name + '.png'); await page.screenshot({ path: file }); screenshots.push(file);
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= innerWidth), 'page overflow: ' + name);
|
||||
assert.ok(await dialog().evaluate(node => node.scrollWidth <= node.clientWidth + 1), 'dialog overflow: ' + name);
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await open();
|
||||
await dialog().getByRole('button', { name: '添加连接', exact: true }).click(); const form = dialog().getByRole('form', { name: '编辑模型连接', exact: true });
|
||||
await form.getByLabel('连接名称', { exact: true }).fill('本机 Messages 验收'); await form.getByLabel('接口协议', { exact: true }).selectOption('anthropic-messages');
|
||||
await form.getByLabel('接口根地址', { exact: true }).fill(baseUrl); await form.getByLabel('请求模型名称', { exact: true }).fill('claude-owned-ui');
|
||||
await form.getByLabel('临时 API Key', { exact: true }).fill('owned-ui-key'); await form.getByLabel('Messages 鉴权', { exact: true }).selectOption('bearer');
|
||||
await form.getByLabel('型号核对', { exact: true }).selectOption('require-match'); await form.getByLabel('流式能力', { exact: true }).selectOption('supported'); await form.getByLabel('使用流式输出', { exact: true }).check();
|
||||
await expect(form.getByLabel('请求地址预览', { exact: true })).toContainText(baseUrl + '/v1/messages'); await shot('desktop-editor');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await form.scrollIntoViewIfNeeded(); await shot('narrow-editor');
|
||||
await form.getByLabel('接口根地址', { exact: true }).fill(baseUrl + '/v1/messages'); await form.getByRole('button', { name: '保存模型连接', exact: true }).click();
|
||||
await expect(dialog().locator('.mc-notice')).toContainText('根地址'); await expect(form.getByLabel('连接名称', { exact: true })).toHaveValue('本机 Messages 验收');
|
||||
await form.getByLabel('接口根地址', { exact: true }).fill(baseUrl + '/v1'); await form.getByRole('button', { name: '保存模型连接', exact: true }).click(); await expect(form).toHaveCount(0);
|
||||
assert.equal(requests.length, 0); mark('root and /v1 preview agree, full endpoint rejects with input retained; save makes no model request');
|
||||
await page.reload(); await open(); const card = dialog().locator('.mc-connection').filter({ hasText: '本机 Messages 验收' });
|
||||
await expect(card).toContainText('Anthropic Messages · 流式开启'); await expect(card).toContainText('Bearer'); await expect(card).toContainText(baseUrl + '/v1/messages');
|
||||
let center = await host.api('/model-center'), connection = center.connections.find(c => c.name === '本机 Messages 验收'); assert.equal(connection.messagesAuth, 'bearer');
|
||||
await card.getByRole('button', { name: '用于新任务', exact: true }).click(); assert.equal(requests.length, 0); await page.reload(); await open();
|
||||
await expect(dialog().getByLabel('新任务默认模型', { exact: true })).toHaveValue('connection:' + connection.id); await shot('narrow-saved');
|
||||
mark('saved protocol, auth and selected connection survive reload and remain operable at 390px');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 }); await card.getByRole('button', { name: '原生参数', exact: true }).click();
|
||||
const native = dialog().getByRole('form', { name: '原生请求参数', exact: true }); await expect(native.getByLabel('输出上限字段')).toHaveValue('max_tokens');
|
||||
await expect(native.getByLabel('输出上限字段').locator('option')).toHaveCount(1); await native.getByLabel('原生参数 JSON').fill('{"reasoning_effort":"high"}');
|
||||
await native.getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(native.getByRole('alert')).toContainText('尚未接入');
|
||||
await native.getByLabel('原生参数 JSON').fill('{"temperature":0.3,"top_k":12}'); await native.getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(native).toHaveCount(0);
|
||||
assert.equal(requests.length, 0); mark('native settings reject other-protocol fields and keep a single hard max_tokens output field');
|
||||
for (const [scope, title] of [['text', '文本'], ['tools', '工具'], ['streaming', '流式']]) {
|
||||
await card.getByLabel('本机 Messages 验收检测范围').selectOption(scope); await card.getByRole('button', { name: '验证响应(调用模型)', exact: true }).click();
|
||||
await expect(dialog().locator('.mc-notice')).toContainText(title + '检测通过');
|
||||
}
|
||||
assert.equal(requests.length, 3); assert.ok(requests.every(r => r.body.top_k === 12 && r.body.temperature === 0.3)); mark('actual local JSON text/tool and native SSE capability probes use the saved protocol and auth');
|
||||
await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
await host.api('/sessions/' + f.session.id + '/messages', { content: '读取本项目文件一次后结束。' });
|
||||
await until(async () => !['running', 'queued'].includes((await host.api('/sessions/' + f.session.id)).status), 20000);
|
||||
const session = await host.api('/sessions/' + f.session.id); assert.equal(session.status, 'completed');
|
||||
assert.equal(requests.length, 5); assert.ok(requests[4].body.messages.some(m => m.content.some(b => b.type === 'tool_result' && b.tool_use_id === 'call_owned_ui')));
|
||||
await open(); await dialog().getByRole('button', { name: '请求账本', exact: true }).click();
|
||||
await expect(dialog().locator('.mc-attempts')).toContainText('请求与返回型号一致'); await expect(dialog().locator('.mc-attempts')).toContainText(baseUrl + '/v1/messages');
|
||||
await expect(dialog().locator('.mc-attempts')).toContainText('70 输入 / 8 输出 tokens'); await dialog().locator('.mc-attempts article').first().scrollIntoViewIfNeeded(); await shot('desktop-ledger');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await dialog().locator('.mc-attempts article').first().scrollIntoViewIfNeeded(); await shot('narrow-ledger'); mark('compiled native tool roundtrip completes with original call ID, cache-inclusive usage and observed request/response URLs');
|
||||
await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click(); mode = 'mismatch';
|
||||
const second = await host.api('/sessions', { projectId: f.project.id }); await host.api('/sessions/' + second.id + '/messages', { content: '有界身份异常夹具。' });
|
||||
await until(async () => !['running', 'queued'].includes((await host.api('/sessions/' + second.id)).status), 20000);
|
||||
center = await host.api('/model-center'); assert.equal(center.attempts[0].modelIdentity.status, 'mismatch'); assert.equal(requests.length, 6);
|
||||
await open(); await dialog().getByRole('button', { name: '请求账本', exact: true }).click(); await expect(dialog().locator('.mc-attempts')).toContainText('请求与返回型号不一致'); await shot('narrow-mismatch');
|
||||
mark('strict mismatch is visible with original GPT identifier and one request, no retry or tool dispatch');
|
||||
// A separate owned connection avoids bypassing the intentionally blocked identity route.
|
||||
mode = 'error'; const original = center.connections.find(c => c.id === connection.id);
|
||||
connection = await host.api('/model-center/connections', { ...original, id: undefined, name: '本机 Messages 错误验收', apiKey: 'owned-ui-key' });
|
||||
// The API returns the authoritative center; resolve by its explicit owned name.
|
||||
center = await host.api('/model-center'); connection = center.connections.find(c => c.name === '本机 Messages 错误验收');
|
||||
await expect(dialog().getByRole('button', { name: '刷新模型中心', exact: true })).toBeEnabled(); await dialog().getByRole('button', { name: '刷新模型中心', exact: true }).click();
|
||||
await dialog().getByRole('button', { name: '模型连接', exact: true }).click(); const errorCard = dialog().locator('.mc-connection').filter({ hasText: '本机 Messages 错误验收' });
|
||||
await errorCard.getByRole('button', { name: '验证响应(调用模型)', exact: true }).click(); await expect(dialog().locator('.mc-notice')).toContainText('模型服务请求达到限额');
|
||||
await dialog().locator('.mc-notice summary').click(); await expect(dialog().getByLabel('上游错误原文')).toContainText('owned Messages limit');
|
||||
await expect(dialog().locator('.mc-notice')).toContainText('req_owned_limit'); await expect(dialog().locator('.mc-notice')).toContainText(baseUrl + '/v1/messages'); await shot('narrow-error');
|
||||
assert.equal(requests.length, 7); mark('429 preserves native type, raw message, request ID, URL and inputs without automatic replay');
|
||||
assert.deepEqual(errors, []);
|
||||
result = { success: true, appRoot, testedVersion: (await host.api('/health')).version, directory, checks, screenshots, errors, requests, realModelRequests: 0,
|
||||
scope: 'Compiled UI and actual owned localhost API; no configured-model capability or model-weight claim.' };
|
||||
} catch (cause) { result = { success: false, directory, checks, screenshots, errors, error: cause.stack, requests, realModelRequests: 0 }; await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {}); process.exitCode = 1; }
|
||||
finally {
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify({ success: result.success, checks: checks.length, directory, error: result.error }));
|
||||
await browser.close(); for (const dispose of disposers.reverse()) await dispose(); for (const socket of sockets) socket.destroy(); await new Promise(resolve => modelServer.close(resolve));
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
// Offline only: no credentials, process execution or network access.
|
||||
const repo=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..'),root=path.join(repo,'docs/apikey-route-l-evidence');
|
||||
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const read=file=>JSON.parse(fs.readFileSync(path.join(root,file),'utf8'));
|
||||
const manifestBytes=fs.readFileSync(path.join(root,'manifest-final.json')),manifest=JSON.parse(manifestBytes);
|
||||
const paths=new Set();let total=0;
|
||||
for(const entry of manifest.entries){
|
||||
assert.equal(entry.path.includes('\\'),false);assert.equal(entry.path.split('/').includes('..'),false);assert.equal(paths.has(entry.path),false);paths.add(entry.path);
|
||||
const file=path.resolve(root,entry.path);assert.ok(file.startsWith(root+path.sep));const bytes=fs.readFileSync(file);
|
||||
assert.equal(bytes.length,entry.bytes,entry.path);assert.equal(sha(bytes),entry.sha256,entry.path);total+=bytes.length;
|
||||
}
|
||||
assert.equal(manifest.count,paths.size);assert.equal(manifest.bytes,total);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.productChanged,false);
|
||||
assert.equal(manifest.supplierMappingConfirmed,false);
|
||||
const jRoot=path.join(repo,'docs/live-model-identity-j-evidence');
|
||||
assert.equal(sha(fs.readFileSync(path.join(jRoot,'manifest.json'))),manifest.originalJManifestSha256);
|
||||
const original=read('observations/original-analysis.json');
|
||||
assert.equal(sha(fs.readFileSync(path.join(jRoot,'request.json'))),original.request.sha256);
|
||||
assert.equal(sha(fs.readFileSync(path.join(jRoot,'response.sse'))),original.response.sha256);
|
||||
assert.equal(original.request.model,'claude-opus-5-5');assert.equal(original.route.poolId,null);assert.equal(original.route.ordinal,1);
|
||||
assert.equal(original.route.computedUrl,'https://api.apikey.fan/chat/completions');
|
||||
assert.deepEqual(original.response.reportedModels,['gpt-6.1-sol']);assert.equal(original.response.redacted,false);assert.equal(original.response.terminalDone,true);
|
||||
const replay=read('observations/provider-replay.json');assert.equal(replay.success,true);assert.equal(replay.requests,3);assert.equal(replay.realModelRequests,0);
|
||||
assert.equal(replay.cases.length,3);
|
||||
for(const [index,item]of replay.cases.entries()){
|
||||
assert.equal(item.passed,true);assert.equal(item.wire.bodySha256,original.request.sha256);assert.equal(item.wire.model,original.request.model);
|
||||
assert.equal(item.wire.ownedAuthorizationMatched,true);assert.equal(item.status,index===2?'matched':'mismatch');
|
||||
if(index<2)assert.equal(item.responseSha256,original.response.sha256);
|
||||
}
|
||||
const vendor=read('observations/vendor-observation.json');assert.equal(vendor.authenticated,false);assert.equal(vendor.modelRequests,0);
|
||||
const mime=read('observations/screenshot-mime-correction.json');
|
||||
assert.equal(mime.actualMediaType,'image/jpeg');
|
||||
const jpeg=fs.readFileSync(path.join(root,mime.correctedPath)),initial=fs.readFileSync(path.join(root,mime.originalPath));
|
||||
assert.equal(jpeg.subarray(0,3).toString('hex'),'ffd8ff');assert.deepEqual(jpeg,initial);assert.equal(sha(jpeg),mime.sha256);
|
||||
assert.equal(sha(fs.readFileSync(path.join(root,'manifest.json'))),manifest.originalManifestSha256);
|
||||
const scriptsDoc=fs.readFileSync(path.join(root,'observations/vendor-api-scripts-ax.txt'),'utf8');
|
||||
assert.ok(scriptsDoc.includes('https://api.apikey.fan/v1/messages'));assert.ok(scriptsDoc.includes('claude-opus-5-5'));
|
||||
assert.ok(scriptsDoc.includes('https://api.apikey.fan/v1/chat/completions'));assert.ok(scriptsDoc.includes('gpt-6.1-sol'));
|
||||
for(const route of read('observations/public-route-checks.json').results){assert.equal(route.method,'GET');assert.equal(route.authorizationSent,false);assert.equal(route.status,404);}
|
||||
const review=read('observations/source-review.json');assert.equal(review.sourceCommit,manifest.sourceCommit);assert.equal(review.findings.anthropicMessagesSupported,false);
|
||||
for(const file of review.files){const bytes=fs.readFileSync(path.join(root,'source',file.path));assert.equal(sha(bytes),file.sha256);assert.equal(bytes.length,file.bytes);}
|
||||
const caps=fs.readFileSync(path.join(root,'source/src/shared/contracts/model-capabilities.mjs'),'utf8');
|
||||
assert.ok(caps.includes("MODEL_PROTOCOLS = ['chat-completions', 'responses']"));
|
||||
const retained=read('observations/formal-retention.json');assert.equal(retained.success,true);assert.equal(retained.version,manifest.version);assert.equal(retained.hasKey,true);
|
||||
assert.deepEqual(retained.active,[]);assert.deepEqual(retained.retention,{originalRows:'exact',originalExports:'exact',originalArtifactBytes:'exact',originalModelCenter:'exact'});
|
||||
console.log(JSON.stringify({success:true,files:manifest.count,bytes:total,manifestSha256:sha(manifestBytes),realModelRequests:0,
|
||||
scope:'Offline original bytes and evidence receipts only. Native Messages adapter gap is confirmed; exact supplier mapping, key group and underlying weights remain unproven.'}));
|
||||
@@ -0,0 +1,97 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
import sys
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
if '--help' in sys.argv:
|
||||
print('python scripts/verify-archives.py [--extract-source] - verify current immutable ZIPs; AGENT_WORKBENCH_ARCHIVE_QA_ROOT selects the extraction root.')
|
||||
sys.exit(0)
|
||||
if any(value != '--extract-source' for value in sys.argv[1:]):
|
||||
raise SystemExit('Unsupported argument; use --help.')
|
||||
|
||||
workspace = Path(__file__).resolve().parent.parent
|
||||
release_dir = (workspace / 'releases').resolve()
|
||||
release = json.loads((release_dir / 'latest.json').read_text(encoding='utf-8'))
|
||||
archive_qa_root = Path(os.environ.get('AGENT_WORKBENCH_ARCHIVE_QA_ROOT', workspace / '.qa')).resolve()
|
||||
archive_qa_root.mkdir(parents=True, exist_ok=True)
|
||||
records = []
|
||||
extraction_root = None
|
||||
version = None
|
||||
|
||||
def filesystem_path(path):
|
||||
"""Use the Windows extended path form for source archives with deep dependency trees."""
|
||||
value = str(path)
|
||||
if os.name == 'nt' and not value.startswith('\\\\?\\'):
|
||||
return '\\\\?\\' + value
|
||||
return value
|
||||
|
||||
def extract_archive(bundle, root):
|
||||
for entry in bundle.infolist():
|
||||
target = (root / entry.filename.replace('\\\\', '/')).resolve()
|
||||
if entry.is_dir():
|
||||
os.makedirs(filesystem_path(target), exist_ok=True)
|
||||
continue
|
||||
os.makedirs(filesystem_path(target.parent), exist_ok=True)
|
||||
with bundle.open(entry, 'r') as source, open(filesystem_path(target), 'wb') as destination:
|
||||
shutil.copyfileobj(source, destination)
|
||||
|
||||
for kind, root_key in [('portable', 'portableRoot'), ('source', 'sourceRoot')] + ([('evidence', 'evidenceRoot')] if 'evidenceRoot' in release else []):
|
||||
root = Path(release[root_key]).resolve()
|
||||
archive = Path(release[kind]['path']).resolve()
|
||||
assert root.is_relative_to(release_dir) and archive.is_relative_to(release_dir)
|
||||
archive_bytes = archive.read_bytes()
|
||||
assert len(archive_bytes) == release[kind]['bytes']
|
||||
assert hashlib.sha256(archive_bytes).hexdigest() == release[kind]['sha256']
|
||||
manifest = json.loads((root / 'MANIFEST.json').read_text(encoding='utf-8'))
|
||||
version = version or manifest['version']
|
||||
if kind == 'evidence':
|
||||
assert manifest['app'] == 'agent-workbench-evidence'
|
||||
assert manifest['sourceCommit'] == release['documentation']['sourceCommit']
|
||||
assert hashlib.sha256((root / 'MANIFEST.json').read_bytes()).hexdigest() == release['documentation']['evidenceManifestSha256']
|
||||
else:
|
||||
assert manifest['version'] == version and manifest['kind'] == kind
|
||||
expected = {entry['path']: entry for entry in manifest['files']}
|
||||
assert len(expected) == len(manifest['files'])
|
||||
with zipfile.ZipFile(archive) as bundle:
|
||||
assert bundle.testzip() is None, 'CRC failure'
|
||||
actual = {}
|
||||
for entry in bundle.infolist():
|
||||
name = PurePosixPath(entry.filename.replace('\\', '/'))
|
||||
assert not name.is_absolute() and '..' not in name.parts and name.parts[0] == root.name
|
||||
if entry.is_dir():
|
||||
continue
|
||||
relative = str(PurePosixPath(*name.parts[1:]))
|
||||
assert relative not in actual, 'Duplicate archive entry'
|
||||
actual[relative] = entry
|
||||
assert set(actual) == set(expected) | {'MANIFEST.json'}
|
||||
for relative, entry in actual.items():
|
||||
contents = bundle.read(entry)
|
||||
target = (root / relative).resolve()
|
||||
assert target.is_relative_to(root), relative
|
||||
with open(filesystem_path(target), 'rb') as original:
|
||||
assert contents == original.read(), relative
|
||||
if relative != 'MANIFEST.json':
|
||||
assert len(contents) == expected[relative]['bytes']
|
||||
assert hashlib.sha256(contents).hexdigest() == expected[relative]['sha256']
|
||||
if kind == 'source' and '--extract-source' in sys.argv:
|
||||
isolated = (archive_qa_root / ('source-check-' + uuid.uuid4().hex)).resolve()
|
||||
assert isolated.is_relative_to(archive_qa_root)
|
||||
isolated.mkdir(parents=True, exist_ok=False)
|
||||
for entry in bundle.infolist():
|
||||
target = (isolated / entry.filename.replace('\\', '/')).resolve()
|
||||
assert target.is_relative_to(isolated)
|
||||
extract_archive(bundle, isolated)
|
||||
extraction_root = str(isolated / root.name)
|
||||
records.append({'kind': kind, 'path': str(archive), 'files': len(expected), 'crcPassed': True,
|
||||
'manifestAndDirectoryMatch': True, 'sha256': release[kind]['sha256']})
|
||||
|
||||
result = {'success': True, 'version': version, 'testedAt': datetime.now(timezone.utc).isoformat(),
|
||||
'archives': records, 'extractionRoot': extraction_root}
|
||||
(workspace / '.qa').mkdir(exist_ok=True)
|
||||
(workspace / '.qa/archives-validation.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps(result, ensure_ascii=True))
|
||||
@@ -0,0 +1,141 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench } from '../tests/helpers.mjs';
|
||||
import { createBrowserProject } from '../tests/fixtures/browser-project.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-browser-ui.mjs — independent real Edge browser plans, review, results, images, mobile, stop, late response and restart'); process.exit(0); }
|
||||
const directory = path.resolve('.qa', 'browser-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = createBrowserProject(fixture(t));
|
||||
f.session.title = '工程实际页面验收';
|
||||
const indexFile = path.join(f.projectRoot, 'dist', 'index.html'); fs.writeFileSync(indexFile, fs.readFileSync(indexFile, 'utf8').replace('<h1>', '<p>' + '页面证据 '.repeat(1800) + '</p><h1>'));
|
||||
f.store.save(f.session); f.store.close(); let host = await launchWorkbench(t, f.data), route = '/sessions/' + f.session.id;
|
||||
assert.notEqual(new URL(host.url).port, '4318'); assert.equal(host.bootstrap.config.hasKey, false);
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const passed = [], pageErrors = [], requestFailures = [], screenshots = [], actualJobs = []; let result, deliberateNavigation = false;
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => { if (!deliberateNavigation) requestFailures.push(request.url() + ': ' + request.failure()?.errorText); });
|
||||
const panel = () => page.getByLabel('工程页面验收', { exact: true }), state = () => host.api(route);
|
||||
const mark = message => { passed.push(message); console.log('PASS ' + message); };
|
||||
const screenshot = async name => { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push(file); };
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
async function selectTask() {
|
||||
await page.getByLabel('当前项目').selectOption(f.project.id);
|
||||
await page.locator('.session-list button').filter({ hasText: f.session.title }).click();
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
await expect(panel().getByRole('button', { name: '封存页面计划', exact: true })).toBeEnabled();
|
||||
}
|
||||
async function seal() {
|
||||
const old = (await state()).execution.browserPlans.at(-1)?.id;
|
||||
await panel().getByRole('button', { name: '封存页面计划', exact: true }).click();
|
||||
await expect.poll(async () => (await state()).execution.browserPlans.at(-1)?.id).not.toBe(old);
|
||||
return (await state()).execution.browserPlans.at(-1);
|
||||
}
|
||||
async function start() {
|
||||
await panel().getByRole('button', { name: '运行这份计划', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '执行页面检查', exact: true }).getByRole('button', { name: '确认运行页面检查', exact: true }).click();
|
||||
}
|
||||
async function finished() {
|
||||
await expect.poll(async () => (await state()).execution.browserJobs.at(-1)?.finishedAt, { timeout: 20000 }).toBeTruthy();
|
||||
const job = (await state()).execution.browserJobs.at(-1); actualJobs.push(job); return job;
|
||||
}
|
||||
async function original(job) {
|
||||
let text = '', offset = 0, value;
|
||||
do { value = await host.api(route + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=8000'); text += value.content; offset += value.content.length; } while (value.truncated);
|
||||
assert.equal(sha(text), job.resultSha256); fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text); return JSON.parse(text);
|
||||
}
|
||||
try {
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await selectTask();
|
||||
await expect(panel()).toContainText('Edge · 实际浏览器'); assert.equal((await state()).execution.browserJobs.length, 0);
|
||||
for (let index = 1; index <= 3; index++) await panel().getByRole('button', { name: '添加步骤', exact: true }).click();
|
||||
await panel().getByLabel('步骤 1 操作').selectOption('click'); await panel().getByLabel('步骤 1 元素').fill('#toggle');
|
||||
await panel().getByLabel('步骤 2 操作').selectOption('assert_text'); await panel().getByLabel('步骤 2 元素').fill('#state'); await panel().getByLabel('步骤 2 预期文字').fill('已完成');
|
||||
await panel().getByLabel('步骤 3 操作').selectOption('assert_count'); await panel().getByLabel('步骤 3 元素').fill('button'); await panel().getByLabel('步骤 3 预期数量').fill('1');
|
||||
const plan = await seal(); assert.equal(plan.steps.length, 3); assert.equal(plan.build.files, 2); assert.equal((await state()).execution.browserJobs.length, 0);
|
||||
mark('actual step editor seals the chosen build, viewport and three CSS interactions without opening the page');
|
||||
|
||||
await panel().getByRole('button', { name: '运行这份计划', exact: true }).click();
|
||||
const confirm = page.getByRole('dialog', { name: '执行页面检查', exact: true }); await expect(confirm).toContainText('1280×900'); await expect(confirm).toContainText('#state'); await expect(confirm).toContainText(plan.build.sha256);
|
||||
await confirm.getByRole('button', { name: '取消', exact: true }).click(); assert.equal((await state()).execution.browserJobs.length, 0);
|
||||
mark('review shows exact build identity and planned assertions; cancelling executes no browser work');
|
||||
|
||||
await start(); const success = await finished(); assert.equal(success.status, 'passed', success.summary); assert.equal(success.assertions.total, 2); assert.equal(success.cleanup.exited, true);
|
||||
await expect(panel()).toContainText('实际断言 2/2'); const succeeded = await original(success); assert.equal(succeeded.steps[2].actual, 1);
|
||||
assert.ok(succeeded.pages.at(-1).text.includes('已完成')); await screenshot('browser-desktop-results');
|
||||
mark('real inner Edge clicks the intended element and the two counted assertions display with original Task/Run/build identity');
|
||||
|
||||
await panel().getByRole('button', { name: '查看原执行记录', exact: true }).first().click();
|
||||
const record = () => page.getByRole('dialog', { name: '浏览器原执行记录', exact: true }); await expect(record()).toContainText(success.resultSha256);
|
||||
await expect(record().getByRole('button', { name: '下一段', exact: true })).toBeEnabled();
|
||||
const first = await record().locator('.artifact-preview').textContent(); await record().getByRole('button', { name: '下一段', exact: true }).click();
|
||||
await expect.poll(async () => await record().locator('.artifact-preview').textContent()).not.toBe(first); await expect(record()).toContainText('字符 8000');
|
||||
await record().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
const downloading = page.waitForEvent('download'); await panel().getByRole('button', { name: '下载原结果', exact: true }).first().click();
|
||||
const download = await downloading, downloadPath = await download.path(); assert.equal(sha(fs.readFileSync(downloadPath)), success.resultSha256);
|
||||
mark('original large JSON is paged in the UI and downloads the complete hash-identical result');
|
||||
|
||||
await panel().getByRole('button', { name: '最终截图', exact: true }).first().click(); const image = () => page.getByRole('dialog', { name: '原始页面截图', exact: true });
|
||||
await expect.poll(async () => image().locator('img').evaluate(node => node.naturalWidth)).toBe(1280); await expect(image()).toContainText(success.screenshots.at(-1).sha256);
|
||||
const imageDownload = page.waitForEvent('download'); await image().getByRole('button', { name: '下载原始PNG', exact: true }).click();
|
||||
const png = await imageDownload, pngPath = await png.path(); assert.equal(sha(fs.readFileSync(pngPath)), success.screenshots.at(-1).sha256);
|
||||
await screenshot('browser-desktop-original-image'); await image().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
mark('the original screenshot renders at its actual resolution and downloads identical PNG bytes');
|
||||
|
||||
await panel().getByLabel('步骤 3 预期数量').fill('2'); await seal(); await start(); const failure = await finished();
|
||||
assert.equal(failure.status, 'failed'); const failed = await original(failure); assert.equal(failed.steps[2].actual, 1); assert.equal(failed.assertions.failed, 1);
|
||||
await expect(panel()).toContainText('第3步失败'); await expect(panel().getByRole('button', { name: '运行这份计划', exact: true })).toBeDisabled();
|
||||
mark('a real failed assertion retains actual count one and requires a newly reviewed plan instead of replaying');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await expect(page.locator('.workbench')).toHaveClass(/sidebar-hidden/);
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
await panel().getByLabel('浏览器视口').selectOption('mobile'); await panel().getByLabel('步骤 3 预期数量').fill('1'); await seal(); await start(); const mobile = await finished();
|
||||
assert.equal(mobile.status, 'passed', mobile.summary); assert.equal(mobile.screenshots.at(-1).width, 390);
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390)); await screenshot('browser-mobile-results');
|
||||
await panel().getByRole('button', { name: '最终截图', exact: true }).first().click(); await expect.poll(async () => image().locator('img').evaluate(node => node.naturalWidth)).toBe(390);
|
||||
await screenshot('browser-mobile-original-image'); await image().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
mark('390px forms and result records remain usable and the inner browser captures its real 390px layout');
|
||||
|
||||
await panel().getByRole('button', { name: '添加步骤', exact: true }).click(); await panel().getByLabel('步骤 4 操作').selectOption('assert_count');
|
||||
await panel().getByLabel('步骤 4 元素').fill('.never-present'); await panel().getByLabel('步骤 4 预期数量').fill('1'); await seal(); await start();
|
||||
await expect.poll(async () => (await state()).execution.browserJobs.at(-1)?.steps.length, { timeout: 15000 }).toBe(3);
|
||||
await panel().getByRole('button', { name: '停止浏览器检查', exact: true }).click(); const stopped = await finished();
|
||||
assert.equal(stopped.status, 'interrupted'); assert.equal(stopped.cleanup.exited, true); assert.equal(stopped.steps[0].status, 'passed');
|
||||
assert.equal((await state()).execution.browserJobs.length, 4);
|
||||
mark('stopping an actual in-flight assertion closes its owned browser and preserves prior successful and failed jobs');
|
||||
|
||||
await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
if (await page.getByRole('button', { name: '展开项目栏', exact: true }).count()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
let held, release; const delivered = new Promise(resolve => { release = resolve; });
|
||||
const matcher = '**/browser-jobs/' + mobile.id + '/record?*';
|
||||
await page.route(matcher, pending => { held = pending; return delivered; });
|
||||
// Locate by original sequence: interrupted job, mobile success, failed job, desktop success.
|
||||
await panel().locator('.browser-result').nth(1).getByRole('button', { name: '查看原执行记录', exact: true }).click();
|
||||
await expect.poll(() => !!held).toBeTruthy(); const originalSession = (await state()).id;
|
||||
await page.getByRole('button', { name: '新建任务', exact: true }).first().click();
|
||||
await expect.poll(async () => (await host.api('/sessions?project=' + f.project.id)).length).toBe(2);
|
||||
await expect.poll(async () => page.evaluate(() => localStorage.getItem('awb-session'))).not.toBe(originalSession);
|
||||
await held.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: '原任务迟到错误' }) }); release(); await page.unroute(matcher);
|
||||
await expect(page.getByText('原任务迟到错误', { exact: false })).toHaveCount(0); assert.equal((await state()).id, originalSession);
|
||||
mark('a delayed original-record failure stays with its source and does not appear in a newly selected task');
|
||||
|
||||
const before = (await state()).execution.browserJobs; deliberateNavigation = true; await host.close(); host = await launchWorkbench(t, f.data);
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); deliberateNavigation = false; await selectTask();
|
||||
assert.deepEqual((await state()).execution.browserJobs, before); assert.equal((await state()).execution.task.usage.requests, 0);
|
||||
await panel().getByRole('button', { name: '最终截图', exact: true }).first().click(); await expect.poll(async () => image().locator('img').evaluate(node => node.naturalWidth)).toBe(390);
|
||||
await image().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
mark('actual host restart preserves the four original browser jobs and PNGs without model requests or execution replay');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
result = { success: true, version: host.bootstrap.version, testedAt: new Date().toISOString(), directory, passed, pageErrors, requestFailures,
|
||||
screenshots, taskId: actualJobs[0].taskId, sessionId: f.session.id, jobs: actualJobs.map(job => ({ id: job.id, status: job.status, planId: job.planId,
|
||||
assertions: job.assertions, resultSha256: job.resultSha256, cleanup: job.cleanup, screenshots: job.screenshots })),
|
||||
scope: 'Actual independent Workbench UI plus real Edge static-build executions. The configured production Provider, other projects and keys are not accessed.' };
|
||||
} catch (error) {
|
||||
await screenshot('browser-ui-failure').catch(() => {}); fs.writeFileSync(path.join(directory, 'failure-dom.html'), await page.content().catch(() => ''));
|
||||
result = { success: false, version: host.bootstrap.version, testedAt: new Date().toISOString(), directory, passed, pageErrors, requestFailures, screenshots, error: error.stack }; process.exitCode = 1;
|
||||
} finally { deliberateNavigation = true; await browser.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(`docs/${releaseEvidencePrefix(host.bootstrap.version)}-browser-ui-validation.json`, JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, groups: passed.length, directory, error: result.error }));
|
||||
@@ -0,0 +1,109 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
async function until(predicate, timeout = 20000) {
|
||||
const deadline = Date.now() + timeout;
|
||||
while (!await predicate()) {
|
||||
if (Date.now() >= deadline) throw new Error('Owned browser verification timed out.');
|
||||
await new Promise(resolve => setTimeout(resolve, 30));
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyBrowserApi({ api, url, token, directory, appRoot, contained = false }) {
|
||||
const projectRoot = path.join(directory, 'browser-project'); fs.mkdirSync(path.join(projectRoot, 'dist'), { recursive: true });
|
||||
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned portable browser acceptance\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'dist', 'index.html'), '<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>随包页面验收</title>'
|
||||
+ '<style>body{font:16px sans-serif;margin:20px}input,select,button{display:block;max-width:100%;margin:12px 0}</style><h1>实际随包浏览器</h1>'
|
||||
+ '<input id="query" aria-label="搜索"><select id="filter"><option value="all">全部</option><option value="pending">待办</option></select>'
|
||||
+ '<button id="toggle">切换</button><p id="state">待办</p><script src="/app.js"></script></html>');
|
||||
fs.writeFileSync(path.join(projectRoot, 'dist', 'app.js'), 'document.querySelector("#toggle").addEventListener("click",()=>document.querySelector("#state").textContent="已完成");');
|
||||
const project = await api('/projects', { root: projectRoot }), session = await api('/sessions', { projectId: project.id }), route = '/sessions/' + session.id;
|
||||
await api(route + '/messages', { content: '为自有静态工程创建独立验收任务。' }); let current;
|
||||
await until(async () => { current = await api(route); return current.status === 'awaiting_approval'; });
|
||||
const approval = current.approvals.find(value => value.status === 'pending'); assert.equal(approval.command, 'node --version');
|
||||
await api(route + '/approvals/' + approval.id, { allow: true });
|
||||
await until(async () => { current = await api(route); return current.status === 'needs_review'; });
|
||||
for (const change of current.changes.filter(value => value.status === 'pending')) await api(route + '/changes/' + change.id + '/reject', {});
|
||||
const taskId = (await api(route)).execution.task.id, availability = (await api(route + '/browser')).availability;
|
||||
assert.equal(availability.status, 'available', availability.message); assert.equal(availability.playwrightVersion, '1.64.1');
|
||||
const steps = [{ action: 'fill', selector: '#query', value: '本地页面验收' }, { action: 'select', selector: '#filter', value: 'pending' },
|
||||
{ action: 'assert_count', selector: 'button', count: 1 }, { action: 'assert_text', selector: '#state', text: '待办' },
|
||||
{ action: 'click', selector: '#toggle' }, { action: 'assert_text', selector: '#state', text: '已完成' }];
|
||||
const results = [];
|
||||
for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
||||
const planned = await api(route + '/browser-plans', { steps, viewport }), plan = planned.execution.browserPlans.at(-1);
|
||||
const contract = await api('/artifacts/' + plan.contractArtifactId + '?session=' + session.id + '&limit=16000');
|
||||
assert.equal(contract.truncated, false); assert.equal(sha(contract.content), contract.sha256);
|
||||
const sealed = JSON.parse(contract.content); assert.equal(sealed.id, plan.id); assert.equal(sealed.taskId, taskId);
|
||||
assert.equal(sealed.installation.playwrightVersion, '1.64.1');
|
||||
if (contained) assert.ok(sealed.installation.modulePath.startsWith(fs.realpathSync.native(path.join(appRoot, 'runtime', 'node_modules')) + path.sep));
|
||||
await api(route + '/browser-jobs', { planId: plan.id }); let job;
|
||||
await until(async () => { job = (await api(route)).execution.browserJobs.at(-1); return !!job?.finishedAt; }, 25000);
|
||||
assert.equal(job.status, 'passed', job.summary); assert.equal(job.taskId, taskId); assert.equal(job.assertions.passed, 3);
|
||||
assert.equal(job.cleanup.exited, true); assert.equal(job.cleanup.previewClosed, true); assert.equal(job.cleanup.snapshotDisposed, true);
|
||||
let text = '', offset = 0, originalPage;
|
||||
do { originalPage = await api(route + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=4000'); text += originalPage.content; offset += originalPage.content.length; } while (originalPage.truncated);
|
||||
assert.equal(sha(text), job.resultSha256); const original = JSON.parse(text), actualPage = original.pages.at(-1);
|
||||
assert.equal(original.steps[0].actual, '本地页面验收'); assert.deepEqual(original.steps[1].actual, ['pending']);
|
||||
assert.deepEqual(original.pageErrors, []); assert.deepEqual(original.blockedRequests, []); assert.equal(actualPage.viewport.width, viewport.width);
|
||||
assert.ok(actualPage.text.includes('已完成')); if (viewport.width === 390) assert.ok(actualPage.viewport.scrollWidth <= 390);
|
||||
fs.writeFileSync(path.join(directory, 'browser-original-' + viewport.width + '.json'), text);
|
||||
const image = job.screenshots.at(-1), response = await fetch(url + '/api' + route + '/browser-jobs/' + job.id + '/images/' + image.artifactId,
|
||||
{ headers: { 'X-Workbench-Token': token, Connection: 'close' }, signal: AbortSignal.timeout(15000) });
|
||||
assert.equal(response.status, 200); assert.equal(response.headers.get('content-type'), 'image/png'); const bytes = Buffer.from(await response.arrayBuffer());
|
||||
assert.equal(sha(bytes), image.sha256); assert.equal(bytes.length, image.bytes); assert.equal(bytes.readUInt32BE(16), viewport.width); assert.equal(bytes.readUInt32BE(20), viewport.height);
|
||||
fs.writeFileSync(path.join(directory, 'browser-original-' + viewport.width + '.png'), bytes);
|
||||
const reused = await api(route + '/browser-jobs', { planId: plan.id }); assert.equal(reused.execution.browserJobs.at(-1).id, job.id);
|
||||
assert.equal(reused.execution.browserJobs.length, results.length + 1);
|
||||
results.push({ jobId: job.id, taskId, viewport, status: job.status, assertions: job.assertions, resultSha256: job.resultSha256,
|
||||
buildSha256: job.buildSha256, screenshotSha256: image.sha256, browserExited: job.cleanup.exited, reusedOriginalJob: true });
|
||||
}
|
||||
return { success: true, installation: availability, results, modelRequests: 0,
|
||||
scope: 'Actual API, locked browser runtime, two isolated Edge processes, CSS interactions, original DOM and PNG bytes. The task is created through the offline demonstration; no remote model is used.' };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-browser.mjs [--app-root <source-or-portable-directory>] [--contained]'); return; }
|
||||
const index = process.argv.indexOf('--app-root'); if (index >= 0 && !process.argv[index + 1]) throw new Error('--app-root needs an absolute or relative directory.');
|
||||
const appRoot = fs.realpathSync.native(path.resolve(index >= 0 ? process.argv[index + 1] : '.'));
|
||||
const directory = path.resolve('.qa', 'browser-native-' + randomUUID()), data = path.join(directory, 'data'); fs.mkdirSync(data, { recursive: true });
|
||||
const allowed = /^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all)$/i;
|
||||
const env = { ...Object.fromEntries(Object.entries(process.env).filter(([name]) => allowed.test(name))),
|
||||
AGENT_WORKBENCH_APP_DIR: appRoot, AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' };
|
||||
const runtime = path.join(appRoot, 'runtime', 'node.exe');
|
||||
const child = spawn(fs.existsSync(runtime) ? runtime : process.execPath, [path.join(appRoot, 'dist-server', 'server.mjs')],
|
||||
{ cwd: appRoot, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let url = '', stdout = '', stderr = '', api, result;
|
||||
child.stdout.on('data', bytes => { stdout += bytes.toString('utf8'); for (const line of stdout.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
|
||||
child.stderr.on('data', bytes => { stderr += bytes.toString('utf8'); });
|
||||
try {
|
||||
await until(() => url || child.exitCode !== null, 10000); assert.ok(url, 'Owned host failed: ' + stderr); assert.notEqual(new URL(url).port, '4318');
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json(); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const health = await (await fetch(url + '/api/health')).json(); assert.equal(health.pid, child.pid);
|
||||
api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
||||
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
||||
};
|
||||
result = { ...await verifyBrowserApi({ api, url, token: bootstrap.token, directory, appRoot, contained: process.argv.includes('--contained') }),
|
||||
version: bootstrap.version, directory, appRoot, testedAt: new Date().toISOString(), ownedHostPid: child.pid };
|
||||
} catch (error) { result = { success: false, directory, appRoot, error: error.stack }; process.exitCode = 1; }
|
||||
finally {
|
||||
if (api && child.exitCode === null) {
|
||||
try { await api('/shutdown', {}); await until(() => child.exitCode !== null || child.signalCode !== null, 10000); }
|
||||
catch (error) { result.success = false; result.shutdownError = error.message; process.exitCode = 1; }
|
||||
}
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill();
|
||||
await until(() => child.exitCode !== null || child.signalCode !== null, 5000); result.rootExited = true;
|
||||
fs.writeFileSync(path.join(directory, 'host.log'), stdout + '\n' + stderr);
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('.qa/browser-native-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, directory, results: result.results, modelRequests: 0, rootExited: result.rootExited, error: result.error, shutdownError: result.shutdownError }));
|
||||
}
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1,65 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import { X509Certificate } from 'node:crypto';
|
||||
import assert from 'node:assert/strict';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
|
||||
// This helper checks the real mobile shell against the same host, observation
|
||||
// and original PNG as the actual Electron test. The loopback relay verifies
|
||||
// the gateway CA on every request; it neither installs trust nor bypasses TLS.
|
||||
export async function verifyBrowsingMobile({ controller, session, root, record }) {
|
||||
let browser, relay, cookie = '', gateway;
|
||||
const errors = [];
|
||||
const post=(route,input)=>controller.api(route,{method:'POST',input});
|
||||
try {
|
||||
const status = await controller.api('/companion'); assert.equal(status.enabled, false); assert.ok(status.addresses.length);
|
||||
const address = status.addresses[0].address, reservation = http.createServer();
|
||||
await new Promise(resolve => reservation.listen(0, address, resolve)); const port = reservation.address().port;
|
||||
await new Promise(resolve => reservation.close(resolve));
|
||||
gateway = await post('/companion/start', { address, port });
|
||||
const certificate = await fetch(controller.url + '/api/companion/certificate', { headers: { 'X-Workbench-Token': controller.token } });
|
||||
assert.equal(certificate.status, 200); const ca = new X509Certificate(Buffer.from(await certificate.arrayBuffer())).toString();
|
||||
const pair = await post('/companion/pair', { projectIds: [session.projectId], permissions: { observe: true, approve: true, stop: true }, expiresInHours: 1 });
|
||||
const target = new URL(gateway.origin);
|
||||
relay = http.createServer((request, response) => {
|
||||
const forwarded = https.request(new URL(request.url, target), { method: request.method, ca, servername: '', headers: {
|
||||
Host: target.host, Origin: target.origin, Connection: 'close', ...(cookie ? { Cookie: cookie } : {}),
|
||||
...(request.headers['content-type'] ? { 'Content-Type': request.headers['content-type'] } : {}) } }, incoming => {
|
||||
if (incoming.headers['set-cookie']?.length) cookie = incoming.headers['set-cookie'][0].split(';')[0];
|
||||
const headers = { ...incoming.headers }; delete headers['set-cookie']; response.writeHead(incoming.statusCode, headers); incoming.pipe(response);
|
||||
});
|
||||
forwarded.on('error', () => { if (response.headersSent) response.destroy(); else { response.writeHead(503); response.end('Owned TLS relay failed'); } });
|
||||
request.pipe(forwarded);
|
||||
});
|
||||
await new Promise(resolve => relay.listen(0, '127.0.0.1', resolve));
|
||||
const url = 'http://127.0.0.1:' + relay.address().port;
|
||||
browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true, serviceWorkers: 'block' });
|
||||
const mobile = await context.newPage(); mobile.on('pageerror', error => errors.push(error.message));
|
||||
await mobile.goto(url + '/#pair=' + new URLSearchParams(new URL(pair.url).hash.slice(1)).get('pair'));
|
||||
await mobile.getByRole('button', { name: '连接这台电脑', exact: true }).click();
|
||||
await mobile.locator('.mobile-task-row').filter({ hasText: session.title }).click();
|
||||
const observation = mobile.locator('.mobile-input-image').filter({ has: mobile.getByRole('heading', { name: '电脑网页原观察', exact: true }) });
|
||||
await expect(observation).toContainText('SHA-256 已核对');
|
||||
await observation.getByText('查看原页面截图', { exact: true }).click(); await expect(observation.getByRole('img')).toBeVisible();
|
||||
assert.equal(await mobile.evaluate(() => document.documentElement.scrollWidth > innerWidth), false);
|
||||
await mobile.screenshot({ path: path.join(root, 'phone-original-browser-observation.png'), fullPage: true });
|
||||
record('390px mobile shell verifies and displays the same actual Electron original PNG through CA-verified TLS', { nativeIPhone: 'pending', trustChanged: false });
|
||||
await mobile.route('**/browser-observations/*/image', async route => { const response = await route.fetch(); await route.fulfill({ response, headers: { ...response.headers(), 'x-content-sha256': '0'.repeat(64) } }); });
|
||||
await mobile.reload(); await mobile.locator('.mobile-task-row').filter({ hasText: session.title }).click();
|
||||
await expect(observation).toContainText('原网页截图SHA-256核对失败'); await expect(observation.getByRole('img')).toHaveCount(0);
|
||||
await mobile.screenshot({ path: path.join(root, 'phone-browser-digest-rejected.png'), fullPage: true });
|
||||
record('mobile digest mismatch shows failure and never displays unverified screenshot');
|
||||
const devices = (await controller.api('/companion')).devices; assert.equal(devices.length, 1);
|
||||
await post('/companion/devices/' + devices[0].id + '/revoke', { revision: devices[0].revision });
|
||||
await mobile.getByRole('button', { name: '刷新任务', exact: true }).click();
|
||||
await expect(mobile.getByRole('heading', { name: '接续电脑上的任务', exact: true })).toBeVisible(); await expect(mobile.getByRole('img')).toHaveCount(0);
|
||||
record('mobile device revocation removes original browser observation and task content'); assert.deepEqual(errors, []);
|
||||
} finally {
|
||||
if (browser) await browser.close();
|
||||
if (relay) await new Promise(resolve => { relay.closeAllConnections(); relay.close(resolve); });
|
||||
if (gateway) await post('/companion/close', {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { batchFixture, assertBatchBytes } from '../tests/fixtures/change-set-project.mjs';
|
||||
|
||||
const root = path.resolve('.qa', 'batch-ui-' + randomUUID()), data = path.join(root, 'data'); fs.mkdirSync(root, { recursive: true });
|
||||
fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const store = new Store(data), cases = {};
|
||||
for (const name of ['normal', 'conflict', 'recovery', 'mobile', 'delayed']) {
|
||||
const projectRoot = path.join(root, 'batch-ui-' + name); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned UI fixture\n');
|
||||
const project = store.addProject(projectRoot), session = store.createSession(project.id, '批次验收 ' + name), f = { projectRoot, project, session, store };
|
||||
const b = batchFixture(f, name === 'recovery' ? { faultInjector: (phase, set, index) => {
|
||||
if (phase === 'replaced' && index === 1) { fs.writeFileSync(path.join(projectRoot, 'src/one.mjs'), 'Owner recovery content\n'); throw new Error('Owned UI lost result'); }
|
||||
} } : {});
|
||||
cases[name] = { ...f, b };
|
||||
if (name === 'mobile') {
|
||||
const run = store.core.run.get(session.runId); session.status = 'paused'; store.finishRun(session, run, 'step_limit');
|
||||
}
|
||||
if (name === 'recovery' || name === 'mobile') {
|
||||
const set = await b.manager.prepare(session, b.ids);
|
||||
if (name === 'recovery') await (await b.manager.start(session, set.id, set.fingerprint)).promise;
|
||||
}
|
||||
}
|
||||
store.close();
|
||||
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }; delete env.AGENT_WORKBENCH_API_KEY;
|
||||
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let output = '', errors = '', url = ''; child.stdout.setEncoding('utf8'); child.stderr.setEncoding('utf8');
|
||||
child.stdout.on('data', value => { output += value; for (const line of output.split('\n')) { try { const item = JSON.parse(line); if (item.ready) url = item.url; } catch {} } });
|
||||
child.stderr.on('data', value => { errors += value; });
|
||||
const wait = async predicate => { const deadline = Date.now() + 15000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('UI fixture timed out: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||||
await wait(() => url); const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.equal(bootstrap.version, JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version, 'The UI host must match this package version.'); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const proofPrefix = releaseEvidencePrefix(bootstrap.version) + '-batch';
|
||||
const screenshotPrefix = bootstrap.version === '1.4.0' ? '1.4-batch' : proofPrefix;
|
||||
const api = async (route, input) => { const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }) }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message)); page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') requestFailures.push(request.failure()?.errorText); });
|
||||
const dialog = () => page.getByRole('dialog', { name: '多文件变更集审查', exact: true });
|
||||
const session = f => api('/sessions/' + f.session.id);
|
||||
const current = async f => (await session(f)).execution.changeSets.at(-1);
|
||||
async function choose(name) {
|
||||
const f = cases[name];
|
||||
const selector = page.getByLabel('当前项目', { exact: true }), expand = page.getByRole('button', { name: '展开项目栏', exact: true });
|
||||
if ((await page.viewportSize()).width <= 620) await expect(expand).toBeVisible();
|
||||
if (!await selector.isVisible()) await expand.click();
|
||||
await expect(selector).toBeVisible(); await selector.selectOption(f.project.id);
|
||||
await expect(page.locator('.session-item.selected')).toContainText('批次验收 ' + name);
|
||||
if ((await page.viewportSize()).width <= 940) {
|
||||
if (await page.locator('.project-sidebar').isVisible()) await page.getByRole('button', { name: '收起项目栏', exact: true }).click();
|
||||
await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
||||
} else await page.locator('.inspector-tabs').getByRole('button', { name: /^改动/ }).click();
|
||||
await expect(page.locator('.changes-inspector')).toBeVisible(); return f;
|
||||
}
|
||||
async function seal() {
|
||||
await page.getByRole('button', { name: '组合审查', exact: true }).click();
|
||||
await page.getByRole('button', { name: '封存审查范围', exact: true }).click(); await expect(dialog()).toBeVisible();
|
||||
}
|
||||
async function viewport(name) {
|
||||
const bounds = await page.evaluate(() => {
|
||||
const modal = document.querySelector('[role="dialog"]'), close = modal?.querySelector('[aria-label="关闭窗口"]');
|
||||
const rect = modal?.getBoundingClientRect(), closeRect = close?.getBoundingClientRect();
|
||||
return { scrollY, width: innerWidth, height: innerHeight, scrollWidth: document.documentElement.scrollWidth, scrollHeight: document.documentElement.scrollHeight,
|
||||
modal: rect ? { top: rect.top, bottom: rect.bottom, width: rect.width, scrollWidth: modal.scrollWidth } : null,
|
||||
close: closeRect ? { top: closeRect.top, bottom: closeRect.bottom } : null };
|
||||
});
|
||||
assert.equal(bounds.scrollY, 0); assert.ok(bounds.scrollWidth <= bounds.width + 1 && bounds.scrollHeight <= bounds.height + 1, JSON.stringify(bounds));
|
||||
if (bounds.modal) assert.ok(bounds.modal.top >= 0 && bounds.modal.bottom <= bounds.height + 1 && bounds.modal.scrollWidth <= bounds.modal.width + 1 && bounds.close.bottom <= bounds.height, JSON.stringify(bounds));
|
||||
const destination = '.qa/screenshots/' + screenshotPrefix + '-' + name + '.png'; await page.screenshot({ path: destination, scale: 'css' }); screenshots.push(destination);
|
||||
}
|
||||
try {
|
||||
await page.goto(url, { waitUntil: 'networkidle' });
|
||||
let f = await choose('normal'); await seal();
|
||||
await expect(page.getByRole('button', { name: '应用这项改动', exact: true })).toBeDisabled();
|
||||
for (const relative of Object.keys(f.b.after)) { await dialog().getByRole('button', { name: '查看批次差异 ' + relative, exact: true }).click(); await expect(dialog().getByLabel('文件差异 ' + relative)).toBeVisible(); }
|
||||
assertBatchBytes(assert, f, f.b.before); await viewport('desktop-review'); passed.push('Desktop batch scope, per-file diffs, sealed single-write guard and pre-apply bytes');
|
||||
await dialog().getByRole('button', { name: '确认应用全部 3 项', exact: true }).click(); await wait(async () => (await current(f)).status === 'completed');
|
||||
assertBatchBytes(assert, f, f.b.after); await expect(dialog().getByText('整批文件的实际建议字节已核对,全部应用完成。')).toBeVisible(); passed.push('Three-file reviewed application preserves exact BOM/CRLF/LF and creates the new file');
|
||||
await dialog().getByRole('button', { name: '回退本批已写入文件', exact: true }).click(); await dialog().getByRole('button', { name: '确认回退本批次', exact: true }).click();
|
||||
await wait(async () => (await current(f)).status === 'rolled_back'); assertBatchBytes(assert, f, f.b.before); passed.push('Explicit batch rollback restores exact originals and removes only the reviewed new file');
|
||||
await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
f = await choose('conflict'); await seal(); fs.mkdirSync(path.join(f.projectRoot, 'docs')); fs.writeFileSync(path.join(f.projectRoot, 'docs/new.md'), 'Owner conflict\n');
|
||||
await dialog().getByRole('button', { name: '确认应用全部 3 项', exact: true }).click(); await wait(async () => (await current(f)).status === 'stale');
|
||||
assertBatchBytes(assert, f, { ...f.b.before, 'docs/new.md': Buffer.from('Owner conflict\n') });
|
||||
await expect(dialog().getByRole('button', { name: '确认应用全部 3 项', exact: true })).toHaveCount(0); await viewport('desktop-conflict'); passed.push('A last-file owner conflict blocks the whole batch and shows the real recovery direction');
|
||||
await dialog().getByRole('button', { name: '撤销未执行批次', exact: true }).click(); await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
f = await choose('recovery'); await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
||||
await dialog().getByRole('button', { name: '核对实际文件', exact: true }).click(); await expect(dialog().getByText('外部改动', { exact: true })).toBeVisible();
|
||||
await dialog().getByRole('button', { name: '回退本批已写入文件', exact: true }).click(); await dialog().getByRole('button', { name: '确认回退本批次', exact: true }).click();
|
||||
await expect(dialog().getByRole('button', { name: '保留外部改动,结束恢复', exact: true })).toBeEnabled(); await viewport('desktop-recovery'); passed.push('Durable partial recovery reads actual hashes and preserves owner content during rollback');
|
||||
await dialog().getByRole('button', { name: '保留外部改动,结束恢复', exact: true }).click(); fs.writeFileSync(path.join(f.projectRoot, 'src/one.mjs'), 'Owner changed after review\n');
|
||||
await dialog().getByRole('button', { name: '确认保留外部改动', exact: true }).click(); await expect(page.locator('.toast')).toContainText('当前文件变化');
|
||||
assert.equal((await current(f)).requiresVerification, true); passed.push('Changed observation fingerprints reject stale acknowledgement without releasing the project');
|
||||
await dialog().getByRole('button', { name: '返回审查', exact: true }).click(); await dialog().getByRole('button', { name: '核对实际文件', exact: true }).click();
|
||||
await dialog().getByRole('button', { name: '保留外部改动,结束恢复', exact: true }).click(); await dialog().getByRole('button', { name: '确认保留外部改动', exact: true }).click();
|
||||
await wait(async () => (await current(f)).status === 'resolved_external'); assert.equal(fs.readFileSync(path.join(f.projectRoot, 'src/one.mjs'), 'utf8'), 'Owner changed after review\n');
|
||||
passed.push('Explicit owner-content acknowledgement ends recovery without claiming successful application'); await dialog().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
await page.setViewportSize({ width: 390, height: 844 }); f = await choose('mobile');
|
||||
await expect(page.locator('.toast')).toHaveCount(0); passed.push('Switching projects clears the previous task notification while retaining its durable error and recovery history');
|
||||
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
||||
await viewport('mobile-review'); await dialog().getByRole('button', { name: '确认应用全部 3 项', exact: true }).click(); await wait(async () => (await current(f)).status === 'completed');
|
||||
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toBeVisible();
|
||||
if (!await page.locator('.changes-inspector').isVisible()) await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
||||
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
||||
await dialog().getByRole('button', { name: '回退本批已写入文件', exact: true }).click(); await viewport('mobile-rollback');
|
||||
await dialog().getByRole('button', { name: '确认回退本批次', exact: true }).click(); await wait(async () => (await current(f)).status === 'rolled_back'); assertBatchBytes(assert, f, f.b.before);
|
||||
await page.keyboard.press('Escape'); await expect(dialog()).toHaveCount(0); await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toHaveCount(0);
|
||||
passed.push('390px step-limit review, same-task continuation entry, rollback confirmation and keyboard close fit the viewport');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 }); f = await choose('delayed');
|
||||
let release, held = false; const gate = new Promise(resolve => { release = resolve; });
|
||||
await page.route('**/api/sessions/' + f.session.id + '/change-sets', async route => { held = true; await gate; await route.continue(); });
|
||||
await page.getByRole('button', { name: '组合审查', exact: true }).click(); await page.getByRole('button', { name: '封存审查范围', exact: true }).click(); await wait(() => held);
|
||||
await page.keyboard.press('Escape'); await choose('normal'); release(); await wait(async () => (await current(f))?.status === 'prepared');
|
||||
await expect(dialog()).toHaveCount(0); await expect(page.locator('.session-item.selected')).toContainText('批次验收 normal');
|
||||
const delayed = await current(f); await api('/sessions/' + f.session.id + '/change-sets/' + delayed.id + '/discard', { fingerprint: delayed.fingerprint });
|
||||
passed.push('A delayed preparation response never opens a different project or changes its selected session');
|
||||
f = await choose('delayed'); let failRelease, failureHeld = false;
|
||||
const failedGate = new Promise(resolve => { failRelease = resolve; }), failedUrl = '/api/sessions/' + f.session.id + '/change-sets';
|
||||
await page.route('**' + failedUrl, async route => { failureHeld = true; await failedGate; await route.fulfill({ status: 409, contentType: 'application/json', body: JSON.stringify({ error: '旧任务的延迟批次准备失败' }) }); });
|
||||
await page.getByRole('button', { name: '组合审查', exact: true }).click(); await page.getByRole('button', { name: '封存审查范围', exact: true }).click(); await wait(() => failureHeld);
|
||||
await page.keyboard.press('Escape'); await choose('normal');
|
||||
const failedResponse = page.waitForResponse(response => response.url().endsWith(failedUrl) && response.status() === 409);
|
||||
failRelease(); await (await failedResponse).finished();
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
||||
await expect(page.locator('.toast')).toHaveCount(0); await expect(dialog()).toHaveCount(0); await expect(page.locator('.session-item.selected')).toContainText('批次验收 normal');
|
||||
passed.push('A failed batch response after its component unmounts cannot display an error in the newly selected project');
|
||||
assert.equal(pageErrors.length, 0, pageErrors.join('\n')); assert.equal(requestFailures.length, 0, requestFailures.join('\n'));
|
||||
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), cases: passed, pageErrors, requestFailures, screenshots, projectRoot: root,
|
||||
scope: 'Real independent demo host and durable batch states; UI fixtures include actual owner conflict/rollback results. No real model credentials or user project were used.' };
|
||||
fs.writeFileSync('docs/' + proofPrefix + '-ui-validation.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: '.qa/screenshots/' + screenshotPrefix + '-failure.png', scale: 'css' }); fs.writeFileSync('.qa/' + proofPrefix + '-ui-failure.json', JSON.stringify({ passed, message: error.message, pageErrors, requestFailures }, null, 2)); throw error;
|
||||
} finally {
|
||||
await browser.close(); try { await api('/shutdown', {}); await wait(() => child.exitCode !== null); } catch { if (child.exitCode === null) child.kill(); }
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import { X509Certificate, randomUUID, createHash } from 'node:crypto';
|
||||
import assert from 'node:assert/strict';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-companion-ui.mjs [--app-root <package>] - isolated desktop/mobile UI; an explicit package runs its contained Node and actual server. Loopback relay verifies the gateway CA; does not modify OS trust or substitute for iPhone acceptance.'); process.exit(0); }
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length && (args.length !== 2 || args[0] !== '--app-root' || !path.isAbsolute(args[1]))) throw new Error('Use --app-root with an absolute package directory, or no arguments.');
|
||||
const appRoot = fs.realpathSync.native(args.length ? args[1] : process.cwd());
|
||||
const nodePath = args.length ? path.join(appRoot, 'runtime', 'node.exe') : process.execPath;
|
||||
assert.ok(fs.existsSync(nodePath));
|
||||
if (args.length) assert.ok(fs.existsSync(path.join(appRoot, 'runtime', 'node_modules', 'npm', 'bin', 'npm-cli.js')), 'The contained engineering runner requires bundled npm.');
|
||||
const directory = path.resolve('.qa', 'companion-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: action => disposers.push(action) }, f = fixture(t);
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ name: 'owned-companion-ui', private: true, type: 'module', scripts: { test: 'node --test fixture.test.mjs' } }));
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'fixture.test.mjs'), "import test from 'node:test';import assert from 'node:assert/strict';test('owned mobile fixture',()=>assert.equal(1+1,2));\n");
|
||||
f.store.close();
|
||||
const host = await launchWorkbench(t, f.data, { Path: path.dirname(nodePath) + path.delimiter + (process.env.Path || process.env.PATH || '') }, { appRoot, nodePath });
|
||||
assert.equal(fs.realpathSync.native(host.bootstrap.runtimeRoot), appRoot, 'The isolated host must actually serve the selected package.');
|
||||
await host.api('/sessions/' + f.session.id + '/messages', { content: '手机伴随端验收:读取项目、准备原差异并等待手机审批。明确为本地离线夹具。' });
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id)).status === 'awaiting_approval');
|
||||
const before = await host.api('/sessions/' + f.session.id), browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const desktop = await browser.newPage({ viewport: { width: 1440, height: 1000 } }), errors = [], passed = [], screenshots = [];
|
||||
desktop.on('pageerror', error => errors.push(error.message));
|
||||
const mark = text => { passed.push(text); console.log('PASS ' + text); };
|
||||
async function shot(page, name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, fullPage: !(await page.locator(':modal').count()) }); screenshots.push(file); }
|
||||
let mobile, relay, gatewayStatus, cookie = '', disconnected = false;
|
||||
try {
|
||||
await desktop.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await desktop.goto(host.url); await desktop.getByRole('button', { name: '手机连接', exact: true }).click();
|
||||
const panel = desktop.getByRole('dialog', { name: '手机连接', exact: true }); await expect(panel).toBeVisible();
|
||||
await expect(panel).toContainText('默认关闭');
|
||||
const initial = await host.api('/companion'); assert.ok(initial.addresses.length, 'Requires an actual owned private IPv4 interface.');
|
||||
const reservation = http.createServer(); await new Promise(resolve => reservation.listen(0, initial.addresses[0].address, resolve));
|
||||
const port = reservation.address().port; await new Promise(resolve => reservation.close(resolve));
|
||||
await panel.getByLabel('手机连接端口', { exact: true }).fill(String(port));
|
||||
await panel.getByRole('button', { name: '开启 HTTPS', exact: true }).click();
|
||||
await expect(panel).toContainText('手机连接已开启', { timeout: 20000 });
|
||||
await expect(panel).toContainText('证书信任设置'); await shot(desktop, 'desktop-certificate');
|
||||
const downloadEvent = desktop.waitForEvent('download'); await panel.getByRole('button', { name: '下载手机根证书', exact: true }).click();
|
||||
const download = await downloadEvent, certFile = path.join(directory, 'fixture-mobile-ca.cer'); await download.saveAs(certFile);
|
||||
const ca = new X509Certificate(fs.readFileSync(certFile)).toString();
|
||||
assert.equal(new X509Certificate(ca).ca, true); mark('desktop explicitly enables HTTPS and downloads the actual public CA without changing OS trust');
|
||||
await panel.getByLabel('允许审批命令', { exact: true }).check(); await panel.getByLabel('允许停止任务', { exact: true }).check();
|
||||
await panel.getByRole('button', { name: '生成一次性配对码', exact: true }).click(); await expect(panel.getByAltText('手机一次性配对二维码')).toBeVisible();
|
||||
const pairUrl = await panel.locator('.companion-pair-url').textContent(); await shot(desktop, 'desktop-pairing');
|
||||
gatewayStatus = await host.api('/companion');
|
||||
// Browser rendering uses an owned loopback relay because this test must not
|
||||
// silently install a root or bypass a TLS warning. The relay verifies the
|
||||
// real gateway certificate for every request and holds only its fixture
|
||||
// Secure cookie in memory. Native Safari cookie/trust/PWA remains device QA.
|
||||
const target = new URL(gatewayStatus.origin);
|
||||
relay = http.createServer((request, response) => {
|
||||
if (disconnected && request.url.startsWith('/mobile-api/')) { response.writeHead(503, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: '自有断线夹具' })); return; }
|
||||
const headers = { Host: target.host, Origin: target.origin, Connection: 'close', ...(cookie ? { Cookie: cookie } : {}),
|
||||
...(request.headers['content-type'] ? { 'Content-Type': request.headers['content-type'] } : {}) };
|
||||
const forwarded = https.request(new URL(request.url, target), { method: request.method, headers, ca, servername: '' }, incoming => {
|
||||
const set = incoming.headers['set-cookie']; if (set?.length) cookie = set[0].split(';')[0];
|
||||
const outgoing = { ...incoming.headers }; delete outgoing['set-cookie']; response.writeHead(incoming.statusCode, outgoing); incoming.pipe(response);
|
||||
});
|
||||
forwarded.on('error', () => { if (!response.headersSent) { response.writeHead(503, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: '自有TLS转发中断' })); } else response.destroy(); });
|
||||
request.pipe(forwarded);
|
||||
});
|
||||
await new Promise(resolve => relay.listen(0, '127.0.0.1', resolve));
|
||||
const relayUrl = 'http://127.0.0.1:' + relay.address().port;
|
||||
const mobileContext = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true });
|
||||
mobile = await mobileContext.newPage(); mobile.on('pageerror', error => errors.push(error.message));
|
||||
await mobile.goto(relayUrl + '/#pair=' + new URLSearchParams(new URL(pairUrl).hash.slice(1)).get('pair'));
|
||||
await expect(mobile.getByRole('heading', { name: '接续电脑上的任务', exact: true })).toBeVisible();
|
||||
assert.equal(new URL(mobile.url()).hash, '');
|
||||
await mobile.getByLabel('设备名称', { exact: true }).fill('验收 iPhone 观察窗口');
|
||||
await mobile.getByRole('button', { name: '连接这台电脑', exact: true }).click();
|
||||
await expect(mobile.getByRole('heading', { name: '项目任务', exact: true })).toBeVisible(); await shot(mobile, 'mobile-task-list');
|
||||
await expect(panel).toContainText('验收 iPhone 观察窗口', { timeout: 10000 });
|
||||
mark('390px real gateway pairing clears fragment, displays scoped task list and registers the same device in desktop');
|
||||
await mobile.locator('.mobile-task-row').filter({ hasText: before.title }).click();
|
||||
await expect(mobile.getByRole('heading', { name: '等待你的确认', exact: true })).toBeVisible();
|
||||
await mobile.getByText('原任务与运行', { exact: true }).click(); await expect(mobile.locator('.mobile-task-title code')).toContainText(before.taskId);
|
||||
assert.ok(await mobile.locator('body').evaluate(body => body.scrollWidth <= innerWidth)); await shot(mobile, 'mobile-original-approval');
|
||||
await mobile.getByRole('button', { name: '核对并批准', exact: true }).click();
|
||||
const confirmation = mobile.getByRole('dialog'); await expect(confirmation).toContainText(before.runId);
|
||||
await confirmation.getByRole('button', { name: '确认', exact: true }).click();
|
||||
await until(async () => ['needs_review', 'completed'].includes((await host.api('/sessions/' + f.session.id)).status));
|
||||
await expect(mobile.locator('.mobile-receipt')).toContainText('已批准原请求');
|
||||
const after = await host.api('/sessions/' + f.session.id); assert.equal(after.taskId, before.taskId); assert.equal(after.runId, before.runId);
|
||||
assert.equal(after.events.filter(item => item.tool === 'run_command').length, 1); mark('phone UI confirms original Task/Run approval and the existing Runtime executes one command');
|
||||
await mobile.getByRole('button', { name: /^文件差异/ }).click();
|
||||
await mobile.getByRole('button', { name: /WORKBENCH-NOTES.md/ }).click(); await expect(mobile.getByRole('dialog')).toContainText('SHA-256 已核对');
|
||||
await shot(mobile, 'mobile-original-diff'); await mobile.getByRole('button', { name: '返回任务', exact: true }).click();
|
||||
await host.api('/sessions/' + f.session.id + '/changes/' + after.changes.at(-1).id + '/apply', {});
|
||||
assert.equal(fs.existsSync(path.join(f.projectRoot, 'WORKBENCH-NOTES.md')), true);
|
||||
await host.api('/sessions/' + f.session.id + '/check-plans', { names: ['test'] });
|
||||
const planned = await host.api('/sessions/' + f.session.id); await host.api('/sessions/' + f.session.id + '/check-jobs', { planId: planned.execution.plans.at(-1).id });
|
||||
await until(async () => ['passed', 'failed', 'unverified'].includes((await host.api('/sessions/' + f.session.id)).execution.checks.at(-1)?.status), 30000);
|
||||
const checked = await host.api('/sessions/' + f.session.id); assert.equal(checked.execution.checks.at(-1).status, 'passed');
|
||||
await host.api('/sessions/' + f.session.id + '/delivery-reports', {});
|
||||
await mobile.getByRole('button', { name: '检查与报告', exact: true }).click();
|
||||
await expect(mobile.locator('.mobile-checks')).toContainText('通过', { timeout: 12000 });
|
||||
await mobile.getByRole('button', { name: /^交付报告 [\d,]+ bytes/ }).click(); await expect(mobile.getByRole('dialog')).toContainText('SHA-256 已核对');
|
||||
await shot(mobile, 'mobile-original-report'); await mobile.getByRole('button', { name: '返回任务', exact: true }).click();
|
||||
mark('original paginated diff and counted check/report display verified content from the same Task');
|
||||
disconnected = true; await mobile.getByRole('button', { name: '刷新任务', exact: true }).click();
|
||||
await expect(mobile.locator('.mobile-connection')).toContainText('连接中断'); await shot(mobile, 'mobile-disconnected');
|
||||
disconnected = false; await mobile.getByRole('button', { name: '重新同步', exact: true }).click(); await expect(mobile.locator('.mobile-connection')).toContainText('已同步');
|
||||
assert.equal((await host.api('/sessions/' + f.session.id)).events.filter(item => item.tool === 'run_command').length, 1);
|
||||
await mobile.reload(); await expect(mobile.getByRole('heading', { name: '项目任务', exact: true })).toBeVisible();
|
||||
mark('disconnect and refresh recover from persistent seq without submitting a new command or Run');
|
||||
const cacheEntries = await mobile.evaluate(async () => { const names = await caches.keys(); return (await Promise.all(names.map(async name => (await (await caches.open(name)).keys()).map(request => new URL(request.url).pathname)))).flat(); });
|
||||
assert.ok(cacheEntries.length > 0 && cacheEntries.every(route => !route.startsWith('/mobile-api/'))); mark('PWA service worker stores the shell and no API/project response');
|
||||
await mobile.setViewportSize({ width: 768, height: 1024 }); await shot(mobile, 'tablet-task-list');
|
||||
await desktop.setViewportSize({ width: 390, height: 844 }); await panel.evaluate(node => { node.scrollTop = 0; }); await shot(desktop, 'desktop-management-narrow');
|
||||
assert.ok(await desktop.locator('body').evaluate(body => body.scrollWidth <= innerWidth));
|
||||
await panel.getByRole('button', { name: '撤销', exact: true }).click();
|
||||
await mobile.getByRole('button', { name: '刷新任务', exact: true }).click(); await expect(mobile.getByRole('heading', { name: '接续电脑上的任务', exact: true })).toBeVisible();
|
||||
await expect(mobile.locator('.mobile-task-row')).toHaveCount(0); await shot(mobile, 'mobile-revoked');
|
||||
assert.equal((await host.api('/sessions/' + f.session.id)).events.filter(item => item.tool === 'run_command').length, 1);
|
||||
mark('desktop revocation removes mobile project content immediately on the next read while preserving the original completed tool');
|
||||
assert.deepEqual(errors, []);
|
||||
const result = { success: true, version: host.bootstrap.version, appRoot, nodePath, nodeSha256: createHash('sha256').update(fs.readFileSync(nodePath)).digest('hex'), containedRuntime: args.length > 0,
|
||||
passed, screenshots, errors, viewports: [1440, 390, 768], browser: 'Microsoft Edge via Playwright; owned loopback QA relay with actual CA verification',
|
||||
nativeSafariTrustCookiePWA: 'pending-real-iPhone', osTrustChanged: false, certificateWarningsBypassed: false, realModelRequests: 0, taskId: before.taskId, runId: before.runId };
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
if (mobile) await shot(mobile, 'failure-mobile').catch(() => {}); await shot(desktop, 'failure-desktop').catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.message, stack: error.stack, passed, screenshots, errors, realModelRequests: 0 }, null, 2)); throw error;
|
||||
} finally {
|
||||
console.log('EVIDENCE ' + directory); await browser.close();
|
||||
if (relay) await new Promise(resolve => { relay.closeAllConnections(); relay.close(resolve); });
|
||||
await host.api('/companion/close', {}).catch(() => {});
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { RequestPlanner } from '../src/core/context/request-plan.mjs';
|
||||
import { toolDefinitions } from '../src/server/provider.mjs';
|
||||
import { call } from '../tests/helpers.mjs';
|
||||
|
||||
const root = path.resolve('.qa', 'context-ui-' + randomUUID()), data = path.join(root, 'data');
|
||||
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const store = new Store(data), cases = {};
|
||||
for (const name of ['long', 'other', 'paused', 'paused-repeat', 'working']) {
|
||||
const projectRoot = path.join(root, 'context-' + name); fs.mkdirSync(projectRoot);
|
||||
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned context UI fixture\n');
|
||||
const project = store.addProject(projectRoot), session = store.createSession(project.id, '上下文验收 ' + name);
|
||||
const goal = '这是独立界面验收样本。完整任务要求与实际原记录保留,读取工件不得重新执行工具。';
|
||||
store.message(session, 'user', goal); session.providerMessages.push({ role: 'user', content: goal });
|
||||
const run = store.beginRun(session, goal, false, store.config), planner = new RequestPlanner(store);
|
||||
if (name === 'paused' || name === 'paused-repeat') {
|
||||
session.status = 'paused'; store.message(session, 'system', name === 'paused' ? '完整任务要求超过请求上下文预算,请核对配置后继续。' : '模型重复读取已覆盖工件区间,原任务已暂停,请核对后继续。');
|
||||
if (name === 'paused-repeat') store.core.task.get(run.taskId).readProgress = { schemaVersion: 1, reads: [], repeated: 3, changingIndexes: [], warning: '重复读取已覆盖区间', pause: true };
|
||||
store.finishRun(session, run, name === 'paused' ? 'context_limit' : 'no_progress'); cases[name] = { project, session, projectRoot, runs: [run.id] }; continue;
|
||||
}
|
||||
if (name === 'long') for (let index = 0; index < 18; index++) {
|
||||
const tool = call('ui-original-' + index, 'read_file', { path: 'README.md', fixture: 'Preserve exact historical JSON.' });
|
||||
const event = { id: randomUUID(), toolCallId: tool.id, tool: 'read_file', kind: 'tool', phase: 'queued', label: '读取文件',
|
||||
createdAt: new Date().toISOString(), startedAt: new Date().toISOString(), input: { path: 'README.md' }, runId: run.id, taskId: run.taskId };
|
||||
session.events.push(event); store.registerInvocation(session, tool, run, event);
|
||||
store.beginInvocation(session, tool, run, event); store.invocationState(session, event, 'running');
|
||||
const result = { status: 'completed', path: 'README.md', content: ('完整原记录🙂 ' + index + '\n').repeat(1200) };
|
||||
event.phase = 'completed'; event.output = '已保存完整原记录。'; event.finishedAt = new Date().toISOString();
|
||||
store.finishInvocation(session, event, result);
|
||||
session.providerMessages.push({ role: 'assistant', content: 'Saved fixture read.', tool_calls: [tool] },
|
||||
{ role: 'tool', tool_call_id: tool.id, content: JSON.stringify(result) });
|
||||
}
|
||||
if (name === 'working') {
|
||||
const content = 'export const actual = "当前任务真实代码🙂";\n';
|
||||
fs.writeFileSync(path.join(projectRoot, 'actual.mjs'), content);
|
||||
const values = [{ tool: 'read_file', args: { path: 'actual.mjs' }, result: { path: 'actual.mjs', content,
|
||||
sha256: createHash('sha256').update(content).digest('hex'), start_line: 1, end_line: 2, total_lines: 2, truncated: false } },
|
||||
{ tool: 'query_code', args: { query: 'large' }, result: { results: 'Large owned mixed group '.repeat(16000) } }];
|
||||
const calls = values.map(value => call(randomUUID(), value.tool, value.args)), messages = [];
|
||||
for (const [index, invocation] of calls.entries()) {
|
||||
const event = store.event(session, { kind: 'tool', tool: invocation.function.name, toolCallId: invocation.id,
|
||||
runId: run.id, taskId: run.taskId, phase: 'queued', startedAt: new Date().toISOString(), label: '工作依据样本' });
|
||||
store.registerInvocation(session, invocation, run, event); store.beginInvocation(session, invocation, run, event);
|
||||
store.finishInvocation(session, event, values[index].result);
|
||||
messages.push({ role: 'tool', tool_call_id: invocation.id, content: JSON.stringify(values[index].result) });
|
||||
}
|
||||
session.providerMessages.push({ role: 'assistant', content: '', tool_calls: calls }, ...messages); store.save(session);
|
||||
}
|
||||
const step = store.beginStep(session, run), first = planner.prepare(session, run, store.config, {
|
||||
system: goal + '\n' + JSON.stringify({ project_id: project.id, session_id: session.id, task_id: run.taskId }), tools: toolDefinitions, stepId: step.id });
|
||||
const response = { message: { role: 'assistant', content: '独立界面样本已保存;没有请求真实模型。' }, usage: { prompt_tokens: 2400, completion_tokens: 300 } };
|
||||
planner.finish(session, first.record, { response, attempts: 1 }); session.providerMessages.push(response.message);
|
||||
store.message(session, 'assistant', response.message.content); store.finishStep(session, step); session.status = 'completed'; store.finishRun(session, run);
|
||||
const runs = [run.id], plans = [first.record.id];
|
||||
if (name === 'long') {
|
||||
const resumed = store.beginRun(session, '', true, store.config), nextStep = store.beginStep(session, resumed);
|
||||
const next = planner.prepare(session, resumed, store.config, { system: goal, tools: toolDefinitions, stepId: nextStep.id });
|
||||
planner.finish(session, next.record, { response, attempts: 1 }); store.finishStep(session, nextStep);
|
||||
session.status = 'completed'; store.finishRun(session, resumed); runs.push(resumed.id); plans.push(next.record.id);
|
||||
}
|
||||
cases[name] = { project, session, projectRoot, runs, plans };
|
||||
}
|
||||
store.close();
|
||||
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: process.cwd(), AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' };
|
||||
delete env.AGENT_WORKBENCH_API_KEY;
|
||||
const child = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: process.cwd(), env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let output = '', errors = '', url = '';
|
||||
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const item = JSON.parse(line); if (item.ready) url = item.url; } catch {} } });
|
||||
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
||||
const until = async predicate => { const deadline = Date.now() + 20000; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Context UI fixture timeout: ' + errors); await new Promise(resolve => setTimeout(resolve, 40)); } };
|
||||
await until(() => url || child.exitCode !== null); assert.ok(url, errors);
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
||||
assert.equal(bootstrap.version, JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version, 'The UI host must match this package version.'); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const api = async route => { const response = await fetch(url + '/api' + route, { headers: { 'X-Workbench-Token': bootstrap.token, Connection: 'close' } }); const value = await response.json(); assert.ok(response.ok, value.error); return value; };
|
||||
const proofPrefix = bootstrap.version === '1.6.0' ? 'workspaces-context' : releaseEvidencePrefix(bootstrap.version);
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const passed = [], pageErrors = [], requestFailures = [], screenshots = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') requestFailures.push(request.failure()?.errorText); });
|
||||
const mark = value => { passed.push(value); console.log('PASS ' + value); };
|
||||
const panel = () => page.getByLabel('请求上下文', { exact: true }), history = () => page.getByRole('dialog', { name: '完整历史记录', exact: true });
|
||||
async function screenshot(name) {
|
||||
const file = '.qa/screenshots/' + proofPrefix + '-' + name + '.png'; await page.screenshot({ path: file, fullPage: true }); screenshots.push(file);
|
||||
}
|
||||
async function choose(name) {
|
||||
const select = page.getByLabel('当前项目', { exact: true });
|
||||
if (await select.inputValue() !== cases[name].project.id) {
|
||||
if (!await select.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await expect(select).toBeVisible(); await select.selectOption(cases[name].project.id);
|
||||
}
|
||||
await expect(page.locator('.session-item.selected')).toContainText('上下文验收 ' + name);
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
await expect(page.getByLabel('任务记录', { exact: true })).toBeVisible();
|
||||
}
|
||||
async function lateFailure(pattern, trigger, label) {
|
||||
let release, entered = false, finished = false;
|
||||
const gate = new Promise(resolve => { release = resolve; });
|
||||
await page.route(pattern, async route => { entered = true; await gate; await route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: label }) }); finished = true; });
|
||||
await trigger(); await until(() => entered);
|
||||
if (await history().isVisible()) await page.keyboard.press('Escape');
|
||||
await choose('other'); release(); await until(() => finished);
|
||||
await page.waitForTimeout(350); await expect(page.locator('.toast')).toHaveCount(0);
|
||||
await expect(panel()).toContainText('2,400 tokens');
|
||||
await page.unroute(pattern); mark(label + ' stays in its original scope after switching projects');
|
||||
}
|
||||
try {
|
||||
await page.goto(url); await page.waitForLoadState('networkidle'); await choose('long');
|
||||
await expect(panel()).toContainText('2,400 tokens'); await expect(panel()).toContainText('16,000 tokens');
|
||||
await expect(panel().getByRole('button', { name: '检索完整历史', exact: true })).toBeVisible();
|
||||
await panel().scrollIntoViewIfNeeded(); await screenshot('desktop-plan'); mark('desktop request estimate, actual usage, complete goal and history identity are visible');
|
||||
await panel().getByRole('button', { name: '检索完整历史', exact: true }).click();
|
||||
await expect(history().locator('.context-history-group')).toHaveCount(12);
|
||||
await history().getByRole('button', { name: '下一页', exact: true }).click();
|
||||
await expect(history().locator('.artifact-pagination')).toContainText('13–18 / 18');
|
||||
await history().getByRole('button', { name: '上一页', exact: true }).click();
|
||||
await expect(history().locator('.artifact-pagination')).toContainText('1–12 / 18'); await screenshot('desktop-history');
|
||||
await history().getByRole('button', { name: '读取工具结果', exact: true }).first().click();
|
||||
const artifact = page.getByRole('dialog', { name: '执行结果工件', exact: true });
|
||||
await expect(artifact.locator('pre')).toContainText('完整原记录');
|
||||
await page.keyboard.press('Escape'); mark('paged complete history opens actual tool result artifacts without executing tools');
|
||||
await page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[0]);
|
||||
await expect(panel()).toContainText('2,400 tokens');
|
||||
const selected = await api('/sessions/' + cases.long.session.id + '/context-plans?run=' + cases.long.runs[0]);
|
||||
await expect(panel().locator('.context-sha')).toHaveAttribute('title', selected[0].requestSha256);
|
||||
await page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[1]); mark('historical and resumed Runs retain their own request records');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await choose('long');
|
||||
await panel().getByRole('button', { name: '检索完整历史', exact: true }).click();
|
||||
await expect(history().locator('.context-history-group')).toHaveCount(12); await screenshot('mobile-history');
|
||||
const bounds = await history().boundingBox(); assert.ok(bounds.x >= 0 && bounds.x + bounds.width <= 390 && bounds.y + bounds.height <= 844);
|
||||
await history().getByRole('button', { name: '读取原记录', exact: true }).first().click(); await expect(artifact.locator('pre')).toContainText('Preserve exact historical JSON');
|
||||
await screenshot('mobile-original'); await page.keyboard.press('Escape');
|
||||
await expect(panel()).toBeVisible();
|
||||
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click(); await expect(artifact.locator('pre')).toContainText('history_index');
|
||||
await screenshot('mobile-request-body'); await page.keyboard.press('Escape');
|
||||
await expect(panel()).toBeVisible();
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth) <= 390); mark('390px history, original JSON and sent-body dialogs fit and retain close controls');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('long');
|
||||
const currentPlanId = cases.long.plans[1];
|
||||
await lateFailure('**/context-plans/' + currentPlanId + '/history?*', () => panel().getByRole('button', { name: '检索完整历史', exact: true }).click(), 'late old history failure');
|
||||
await choose('long');
|
||||
await lateFailure('**/context-plans?run=' + cases.long.runs[0], () => page.getByLabel('选择运行记录', { exact: true }).selectOption(cases.long.runs[0]), 'late old Run failure');
|
||||
await choose('long');
|
||||
const currentPlan = await api('/sessions/' + cases.long.session.id + '/context-plans/' + currentPlanId);
|
||||
await lateFailure('**/artifacts/' + currentPlan.requestArtifactId + '?*', () => panel().getByRole('button', { name: '查看发送内容', exact: true }).click(), 'late old request-body failure');
|
||||
await choose('paused'); await expect(page.locator('.run-stop')).toContainText('完整任务要求超过请求上下文预算');
|
||||
await expect(page.getByRole('button', { name: '继续任务', exact: true })).toBeVisible(); mark('context-limit pause explains the intact requirement and explicit continuation');
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await expect(page.getByLabel('当前项目', { exact: true })).toBeHidden();
|
||||
await choose('paused-repeat'); await expect(page.locator('.run-stop')).toContainText('重复读取未推进任务');
|
||||
await page.locator('.run-stop').scrollIntoViewIfNeeded();
|
||||
assert.equal(await page.evaluate(() => window.innerWidth), 390);
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
|
||||
await screenshot('mobile-read-pause');
|
||||
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续任务', exact: true })).toBeVisible();
|
||||
const pausedTaskId = (await api('/sessions/' + cases['paused-repeat'].session.id)).execution.task.id;
|
||||
await page.getByRole('button', { name: '继续任务', exact: true }).click();
|
||||
await until(async () => (await api('/sessions/' + cases['paused-repeat'].session.id)).execution.runs.length === 2);
|
||||
const resumed = await api('/sessions/' + cases['paused-repeat'].session.id);
|
||||
assert.equal(resumed.execution.task.id, pausedTaskId); assert.equal(resumed.execution.run.resumeOf, cases['paused-repeat'].runs[0]);
|
||||
mark('390px repeated-read pause explains the cause and explicit continuation retains the same Task');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 }); await choose('working');
|
||||
await expect(panel()).toContainText('当前任务依据'); await expect(panel()).toContainText('1 项代码与执行结果');
|
||||
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
|
||||
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
|
||||
await expect(artifact.locator('pre')).toContainText('result_artifact_id'); await screenshot('desktop-working-evidence');
|
||||
await page.keyboard.press('Escape'); mark('desktop working evidence shows actual code, original result provenance and the current Task');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await choose('working');
|
||||
await panel().getByRole('button', { name: '查看工作依据', exact: true }).click();
|
||||
await expect(artifact.locator('pre')).toContainText('当前任务真实代码');
|
||||
const workingBounds = await artifact.boundingBox(); assert.ok(workingBounds.x >= 0 && workingBounds.x + workingBounds.width <= 390);
|
||||
await screenshot('mobile-working-evidence'); await page.keyboard.press('Escape'); await expect(panel()).toBeVisible();
|
||||
await panel().getByRole('button', { name: '查看发送内容', exact: true }).click();
|
||||
await expect(artifact.locator('pre')).toContainText('working_set'); await page.keyboard.press('Escape');
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
|
||||
mark('390px working evidence and sent request stay readable and closing the artifact retains task navigation');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), passed, pageErrors, requestFailures, screenshots, root, data,
|
||||
mode: 'saved synthetic context evidence; no real model or project commands executed by this UI verifier', browser: browser.version() };
|
||||
fs.writeFileSync('.qa/' + proofPrefix + '-ui-validation.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
await screenshot('failure').catch(() => {});
|
||||
fs.writeFileSync('.qa/' + proofPrefix + '-ui-validation.json', JSON.stringify({ success: false, version: bootstrap.version, passed, pageErrors, requestFailures, error: error.message, screenshots }, null, 2) + '\n');
|
||||
throw error;
|
||||
} finally {
|
||||
await browser.close();
|
||||
const response = await fetch(url + '/api/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, body: '{}' }).catch(() => null);
|
||||
if (response) assert.ok(response.ok);
|
||||
await until(() => child.exitCode !== null || child.signalCode !== null).catch(() => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
|
||||
}
|
||||
+301
@@ -0,0 +1,301 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fixture, compatibleFixture, call, assertToolPairs } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
import { Runtime } from '../src/server/runtime.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { originalEngineRun, verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
|
||||
import { resolvePurgeSource, resolveOriginalPurgeSource, ORIGINAL_PURGE_KIND, assertPurgeModelInput, nativeSystemTexts, PURGE_DEFAULT_PROMPT } from '../src/core/engines/purge-strategy.mjs';
|
||||
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { desktopProjectQa } from './desktop-project-qa.mjs';
|
||||
import { specification } from '../tests/fixtures/engineering-specification.mjs';
|
||||
import { responsesEngineeringHttp } from '../tests/fixtures/responses-engineering-http.mjs';
|
||||
import { nativeContextSpecification } from '../tests/fixtures/native-context-project.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
assert.ok(args.every(value => ['--help', '--purge-comparison', '--desktop', '--responses', '--context-16k','--context-32k','--original-purge'].includes(value)), 'Unknown option; use --help.');
|
||||
if (args.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-projects.mjs [--purge-comparison | --desktop | --context-16k | --context-32k] [--responses] [--original-purge] — actual pinned Harness and owned HTTP model fixture; default verifies Node/React/Python with approvals, Edge and Git. Context modes use a new copy of the sealed three-module/nine-test pilot, original failure/log paging and a complete reviewed batch at the explicit limit. Original Purge is separate from the independent prompt and its fixed instructions need the 32K profile. --responses uses actual Responses SSE and ModelHub routing. Set AGENT_WORKBENCH_DEEPSEEK_ROOT and AGENT_WORKBENCH_TEST_PYTHON; context mode needs no Python. No configured-model requests.'); process.exit(0);
|
||||
}
|
||||
const comparisonMode = args.includes('--purge-comparison'), desktopMode = args.includes('--desktop');
|
||||
assert.ok(!(args.includes('--context-16k')&&args.includes('--context-32k')),'Choose one explicit native context limit.');
|
||||
const contextMode = args.includes('--context-16k')||args.includes('--context-32k');assert.ok(!contextMode || !comparisonMode && !desktopMode, 'Native context acceptance is a separate run.');
|
||||
const contextLimit=args.includes('--context-32k')?32000:16000;
|
||||
const originalPurgeMode=args.includes('--original-purge');assert.ok(!originalPurgeMode||contextMode,'Original Purge engineering acceptance uses a separate owned context case.');
|
||||
const responsesMode = args.includes('--responses'); assert.ok(!responsesMode || !comparisonMode, 'Protocol acceptance and Purge comparison remain separate.');
|
||||
assert.ok(!comparisonMode || !desktopMode, 'Desktop engineering and prompt comparison are separate acceptance runs.');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version;
|
||||
const directory = path.resolve('.qa', (contextMode ? 'controlled-context-' : desktopMode ? 'desktop-projects-' : comparisonMode ? 'controlled-purge-' : 'controlled-projects-') + randomUUID()), results = [], cases = [];
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT);
|
||||
assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
const originalSource=originalPurgeMode?resolveOriginalPurgeSource(process.cwd(),process.env.AGENT_WORKBENCH_PURGE_178_ROOT,installation):null;
|
||||
if(originalSource)assert.equal(originalSource.status,'available',originalSource.message);
|
||||
const originalStrategy=originalSource?{kind:ORIGINAL_PURGE_KIND,root:originalSource.root,sourceSha256:originalSource.sourceSha256,runtimeSha256:originalSource.runtimeSha256,prompt:PURGE_DEFAULT_PROMPT}:null;
|
||||
const python = process.env.AGENT_WORKBENCH_TEST_PYTHON;
|
||||
if (!comparisonMode && !contextMode) assert.ok(python && path.isAbsolute(python) && fs.existsSync(python), 'Provide the existing isolated Python 3.10+ interpreter.');
|
||||
const sources = comparisonMode ? ['1.1.78', '1.1.81'].map((expected, index) => {
|
||||
const root = process.env[index === 0 ? 'AGENT_WORKBENCH_PURGE_178_ROOT' : 'AGENT_WORKBENCH_PURGE_181_ROOT'];
|
||||
assert.ok(root && path.isAbsolute(root), 'Provide both existing Purge source directories.');
|
||||
const source = resolvePurgeSource(process.cwd(), root); assert.equal(source.status, 'available', source.message); assert.equal(source.version, expected); return source;
|
||||
}) : [];
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
||||
function inputs() {
|
||||
const files = ['package.json', 'package-lock.json'];
|
||||
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file);
|
||||
} };
|
||||
for (const root of ['src', 'scripts', 'tests', 'benchmarks', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
|
||||
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
||||
}
|
||||
const frozen = inputs(), mark = message => { cases.push(message); console.log('PASS ' + message); };
|
||||
async function runSample(kind, { sampleName = kind, strategy = null, comparison = null } = {}) {
|
||||
const startedAt = Date.now(), label = sampleName;
|
||||
const disposers = [], t = { after: value => disposers.push(value) }, f = fixture(t), sample = path.join(directory, sampleName);
|
||||
fs.mkdirSync(sample); let runtime, group, result, desktopQa, model, cycledRequest = false, cycledApproval = false; const reviews = [], modelRequests = [];
|
||||
try {
|
||||
const baseline = contextMode ? 'docs/live-model-purge-1.24-evidence/source-baselines/r1-off' : kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
||||
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
write(f, '.gitignore', 'node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
|
||||
write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n');
|
||||
git(f, ['init', '--initial-branch=main']); git(f, ['config', 'core.autocrlf', 'false']); git(f, ['config', 'user.name', 'Workbench QA']); git(f, ['config', 'user.email', 'qa@workbench.invalid']);
|
||||
git(f, ['add', '--', '.']); git(f, ['commit', '--no-gpg-sign', '-m', 'Seal ' + kind + ' controlled-engine acceptance']);
|
||||
const baselineTree = git(f, ['rev-parse', 'HEAD^{tree}']).trim();
|
||||
write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']);
|
||||
write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n');
|
||||
const spec = contextMode ? nativeContextSpecification : specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
|
||||
const original = ownerBytes(f), protectedNames = git(f, ['ls-files', '-z']).split('\0').filter(file => file && !writable.includes(file) && !file.startsWith('owner-'));
|
||||
const protectedFiles = Object.fromEntries(protectedNames.map(file => [file, fs.readFileSync(path.join(f.projectRoot, file))]));
|
||||
const verify = root => { for (const [file, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(root, file)), bytes, 'Protected ' + file); };
|
||||
const install = (root, label) => {
|
||||
const child = spawnSync(process.execPath, [npmCli, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: root, windowsHide: true, timeout: 120000, maxBuffer: 8 * 1024 * 1024 });
|
||||
fs.writeFileSync(path.join(sample, label + '-npm-ci-log.txt'), child.stdout ?? Buffer.alloc(0)); fs.writeFileSync(path.join(sample, label + '-npm-ci-stderr-log.txt'), child.stderr ?? Buffer.alloc(0));
|
||||
if (child.error) throw child.error; assert.equal(child.status, 0, 'Inspect the original offline installation logs.'); verify(root);
|
||||
};
|
||||
if (kind === 'react') install(f.projectRoot, 'parent');
|
||||
let phase = 'new';
|
||||
model = await (responsesMode ? responsesEngineeringHttp : compatibleFixture)(t, async (route, input) => {
|
||||
assert.equal(route, '/v1/chat/completions'); assert.equal(input.model, 'owned-controlled-engineering');
|
||||
assertToolPairs(input.messages);
|
||||
const worker = [...f.store.core.worker.values()][0], session = f.store.session(worker.sessionId);
|
||||
if (desktopQa && !cycledRequest) { cycledRequest = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-owned-model-request'); }
|
||||
const checks = [...f.store.core.check_job.values()].filter(job => job.taskId === worker.taskId), job = checks.at(-1);
|
||||
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === worker.taskId).at(-1);
|
||||
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Fixed local acceptance repair; retain all tests and declarations.' }));
|
||||
let calls = [];
|
||||
if(contextMode){
|
||||
const pending=session.changes.filter(change=>change.status==='pending');
|
||||
const batch=[...f.store.core.change_set.values()].findLast(set=>set.taskId===worker.taskId&&set.status==='prepared');
|
||||
if(pending.length){if(!batch)calls=[call(randomUUID(),'prepare_change_set',{change_ids:pending.map(change=>change.id)})];}
|
||||
else if(phase==='new'){
|
||||
calls=[...writable,'test/expense.test.mjs'].map(file=>call(randomUUID(),'read_file',{path:file,limit:16000}));phase='observed';
|
||||
}else if(phase!=='fixed'&&job){
|
||||
assert.equal(job.status,'failed');assert.equal(job.results[0].tests.failed,9);
|
||||
if(phase!=='failure-log-read'){
|
||||
calls=[call(randomUUID(),'read_artifact',{artifact_id:job.results[0].logArtifactId,offset:0,limit:16000})];phase='failure-log-read';
|
||||
}else{calls=propose(spec.fixes);phase='fixed';}
|
||||
}else if(phase==='fixed'&&job?.status==='passed')assert.equal(job.results[0].tests.passed,9);
|
||||
else if(!plan||job?.planId===plan.id)calls=[call(randomUUID(),'plan_checks',{names:spec.checks})];
|
||||
else calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
|
||||
}
|
||||
else if (session.changes.some(change => change.status === 'pending')) { /* Explicit review follows native completion. */ }
|
||||
else if (phase === 'new' && spec.initial) { calls = propose(spec.initial); phase = 'initial'; }
|
||||
else if (phase !== 'fixed' && job) {
|
||||
assert.equal(job.status, 'failed', job.summary); assert.ok(job.results.some(result => result.tests?.failed > 0)); calls = propose(spec.fixes); phase = 'fixed';
|
||||
} else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, spec.tests);
|
||||
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: spec.checks })];
|
||||
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
|
||||
modelRequests.push({ requestBody: input, callIds: calls.map(call => call.id), phase,
|
||||
responsePlan: calls.map(call => ({ name: call.function.name, arguments: { ...JSON.parse(call.function.arguments),
|
||||
...(call.function.name === 'run_checks' ? { plan_id: 'owned-current-check-plan' } : {}) } })) });
|
||||
return { choices: [{ message: { role: 'assistant', content: calls.length ? '' : 'Owned native run complete; host proof is still required.', ...(calls.length ? { tool_calls: calls } : {}) },
|
||||
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } };
|
||||
});
|
||||
f.store.saveConfig({ mode: 'compatible', model: 'owned-controlled-engineering', baseUrl: model.baseUrl, contextLimit: contextMode ? contextLimit : 64000,
|
||||
...(contextMode ? {maxOutputTokens:2048} : {}) });
|
||||
const provider = new Provider(); provider.setKey('self-owned-controlled-model-fixture');
|
||||
if (responsesMode) {
|
||||
const hub = provider.modelHub(f.store), connection = hub.upsert({ name: 'Owned Responses engineering', protocol: 'responses', model: 'owned-controlled-engineering', baseUrl: model.baseUrl,
|
||||
requiresKey: false, contextLimit: contextMode ? contextLimit : 64000, ...(contextMode ? {maxOutputTokens:2048} : {}), streaming: true, capabilityDeclarations: { streaming: 'supported' } });
|
||||
hub.select({ type: 'connection', id: connection.id });
|
||||
}
|
||||
runtime = new Runtime(f.store, provider); f.onCleanup(() => runtime.stopAll());
|
||||
runtime.controlledEngines = new EngineRunner(f.store, provider, { directoryRoot: path.join(f.root, 'owned-engine-runs') });
|
||||
runtime.checkRunner().pythonExecutable = python; const manager = runtime.workerManager();
|
||||
if (desktopMode) desktopQa = await desktopProjectQa({ store: f.store, provider, runtime, sample, kind });
|
||||
group = await manager.plan(f.session, { goal: 'Deliver ' + kind + ' with the pinned controlled Harness, retained failures, explicit review, parent validation and local Git.',
|
||||
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256, ...(strategy ? { strategy } : {}) },
|
||||
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
|
||||
children: [{ label: 'engine-' + kind, goal: 'Repair and validate the complete ' + kind + ' functional suite without replacing tests.', paths: writable }] });
|
||||
await (await manager.prepare(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'ready', group.summary);
|
||||
const child = group.children[0], childRoot = f.store.project(child.projectId).root;
|
||||
assert.notEqual(childRoot, f.projectRoot); assert.deepEqual(ownerBytes(f), original); verify(childRoot); if (kind === 'react') install(childRoot, 'child');
|
||||
if (desktopQa) await desktopQa.select(f.store.session(child.sessionId));
|
||||
mark(label + ': pinned native engine and one owned Git worktree preserve parent staging and protected bytes');
|
||||
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
|
||||
const session = f.store.session(child.sessionId), ids = session.changes.filter(change => change.status === 'pending').map(change => change.id);
|
||||
if (ids.length) {
|
||||
assert.equal(group.status, 'awaiting_review', group.summary);
|
||||
const nativeBatch=contextMode&&[...f.store.core.change_set.values()].findLast(set=>set.taskId===child.taskId&&set.status==='prepared');
|
||||
const set = nativeBatch || (desktopQa ? await desktopQa.review(session, ids) : await runtime.changeSetManager().prepare(session, ids));
|
||||
if(contextMode)assert.deepEqual(set.files.map(file=>file.path).sort(),writable);
|
||||
reviews.push({ operation: 'review child batch', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(file => file.path) });
|
||||
if (!desktopQa) await (await runtime.changeSetManager().start(session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary);
|
||||
}
|
||||
const started = manager.start(f.session, group.id, group.fingerprint); let done = false, failure;
|
||||
started.promise.then(() => { done = true; }, error => { failure = error; done = true; });
|
||||
while (!done) {
|
||||
for (const approval of session.approvals.filter(approval => approval.status === 'pending')) {
|
||||
assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks');
|
||||
const plan = [...f.store.core.check_plan.values()].filter(plan => plan.taskId === child.taskId).at(-1);
|
||||
assert.deepEqual(plan.checks.map(check => check.name), spec.checks);
|
||||
reviews.push({ operation: 'approve original check', id: approval.id, taskId: child.taskId, planId: plan.id, contractSha256: plan.contractSha256 });
|
||||
if (desktopQa) { if (!cycledApproval) { cycledApproval = true; await desktopQa.cycle(session, 'close-and-reopen-window-during-original-approval'); } await desktopQa.approve(session, approval); }
|
||||
else runtime.decide(session.id, approval.id, true);
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 30));
|
||||
}
|
||||
if (failure) throw failure;
|
||||
const native = [...f.store.core.engine_run.values()].filter(job => job.taskId === child.taskId).at(-1);
|
||||
if(session.status==='needs_review'&&native.reviewHandoff){
|
||||
assert.equal(native.status,'paused');assert.equal(native.verified,false);assert.equal(native.requiresVerification,false);
|
||||
assert.equal(native.reviewHandoff.kind,'change_set');assert.equal(native.native.cleanup.treeConfirmed,true);
|
||||
}else assert.equal(native.status, 'completed', native.error);
|
||||
}
|
||||
assert.equal(group.status, 'ready_to_collect', group.summary); assert.deepEqual(ownerBytes(f), original); verify(childRoot);
|
||||
const childChecks = [...f.store.core.check_job.values()].filter(job => job.taskId === child.taskId);
|
||||
assert.ok(childChecks.some(job => job.status === 'failed' && job.results.some(result => result.tests?.failed > 0)));
|
||||
assert.equal(childChecks.at(-1).results[0].tests.passed, spec.tests); assert.equal(childChecks.at(-1).status, 'passed');
|
||||
const engineRuns = [...f.store.core.engine_run.values()].filter(job => job.taskId === child.taskId), childSession = f.store.session(child.sessionId);
|
||||
for (const job of engineRuns) {
|
||||
originalEngineRun(f.store, childSession, job);
|
||||
if(job.reviewHandoff){assert.equal(job.status,'paused');assert.equal(job.verified,false);}else assert.equal(job.status, 'completed', job.error);
|
||||
assert.equal(job.native.cleanup.treeConfirmed, true);
|
||||
if(originalPurgeMode){
|
||||
const warning='(node:'+job.native.process.pid+') [PERM0001] SecurityWarning: The flag --allow-addons must be used with extreme caution. It could invalidate the permission model.\n(Use `node --trace-warnings ...` to show where the warning was created)\n';
|
||||
const expected=warning+['package.json','lib/index.js'].map(file=>'AWB_MODULE_REFUSED '+path.join(installation.root,'packages/context/agent-instructions',file)+'\n').join('');
|
||||
assert.equal(job.native.stderr,expected);assert.equal(job.native.originalPurge.readiness.originalApplyCalled,true);
|
||||
assert.equal(job.native.originalPurge.nativeLoads.length,2);assert.ok(job.native.originalPurge.durabilityReceipts.length>0);
|
||||
}else assert.equal(job.native.stderr, '');
|
||||
if(!job.reviewHandoff)assert.equal(job.native.activity.endEvents[0].data.reason.kind, 'completed');
|
||||
for (const request of job.requests) {
|
||||
const saved = verifyEngineModelInput(f.store, childSession, job, request);
|
||||
if (strategy) { assertPurgeModelInput(saved.native, job.installation.strategy);
|
||||
if(originalPurgeMode)assert.equal(job.native.originalPurge.strategySha256,job.installation.strategy.sha256);
|
||||
else assert.equal(job.native.readiness.promptExperimentSha256, job.installation.strategy.sha256); }
|
||||
else assert.ok(nativeSystemTexts(saved.native).every(text => !text.includes('[Agent Workbench engineering prompt experiment]')));
|
||||
}
|
||||
}
|
||||
assert.equal(engineRuns.at(-1).status,'completed','The final native Run must complete after explicit review continuation.');
|
||||
assert.equal(engineRuns.at(-1).verified,true);
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.requests, modelRequests.length);
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, modelRequests.length * 168);
|
||||
if (responsesMode) {
|
||||
const attempts = provider.modelHub(f.store).attempts;
|
||||
assert.equal(attempts.length, model.requests.length); assert.ok(attempts.every(value => value.protocol === 'responses' && value.stream?.terminal && value.responseArtifactId));
|
||||
fs.writeFileSync(path.join(sample, 'original-responses-requests.json'), JSON.stringify(model.requests, null, 2));
|
||||
fs.writeFileSync(path.join(sample, 'original-responses-attempts.json'), JSON.stringify(attempts, null, 2));
|
||||
}
|
||||
mark(label + ': actual Harness uses owned model/tool bridges, retains failing tests, and completes reviewed repairs with one accounting entry per HTTP request');
|
||||
if(contextMode){
|
||||
const bounded=engineRuns.flatMap(job=>job.requests.map(request=>verifyEngineModelInput(f.store,childSession,job,request))).filter(input=>input.view.mode==='bounded');
|
||||
if(contextLimit===16000)assert.ok(bounded.length>0,'The tighter original context profile must exercise the bounded bridge.');
|
||||
assert.ok(bounded.every(input=>input.view.beforeInputTokens>input.view.inputBudget&&input.plan.estimatedInputTokens<=input.view.inputBudget));
|
||||
const sent=JSON.stringify(engineRuns.flatMap(job=>job.requests.map(request=>verifyEngineModelInput(f.store,childSession,job,request).body)));assert.ok(sent.includes('logArtifactId'));assert.ok(sent.includes('failed'));
|
||||
const failed=childChecks.find(job=>job.status==='failed'),log=f.store.core.artifact.get(failed.results[0].logArtifactId);
|
||||
assert.ok(f.store.artifactStore.read(log).includes('fail 9'));assert.ok(engineRuns.some(job=>job.calls.some(call=>call.name==='read_artifact')));
|
||||
assert.ok(engineRuns.some(job=>job.calls.some(call=>call.name==='prepare_change_set')));assert.ok(engineRuns.every(job=>!job.inputRejections?.length));
|
||||
mark(contextLimit/1000+'K: original nine failures and paged log retained; '+bounded.length+' native requests used verified compaction; all three proposals form one reviewed batch without replay');
|
||||
}
|
||||
await (await manager.collect(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'collected', group.summary);
|
||||
const collection = await manager.readCollection(f.session, group); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
|
||||
await (await manager.prepareMerge(f.session, group.id, group.fingerprint)).promise; assert.equal(group.status, 'merge_prepared', group.summary);
|
||||
const merge = f.store.core.change_set.get(group.merge.changeSetId); reviews.push({ operation: 'review parent batch', id: merge.id, fingerprint: merge.fingerprint });
|
||||
if (desktopQa) assert.equal((await desktopQa.review(f.session, merge.files.map(file => file.changeId))).id, merge.id);
|
||||
else await (await runtime.changeSetManager().start(f.session, merge.id, merge.fingerprint)).promise;
|
||||
assert.equal(merge.status, 'completed', merge.summary);
|
||||
const checkPlan = await runtime.checkRunner().plan(f.session, spec.checks), check = await (await runtime.checkRunner().start(f.session, checkPlan.id)).promise;
|
||||
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify(f.projectRoot);
|
||||
mark(label + ': separately reviewed parent merge passes all ' + spec.tests + ' functional tests and actual build/compile');
|
||||
if (desktopQa) await desktopQa.capture(f.session, 'parent-checks');
|
||||
const browserJobs = [];
|
||||
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
||||
const steps = [ { action: 'assert_count', selector: '.task-list article', count: 3 },
|
||||
{ action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '8小时' : '8\n小时' },
|
||||
{ action: 'select', selector: 'select[aria-label="任务状态"]', value: 'pending' }, { action: 'assert_count', selector: '.task-list article', count: 2 },
|
||||
{ action: 'fill', selector: 'input[aria-label="任务搜索"]', value: ' 编写 ' }, { action: 'assert_count', selector: '.task-list article', count: 1 },
|
||||
{ action: 'fill', selector: 'input[aria-label="任务搜索"]', value: '' }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'all' },
|
||||
{ action: 'click', selector: '.task-list article[data-task="api"] button' }, { action: 'assert_text', selector: '.task-list article[data-task="api"] .tag', text: '已完成' },
|
||||
{ action: 'assert_text', selector: '.task-list article[data-task="tests"] .tag', text: '待完成' }, { action: 'assert_text', selector: '.summary > div:nth-child(2) strong', text: '2' },
|
||||
{ action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '5小时' : '5\n小时' } ];
|
||||
const plan = await runtime.browserManager().plan(f.session, { viewport, steps }), job = await runtime.browserManager().start(f.session, plan.id).promise;
|
||||
assert.equal(job.status, 'passed', job.summary); assert.equal(job.cleanup.exited, true); browserJobs.push(job);
|
||||
for (const image of job.screenshots) fs.writeFileSync(path.join(sample, 'page-' + viewport.width + '-' + image.artifactId + '.png'), f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId)));
|
||||
}
|
||||
if (kind === 'react') mark('react: real Edge desktop/390px filtering, trimmed search, one-task toggle and complete totals pass');
|
||||
const deliveryPlan = await runtime.gitDelivery().plan(f.session, { message: 'Reviewed ' + kind + ' controlled-engine delivery' }), delivery = await (await runtime.gitDelivery().start(f.session, deliveryPlan.id)).promise;
|
||||
assert.equal(delivery.status, 'completed', delivery.summary); await manager.finalize(f.session, group.id, group.fingerprint); assert.equal(group.status, 'completed');
|
||||
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n');
|
||||
assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
|
||||
mark(label + ': local Git contains only reviewed functional changes and preserves original staged/unstaged owner bytes');
|
||||
if (desktopQa) { await desktopQa.capture(f.session, 'git-completed'); assert.equal(cycledRequest, true); assert.equal(cycledApproval, true); mark(label + ': real Electron reviews/approvals and window cycles retain Task/Run/eventSeq/results at 1440px and 390px'); }
|
||||
const requests = engineRuns.flatMap(job => job.requests), metrics = { durationMs: Date.now() - startedAt, baselineTree,
|
||||
contextLimit: group.config.contextLimit, budget: group.budget, model: group.model.model, tests: spec.tests,
|
||||
engineeringFailures: childChecks.filter(job => job.status === 'failed').length,
|
||||
hostApprovals: reviews.filter(review => review.operation === 'approve original check').length, reviews: reviews.length,
|
||||
requests: modelRequests.length, estimatedInputTokens: requests.reduce((sum, request) => sum + request.estimatedInputTokens, 0),
|
||||
requestBytes: requests.reduce((sum, request) => sum + request.requestBytes, 0),
|
||||
systemBytes: modelRequests.reduce((sum, request) => sum + Buffer.byteLength(request.requestBody.messages.find(message => message.role === 'system')?.content ?? ''), 0),
|
||||
reportedFixtureTokens: modelRequests.length * 168, unknownRequests: f.store.core.task.get(child.taskId).usage.unknownRequests,
|
||||
responseSequence: modelRequests.map(({ phase, responsePlan }) => ({ phase, responsePlan })) };
|
||||
result = { success: true, scope: 'actual-native-Harness-with-owned-HTTP-model-fixture', version, kind, sampleName, comparison, metrics, parentTaskId: group.taskId, groupId: group.id,
|
||||
child: f.store.core.worker.get(child.id), engineRuns, checks: [...f.store.core.check_job.values()], parentCheck: check, browserJobs, commit: delivery.commit,
|
||||
writable, reviews, modelRequests, ...(desktopQa ? { desktop: { events: desktopQa.events, screenshots: desktopQa.screenshots, scope: desktopQa.scope, errors: desktopQa.errors } } : {}), protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
||||
return result;
|
||||
} catch (error) { await desktopQa?.failure(error).catch(() => {}); result = { success: false, version, kind, sampleName, comparison, error: error.stack, group, reviews, modelRequests, ownedFixtureErrors:model?.errors??[], engineRuns: [...f.store.core.engine_run.values()], checks: [...f.store.core.check_job.values()], realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
await desktopQa?.close();
|
||||
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(sample, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(sample, 'session-' + session.id + '.json'), JSON.stringify({ project: f.store.project(session.projectId), session, execution: f.store.executionSnapshot(session) }, null, 2));
|
||||
fs.writeFileSync(path.join(sample, 'result.json'), JSON.stringify(result, null, 2));
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
}
|
||||
let report;
|
||||
try {
|
||||
const comparisons = [];
|
||||
if (comparisonMode) {
|
||||
const prompt = PURGE_DEFAULT_PROMPT + '\n保留字面值 {{engineering_literal}},依据实际失败证据完成同题修复。';
|
||||
for (const source of sources) for (const repetition of [1, 2]) {
|
||||
const pair = [];
|
||||
for (const enabled of repetition === 1 ? [false, true] : [true, false]) {
|
||||
const comparison = { sourceVersion: source.version, sourceSha256: source.sourceSha256, repetition, enabled };
|
||||
const result = await runSample('node', { sampleName: source.version + '-r' + repetition + '-' + (enabled ? 'enabled' : 'disabled'), comparison,
|
||||
strategy: enabled ? { kind: 'purge-prompt-experiment', root: source.root, sourceSha256: source.sourceSha256, prompt } : null });
|
||||
results.push(result); pair.push(result);
|
||||
}
|
||||
const control = pair.find(result => !result.comparison.enabled), experiment = pair.find(result => result.comparison.enabled);
|
||||
for (const key of ['baselineTree', 'contextLimit', 'budget', 'model', 'tests', 'responseSequence', 'hostApprovals', 'engineeringFailures', 'reportedFixtureTokens', 'requests']) assert.deepEqual(experiment.metrics[key], control.metrics[key], 'Comparable original ' + key);
|
||||
assert.deepEqual(experiment.writable, control.writable); assert.deepEqual(experiment.protectedFiles, control.protectedFiles);
|
||||
assert.ok(experiment.metrics.systemBytes > control.metrics.systemBytes);
|
||||
comparisons.push({ sourceVersion: source.version, repetition, control: control.sampleName, experiment: experiment.sampleName,
|
||||
estimatedInputTokenDelta: experiment.metrics.estimatedInputTokens - control.metrics.estimatedInputTokens,
|
||||
requestByteDelta: experiment.metrics.requestBytes - control.metrics.requestBytes, systemByteDelta: experiment.metrics.systemBytes - control.metrics.systemBytes,
|
||||
durationDeltaMs: experiment.metrics.durationMs - control.metrics.durationMs, controlMetrics: control.metrics, experimentMetrics: experiment.metrics,
|
||||
modelBenefit: 'not-measured-owned-response-fixture', defaultEnabled: false });
|
||||
mark(source.version + ' repetition ' + repetition + ': identical original task/baseline/tests/response algorithm and budgets; actual prompt overhead and host approvals recorded');
|
||||
}
|
||||
assert.equal(results.length, 8); assert.equal(comparisons.length, 4);
|
||||
} else for (const kind of contextMode ? ['native-context'] : ['node', 'react', 'python']) results.push(await runSample(kind,{strategy:originalStrategy}));
|
||||
assert.deepEqual(inputs(), frozen);
|
||||
report = { success: true, targetVersion: version, testedVersion: version, protocol: responsesMode ? 'responses-stream' : 'chat-completions', contextLimit:contextMode?contextLimit:64000, originalPurge:originalPurgeMode, directory, installation, cases, results, comparisons, sources, inputs: frozen, realModelRequests: 0,
|
||||
scope: 'Pinned actual Harness and workbench execution with an owned HTTP model fixture. Checks, approvals, failures, parent merges, Edge and Git are real. Model reasoning and Purge benefit remain separate.' };
|
||||
} catch (error) { report = { success: false, targetVersion: version, testedVersion: version, directory, installation, cases, results, inputs: frozen, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
||||
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/' + (contextMode ? (responsesMode ? 'responses-context' : 'controlled-context') : responsesMode ? 'responses-engineering' : desktopMode ? 'desktop-projects' : comparisonMode ? 'controlled-purge' : 'controlled-projects') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, directory, realModelRequests: 0, error: report.error ?? null })); }
|
||||
@@ -0,0 +1,199 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { assertToolPairs, call } from '../tests/helpers.mjs';
|
||||
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const wait = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||
async function until(predicate, timeout = 45000) {
|
||||
const deadline = Date.now() + timeout;
|
||||
while (!await predicate()) { if (Date.now() > deadline) throw new Error('Owned compiled-engine acceptance timed out.'); await wait(50); }
|
||||
}
|
||||
|
||||
// This verifier supplies a declared local response algorithm. The executing
|
||||
// Runtime/Store/permissions/native bridge all belong to the compiled host.
|
||||
export async function verifyControlledApi({ api, url, token, directory, appRoot, contained = false }) {
|
||||
const origin = new URL(url);
|
||||
assert.equal(origin.hostname, '127.0.0.1'); assert.notEqual(origin.port, '4318');
|
||||
const expectedVersion = JSON.parse(fs.readFileSync(path.join(appRoot, fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json'), 'utf8')).version;
|
||||
assert.ok(['1.14.0', '1.15.0', '1.16.0', '1.17.0', '1.18.0', '1.19.0', '1.20.0', '1.20.1', '1.20.2', '1.20.3', '1.21.0', '1.22.0', '1.22.1', '1.23.0', '1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.8', '1.24.9', '1.25.0'].includes(expectedVersion));
|
||||
const bootstrap = await api('/bootstrap'); assert.equal(bootstrap.version, expectedVersion);
|
||||
assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const root = path.join(directory, 'compiled-controlled-node'), hooks = path.join(directory, 'disabled-hooks');
|
||||
fs.mkdirSync(root); fs.mkdirSync(hooks);
|
||||
fs.cpSync(path.join(appRoot, 'benchmarks/workers-delivery-v1/node'), root, { recursive: true });
|
||||
fs.writeFileSync(path.join(root, '.gitignore'), 'dist/\nnode_modules/\n.awb-*\n');
|
||||
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) fs.writeFileSync(path.join(root, name), 'Owner baseline\n');
|
||||
const git = args => execFileSync('git', ['-c', 'core.hooksPath=' + hooks, ...args], { cwd: root, env: gitEnvironment(), encoding: 'utf8', windowsHide: true, timeout: 15000 });
|
||||
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['config', 'user.name', 'Workbench QA']); git(['config', 'user.email', 'qa@workbench.invalid']);
|
||||
git(['add', '--', '.']); git(['commit', '--no-gpg-sign', '-m', 'Seal compiled controlled-engine acceptance']);
|
||||
const initialHead = git(['rev-parse', 'HEAD']).trim(), writable = ['src/add.mjs', 'src/range.mjs'];
|
||||
const protectedFiles = Object.fromEntries(git(['ls-files', '-z']).split('\0').filter(name => name && !writable.includes(name) && !name.startsWith('owner-'))
|
||||
.map(name => [name, fs.readFileSync(path.join(root, name))]));
|
||||
fs.writeFileSync(path.join(root, 'owner-staged.txt'), 'Owner staged change\n'); git(['add', '--', 'owner-staged.txt']);
|
||||
fs.writeFileSync(path.join(root, 'owner-staged.txt'), 'Owner working copy after staging\n'); fs.writeFileSync(path.join(root, 'owner-unstaged.txt'), 'Owner unstaged change\n');
|
||||
fs.writeFileSync(path.join(root, 'owner-untracked.txt'), 'Owner untracked file\n');
|
||||
const owner = () => ({ staged: git(['show', ':owner-staged.txt']), stagedDiff: git(['diff', '--cached', '--binary', '--', 'owner-staged.txt']),
|
||||
files: Object.fromEntries(['owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(root, name)).toString('base64')])) });
|
||||
const originalOwner = owner(), verifyProtected = target => { for (const [name, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(target, name)), bytes, name); };
|
||||
const requests = [], reviews = [], cases = [], originalRuns = [], exports = [];
|
||||
const mark = text => { cases.push(text); console.log('PASS ' + text); };
|
||||
let phase = 'new', childRoute, parentRoute, group;
|
||||
const modelServer = http.createServer(async (request, response) => {
|
||||
try {
|
||||
assert.equal(request.url, '/v1/chat/completions'); assert.equal(request.headers.authorization, 'Bearer owned-compiled-engine-fixture');
|
||||
const chunks = []; for await (const chunk of request) chunks.push(chunk);
|
||||
const input = JSON.parse(Buffer.concat(chunks)); assert.equal(input.model, 'owned-compiled-engineering'); assertToolPairs(input.messages);
|
||||
const session = await api(childRoute), job = session.execution.checks.at(-1), plan = session.execution.plans.at(-1);
|
||||
let calls = [];
|
||||
if (session.changes.some(value => value.status === 'pending')) { /* Host review follows native completion. */ }
|
||||
else if (phase !== 'fixed' && job) {
|
||||
assert.equal(job.status, 'failed'); assert.ok(job.results.some(value => value.tests?.failed > 0)); phase = 'fixed';
|
||||
const fixes = { 'src/add.mjs': 'export function add(left, right) {\n if (!Number.isFinite(left) || !Number.isFinite(right)) throw new TypeError("finite numeric inputs required");\n return left + right;\n}\n',
|
||||
'src/range.mjs': 'export function inclusiveRange(start, end) {\n if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end < start || end - start >= 10000) throw new RangeError("bounded ascending integer range required");\n return Array.from({ length: end - start + 1 }, (_, index) => start + index);\n}\n' };
|
||||
calls = Object.entries(fixes).map(([name, content]) => call(randomUUID(), 'propose_file', { path: name, content, reason: 'Declared local acceptance repair; retain protected tests and configuration.' }));
|
||||
} else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, 8);
|
||||
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: ['test', 'build'] })];
|
||||
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
|
||||
requests.push({ input, phase, calls });
|
||||
response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: calls.length ? '' : 'Owned compiled native run complete; host proof remains required.',
|
||||
...(calls.length ? { tool_calls: calls } : {}) }, finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 123, completion_tokens: 45, total_tokens: 168 } }));
|
||||
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
|
||||
});
|
||||
await new Promise(resolve => modelServer.listen(0, '127.0.0.1', resolve));
|
||||
try {
|
||||
await api('/config', { mode: 'compatible', model: 'owned-compiled-engineering', baseUrl: 'http://127.0.0.1:' + modelServer.address().port + '/v1', apiKey: 'owned-compiled-engine-fixture', contextLimit: 64000 });
|
||||
const project = await api('/projects', { root }), parent = await api('/sessions', { projectId: project.id }); parentRoute = '/sessions/' + parent.id;
|
||||
const installation = await api('/engines/deepseek/controlled-availability'); assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
assert.equal(installation.version, '0.2.1-rc.2');
|
||||
if (contained) assert.equal(fs.realpathSync.native(installation.node.path), fs.realpathSync.native(path.join(appRoot, 'runtime/node.exe')));
|
||||
const source = await api('/engines/purge/source'); assert.equal(source.status, 'available', source.message); assert.equal(source.defaultEnabled, false);
|
||||
const planned = await api(parentRoute + '/collaborations', { goal: 'Deliver compiled controlled-engine Node library with retained failures, review, checks and Git.',
|
||||
engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256,
|
||||
strategy: { kind: 'purge-prompt-experiment', root: source.root, sourceSha256: source.sourceSha256, prompt: '保留原测试与 {{compiled_literal}},按实际失败修复已允许源码,等待宿主审批。' } },
|
||||
budget: { maxTokens: 200000, maxSteps: 64, maxToolCalls: 128, maxDurationMs: 600000 },
|
||||
children: [{ label: 'engine-compiled-node', goal: 'Repair two source modules and pass the original 8 tests and build.', paths: writable }] });
|
||||
group = planned.group;
|
||||
const groupState = async () => (await api(parentRoute + '/collaborations')).groups.find(value => value.id === group.id);
|
||||
const operation = name => api(parentRoute + '/collaborations/' + group.id + '/' + name, { fingerprint: group.fingerprint });
|
||||
await operation('prepare'); await until(async () => { group = await groupState(); return !['preparing', 'planned'].includes(group.status); });
|
||||
assert.equal(group.status, 'ready', group.summary); const child = group.children[0]; childRoute = '/sessions/' + child.sessionId;
|
||||
const childProject = (await api('/bootstrap')).projects.find(value => value.id === child.projectId);
|
||||
assert.notEqual(childProject.root, root); verifyProtected(childProject.root); assert.deepEqual(owner(), originalOwner); assert.equal(requests.length, 0);
|
||||
mark('compiled host seals actual Harness/source identities and prepares one owned worktree while retaining parent bytes and staging');
|
||||
const apply = async (route, set) => { reviews.push({ action: 'apply reviewed change set', id: set.id, fingerprint: set.fingerprint, paths: set.files.map(value => value.path) });
|
||||
await api(route + '/change-sets/' + set.id + '/apply', { fingerprint: set.fingerprint });
|
||||
let state; await until(async () => { state = await api(route + '/change-sets/' + set.id); return !['prepared', 'applying'].includes(state.status); }); assert.equal(state.status, 'completed', state.summary); };
|
||||
for (let turn = 0; turn < 4 && group.status !== 'ready_to_collect'; turn++) {
|
||||
let current = await api(childRoute); const pending = current.changes.filter(value => value.status === 'pending');
|
||||
if (pending.length) { assert.equal(group.status, 'awaiting_review'); const next = await api(childRoute + '/change-sets', { changeIds: pending.map(value => value.id) }); await apply(childRoute, next.execution.changeSets.at(-1)); }
|
||||
await operation('start');
|
||||
await until(async () => {
|
||||
current = await api(childRoute);
|
||||
for (const approval of current.approvals.filter(value => value.status === 'pending')) {
|
||||
const event = current.events.find(value => value.id === approval.eventId); assert.equal(event.tool, 'run_checks');
|
||||
const plan = current.execution.plans.at(-1); assert.deepEqual(plan.checks.map(value => value.name), ['test', 'build']);
|
||||
reviews.push({ action: 'approve original check', id: approval.id, taskId: approval.taskId, invocationId: approval.invocationId, planId: plan.id, contractSha256: plan.contractSha256 });
|
||||
await api(childRoute + '/approvals/' + approval.id, { allow: true });
|
||||
}
|
||||
group = await groupState(); return ['ready_to_collect', 'awaiting_review', 'paused', 'unknown', 'failed', 'stale'].includes(group.status);
|
||||
}, 120000);
|
||||
const run = (await api(childRoute + '/engine-runs')).at(-1); assert.equal(run.status, 'completed', run.error); assert.equal(run.nativeSummary.treeConfirmed, true);
|
||||
const response = await fetch(url + '/api' + childRoute + '/engine-runs/' + run.id + '/download', { headers: { 'X-Workbench-Token': token, Connection: 'close' } });
|
||||
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), run.reportSha256);
|
||||
const original = JSON.parse(bytes); assert.equal(original.native.stderr, ''); assert.equal(original.native.cleanup.treeConfirmed, true);
|
||||
assert.equal(original.native.activity.endEvents[0].data.reason.kind, 'completed');
|
||||
assert.equal(original.native.readiness.promptExperimentSha256, run.installation.strategy.sha256);
|
||||
const saved = path.join(directory, 'compiled-engine-original-' + run.id + '.json'); fs.writeFileSync(saved, bytes); originalRuns.push({ run, original, reportPath: saved, sha256: sha(bytes) });
|
||||
}
|
||||
assert.equal(group.status, 'ready_to_collect', group.summary); const current = await api(childRoute);
|
||||
assert.ok(current.execution.checks.some(value => value.status === 'failed' && value.results.some(result => result.tests?.failed > 0)));
|
||||
assert.equal(current.execution.checks.at(-1).status, 'passed'); assert.equal(current.execution.checks.at(-1).results[0].tests.passed, 8);
|
||||
assert.equal(current.execution.task.usage.requests, requests.length); assert.equal(current.execution.task.usage.totalTokens, requests.length * 168);
|
||||
for (const request of requests) assert.equal(request.input.messages.filter(value => value.role === 'system').map(value => value.content).join('').split(group.children[0].engine.strategy.section.text).length - 1, 1);
|
||||
verifyProtected(childProject.root); assert.deepEqual(owner(), originalOwner);
|
||||
mark('compiled native execution retains failing tests, requires original check approvals, seals literal prompt/model/receipts/results and accounts each owned HTTP request once');
|
||||
await operation('collect'); await until(async () => { group = await groupState(); return group.status !== 'collecting'; }); assert.equal(group.status, 'collected', group.summary);
|
||||
const collection = await api(parentRoute + '/collaborations/' + group.id + '/collection'); assert.deepEqual(collection.conflicts, []); assert.deepEqual(collection.paths, writable);
|
||||
await operation('merge'); await until(async () => { group = await groupState(); return ['merge_prepared', 'unknown', 'stale'].includes(group.status); }); assert.equal(group.status, 'merge_prepared', group.summary);
|
||||
await apply(parentRoute, (await api(parentRoute + '/change-sets/' + group.merge.changeSetId)));
|
||||
const checkPlan = (await api(parentRoute + '/check-plans', { names: ['test', 'build'] })).execution.plans.at(-1);
|
||||
const accepted = (await api(parentRoute + '/check-jobs', { planId: checkPlan.id })).execution.checks.at(-1); let check;
|
||||
await until(async () => { check = await api(parentRoute + '/check-jobs/' + accepted.id); return !['queued', 'running'].includes(check.status); });
|
||||
assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, 8); verifyProtected(root);
|
||||
mark('compiled parent review and immutable merge pass the original 8 functional tests and actual ESM build');
|
||||
const deliveryPlan = (await api(parentRoute + '/delivery-plans', { message: 'feat: deliver compiled controlled-engine Node repair' })).execution.deliveryPlans.at(-1);
|
||||
assert.deepEqual(deliveryPlan.paths, writable);
|
||||
const deliveryId = (await api(parentRoute + '/delivery-jobs', { planId: deliveryPlan.id })).execution.deliveries.at(-1).id; let delivery;
|
||||
await until(async () => { delivery = await api(parentRoute + '/delivery-jobs/' + deliveryId); return ['completed', 'failed', 'cancelled', 'unknown'].includes(delivery.status); });
|
||||
assert.equal(delivery.status, 'completed', delivery.summary); assert.equal(git(['rev-parse', 'HEAD^']).trim(), initialHead);
|
||||
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split('\n'), writable);
|
||||
assert.deepEqual(owner(), originalOwner); verifyProtected(root); await operation('finalize'); group = await groupState(); assert.equal(group.status, 'completed', group.summary);
|
||||
for (const route of [parentRoute, childRoute]) { const value = await api(route); const file = path.join(directory, value.id + '-session.json'); fs.writeFileSync(file, JSON.stringify(value, null, 2)); exports.push({ id: value.id, file, sha256: sha(fs.readFileSync(file)) }); }
|
||||
mark('compiled Git delivery contains only reviewed modules, preserves staged/unstaged/untracked owner bytes and completes the parent with sealed original evidence');
|
||||
return { success: true, version: bootstrap.version, appRoot, contained, cases, root, group, requests, reviews, originalRuns, exports,
|
||||
protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([name, bytes]) => [name, sha(bytes)])), owner: originalOwner, installation, purgeSource: source,
|
||||
checks: check, delivery, fixtureModelRequests: requests.length, realModelRequests: 0,
|
||||
scope: 'Actual compiled host, local native Harness, checks/reviews/Git and an explicitly owned HTTP response fixture. No configured model or upstream Purge plugin executes; real inference benefit remains unmeasured.' };
|
||||
} catch (error) {
|
||||
const sessions = [];
|
||||
for (const route of [parentRoute, childRoute].filter(Boolean)) { try { sessions.push(await api(route)); } catch {} }
|
||||
fs.writeFileSync(path.join(directory, 'compiled-api-failure.json'), JSON.stringify({ success: false, error: error.stack, cases, group, requests, reviews, originalRuns, sessions, realModelRequests: 0 }, null, 2));
|
||||
throw error;
|
||||
} finally {
|
||||
if (group?.status === 'running') { try { await api(parentRoute + '/collaborations/' + group.id + '/stop', { fingerprint: group.fingerprint }); } catch {} }
|
||||
modelServer.closeAllConnections(); await new Promise(resolve => modelServer.close(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyControlledRelease(appRoot = process.cwd()) {
|
||||
appRoot = fs.realpathSync.native(appRoot);
|
||||
// Keep the synthetic Git project independent of the installation/extraction
|
||||
// depth. The production workspace path guard remains responsible for users'
|
||||
// actual project roots. Retain this exclusive directory as original evidence.
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'awb-controlled-'));
|
||||
const node = fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
|
||||
const env = { ...gitEnvironment(), ...Object.fromEntries(['AGENT_WORKBENCH_DEEPSEEK_ROOT', 'AGENT_WORKBENCH_PURGE_ROOT'].filter(name => process.env[name]).map(name => [name, process.env[name]])),
|
||||
AGENT_WORKBENCH_APP_DIR: appRoot, AGENT_WORKBENCH_DATA_DIR: path.join(directory, 'data'), AGENT_WORKBENCH_PORT: '0' };
|
||||
const child = spawn(node, [path.join(appRoot, 'dist-server/server.mjs')], { cwd: appRoot, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let output = '', errors = '', url = '', result, bootstrap;
|
||||
child.stdout.on('data', bytes => { output += bytes.toString('utf8'); for (const line of output.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
|
||||
child.stderr.on('data', bytes => { errors += bytes.toString('utf8'); });
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', signal: AbortSignal.timeout(30000),
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const value = await response.json(); assert.equal(response.ok, true, value.error); return value;
|
||||
};
|
||||
try {
|
||||
await until(() => url || child.exitCode !== null, 15000); assert.ok(url, errors);
|
||||
bootstrap = await (await fetch(url + '/api/bootstrap', { signal: AbortSignal.timeout(5000) })).json();
|
||||
const health = await api('/health'); assert.equal(health.pid, child.pid);
|
||||
result = await verifyControlledApi({ api, url, token: bootstrap.token, directory, appRoot, contained: node !== process.execPath });
|
||||
} catch (error) { result = { success: false, appRoot, directory, error: error.stack, realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
try { if (url && bootstrap && child.exitCode === null) { await api('/shutdown', {}); await until(() => child.exitCode !== null || child.signalCode !== null, 15000); } }
|
||||
finally {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
if (process.platform === 'win32') { try { execFileSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, timeout: 10000 }); } catch { child.kill(); } } else child.kill();
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'host-stdout-log.txt'), output); fs.writeFileSync(path.join(directory, 'host-stderr-log.txt'), errors);
|
||||
Object.assign(result ??= { success: false }, { directory, pid: child.pid, rootExited: child.exitCode !== null || child.signalCode !== null, exit: { code: child.exitCode, signal: child.signalCode } });
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2));
|
||||
console.log('EVIDENCE ' + directory);
|
||||
}
|
||||
}
|
||||
assert.equal(result.rootExited, true); assert.equal(result.exit.code, 0); return result;
|
||||
}
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 1 && args[0] === '--help') { console.log('node scripts/verify-controlled-release.mjs [--app-root DIR] - run an actual compiled local host/native Harness/owned model/Node repair/check/review/Git/prompt acceptance in independent data. Requires explicit Harness and Purge source paths.'); process.exit(0); }
|
||||
assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]), 'Unsupported argument; use --help.');
|
||||
const result = await verifyControlledRelease(args[1]); console.log(JSON.stringify({ success: result.success, version: result.version, directory: result.directory, cases: result.cases.length,
|
||||
rootExited: result.rootExited, fixtureModelRequests: result.fixtureModelRequests, realModelRequests: 0 }));
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture } from '../tests/helpers.mjs';
|
||||
import { workerHttp } from '../tests/fixtures/workers-http.mjs';
|
||||
import { EngineRunner } from '../src/core/engines/engine-runner.mjs';
|
||||
import { resolveControlledInstallation } from '../src/core/engines/controlled-installation.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
import { desktopProjectQa } from './desktop-project-qa.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-controlled-ui.mjs [--desktop] — real Edge or Electron desktop/390px, actual fixed Harness, owned in-memory model fixture, engine selection, reports, approvals, stop and explicit unknown recovery. No configured-model requests.'); process.exit(0);
|
||||
}
|
||||
assert.ok(process.argv.slice(2).every(value => value === '--desktop')); assert.ok(process.argv.length <= 3);
|
||||
const desktopMode = process.argv.includes('--desktop');
|
||||
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', (desktopMode ? 'desktop-controlled-ui-' : 'controlled-ui-') + randomUUID());
|
||||
const installation = resolveControlledInstallation(process.cwd(), process.env.AGENT_WORKBENCH_DEEPSEEK_ROOT); assert.equal(installation.controlledStatus, 'available', installation.message);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex'), disposers = [], t = { after: value => disposers.push(value) };
|
||||
const f = fixture(t), host = await workerHttp(f), original = ownerBytes(f), assertions = [], screenshots = [], pageErrors = [], requestFailures = [], expectedRejections = [];
|
||||
f.store.saveConfig({ contextLimit: 64000 });
|
||||
host.runtime.controlledEngines = new EngineRunner(f.store, host.provider, { directoryRoot: path.join(f.root, 'engine-runs') });
|
||||
function inputs() {
|
||||
const files = ['package.json', 'package-lock.json'];
|
||||
const walk = root => { for (const entry of fs.readdirSync(root, { withFileTypes: true })) { const file = path.join(root, entry.name); if (entry.isDirectory()) walk(file); else if (entry.isFile()) files.push(file); } };
|
||||
for (const root of ['src', 'scripts', 'dist-client', 'dist-server', ...(desktopMode ? ['dist-desktop'] : [])]) walk(root);
|
||||
return files.sort().map(file => ({ path: file, sha256: hash(fs.readFileSync(file)) }));
|
||||
}
|
||||
const frozen = inputs(), desktop = desktopMode ? await desktopProjectQa({ store: f.store, provider: host.provider, runtime: host.runtime, sample: directory, kind: 'controlled-ui' }) : null;
|
||||
const browser = desktopMode ? null : await chromium.launch({ channel: 'msedge', headless: true }), page = desktop?.page ?? await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const resize = size => desktop ? desktop.resize(size.width, size.height) : page.setViewportSize(size);
|
||||
page.on('pageerror', error => pageErrors.push(error.message)); page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const panel = () => page.getByLabel('隔离协作控制台', { exact: true }), runPanel = () => page.getByLabel('受控引擎工程运行', { exact: true });
|
||||
const mark = text => { assertions.push(text); console.log('PASS ' + text); };
|
||||
async function screenshot(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push({ file, sha256: hash(fs.readFileSync(file)) }); }
|
||||
async function closeInspector() {
|
||||
const close = page.getByRole('button', { name: '关闭检视区', exact: true });
|
||||
if (await page.evaluate(() => innerWidth) > 940) { await expect(close).toHaveCount(0); return; }
|
||||
await expect(close).toHaveCount(1);
|
||||
if (await close.isVisible()) { await close.click(); await expect(page.locator('.inspector')).toBeHidden(); }
|
||||
}
|
||||
async function openWorkers() { await closeInspector(); await page.getByRole('button', { name: '隔离协作', exact: true }).click(); await expect(panel()).toBeVisible(); await expect(panel().getByRole('button', { name: '刷新', exact: true })).toBeEnabled(); }
|
||||
async function openEngine() { await closeInspector(); await page.getByRole('button', { name: '本地代理引擎', exact: true }).click(); await expect(runPanel()).toBeVisible(); }
|
||||
async function parentFromChild() { await openWorkers(); await panel().getByRole('button', { name: '返回父目标', exact: true }).click(); await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session'))).toBe(f.session.id); await openWorkers(); }
|
||||
async function start() { await panel().getByRole('button', { name: /开始引擎任务|继续未完成子任务/, exact: true }).click(); await page.getByRole('dialog', { name: '确认受限子执行', exact: true }).getByRole('button', { name: '确认开始', exact: true }).click(); }
|
||||
async function applyOne() {
|
||||
if (await page.getByRole('button', { name: '查看改动', exact: true }).isVisible()) await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
||||
else await page.locator('.inspector-tabs').getByRole('button', { name: /^改动/ }).click();
|
||||
await page.getByRole('button', { name: '组合审查', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '选择要一起审查的提案', exact: true }).getByRole('button', { name: '封存审查范围', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: '多文件变更集审查', exact: true }); await review.getByRole('button', { name: '确认应用全部 1 项', exact: true }).click(); await expect(review).toContainText('全部应用完成'); await page.keyboard.press('Escape');
|
||||
}
|
||||
let result, releaseHeld;
|
||||
try {
|
||||
if (desktop) { await desktop.select(f.session); await resize({ width: 1440, height: 1000 }); }
|
||||
else { await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id }); await page.goto(host.url); await page.waitForLoadState('networkidle'); }
|
||||
await openWorkers();
|
||||
await panel().getByLabel('执行方式', { exact: true }).selectOption('deepseek');
|
||||
const rootField = panel().getByLabel('Harness 安装目录', { exact: true }); await rootField.fill(path.join(f.root, 'missing-installation'));
|
||||
await panel().getByRole('button', { name: '检测受控安装', exact: true }).click(); await expect(panel()).toContainText('安装尚不可用');
|
||||
await expect(panel().getByRole('button', { name: '封存分工计划', exact: true })).toBeDisabled();
|
||||
await rootField.fill(installation.root); await panel().getByRole('button', { name: '检测受控安装', exact: true }).click(); await expect(panel()).toContainText('受控安装可用');
|
||||
await expect(panel().locator('fieldset')).toHaveCount(1); assert.equal(host.requests.length, 0);
|
||||
await panel().getByLabel('父目标', { exact: true }).fill('受控原生引擎与正式审查界面验收');
|
||||
await panel().getByLabel('任务名称', { exact: true }).fill('engine-1'); await panel().getByLabel('独立子目标', { exact: true }).fill('准备范围内提案,审批工程检查并保留原运行记录');
|
||||
await panel().getByLabel('允许修改的文件或目录', { exact: true }).fill('src/a.mjs');
|
||||
await panel().getByRole('button', { name: '封存分工计划', exact: true }).click(); await expect(panel()).toContainText('计划已封存');
|
||||
const group = [...f.store.core.collaboration.values()].at(-1), child = group.children[0]; assert.equal(group.children.length, 1); assert.equal(child.engine.controlledSha256, installation.controlledSha256);
|
||||
assert.deepEqual(ownerBytes(f), original); mark('desktop engine selector rejects a missing source, pins the actual installation and seals one original scope without model calls');
|
||||
await panel().getByRole('button', { name: '创建引擎工作区', exact: true }).click(); await expect(page.getByRole('dialog', { name: '确认创建隔离工作区', exact: true })).toContainText('1 个独立 Git 工作区');
|
||||
await page.getByRole('dialog', { name: '确认创建隔离工作区', exact: true }).getByRole('button', { name: '确认创建', exact: true }).click(); await expect.poll(() => group.status, { timeout: 20000 }).toBe('ready');
|
||||
await start(); await expect.poll(() => group.status, { timeout: 30000 }).toBe('awaiting_review');
|
||||
const first = [...f.store.core.engine_run.values()].find(job => job.taskId === child.taskId); assert.equal(first.status, 'completed', first.error); assert.equal(first.native.cleanup.treeConfirmed, true);
|
||||
await expect(panel().locator('.worker-lane')).toHaveCount(1); await expect(panel()).toContainText('DeepSeek Harness'); await screenshot('desktop-controlled-parent');
|
||||
mark('actual Harness runs the owned bridge and returns reviewable proposals after confirmed process-tree closure');
|
||||
await resize({ width: 390, height: 844 }); await panel().locator('.worker-console-heading').scrollIntoViewIfNeeded();
|
||||
const refresh = panel().getByRole('button', { name: '刷新', exact: true });
|
||||
const bounds = await refresh.boundingBox(); assert.ok(bounds.width >= 76 && bounds.height >= 42, JSON.stringify(bounds)); assert.equal(await refresh.evaluate(node => getComputedStyle(node).whiteSpace), 'nowrap');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390)); await screenshot('mobile-controlled-parent');
|
||||
mark('390px keeps the real single-engine scope, header and a single-line 42px refresh action without horizontal overflow');
|
||||
await panel().getByRole('button', { name: '审查子任务改动', exact: true }).click(); await expect.poll(() => page.evaluate(() => localStorage.getItem('awb-session'))).toBe(child.sessionId);
|
||||
await openEngine(); await expect(runPanel()).toContainText('运行已核验');
|
||||
await runPanel().getByRole('button', { name: '查看原输入', exact: true }).click(); const inputPreview = page.getByRole('dialog', { name: '引擎原执行输入', exact: true });
|
||||
await expect(inputPreview).toContainText(first.inputSha256); await expect(inputPreview.getByLabel('引擎运行原记录', { exact: true })).toContainText(child.taskId); await screenshot('mobile-controlled-input');
|
||||
await inputPreview.getByRole('button', { name: '下一页', exact: true }).click(); await expect(inputPreview.getByRole('button', { name: '回到首页', exact: true })).toBeEnabled(); await page.keyboard.press('Escape');
|
||||
let reportFile;
|
||||
if (desktop) {
|
||||
reportFile = path.join(directory, 'agent-workbench-engine-run-' + first.id + '.json');
|
||||
const download = await desktop.download(reportFile, () => runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click());
|
||||
assert.equal(download.suggestedFilename, path.basename(reportFile));
|
||||
} else {
|
||||
const downloading = page.waitForEvent('download'); await runPanel().getByRole('button', { name: '下载完整报告', exact: true }).click();
|
||||
const download = await downloading; reportFile = path.join(directory, download.suggestedFilename()); await download.saveAs(reportFile);
|
||||
}
|
||||
assert.equal(hash(fs.readFileSync(reportFile)), first.reportSha256);
|
||||
mark('mobile child run view shows actual counts and closure, pages original inputs and downloads the exact original report bytes');
|
||||
let heldRoute; const latePattern = '**/api/sessions/' + child.sessionId + '/engine-runs/' + first.id + '/report**';
|
||||
await page.route(latePattern, async route => { heldRoute = route; await new Promise(resolve => { releaseHeld = resolve; }); expectedRejections.push('owned previous-run 500 response'); await route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"Owned late engine record error"}' }); });
|
||||
await runPanel().getByRole('button', { name: '查看原报告', exact: true }).click(); await expect.poll(() => !!heldRoute).toBe(true); await page.keyboard.press('Escape');
|
||||
releaseHeld(); await expect.poll(() => expectedRejections.length).toBe(1); await page.unroute(latePattern); await expect(page.getByText('Owned late engine record error', { exact: true })).toHaveCount(0);
|
||||
mark('closing a run view ignores an old delayed record failure and keeps the selected task unchanged');
|
||||
await applyOne(); await parentFromChild(); await start(); await expect.poll(() => f.store.session(child.sessionId).status, { timeout: 30000 }).toBe('awaiting_approval');
|
||||
await panel().getByRole('button', { name: '查看并审批检查', exact: true }).click(); await expect(page.locator('.approval-card')).toContainText('npm'); await expect(page.locator('.inspector')).toBeHidden(); await screenshot('mobile-controlled-approval'); await page.locator('.approval-card').getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
await expect.poll(() => group.status, { timeout: 30000 }).toBe('ready_to_collect'); await resize({ width: 1440, height: 1000 });
|
||||
await openEngine(); await expect(runPanel().locator('[data-engine-run]')).toHaveCount(2); await screenshot('desktop-controlled-completed'); await page.keyboard.press('Escape');
|
||||
assert.equal(f.store.core.task.get(child.taskId).usage.requests, host.requests.length); assert.equal(f.store.core.task.get(child.taskId).usage.totalTokens, host.requests.length * 168);
|
||||
mark('explicit child file review and check approval create a new native Run, retain previous evidence and account actual model requests once');
|
||||
await parentFromChild(); await panel().getByRole('button', { name: '核对成果', exact: true }).click(); await expect.poll(() => group.status, { timeout: 15000 }).toBe('collected');
|
||||
await panel().getByRole('button', { name: '查看原成果记录', exact: true }).click(); await expect(page.getByRole('dialog', { name: '原成果核对记录', exact: true })).toContainText('src/a.mjs'); await page.keyboard.press('Escape');
|
||||
await panel().getByRole('button', { name: '结束协作并保留现场', exact: true }).click(); await page.getByRole('dialog', { name: '确认保留现场', exact: true }).getByRole('button', { name: '确认保留现场结束', exact: true }).click(); await expect.poll(() => group.status).toBe('resolved_external');
|
||||
mark('parent collection reads the original controlled-worker proof and explicit retention preserves every owner byte');
|
||||
const stopped = await host.manager.plan(f.session, { goal: '原生引擎停止与原输入核对', engine: { id: 'deepseek-harness', root: installation.root, controlledSha256: installation.controlledSha256 },
|
||||
children: [{ label: 'stopped-engine', goal: '停止之后不执行迟到工具', paths: ['src/stopped.mjs'] }] });
|
||||
await (await host.manager.prepare(f.session, stopped.id, stopped.fingerprint)).promise;
|
||||
const complete = host.provider.complete; let held = 0;
|
||||
host.provider.complete = async (_config, _messages, _project, signal) => new Promise((_resolve, reject) => { held++; signal.addEventListener('abort', () => reject(Object.assign(new Error('Owned held native model stopped'), { name: 'AbortError' })), { once: true }); });
|
||||
await panel().getByRole('button', { name: '刷新', exact: true }).click(); await panel().getByLabel('选择协作记录', { exact: true }).selectOption(stopped.id); await start(); await expect.poll(() => held, { timeout: 30000 }).toBe(1);
|
||||
await panel().getByRole('button', { name: '停止全部', exact: true }).click(); await expect.poll(() => stopped.status, { timeout: 20000 }).toBe('paused'); host.provider.complete = complete;
|
||||
const stoppedJob = [...f.store.core.engine_run.values()].find(job => job.taskId === stopped.children[0].taskId);
|
||||
assert.equal(stoppedJob.status, 'cancelled'); assert.equal(stoppedJob.native.cleanup.treeConfirmed, true); assert.equal(stoppedJob.calls.length, 0); assert.equal(stopped.usage.unknownRequests, 1);
|
||||
mark('actual native stop closes the owned process tree, preserves one unknown model attempt and dispatches no late tool');
|
||||
stopped.status = 'unknown'; stopped.requiresVerification = true;
|
||||
const stoppedWorker = f.store.core.worker.get(stopped.children[0].id); stoppedWorker.status = 'unknown'; stoppedWorker.requiresVerification = true; stoppedWorker.ownerPid = null;
|
||||
stoppedJob.status = 'unknown'; stoppedJob.requiresVerification = true; stoppedJob.verified = false;
|
||||
f.store.save(f.store.session(stoppedWorker.sessionId)); f.store.save(f.session);
|
||||
const originalReport = f.store.artifactStore.read(f.store.core.artifact.get(stoppedJob.reportArtifactId)), originalOutcome = structuredClone(stoppedJob.originalOutcome), previousUsage = structuredClone(stopped.usage);
|
||||
await resize({ width: 390, height: 844 }); await panel().getByRole('button', { name: '刷新', exact: true }).click(); await expect(panel()).toContainText('结果待核对');
|
||||
await panel().getByRole('button', { name: '核对原输入与现场', exact: true }).click(); await expect.poll(() => !!stopped.evidence, { timeout: 15000 }).toBe(true);
|
||||
await expect(panel().getByRole('button', { name: '确认核对并继续', exact: true })).toBeDisabled();
|
||||
await panel().getByRole('button', { name: '查看核对记录', exact: true }).click(); await expect(page.getByRole('dialog', { name: '原输入与现场核对', exact: true })).toContainText(stoppedJob.inputSha256); await screenshot('mobile-controlled-recovery'); await page.keyboard.press('Escape');
|
||||
await panel().getByRole('button', { name: '确认核对并继续', exact: true }).click(); await page.getByRole('dialog', { name: '确认原结果核对', exact: true }).getByRole('button', { name: '确认核对并继续', exact: true }).click(); await expect.poll(() => stopped.requiresVerification).toBe(false);
|
||||
assert.equal(stoppedJob.status, 'resolved_external'); assert.deepEqual(stoppedJob.originalOutcome, originalOutcome); assert.equal(f.store.artifactStore.read(f.store.core.artifact.get(stoppedJob.reportArtifactId)), originalReport); assert.deepEqual(stopped.usage, previousUsage); assert.deepEqual(ownerBytes(f), original);
|
||||
mark('390px recovery requires reading the owned observation before explicit review and preserves original outcome/report and cumulative budget');
|
||||
assert.deepEqual(inputs(), frozen); assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
result = { passed: true, targetVersion: version, testedVersion: version, assertions, screenshots, pageErrors, requestFailures, expectedRejections, installation,
|
||||
engineRuns: [...f.store.core.engine_run.values()], groups: [group.id, stopped.id], inputs: frozen, realModelRequests: 0, ...(desktop ? { desktopEvents: desktop.events } : {}),
|
||||
scope: 'Actual native Harness and ' + (desktop ? 'Electron with fixed main/preload and own in-process Node host' : 'Edge') + ' with an owned predefined in-memory model fixture. Unknown recovery is a declared fixture transition; actual host-crash behavior is independently tested.' };
|
||||
} catch (error) { result = { passed: false, targetVersion: version, testedVersion: version, error: error.stack, assertions, screenshots, pageErrors, requestFailures, realModelRequests: 0 }; await screenshot('controlled-ui-failure').catch(() => {}); throw error; }
|
||||
finally {
|
||||
releaseHeld?.(); await browser?.close(); await desktop?.close(); await host.runtime.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
fs.cpSync(f.root, path.join(directory, 'original-fixture'), { recursive: true });
|
||||
for (const session of f.store.sessions.values()) fs.writeFileSync(path.join(directory, 'session-' + session.id + '.json'), JSON.stringify(await host.api('/sessions/' + session.id + '/export'), null, 2));
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/' + (desktopMode ? 'desktop-controlled-ui' : 'controlled-ui') + '-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') }));
|
||||
for (const dispose of disposers.reverse()) await dispose(); console.log('EVIDENCE ' + directory);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash,randomUUID} from 'node:crypto';
|
||||
import {fixture,compatibleFixture,launchWorkbench,until,call,completion} from '../tests/helpers.mjs';
|
||||
import {createDeliveryProject,deliveryGit} from '../tests/fixtures/delivery-project.mjs';
|
||||
|
||||
if(process.argv.includes('--help')){console.log('node scripts/verify-delivery-progress.mjs [--responses] [--app-root ABSOLUTE_ROOT] — compiled host, owned HTTP fixture, real tests/review/continuation/Git; no configured-model requests.');process.exit(0);}
|
||||
const args=process.argv.slice(2),responses=args.includes('--responses'),rootIndex=args.indexOf('--app-root'),appRoot=rootIndex<0?process.cwd():path.resolve(args[rootIndex+1]);
|
||||
const directory=path.resolve('.qa','delivery-progress-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const cleanups=[],t={after:fn=>cleanups.push(fn)},f=fixture(t),snapshots=[],checks=[];
|
||||
createDeliveryProject(f);const parent=deliveryGit(f,['rev-parse','HEAD']).trim(),protectedIndex=deliveryGit(f,['show',':owner-staged.txt']);
|
||||
const protectedFiles=['owner-staged.txt','owner-unstaged.txt','owner-untracked.txt'].map(file=>[file,fs.readFileSync(path.join(f.projectRoot,file))]);
|
||||
f.store.close();let requests=0,firstPhase=0,host,result;
|
||||
const marker='已应用修改后的当前交付依据(每次请求重新核对,非操作授权):\n';
|
||||
try{
|
||||
const upstream=await compatibleFixture(t,(route,body)=>{
|
||||
const messages=body.messages??body.input,allText=messages.map(value=>typeof value.content==='string'?value.content:JSON.stringify(value.content??'')).join('\n');
|
||||
const position=allText.indexOf(marker),progress=position<0?null:JSON.parse(allText.slice(position+marker.length).split('\n')[0]);requests++;
|
||||
let calls;
|
||||
if(progress){snapshots.push(progress);assert.ok(progress.files.every(file=>file.current));assert.ok(progress.next_action,'known facts must permit the next fixture action');
|
||||
calls=[call('next-'+requests,progress.next_action.tool,progress.next_action.arguments)];
|
||||
}else{
|
||||
firstPhase++;
|
||||
if(firstPhase===1)calls=[call('old-plan','plan_checks',{names:['test','build']})];
|
||||
else if(firstPhase===2){
|
||||
const original=messages.filter(value=>value.role==='tool'||value.type==='function_call_output').map(value=>{try{return JSON.parse(value.content??value.output);}catch{return null;}}).findLast(value=>value?.id&&Array.isArray(value.checks));
|
||||
assert.ok(original);calls=[call('old-check','run_checks',{plan_id:original.id})];
|
||||
}else if(firstPhase===3)calls=[call('fix-add','propose_file',{path:'src/add.mjs',content:"import {sum} from './math.mjs';\nexport const add=sum;\n",reason:'Owned repair after original failing tests'}),
|
||||
call('fix-math','propose_file',{path:'src/math.mjs',content:'export const sum=(a,b)=>a+b;\n',reason:'Owned repair'})];
|
||||
else if(firstPhase===4){
|
||||
const ids=messages.filter(value=>value.role==='tool'||value.type==='function_call_output').map(value=>{try{return JSON.parse(value.content??value.output)?.change_id;}catch{return null;}}).filter(Boolean);
|
||||
assert.equal(ids.length,2);calls=[call('seal','prepare_change_set',{change_ids:ids})];
|
||||
}else throw new Error('Fixture must not poll for state after the recorded application.');
|
||||
}
|
||||
const response=completion(calls);response.model='owned-delivery-progress';
|
||||
if(!responses)return {body:response};
|
||||
return {body:{id:'owned-response-'+requests,model:response.model,status:'completed',output:calls.map(tool=>({type:'function_call',id:'item-'+tool.id,call_id:tool.id,name:tool.function.name,arguments:tool.function.arguments})),
|
||||
usage:{input_tokens:13,output_tokens:7,total_tokens:20}}};
|
||||
});
|
||||
host=await launchWorkbench(t,f.data,{}, {appRoot});assert.notEqual(new URL(host.url).port,'4318');
|
||||
const connection=await host.api('/model-center/connections',{name:'Owned progress fixture',model:'owned-delivery-progress',baseUrl:upstream.baseUrl,requiresKey:false,
|
||||
protocol:responses?'responses':'chat-completions',contextLimit:32000,maxOutputTokens:512,modelIdentityPolicy:'require-match'});
|
||||
await host.api('/model-center/select',{type:'connection',id:connection.id});const route='/sessions/'+f.session.id;
|
||||
const settle=async()=>{let state;await until(async()=>{state=await host.api(route);for(const approval of state.approvals.filter(value=>value.status==='pending'))await host.api(route+'/approvals/'+approval.id,{allow:true});
|
||||
return !['running','awaiting_approval','stopping'].includes(state.status);},60000);return state;};
|
||||
await host.api(route+'/messages',{content:'修复加法模块,通过原测试与构建,并准备审查后的本地Git交付。'});
|
||||
let state=await settle();assert.equal(state.status,'needs_review',JSON.stringify(state.failure));assert.equal(requests,4);
|
||||
assert.equal(state.execution.checks[0].status,'failed');assert.equal(state.execution.checks[0].results[0].tests.failed,3);
|
||||
const taskId=state.taskId,batch=state.execution.changeSets.at(-1);assert.equal(batch.status,'prepared');
|
||||
await host.api(route+'/change-sets/'+batch.id+'/apply',{fingerprint:batch.fingerprint});
|
||||
await until(async()=>(await host.api(route)).execution.changeSets.at(-1).status==='completed',20000);assert.equal(requests,4);
|
||||
checks.push('original three failed tests, complete batch review and application with no automatic model continuation');
|
||||
await host.api(route+'/continue-reviewed',{});state=await settle();assert.equal(state.status,'needs_review',JSON.stringify(state.failure));
|
||||
assert.equal(state.taskId,taskId);assert.equal(state.execution.run.stopReason,'git_review');assert.equal(requests,7);
|
||||
assert.deepEqual(snapshots.map(value=>value.stage),['needs_current_check_plan','ready_for_checks','ready_for_git_preview']);
|
||||
const passed=state.execution.checks.at(-1);assert.equal(passed.status,'passed');assert.equal(passed.results[0].tests.passed,3);
|
||||
assert.equal(deliveryGit(f,['rev-parse','HEAD']).trim(),parent);checks.push('current applied SHA and stale checks drive new plan, approval, original tests/build and Git preview without state polling');
|
||||
await host.api(route+'/delivery-jobs',{planId:state.execution.deliveryPlans.at(-1).id});
|
||||
await until(async()=>{state=await host.api(route);return !!state.execution.deliveries.at(-1)?.reportArtifactId;},30000);
|
||||
const job=state.execution.deliveries.at(-1);assert.equal(job.status,'completed');assert.deepEqual(job.paths.slice().sort(),['src/add.mjs','src/math.mjs']);
|
||||
assert.equal(deliveryGit(f,['show',':owner-staged.txt']),protectedIndex);for(const [file,bytes]of protectedFiles)assert.deepEqual(fs.readFileSync(path.join(f.projectRoot,file)),bytes);
|
||||
const report=await host.api('/artifacts/'+job.reportArtifactId+'?session='+state.id+'&limit=64000');assert.equal(report.truncated,false);
|
||||
const raw=JSON.parse(report.content);assert.equal(raw.status,'verified_committed');assert.equal(raw.task.id,taskId);
|
||||
assert.ok(raw.runs.flatMap(run=>run.consumption).every(receipt=>receipt.modelIdentity.status==='matched'));
|
||||
checks.push('explicit Git confirmation publishes only reviewed files and preserves owner index/worktree with identity-bearing report');
|
||||
result={success:true,version:host.bootstrap.version,protocol:responses?'responses':'chat-completions',requests,checks,snapshots,commit:job.commit,
|
||||
reportSha256:createHash('sha256').update(report.content).digest('hex'),realModelRequests:0};
|
||||
}catch(error){result={success:false,error:error.stack,requests,checks,snapshots,realModelRequests:0};throw error;}
|
||||
finally{fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));console.log(JSON.stringify({success:result.success,directory,checks:checks.length,requests}));for(const action of cleanups.reverse())await action();}
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { requestMetrics } from '../src/shared/contracts/model-request.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-delivery-reliability-evidence.mjs — offline verify sealed G bytes, exact original/fixed context, frozen regression, UI/rebuild/installation receipts and preserved state. No model or host calls.');
|
||||
process.exit(0);
|
||||
}
|
||||
assert.equal(process.argv.length, 2);
|
||||
const root = path.resolve('docs/delivery-reliability-g-evidence');
|
||||
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const read = file => JSON.parse(fs.readFileSync(path.join(root, file), 'utf8'));
|
||||
const manifestBytes = fs.readFileSync(path.join(root, 'manifest.json')), manifest = JSON.parse(manifestBytes);
|
||||
assert.equal(manifest.schemaVersion, 1); assert.equal(manifest.version, '1.24.1');
|
||||
assert.match(manifest.sourceCommit, /^[a-f0-9]{40}$/);
|
||||
const paths = new Set(); let bytes = 0;
|
||||
for (const entry of manifest.entries) {
|
||||
assert.ok(!paths.has(entry.path)); paths.add(entry.path);
|
||||
const target = path.resolve(root, entry.path); assert.ok(target.startsWith(root + path.sep));
|
||||
const raw = fs.readFileSync(target); assert.equal(raw.length, entry.bytes, entry.path); assert.equal(hash(raw), entry.sha256, entry.path); bytes += raw.length;
|
||||
}
|
||||
assert.equal(hash(fs.readFileSync('docs/live-original-purge-e-evidence/manifest.json')), manifest.originalEManifestSha256);
|
||||
assert.equal(hash(fs.readFileSync('docs/original-purge-release-f-evidence/manifest.json')), manifest.originalFManifestSha256);
|
||||
const tap = fs.readFileSync(path.join(root, manifest.checks.regression), 'utf8');
|
||||
const total = Number(tap.match(/ℹ tests (\d+)/)?.[1]), passed = Number(tap.match(/ℹ pass (\d+)/)?.[1]), failed = Number(tap.match(/ℹ fail (\d+)/)?.[1]);
|
||||
assert.equal(total, 155); assert.equal(total, passed); assert.equal(failed, 0);
|
||||
function strings(value, output = [], depth = 0) {
|
||||
if (depth > 12) return output;
|
||||
if (typeof value === 'string') {
|
||||
output.push(value);
|
||||
for (const text of new Set([value, ...value.split('\n').filter(line => /^\s*[\[{]/.test(line))])) {
|
||||
try { strings(JSON.parse(text), output, depth + 1); } catch {}
|
||||
}
|
||||
} else if (value && typeof value === 'object') for (const item of Object.values(value)) strings(item, output, depth + 1);
|
||||
return output;
|
||||
}
|
||||
const before = read('context/before/result.json'), after = read('context/final/result.json');
|
||||
assert.equal(before.manifestSha256, manifest.originalEManifestSha256); assert.equal(after.manifestSha256, before.manifestSha256);
|
||||
assert.equal(after.realModelRequests, 0); assert.equal(before.realModelRequests, 0);
|
||||
assert.deepEqual(before.reports.map(report => report.sources.filter(source => source.retained).length), [6, 5, 5, 4, 2, 2]);
|
||||
for (const [index, report] of after.reports.entries()) {
|
||||
assert.equal(report.sources.length, 6); assert.ok(report.sources.every(source => source.retained));
|
||||
assert.equal(report.inputBudget, before.reports[index].inputBudget); assert.equal(report.softTarget, 16000);
|
||||
const original = read('context/before/request-' + report.request + '.json');
|
||||
const fixed = read('context/final/request-' + report.request + '.json');
|
||||
assert.equal(requestMetrics(original.body).estimatedInputTokens, before.reports[index].originalInputTokens);
|
||||
const metrics = requestMetrics(fixed.body); assert.equal(metrics.estimatedInputTokens, report.inputTokens); assert.ok(metrics.estimatedInputTokens <= report.inputBudget);
|
||||
assert.equal(metrics.sha256, fixed.plan.requestSha256);
|
||||
assert.deepEqual(fixed.body.messages[0], original.body.messages[0], 'Complete original fixed goal/context');
|
||||
assert.deepEqual(fixed.body.variants.map(route => route.body.tools), original.body.variants.map(route => route.body.tools), 'Original complete tools');
|
||||
for (const source of report.sources) {
|
||||
const raw = fs.readFileSync(path.join('docs/live-original-purge-e-evidence/baselines/react', source.path), 'utf8');
|
||||
assert.equal(raw.length, source.characters); assert.ok(strings(fixed.body.messages).includes(raw), source.path);
|
||||
}
|
||||
}
|
||||
for (const key of ['sourceUI', 'portableUI']) {
|
||||
const ui = read(manifest.checks[key]); assert.equal(ui.success, true); assert.equal(ui.version, manifest.version);
|
||||
assert.equal(ui.realModelRequests, 0); assert.equal(ui.fixtureRequests, 5); assert.equal(ui.checks.length, 10); assert.deepEqual(ui.errors, []);
|
||||
for (const file of ui.screenshots) assert.ok(paths.has(path.posix.join(path.posix.dirname(manifest.checks[key]), path.basename(file))));
|
||||
}
|
||||
const release = read(manifest.checks.release), rebuild = read(manifest.checks.rebuild), archives = read(manifest.checks.archives), portable = read(manifest.checks.portable), install = read(manifest.checks.installer);
|
||||
assert.equal(release.documentation.sourceCommit, manifest.sourceCommit);
|
||||
assert.equal(rebuild.success, true); assert.equal(rebuild.sourceCommit, manifest.sourceCommit); assert.equal(rebuild.sourceZip.sha256, release.source.sha256);
|
||||
assert.equal(rebuild.compiled.length, 22); assert.deepEqual(rebuild.mismatches, []);
|
||||
assert.equal(archives.success, true); assert.equal(archives.version, manifest.version); assert.equal(archives.archives.length, 3);
|
||||
for (const archive of [release.portable, release.source, release.evidence]) assert.ok(archives.archives.some(record => record.sha256 === archive.sha256));
|
||||
assert.equal(portable.success, true); assert.equal(portable.version, manifest.version); assert.equal(portable.realModel, false);
|
||||
assert.ok(portable.actualRea.length > 0); assert.ok(portable.actualEngine); assert.ok(portable.actualControlled); assert.ok(portable.actualBrowser);
|
||||
assert.equal(install.success, true); assert.equal(install.version, manifest.version); assert.equal(install.realModelRequests, 0);
|
||||
assert.equal(install.results.length, 12); assert.ok(install.results.every(result => result.passed));
|
||||
for (const key of ['formalInstall', 'formalFinal']) {
|
||||
const record = read(manifest.checks[key]); assert.equal(record.success, true); assert.equal(record.version, manifest.version);
|
||||
assert.equal(record.sourceCommit, manifest.sourceCommit); assert.equal(record.pid, 36552); assert.equal(record.hasKey, true);
|
||||
assert.equal(record.retention.originalRows, 'exact'); assert.equal(record.retention.originalExports, 'exact');
|
||||
assert.equal(record.retention.originalArtifactBytes, 'exact'); assert.equal(record.retention.originalModelCenter, 'exact');
|
||||
assert.equal(record.operation.state, 'completed'); assert.equal(record.inspectorClosed, true); assert.equal(record.realModelRequests, 0);
|
||||
}
|
||||
console.log(JSON.stringify({ success: true, version: manifest.version, sourceCommit: manifest.sourceCommit,
|
||||
files: paths.size, bytes, manifestSha256: hash(manifestBytes), regression: { total, passed, failed },
|
||||
originalRetained: before.reports.map(report => report.sources.filter(source => source.retained).length), fixedRetained: after.reports.map(report => report.sources.filter(source => source.retained).length),
|
||||
compiledFiles: rebuild.compiled.length, uiGroups: 20, realModelRequests: 0, scope: 'Offline sealed-byte and actual-receipt verification. No new model calls, task execution, or claim of real-model/Purge success.' }));
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, compatibleFixture, launchWorkbench, call, completion, until, assertToolPairs } from '../tests/helpers.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-delivery-reliability-ui.mjs OUTPUT [APP_ROOT] — declared local HTTP model fixture, actual review/continue/check UI and saved budget cases; no real model. OUTPUT must be new.');
|
||||
process.exit(0);
|
||||
}
|
||||
assert.ok(process.argv[2], 'Choose a new output directory.');
|
||||
const output = path.resolve(process.argv[2]);
|
||||
assert.ok(!fs.existsSync(output), 'Keep prior evidence; choose a new directory.');
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
const appRoot = path.resolve(process.argv[3] ?? '.');
|
||||
const packageIdentity = JSON.parse(fs.readFileSync(path.join(appRoot,
|
||||
fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json')));
|
||||
const cleanups = [], t = { after: fn => cleanups.push(fn) };
|
||||
const f = fixture(t), cases = {};
|
||||
fs.mkdirSync(path.join(f.projectRoot, 'test'));
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'package.json'), JSON.stringify({ type: 'module', scripts: { test: 'node --test test/*.test.mjs' } }));
|
||||
fs.writeFileSync(path.join(f.projectRoot, 'test/values.test.mjs'), "import test from 'node:test';import assert from 'node:assert/strict';import {left} from '../src/left.mjs';import {right} from '../src/right.mjs';test('left',()=>assert.equal(left,1));test('right',()=>assert.equal(right,2));\n");
|
||||
for (const [name, limitingReason] of [['remaining', 'remaining_token_budget'], ['window', 'context_limit'], ['legacy', null]]) {
|
||||
const root = path.join(f.root, name); fs.mkdirSync(root); fs.writeFileSync(path.join(root, 'README.md'), '# Declared saved-budget UI fixture\n');
|
||||
const project = f.store.addProject(root), session = f.store.createSession(project.id, '预算诊断 ' + name);
|
||||
const goal = '保留原任务和已完成工具;按原拒绝分项核对预算。';
|
||||
const run = f.store.beginRun(session, goal, false, f.store.config);
|
||||
run.budgetPause = { schema: 1, reason: 'engine_input_budget', ...(limitingReason ? { limitingReason } : {}),
|
||||
requiredInputTokens: name === 'window' ? 28000 : 27424, inputBudget: 27552, contextLimit: 32000,
|
||||
outputReservation: 2048, safetyTokens: 2400, runRemainingTokens: 28339, taskRemainingTokens: 28339,
|
||||
remainingTokens: 28339, tokenReservation: { schemaVersion: 1, reservedTokens: 29600, routes: [] } };
|
||||
session.status = 'paused'; f.store.finishRun(session, run, 'engine_input_budget'); cases[name] = { project, session };
|
||||
}
|
||||
f.store.close();
|
||||
let requests = 0, browser, page;
|
||||
const checks = [], errors = [], screenshots = [];
|
||||
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
||||
const screenshot = async name => { const file = path.join(output, name + '.png'); await page.screenshot({ path: file, fullPage: true }); screenshots.push(file); };
|
||||
try {
|
||||
const model = await compatibleFixture(t, (_route, input) => {
|
||||
if (!input?.messages) return { data: [{ id: 'declared-ui-fixture' }] };
|
||||
requests++; assertToolPairs(input.messages);
|
||||
const values = input.messages.filter(message => message.role === 'tool').map(message => JSON.parse(message.content));
|
||||
if (requests === 1) return completion([
|
||||
call('ui-left', 'propose_file', { path: 'src/left.mjs', content: 'export const left = 1;\n', reason: 'Declared fixture left' }),
|
||||
call('ui-right', 'propose_file', { path: 'src/right.mjs', content: 'export const right = 2;\n', reason: 'Declared fixture right' }),
|
||||
]);
|
||||
if (requests === 2) return completion([
|
||||
call('ui-seal', 'prepare_change_set', { change_ids: values.map(value => value.change_id).filter(Boolean) }),
|
||||
call('ui-same-turn-inspect', 'inspect_session', {}),
|
||||
]);
|
||||
if (requests === 3) return completion([call('ui-check-plan', 'plan_checks', { names: ['test'] })]);
|
||||
if (requests === 4) return completion([call('ui-check-run', 'run_checks', { plan_id: values.findLast(value => value.checks && value.fingerprint).id })]);
|
||||
return completion([], '两份模块已通过实际 Node 测试。');
|
||||
});
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot });
|
||||
assert.equal(host.bootstrap.version, packageIdentity.version, 'Compiled host and package must agree.');
|
||||
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'declared-ui-fixture', apiKey: 'SYNTHETIC-QA-ONLY', maxSteps: 6 });
|
||||
browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') errors.push(request.url() + ': ' + request.failure()?.errorText); });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
async function choose(projectId) {
|
||||
const selector = page.getByLabel('当前项目', { exact: true });
|
||||
const expand = page.getByRole('button', { name: '展开项目栏', exact: true });
|
||||
if ((await page.viewportSize()).width <= 620) await expect(expand).toBeVisible();
|
||||
if (!await selector.isVisible()) await expand.click();
|
||||
await expect(selector).toBeVisible();
|
||||
await selector.selectOption(projectId);
|
||||
if ((await page.viewportSize()).width <= 940 && await page.locator('.project-sidebar').isVisible()) await page.getByRole('button', { name: '收起项目栏', exact: true }).click();
|
||||
}
|
||||
await choose(f.project.id);
|
||||
await page.getByLabel('任务内容', { exact: true }).fill('完成双模块工程,并在审查应用后继续实际测试。');
|
||||
await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
const route = '/sessions/' + f.session.id;
|
||||
let reviewed;
|
||||
await until(async () => { reviewed = await host.api(route); return reviewed.status === 'needs_review'; }, 30000);
|
||||
assert.equal(requests, 2); assert.equal(reviewed.execution.run.stopReason, 'change_set_review');
|
||||
assert.equal(reviewed.events.find(event => event.toolCallId === 'ui-same-turn-inspect').phase, 'completed');
|
||||
assert.equal(fs.existsSync(path.join(f.projectRoot, 'src')), false);
|
||||
mark('Actual compiled HTTP model loop settles both tools and hands off after exactly two fixture requests without writing files');
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
await expect(page.getByLabel('任务记录', { exact: true })).toContainText('完整文件批次已封存,等待审查与应用');
|
||||
await screenshot('desktop-review-handoff');
|
||||
await page.reload(); await page.waitForLoadState('networkidle');
|
||||
assert.equal((await host.api(route)).execution.task.id, reviewed.execution.task.id); assert.equal(requests, 2);
|
||||
mark('Review handoff persists across refresh without another model request');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await choose(f.project.id);
|
||||
await page.getByRole('button', { name: '查看改动', exact: true }).click();
|
||||
await page.getByRole('button', { name: '审查变更集', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '多文件变更集审查', exact: true });
|
||||
await expect(dialog).toContainText('src/left.mjs'); await expect(dialog).toContainText('src/right.mjs');
|
||||
await screenshot('mobile-batch-review');
|
||||
await dialog.getByRole('button', { name: '确认应用全部 2 项', exact: true }).click();
|
||||
await until(async () => (await host.api(route)).execution.changeSets.at(-1).status === 'completed', 30000);
|
||||
await page.keyboard.press('Escape'); await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toBeVisible();
|
||||
await page.getByRole('button', { name: '关闭检视区', exact: true }).click();
|
||||
assert.equal(requests, 2); mark('390px reviewed whole-batch application waits for explicit continuation');
|
||||
await page.getByRole('button', { name: '继续原任务', exact: true }).click();
|
||||
await until(async () => (await host.api(route)).approvals.some(approval => approval.status === 'pending'), 30000);
|
||||
await screenshot('mobile-check-approval');
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
let done;
|
||||
await until(async () => { done = await host.api(route); return done.status === 'completed'; }, 30000);
|
||||
assert.equal(done.execution.task.id, reviewed.execution.task.id); assert.equal(done.execution.checks[0].results[0].tests.passed, 2);
|
||||
assert.equal(requests, 5);
|
||||
assert.equal(done.events.filter(event => ['propose_file', 'prepare_change_set'].includes(event.tool)).length, 3);
|
||||
mark('Explicit same-Task continuation runs two actual Node assertions and never replays proposals or sealing');
|
||||
for (const width of [1440, 390]) {
|
||||
await page.setViewportSize({ width, height: width === 390 ? 844 : 1000 });
|
||||
for (const [name, title] of [['remaining', '原生引擎剩余预算不足'], ['window', '原生引擎上下文窗口不足'], ['legacy', '原生引擎输入或剩余预算不足']]) {
|
||||
await choose(cases[name].project.id);
|
||||
await expect(page.locator('.session-item.selected')).toContainText('预算诊断 ' + name);
|
||||
const panel = page.getByLabel('任务预算诊断', { exact: true });
|
||||
await expect(panel.getByRole('heading', { name: title, exact: true })).toBeVisible();
|
||||
if (name === 'remaining') { await expect(panel).toContainText('输入可容纳于当前上下文窗口'); await expect(panel).toContainText('29,600'); await expect(panel).toContainText('28,339'); }
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 2));
|
||||
await screenshot(width + '-budget-' + name);
|
||||
mark(width + 'px ' + name + ' diagnosis renders actual limit, numbers and historical fallback without overflow');
|
||||
}
|
||||
}
|
||||
assert.deepEqual(errors, []);
|
||||
const result = { success: true, version: (await host.api('/bootstrap')).version, appRoot, checks, errors, screenshots,
|
||||
fixtureRequests: requests, realModelRequests: 0, taskId: done.execution.task.id, runs: done.execution.runs.map(run => ({ id: run.id, status: run.status, stopReason: run.stopReason })),
|
||||
scope: 'Declared local HTTP model fixture with actual engineering tools/checks, persisted review state and seeded diagnostic cases. No real model or user project.', browser: browser.version() };
|
||||
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
||||
console.log(JSON.stringify({ success: true, checks: checks.length, output }));
|
||||
} catch (error) {
|
||||
if (page) await screenshot('failure').catch(() => {});
|
||||
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify({ success: false, checks, errors, screenshots, error: error.stack, requests }, null, 2) + '\n', { flag: 'wx' });
|
||||
throw error;
|
||||
} finally { if (browser) await browser.close(); for (const cleanup of cleanups.reverse()) await cleanup(); }
|
||||
@@ -0,0 +1,226 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { spawn, execFileSync } from 'node:child_process';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { GitDelivery } from '../src/core/git-delivery.mjs';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { checkedDelivery } from '../tests/fixtures/delivery-project.mjs';
|
||||
|
||||
// Owns an independent host and synthetic projects. It never connects to 4318.
|
||||
const workspace = process.cwd(), root = path.resolve('.qa', 'delivery-ui-' + randomUUID()), data = path.join(root, 'data');
|
||||
const projectRoot = path.join(root, '四模块交付工程'), recoveryRoot = path.join(root, '提交恢复工程');
|
||||
const allowed = ['src/normalize.mjs', 'src/summary.mjs', 'src/render.mjs', 'src/cli.mjs'];
|
||||
const bytesHash = value => createHash('sha256').update(value).digest('hex');
|
||||
fs.mkdirSync(root, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
fs.cpSync(path.join(workspace, 'benchmarks/node-delivery-v1/baseline'), projectRoot, { recursive: true });
|
||||
fs.writeFileSync(path.join(projectRoot, '.gitignore'), 'dist/\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Owner baseline notes\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Owner baseline staged file\n');
|
||||
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', projectRoot, ...args], { env: gitEnvironment(), encoding: 'utf8', windowsHide: true });
|
||||
const list = directory => fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => entry.isDirectory() ? list(path.join(directory, entry.name)) : [path.relative(projectRoot, path.join(directory, entry.name)).replaceAll('\\', '/')]);
|
||||
const baselineFiles = list(projectRoot);
|
||||
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['config', 'user.name', 'Workbench UI QA']); git(['config', 'user.email', 'ui@workbench.invalid']);
|
||||
git(['add', '--', ...baselineFiles]); git(['commit', '--no-gpg-sign', '-m', 'Sealed four-module UI input']);
|
||||
const baselineHead = git(['rev-parse', 'HEAD']).trim();
|
||||
fs.appendFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Keep owner unstaged notes\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Keep owner staged content\n'); git(['add', '--', 'OWNER-STAGED.txt']);
|
||||
fs.appendFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Keep later owner working copy\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'OWNER-UNTRACKED.txt'), 'Keep owner untracked file\n');
|
||||
const protectedFiles = [...baselineFiles.filter(relative => !allowed.includes(relative)), 'OWNER-UNTRACKED.txt'];
|
||||
const protectedHashes = Object.fromEntries(protectedFiles.map(relative => [relative, bytesHash(fs.readFileSync(path.join(projectRoot, relative)))]));
|
||||
const initialHashes = Object.fromEntries(allowed.map(relative => [relative, bytesHash(fs.readFileSync(path.join(projectRoot, relative)))]));
|
||||
const repairs = Object.fromEntries(allowed.map(relative => [relative, fs.readFileSync(path.join(workspace, 'examples/delivery-report', relative), 'utf8')]));
|
||||
const store = new Store(data), primary = store.addProject(projectRoot);
|
||||
fs.mkdirSync(recoveryRoot);
|
||||
fs.writeFileSync(path.join(recoveryRoot, 'README.md'), '# Recovery UI fixture\n');
|
||||
const recoveryProject = store.addProject(recoveryRoot), recoverySession = store.createSession(recoveryProject.id), seedCleanup = [];
|
||||
const f = { root, projectRoot: recoveryRoot, data, store, project: recoveryProject, session: recoverySession, onCleanup: fn => seedCleanup.push(fn) };
|
||||
await checkedDelivery(f);
|
||||
recoverySession.title = 'Git中断核对样本'; store.save(recoverySession);
|
||||
const recoveryAdapter = new GitDelivery(store, { faultInjector: phase => { if (phase === 'committed') throw new Error('Controlled lost acknowledgement after actual publication'); } });
|
||||
const recoveryPlan = await recoveryAdapter.plan(recoverySession);
|
||||
for (const cleanup of seedCleanup) await cleanup(); store.close();
|
||||
const exited = spawn(process.execPath, [path.join(workspace, 'tests/fixtures/delivery-crash-worker.mjs'), data, recoverySession.id, recoveryPlan.id, 'committed'],
|
||||
{ windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let exitOutput = '', exitErrors = '';
|
||||
exited.stdout.on('data', bytes => { exitOutput += bytes.toString('utf8'); }); exited.stderr.on('data', bytes => { exitErrors += bytes.toString('utf8'); });
|
||||
assert.equal(await new Promise(resolve => exited.once('close', resolve)), 73, exitErrors);
|
||||
const crashed = JSON.parse(exitOutput.trim()), recoveredStore = new Store(data), recoveryJob = structuredClone(recoveredStore.owned('delivery_job', crashed.jobId, recoverySession.id));
|
||||
assert.equal(recoveryJob.status, 'unknown'); recoveredStore.close();
|
||||
|
||||
let modelMode = 'initial', modelStage = 0;
|
||||
const model = http.createServer(async (request, response) => {
|
||||
try {
|
||||
const chunks = []; for await (const chunk of request) chunks.push(chunk);
|
||||
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
||||
const last = input.messages.findLastIndex(item => item.role === 'user');
|
||||
const results = input.messages.slice(last + 1).filter(item => item.role === 'tool').map(item => JSON.parse(item.content));
|
||||
// The request intentionally seals old complete tool groups when bounded.
|
||||
// Consume the original fields in its visible feedback, not history length.
|
||||
const feedback = input.messages.filter(item => item.role === 'system').flatMap(item => {
|
||||
for (const line of item.content.split('\n')) try { const value = JSON.parse(line); if (value.data?.kind === 'latest-tool-feedback') return value.data.entries.map(entry => entry.result); } catch {}
|
||||
return [];
|
||||
});
|
||||
const plan = [...results, ...feedback].findLast(value => value.checks && value.fingerprint);
|
||||
let calls = [];
|
||||
if (modelStage === 0) calls = [['inspect_project', {}]];
|
||||
else if (modelStage === 1) calls = [['plan_checks', {}]];
|
||||
else if (modelStage === 2) { assert.ok(plan?.id, 'The actual check plan must remain visible in raw results or original-field feedback.'); calls = [['run_checks', { plan_id: plan.id }]]; }
|
||||
else if (modelStage === 3 && modelMode === 'initial') calls = allowed.map(relative => ['propose_file', { path: relative, content: repairs[relative], base_sha256: initialHashes[relative], reason: 'Controlled four-module repair after actual failure logs' }]);
|
||||
else if (modelStage === 3 && modelMode === 'final') calls = [['plan_delivery', { message: 'feat: prepare verified four-module delivery' }]];
|
||||
else if (modelStage === 4 && modelMode === 'final') calls = [['delivery_report', {}]];
|
||||
modelStage++;
|
||||
response.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: calls.length ? '' : modelMode === 'initial' ? '四个文件已准备提案,等待审查。' : '实际14项测试和构建通过,Git预览已准备,等待确认。',
|
||||
...(calls.length ? { tool_calls: calls.map(([name, args]) => ({ id: randomUUID(), type: 'function', function: { name, arguments: JSON.stringify(args) } })) } : {}) },
|
||||
finish_reason: calls.length ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 40, completion_tokens: 20, total_tokens: 60 } }));
|
||||
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
|
||||
});
|
||||
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
||||
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: workspace, AGENT_WORKBENCH_DATA_DIR: data, AGENT_WORKBENCH_PORT: '0' }; delete env.AGENT_WORKBENCH_API_KEY;
|
||||
const host = spawn(process.execPath, ['dist-server/server.mjs'], { cwd: workspace, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let url = '', hostOutput = '', hostErrors = '';
|
||||
host.stdout.on('data', bytes => { hostOutput += bytes.toString('utf8'); for (const line of hostOutput.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} } });
|
||||
host.stderr.on('data', bytes => { hostErrors += bytes.toString('utf8'); });
|
||||
const until = async predicate => { const deadline = Date.now() + 20000; while (!(await predicate())) { if (Date.now() > deadline) throw new Error('QA host state timeout: ' + hostErrors); await new Promise(resolve => setTimeout(resolve, 20)); } };
|
||||
await until(() => url || host.exitCode !== null); assert.ok(url, hostErrors); assert.notEqual(new URL(url).port, '4318');
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const result = await response.json(); assert.ok(response.ok, result.error); return result;
|
||||
};
|
||||
await api('/config', { ...bootstrap.config, mode: 'compatible', model: 'delivery-ui-controlled-fixture', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', apiKey: 'delivery-ui-volatile-fixture-key' });
|
||||
const browser = await chromium.launch({ executablePath: 'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
|
||||
const page = await context.newPage(), passed = [], pageErrors = [], requestFailures = [], confirmations = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
page.on('request', request => { if (request.method() === 'POST' && /\/delivery-jobs$/.test(new URL(request.url()).pathname)) confirmations.push(request.postDataJSON().planId); });
|
||||
const mark = text => { passed.push(text); console.log('PASS ' + text); };
|
||||
const screenshot = name => page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' });
|
||||
const task = () => page.getByLabel('任务记录', { exact: true });
|
||||
const delivery = () => task().getByLabel('Git交付', { exact: true });
|
||||
const confirm = () => page.getByRole('dialog', { name: '确认本地Git提交', exact: true });
|
||||
const artifact = () => page.getByRole('dialog', { name: '执行结果工件', exact: true });
|
||||
let sessionId = '';
|
||||
const state = () => api('/sessions/' + sessionId);
|
||||
const waitStatus = status => expect.poll(async () => (await state()).status, { timeout: 30000 }).toBe(status);
|
||||
try {
|
||||
await page.goto(url); await page.waitForLoadState('networkidle');
|
||||
await page.getByLabel('当前项目').selectOption(primary.id);
|
||||
await page.getByRole('button', { name: '新建任务', exact: true }).click();
|
||||
await page.getByLabel('任务内容').fill('修复四模块并交付Git报告'); await page.getByRole('button', { name: '发送任务' }).click();
|
||||
await expect.poll(async () => (await api('/sessions')).find(value => value.title === '修复四模块并交付Git报告')?.id).toBeTruthy();
|
||||
sessionId = (await api('/sessions')).find(value => value.title === '修复四模块并交付Git报告').id;
|
||||
await waitStatus('awaiting_approval'); await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitStatus('needs_review');
|
||||
const reviewed = await state(); assert.equal(reviewed.execution.checks[0].results[0].tests.failed, 13);
|
||||
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(task()).toContainText('13');
|
||||
assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineHead);
|
||||
mark('actual failing tests and four source proposals retain the original project and task');
|
||||
await page.locator('.inspector-tabs').getByRole('button', { name: /改动/ }).click();
|
||||
for (const relative of allowed) {
|
||||
await page.locator('.change-list').getByRole('button').filter({ hasText: relative }).click();
|
||||
await page.getByRole('button', { name: '应用这项改动', exact: true }).click();
|
||||
}
|
||||
await waitStatus('paused'); modelMode = 'final'; modelStage = 0; await page.getByRole('button', { name: '继续原任务', exact: true }).click();
|
||||
await waitStatus('awaiting_approval'); await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitStatus('needs_review');
|
||||
const verified = await state(); assert.equal(verified.execution.task.id, reviewed.execution.task.id); assert.equal(verified.execution.checks.at(-1).status, 'passed');
|
||||
assert.equal(verified.execution.checks.at(-1).results[0].tests.passed, 14); assert.equal(verified.execution.deliveries.length, 0);
|
||||
assert.equal(verified.execution.deliveryPlans.at(-1).preparedBy, 'agent');
|
||||
assert.equal(verified.execution.run.stopReason, 'git_review');
|
||||
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(task()).toContainText('测试 14/14 通过');
|
||||
await page.locator('.session-select').filter({ hasText: '修复四模块并交付Git报告' }).click();
|
||||
await expect(task()).toContainText('测试 14/14 通过');
|
||||
await expect(delivery().getByLabel('等待Git确认')).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: '继续原任务', exact: true })).toHaveCount(0);
|
||||
await delivery().getByRole('button', { name: '查看待确认预览', exact: true }).click();
|
||||
await expect(confirm()).toBeVisible(); await expect(confirm()).toContainText('14/14');
|
||||
await screenshot('delivery-git-review-boundary');
|
||||
await confirm().getByRole('button', { name: '返回修改', exact: true }).click();
|
||||
mark('reviewed changes continue the same Task through 14 real tests and model-prepared Git review');
|
||||
|
||||
await delivery().getByLabel('提交说明', { exact: true }).fill('feat: ship the reviewed four-module delivery');
|
||||
await delivery().getByText('Git署名', { exact: true }).click(); await delivery().getByLabel('提交者姓名').fill('UI Reviewer'); await delivery().getByLabel('提交者邮箱').fill('reviewer@workbench.invalid');
|
||||
await delivery().getByRole('button', { name: '预览Git提交', exact: true }).click(); await expect(confirm()).toBeVisible();
|
||||
await expect(confirm()).toContainText('UI Reviewer'); await expect(confirm()).toContainText('14/14'); await expect(confirm()).toContainText(baselineHead.slice(0, 12)); await expect(confirm()).toContainText('不执行仓库hooks');
|
||||
for (const relative of allowed) { await confirm().locator('summary').filter({ hasText: relative }).click(); await expect(confirm().getByLabel('提交差异 ' + relative, { exact: true })).toBeVisible(); }
|
||||
await confirm().locator('.delivery-confirm').evaluate(element => { element.scrollTop = 0; });
|
||||
await screenshot('delivery-desktop-review');
|
||||
await page.keyboard.press('Escape'); await expect(confirm()).not.toBeVisible(); assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineHead);
|
||||
mark('review shows actual paths, diffs, tests, author and policy; cancelling leaves Git unchanged');
|
||||
|
||||
await delivery().getByRole('button', { name: '预览Git提交', exact: true }).click(); await expect(confirm()).toBeVisible();
|
||||
git(['add', '--', 'OWNER-NOTES.md']); const protectedStage = git(['show', ':OWNER-STAGED.txt']), stagedNotes = git(['show', ':OWNER-NOTES.md']);
|
||||
await confirm().getByRole('button', { name: '确认创建本地提交', exact: true }).click();
|
||||
await expect.poll(async () => (await state()).execution.deliveries.at(-1)?.status, { timeout: 20000 }).toBe('stale');
|
||||
await expect(delivery()).toContainText('提交预览后'); assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineHead);
|
||||
assert.equal(git(['show', ':OWNER-STAGED.txt']), protectedStage); assert.equal(git(['show', ':OWNER-NOTES.md']), stagedNotes);
|
||||
mark('an index change after confirmation preview refuses the stale job and preserves both staged owners');
|
||||
|
||||
await delivery().getByRole('button', { name: '预览Git提交', exact: true }).click(); await expect(confirm()).toBeVisible();
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await expect(confirm()).toBeVisible();
|
||||
await confirm().locator('summary').filter({ hasText: 'src/normalize.mjs' }).click();
|
||||
await screenshot('delivery-mobile-review');
|
||||
const box = await confirm().boundingBox(), confirmButton = await confirm().getByRole('button', { name: '确认创建本地提交', exact: true }).boundingBox();
|
||||
assert.ok(box.x >= 0 && box.x + box.width <= 391 && box.y >= 0 && box.y + box.height <= 845);
|
||||
assert.ok(confirmButton.x >= 0 && confirmButton.x + confirmButton.width <= 391);
|
||||
assert.ok(await confirm().evaluate(element => element.scrollWidth <= element.clientWidth + 1));
|
||||
mark('390px confirmation keeps file hashes, diffs and actions inside the modal');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
await confirm().getByRole('button', { name: '确认创建本地提交', exact: true }).dblclick();
|
||||
await expect.poll(async () => (await state()).execution.deliveries.at(-1)?.status, { timeout: 25000 }).toBe('completed');
|
||||
const delivered = await state(), job = delivered.execution.deliveries.at(-1);
|
||||
assert.equal(confirmations.filter(value => value === job.planId).length, 1);
|
||||
assert.equal(job.workspacePreserved, true); assert.equal(job.indexPreserved, true); assert.equal(git(['rev-list', '--count', 'HEAD']).trim(), '2');
|
||||
assert.equal(git(['show', ':OWNER-STAGED.txt']), protectedStage); assert.equal(git(['show', ':OWNER-NOTES.md']), stagedNotes);
|
||||
for (const [relative, expected] of Object.entries(protectedHashes)) assert.equal(bytesHash(fs.readFileSync(path.join(projectRoot, relative))), expected);
|
||||
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split('\n').sort(), allowed.slice().sort());
|
||||
await expect(delivery()).toContainText(job.commit); await expect(delivery()).toContainText('实际提交已核对'); await screenshot('delivery-desktop-result');
|
||||
mark('double confirmation creates one four-file commit, retains owner bytes and publishes report with completion');
|
||||
|
||||
const jobCard = delivery().locator('.delivery-job').filter({ hasText: job.commit });
|
||||
await jobCard.getByRole('button', { name: '查看交付报告', exact: true }).click(); await expect(artifact()).toContainText('检查通过并已提交'); await expect(artifact()).toContainText('14/14');
|
||||
await screenshot('delivery-desktop-report'); await artifact().getByRole('button', { name: '关闭窗口' }).click();
|
||||
for (const [label, id, extension] of [['下载Markdown', job.reportMarkdownArtifactId, '.md'], ['下载JSON', job.reportArtifactId, '.json']]) {
|
||||
const event = page.waitForEvent('download'); await jobCard.getByRole('button', { name: label, exact: true }).click(); const download = await event;
|
||||
const target = path.join(root, 'download' + extension); await download.saveAs(target);
|
||||
const meta = await api('/artifacts/' + id + '?session=' + sessionId); assert.equal(bytesHash(fs.readFileSync(target)), meta.sha256);
|
||||
if (extension === '.json') assert.equal(JSON.parse(fs.readFileSync(target, 'utf8')).status, 'verified_committed');
|
||||
}
|
||||
mark('report viewing and Markdown/JSON downloads preserve verified artifact bytes and truthful acceptance limits');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: '查看任务记录' }).click();
|
||||
await delivery().getByRole('button', { name: '查看最新报告', exact: true }).click(); await expect(artifact()).toBeVisible(); await screenshot('delivery-mobile-report');
|
||||
await artifact().getByRole('button', { name: '关闭窗口' }).click(); await page.getByRole('button', { name: '关闭检视区' }).click();
|
||||
const composer = await page.getByLabel('任务内容').boundingBox(); assert.ok(composer.y >= 0 && composer.y + composer.height <= 844);
|
||||
assert.ok(await page.locator('body').evaluate(element => element.scrollWidth <= 390)); mark('mobile report preview closes cleanly and keeps the task input fixed');
|
||||
|
||||
await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
if (!await page.getByLabel('当前项目').isVisible()) await page.getByRole('button', { name: '展开项目栏' }).click();
|
||||
await page.getByLabel('当前项目').selectOption(recoveryProject.id); await page.locator('.session-select').filter({ hasText: 'Git中断核对样本' }).click();
|
||||
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(delivery()).toContainText('结果待核对');
|
||||
await expect(delivery().getByRole('button', { name: '预览Git提交', exact: true })).toBeDisabled(); await screenshot('delivery-unknown-result');
|
||||
await delivery().getByRole('button', { name: '核对提交结果', exact: true }).click();
|
||||
await expect.poll(async () => (await api('/sessions/' + recoverySession.id)).execution.deliveries[0].status, { timeout: 15000 }).toBe('completed');
|
||||
const recovered = await api('/sessions/' + recoverySession.id);
|
||||
assert.equal(recovered.execution.deliveries[0].commit, recoveryJob.expectedCommit); assert.equal(fs.existsSync(path.join(recoveryRoot, '.git/index.lock')), false);
|
||||
await expect(delivery()).toContainText('没有重新执行提交'); await screenshot('delivery-reconciled-result');
|
||||
mark('unknown Git results block new submission and the explicit reconcile action restores the existing commit');
|
||||
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), browser: browser.version(), passed, pageErrors, requestFailures,
|
||||
projectRoot, projectId: primary.id, sessionId, taskId: reviewed.execution.task.id, commit: job.commit, recoveryCommit: recoveryJob.expectedCommit,
|
||||
testCounts: { baselineFailed: 13, passed: 14 }, model: 'controlled local HTTP fixture', confirmations: confirmations.length };
|
||||
fs.writeFileSync('.qa/delivery-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
await screenshot('delivery-ui-failure').catch(() => {}); fs.writeFileSync('.qa/delivery-ui-failure-dom.html', await page.content().catch(() => ''));
|
||||
fs.writeFileSync('.qa/delivery-ui-validation.json', JSON.stringify({ success: false, error: error.message, passed, pageErrors, requestFailures, root, url }, null, 2));
|
||||
throw error;
|
||||
} finally {
|
||||
await browser.close(); await api('/shutdown', {}).catch(() => {}); await until(() => host.exitCode !== null).catch(() => host.kill());
|
||||
model.closeAllConnections(); await new Promise(resolve => model.close(resolve));
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Owned Windows DPAPI restart and real browser acceptance; no user credentials."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--node', required=True)
|
||||
args = parser.parse_args()
|
||||
from playwright.sync_api import sync_playwright, expect
|
||||
workspace = Path.cwd()
|
||||
root = workspace / '.qa' / ('desktop-credentials-' + str(uuid.uuid4()))
|
||||
app_root, data_root = root / 'app', root / 'data'
|
||||
app_root.mkdir(parents=True)
|
||||
for name in ['dist-client', 'dist-server']:
|
||||
shutil.copytree(workspace / name, app_root / name)
|
||||
shutil.copy2(workspace / 'package.json', app_root / 'package.json')
|
||||
with socket.socket() as probe:
|
||||
probe.bind(('127.0.0.1', 0))
|
||||
port = probe.getsockname()[1]
|
||||
url = 'http://127.0.0.1:' + str(port)
|
||||
node = str(Path(args.node).resolve())
|
||||
allowed = re.compile(r'^(path|pathext|systemroot|systemdrive|windir|comspec|temp|tmp|userprofile|homedrive|homepath|home|appdata|localappdata|programfiles(?:\(x86\))?|commonprogramfiles(?:\(x86\))?|lang|lc_all)$', re.I)
|
||||
environment = {key: value for key, value in os.environ.items() if allowed.match(key)}
|
||||
environment['PATH'] = str(Path(node).parent) + os.pathsep + environment.get('PATH', environment.get('Path', ''))
|
||||
for key in list(environment):
|
||||
if key.lower() == 'path' and key != 'PATH':
|
||||
del environment[key]
|
||||
keys = ['OWNED_UI_DPAPI_' + uuid.uuid4().hex, 'OWNED_UI_LEGACY_' + uuid.uuid4().hex, 'OWNED_UI_VOLATILE_' + uuid.uuid4().hex]
|
||||
requests = []
|
||||
|
||||
class ModelFixture(http.server.BaseHTTPRequestHandler):
|
||||
def respond(self):
|
||||
if self.command == 'POST':
|
||||
self.rfile.read(int(self.headers.get('Content-Length', 0)))
|
||||
valid = self.headers.get('Authorization') in ['Bearer ' + key for key in keys]
|
||||
requests.append({'path': self.path, 'authorizedOwnedKey': valid})
|
||||
result = {'data': [{'id': 'owned-fixture'}]} if self.path.endswith('/models') else {'choices': [{'message': {'role': 'assistant', 'content': 'Owned response OK'}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 5, 'completion_tokens': 4, 'total_tokens': 9}}
|
||||
encoded = json.dumps(result if valid else {'error': {'message': 'Owned fixture credential rejected'}}).encode()
|
||||
self.send_response(200 if valid else 401)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
do_GET = respond
|
||||
do_POST = respond
|
||||
def log_message(self, *unused):
|
||||
pass
|
||||
|
||||
fixture = http.server.ThreadingHTTPServer(('127.0.0.1', 0), ModelFixture)
|
||||
threading.Thread(target=fixture.serve_forever, daemon=True).start()
|
||||
endpoint = 'http://127.0.0.1:' + str(fixture.server_port) + '/v1'
|
||||
marks, errors, images = [], [], []
|
||||
started, token, stage = False, None, 'host start'
|
||||
|
||||
def host(command):
|
||||
child = subprocess.run([node, str(app_root / 'dist-server' / 'host-launcher.mjs'), command, '--app-root', str(app_root), '--data-root', str(data_root), '--node', node, '--port', str(port)], cwd=str(app_root), env=environment, capture_output=True, encoding='utf-8', timeout=75, creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
if child.returncode:
|
||||
raise RuntimeError('Owned host operation failed: ' + child.stderr)
|
||||
return json.loads(child.stdout)
|
||||
|
||||
def api(route, payload=None):
|
||||
req = urllib.request.Request(url + '/api' + route, data=None if payload is None else json.dumps(payload).encode(), headers={'Content-Type': 'application/json', 'X-Workbench-Token': token or '', 'Connection': 'close'})
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return json.load(response)
|
||||
|
||||
def mark(name):
|
||||
marks.append(name)
|
||||
print('PASS ' + name, flush=True)
|
||||
|
||||
def snapshot_rows():
|
||||
with sqlite3.connect((data_root / 'workbench.sqlite').as_uri() + '?mode=ro', uri=True) as db:
|
||||
result = {table: db.execute('SELECT * FROM ' + table + ' ORDER BY rowid').fetchall() for table in ['metadata', 'projects', 'sessions', 'records', 'events', 'outbox']}
|
||||
result['metadata'] = [row for row in result['metadata'] if row[0] != 'runtimeOwner']
|
||||
return result
|
||||
|
||||
def save(success, detail=None):
|
||||
result = {'success': success, 'root': str(root), 'url': url, 'stage': stage, 'originalHost': globals().get('original'), 'freshHost': globals().get('fresh'), 'groups': marks, 'requests': requests, 'images': images, 'pageErrors': errors, 'detail': detail, 'scope': 'Owned CurrentUser DPAPI and local HTTP fixture; no external model request or existing credential is used.'}
|
||||
(root / 'result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
(workspace / '.qa' / 'desktop-credentials-latest.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
try:
|
||||
original = host('start'); started = True
|
||||
token = api('/bootstrap')['token']
|
||||
api('/config', {'mode': 'demo', 'baseUrl': endpoint, 'model': 'owned-legacy'})
|
||||
with sync_playwright() as pw:
|
||||
browser = pw.chromium.launch(channel='msedge', headless=True)
|
||||
page = browser.new_page(viewport={'width': 1440, 'height': 1000})
|
||||
page.on('pageerror', lambda error: errors.append(str(error)))
|
||||
page.goto(url)
|
||||
page.wait_for_load_state('networkidle')
|
||||
assert page.get_by_role('button', name='打开模型中心', exact=True).count() == 1
|
||||
page.get_by_role('button', name='打开模型中心', exact=True).click()
|
||||
dialog = page.get_by_role('dialog', name='模型中心', exact=True)
|
||||
expect(dialog).to_be_visible()
|
||||
stage = 'real connection form and explicit encrypted save'
|
||||
dialog.get_by_role('button', name='添加连接', exact=True).click()
|
||||
editor = dialog.get_by_role('form', name='编辑模型连接', exact=True)
|
||||
editor.get_by_label('连接名称', exact=True).fill('Owned Saved Primary')
|
||||
editor.get_by_label('接口根地址', exact=True).fill(endpoint)
|
||||
editor.get_by_label('实际模型名称', exact=True).fill('owned-primary')
|
||||
editor.get_by_label('本机加密保存本次填写的 API Key', exact=True).check()
|
||||
expect(editor.get_by_role('button', name='保存模型连接', exact=True)).to_be_disabled()
|
||||
editor.get_by_label('临时 API Key', exact=True).fill(keys[0])
|
||||
editor.get_by_role('button', name='保存模型连接', exact=True).click()
|
||||
expect(dialog.locator('.mc-connection').filter(has_text='Owned Saved Primary')).to_contain_text('可从本机加密保存加载')
|
||||
assert keys[0] not in page.content()
|
||||
mark(stage)
|
||||
initial_image = root / 'credentials-new-connection.png'
|
||||
page.screenshot(path=str(initial_image), full_page=True)
|
||||
images.append(str(initial_image))
|
||||
center = api('/model-center')
|
||||
saved = next(value for value in center['connections'] if value['name'] == 'Owned Saved Primary')
|
||||
volatile = api('/model-center/connections', {'name': 'Owned Volatile', 'model': 'owned-volatile', 'baseUrl': endpoint, 'apiKey': keys[2]})
|
||||
stage = 'real legacy credential save without exposing an existing key'
|
||||
dialog.get_by_role('button', name='凭据保存', exact=True).click()
|
||||
form = dialog.get_by_role('form', name='保存加密凭据', exact=True)
|
||||
expect(form).to_be_visible()
|
||||
form.get_by_label('选择保存连接', exact=True).select_option('legacy-default')
|
||||
form.get_by_label('重新填写 API Key', exact=True).fill(keys[1])
|
||||
form.get_by_role('button', name='加密保存凭据', exact=True).click()
|
||||
expect(form.get_by_label('重新填写 API Key', exact=True)).to_have_value('')
|
||||
expect(dialog.locator('.mc-connection').filter(has_text='原模型连接')).to_contain_text('本机加密保存')
|
||||
mark(stage)
|
||||
for width in [1440, 390]:
|
||||
page.set_viewport_size({'width': width, 'height': 1000})
|
||||
expect(dialog.get_by_role('heading', name='本机加密凭据', exact=True)).to_be_visible()
|
||||
assert page.evaluate('document.documentElement.scrollWidth <= innerWidth + 1')
|
||||
image = root / ('credentials-' + str(width) + '.png')
|
||||
page.screenshot(path=str(image), full_page=True)
|
||||
images.append(str(image))
|
||||
mark('real desktop and 390px saved-state controls and layout')
|
||||
browser.close()
|
||||
stage = 'actual host exit and fresh process restores encrypted revisions'
|
||||
api('/model-center/probe', {'id': saved['id']})
|
||||
before_rows = snapshot_rows()
|
||||
before_credentials = api('/model-center/credentials')
|
||||
stopped = host('stop'); started = False
|
||||
assert stopped['processExited']
|
||||
fresh = host('start'); started = True
|
||||
assert fresh['identity']['instanceId'] != original['identity']['instanceId']
|
||||
token = api('/bootstrap')['token']
|
||||
restored = api('/model-center')
|
||||
current = next(value for value in restored['connections'] if value['id'] == saved['id'])
|
||||
assert current['hasKey'] and current['credentialRevision'] == saved['credentialRevision']
|
||||
assert next(value for value in restored['connections'] if value['id'] == volatile['id'])['hasKey'] is False
|
||||
assert next(value for value in restored['connections'] if value['id'] == 'legacy-default')['hasKey']
|
||||
assert snapshot_rows() == before_rows
|
||||
assert api('/model-center/credentials')['connections'] == before_credentials['connections']
|
||||
mark(stage)
|
||||
api('/model-center/probe', {'id': saved['id']})
|
||||
api('/model-center/probe', {'id': 'legacy-default'})
|
||||
assert len(requests) == 3 and all(value['authorizedOwnedKey'] for value in requests)
|
||||
mark('fresh host actually uses both DPAPI-restored keys with owned HTTP responses')
|
||||
stage = 'disable, delete and forget through real controls'
|
||||
with sync_playwright() as pw:
|
||||
browser = pw.chromium.launch(channel='msedge', headless=True)
|
||||
page = browser.new_page(viewport={'width': 1280, 'height': 900})
|
||||
page.goto(url); page.wait_for_load_state('networkidle')
|
||||
page.get_by_role('button', name='打开模型中心', exact=True).click()
|
||||
dialog = page.get_by_role('dialog', name='模型中心', exact=True)
|
||||
dialog.get_by_role('button', name='凭据保存', exact=True).click()
|
||||
card = dialog.locator('.mc-connection').filter(has_text='Owned Saved Primary')
|
||||
card.get_by_role('button', name='关闭自动加载', exact=True).click()
|
||||
expect(card).to_contain_text('关闭自动加载')
|
||||
card.get_by_role('button', name='启用自动加载', exact=True).click()
|
||||
expect(card.get_by_role('button', name='关闭自动加载', exact=True)).to_be_visible()
|
||||
card.get_by_role('button', name='删除本机保存', exact=True).click()
|
||||
expect(card).to_contain_text('仅本次内存')
|
||||
assert next(value for value in api('/model-center')['connections'] if value['id'] == saved['id'])['hasKey']
|
||||
card.get_by_role('button', name='清除该连接凭据', exact=True).click()
|
||||
deadline = time.time() + 10
|
||||
while next(value for value in api('/model-center')['connections'] if value['id'] == saved['id'])['hasKey']:
|
||||
if time.time() > deadline:
|
||||
raise AssertionError('Credential clearing did not settle')
|
||||
time.sleep(0.1)
|
||||
browser.close()
|
||||
mark(stage)
|
||||
stage = 'plaintext leak scan over actual database, WAL, exports and host logs'
|
||||
needles = [needle for key in keys for needle in [key.encode(), key.encode('utf-16le')]]
|
||||
for file in data_root.rglob('*'):
|
||||
if file.is_file():
|
||||
data = file.read_bytes()
|
||||
assert not any(needle in data for needle in needles), str(file)
|
||||
assert not any(key in json.dumps(api('/model-center')) for key in keys)
|
||||
mark(stage)
|
||||
assert not errors
|
||||
save(True)
|
||||
except Exception as error:
|
||||
save(False, str(error))
|
||||
raise
|
||||
finally:
|
||||
fixture.shutdown(); fixture.server_close()
|
||||
if started:
|
||||
host('stop')
|
||||
@@ -0,0 +1,72 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import { randomUUID, randomBytes, createHash } from 'node:crypto';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
||||
import { ownedChild, readState, writeState } from '../src/core/desktop/file-state.mjs';
|
||||
import { observeWindowsProcess } from '../src/core/desktop/process-identity.mjs';
|
||||
import { HOST_REGISTRATION_FILE } from '../src/shared/contracts/desktop-host.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-desktop-host.mjs - real Windows isolated host identity, concurrent start, request stop and restart acceptance.'); process.exit(0); }
|
||||
if (process.argv.length > 2) throw new Error('Use --help.');
|
||||
const root = path.resolve('.qa', 'desktop-host-' + randomUUID()), appRoot = path.join(root, 'app'), dataRoot = path.join(root, 'data');
|
||||
fs.mkdirSync(appRoot, { recursive: true });
|
||||
for (const directory of ['dist-server', 'dist-client']) fs.cpSync(path.resolve(directory), path.join(appRoot, directory), { recursive: true });
|
||||
fs.copyFileSync('package.json', path.join(appRoot, 'package.json'));
|
||||
const freePort = async () => { const server = net.createServer(); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const port = server.address().port; await new Promise(resolve => server.close(resolve)); return port; };
|
||||
const port = await freePort(), options = { appRoot, dataRoot, port, nodePath: process.execPath, timeoutMs: 60000 };
|
||||
const controller = new HostController(options), results = []; let stage = 'concurrent start', started = false, fixture;
|
||||
const record = (name, detail = {}) => { results.push({ name, passed: true, ...detail }); console.log(name + ': passed'); };
|
||||
const until = async action => { const end = Date.now() + 20000; while (!await action()) { if (Date.now() > end) throw new Error('Timed out: ' + stage); await new Promise(resolve => setTimeout(resolve, 100)); } };
|
||||
const save = result => { fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/desktop-host-latest.json', JSON.stringify(result, null, 2)); };
|
||||
const completedTools = id => {
|
||||
const db = new DatabaseSync(path.join(dataRoot, 'workbench.sqlite'), { readOnly: true });
|
||||
try { return db.prepare("SELECT payload FROM records WHERE kind='tool' AND session_id=? ORDER BY id").all(id).map(row => JSON.parse(row.payload)).filter(value => value.state === 'completed'); }
|
||||
finally { db.close(); }
|
||||
};
|
||||
try {
|
||||
const starts = await Promise.all([controller.start(), new HostController(options).start()]); started = true;
|
||||
assert.equal(starts[0].identity.pid, starts[1].identity.pid); assert.equal(starts[0].identity.instanceId, starts[1].identity.instanceId);
|
||||
assert.equal(starts.filter(value => !value.reused).length, 1); record(stage, { pid: starts[0].identity.pid, instanceId: starts[0].identity.instanceId });
|
||||
const original = starts[0].identity;
|
||||
stage = 'independent creation and user observation'; const observed = observeWindowsProcess(original.pid); assert.equal(observed.createdAt, original.createdAt); assert.equal(observed.userSid, original.userSid); record(stage);
|
||||
const status = await controller.status(); assert.equal(status.state, 'ready'); assert.equal(status.windowClose, 'keep-host-running');
|
||||
stage = 'wrong data root and installation rejected'; await assert.rejects(new HostController({ ...options, dataRoot: path.join(root, 'wrong-data') }).start());
|
||||
const other = path.join(root, 'other-app'); fs.cpSync(appRoot, other, { recursive: true }); await assert.rejects(new HostController({ ...options, appRoot: other }).start()); record(stage);
|
||||
stage = 'stale registration and creation rejected'; const registrationFile = ownedChild(dataRoot, '.secrets/' + HOST_REGISTRATION_FILE), registration = readState(registrationFile);
|
||||
for (const changed of [{ createdAt: '2000-01-01T00:00:00.000Z' }, { instanceId: randomUUID() }, { userSid: 'S-1-5-21-1-2-3-1000' }, { secret: randomBytes(32).toString('base64') }]) {
|
||||
writeState(registrationFile, { ...registration, ...changed }); await assert.rejects(new HostController(options).connect()); writeState(registrationFile, registration);
|
||||
} record(stage);
|
||||
stage = 'damaged service index rejected'; const infoFile = ownedChild(dataRoot, 'service-info.json'), info = readState(infoFile); writeState(infoFile, { ...info, runtimeRoot: other }); await assert.rejects(new HostController(options).connect()); writeState(infoFile, info); record(stage);
|
||||
stage = 'running input binding does not silently change'; const serverFile = ownedChild(appRoot, 'dist-server/server.mjs'), bytes = fs.readFileSync(serverFile); fs.appendFileSync(serverFile, '\n// owned changed-on-disk fixture\n');
|
||||
assert.equal((await controller.connect()).identity.runtimeIdentity, original.runtimeIdentity); await assert.rejects(new HostController(options).connect()); fs.writeFileSync(serverFile, bytes); record(stage);
|
||||
stage = 'foreign health service rejected'; const foreignPort = await freePort(), foreign = http.createServer((_req, res) => { res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify({ app: 'agent-workbench', ready: true, pid: process.pid })); });
|
||||
await new Promise(resolve => foreign.listen(foreignPort, '127.0.0.1', resolve));
|
||||
try { await assert.rejects(new HostController({ ...options, port: foreignPort, dataRoot: path.join(root, 'foreign-data') }).start()); } finally { await new Promise(resolve => foreign.close(resolve)); } record(stage);
|
||||
stage = 'completed tools and approvals retained on stop and restart'; const projectRoot = path.join(root, 'project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned desktop lifecycle fixture\n');
|
||||
const project = await controller.api('/projects', { method: 'POST', input: { root: projectRoot } });
|
||||
const session = await controller.api('/sessions', { method: 'POST', input: { projectId: project.id } });
|
||||
await controller.api('/sessions/' + session.id + '/messages', { method: 'POST', input: { content: 'Run the offline demonstration in this owned project.' } });
|
||||
await until(async () => (await controller.api('/sessions/' + session.id)).approvals.some(value => value.status === 'pending'));
|
||||
const before = await controller.api('/sessions/' + session.id + '/export'), completed = completedTools(session.id); assert.ok(completed.length >= 3);
|
||||
const completedHash = createHash('sha256').update(JSON.stringify(completed)).digest('hex');
|
||||
const firstStop = await controller.stop(); started = false; assert.equal(firstStop.processExited, true); assert.equal(observeWindowsProcess(original.pid), null);
|
||||
const second = await controller.start(); started = true; assert.notEqual(second.identity.instanceId, original.instanceId);
|
||||
const restored = await controller.api('/sessions/' + session.id + '/export');
|
||||
assert.equal(createHash('sha256').update(JSON.stringify(completedTools(session.id))).digest('hex'), completedHash);
|
||||
assert.equal(restored.session.events.filter(value => value.kind === 'tool').length, before.session.events.filter(value => value.kind === 'tool').length);
|
||||
assert.equal(restored.session.status, 'paused'); record(stage, { completedTools: completed.length, originalEventSeq: before.session.eventSeq, restoredEventSeq: restored.session.eventSeq });
|
||||
stage = 'actual model HTTP request abort and verified process exit'; let requests = 0, disconnected = false;
|
||||
fixture = http.createServer((req, res) => { requests++; res.on('close', () => { disconnected = true; }); req.resume(); }); await new Promise(resolve => fixture.listen(0, '127.0.0.1', resolve));
|
||||
const pending = controller.api('/config/models', { method: 'POST', input: { baseUrl: 'http://127.0.0.1:' + fixture.address().port + '/v1', apiKey: 'owned-desktop-http-key' }, timeoutMs: 60000 }).catch(error => error);
|
||||
await until(() => requests === 1); const stopped = await controller.stop(); started = false; await pending;
|
||||
assert.equal(stopped.processExited, true); await until(() => disconnected); assert.equal(requests, 1); record(stage, { requests, disconnected, pid: stopped.identity.pid });
|
||||
save({ success: true, root, appRoot, dataRoot, port, results, inference: 'Owned local HTTP fixtures only; no existing model credential is read or persisted.' });
|
||||
} catch (cause) { save({ success: false, root, stage, results, message: cause.message, stack: cause.stack }); throw cause; }
|
||||
finally {
|
||||
if (fixture) { fixture.closeAllConnections(); await new Promise(resolve => fixture.close(resolve)); }
|
||||
if (started) await controller.stop();
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import net from 'node:net';
|
||||
import http from 'node:http';
|
||||
import { spawn, execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { _electron } from 'playwright';
|
||||
import { planDesktopBundle, copyDesktopBundle, buildNativeLauncher } from './bundle-desktop.mjs';
|
||||
import { buildWindowsInstaller } from './build-installer.mjs';
|
||||
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
||||
import { InstallationManager, snapshotPersistentData } from '../src/core/desktop/install-manager.mjs';
|
||||
import { observeWindowsProcess } from '../src/core/desktop/process-identity.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-desktop-install.mjs — isolated actual user-level setup/update/uninstall, full data retention, owned DPAPI/HTTP fixture, active-window and request guards, fault rollback and original PID retention. No configured-model credentials.'); process.exit(0); }
|
||||
assert.equal(process.argv.length, 2);
|
||||
const appRoot = process.cwd(), root = path.resolve('.qa', 'desktop-install-' + randomUUID()), installRoot = path.join(root, 'program'), dataRoot = path.join(root, 'user-data');
|
||||
const packagedSource = process.env.AGENT_WORKBENCH_QA_APP_ROOT ? fs.realpathSync.native(process.env.AGENT_WORKBENCH_QA_APP_ROOT) : null;
|
||||
const version = JSON.parse(fs.readFileSync('package.json')).version, results = [], appId = randomUUID(), hash = value => createHash('sha256').update(value).digest('hex');
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
const reserved = net.createServer(); await new Promise(resolve => reserved.listen(0, '127.0.0.1', resolve)); const port = reserved.address().port; await new Promise(resolve => reserved.close(resolve));
|
||||
const record = (name, detail = {}) => { results.push({ name, passed: true, ...detail }); console.log('PASS ' + name); };
|
||||
const until = async (action, timeout = 60000) => { const end = Date.now() + timeout; while (!await action()) { if (Date.now() > end) throw Error('Timeout: ' + stage); await new Promise(resolve => setTimeout(resolve, 100)); } };
|
||||
const run = promisify(execFile), env = projectEnvironment();
|
||||
let stage = 'prepare owned full releases', controller, application, ownedModel, report, heldRequest, releaseRequest, original, installedUI = null;
|
||||
const launcher = path.resolve('.qa/native-launcher/AgentWorkbench.exe'); buildNativeLauncher(appRoot, launcher); const desktop = planDesktopBundle(appRoot);
|
||||
function payload(name) {
|
||||
const directory = path.join(root, name); fs.mkdirSync(directory);
|
||||
if (packagedSource) {
|
||||
const original = JSON.parse(fs.readFileSync(path.join(packagedSource, 'MANIFEST.json')));
|
||||
assert.equal(original.kind, 'portable'); assert.equal(original.version, version);
|
||||
for (const entry of original.files) { const file = path.join(packagedSource, entry.path); assert.equal(hash(fs.readFileSync(file)), entry.sha256, entry.path); }
|
||||
fs.cpSync(packagedSource, directory, { recursive: true }); fs.unlinkSync(path.join(directory, 'MANIFEST.json'));
|
||||
} else {
|
||||
for (const item of ['dist-client', 'dist-server', 'dist-desktop', 'public', 'examples']) fs.cpSync(path.join(appRoot, item), path.join(directory, item), { recursive: true });
|
||||
fs.mkdirSync(path.join(directory, 'scripts')); fs.copyFileSync('scripts/python-check-runner.py', path.join(directory, 'scripts/python-check-runner.py'));
|
||||
fs.mkdirSync(path.join(directory, 'runtime')); fs.copyFileSync(process.execPath, path.join(directory, 'runtime/node.exe'));
|
||||
fs.cpSync(path.join(path.dirname(process.execPath), 'node_modules/npm'), path.join(directory, 'runtime/node_modules/npm'), { recursive: true });
|
||||
copyDesktopBundle(appRoot, directory, desktop, { launcher });
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'owned-release.txt'), name + '\n');
|
||||
// A valid PE overlay explicitly models a future owned native bootstrap with
|
||||
// different bytes. The updated root entry must follow it and roll back with it.
|
||||
if (name.startsWith('owned-b-')) fs.appendFileSync(path.join(directory, 'AgentWorkbench.exe'), '\nowned-native-bootstrap-update-fixture\n');
|
||||
const walk = relative => fs.readdirSync(path.join(directory, relative), { withFileTypes: true }).flatMap(entry => {
|
||||
const next = path.posix.join(relative, entry.name), file = path.join(directory, next); return entry.isDirectory() ? walk(next) : [{ path: next, bytes: fs.statSync(file).size, sha256: hash(fs.readFileSync(file)) }];
|
||||
});
|
||||
fs.writeFileSync(path.join(directory, 'MANIFEST.json'), JSON.stringify({ app: 'agent-workbench', kind: 'portable', version, platform: 'win32-x64', nodeVersion: process.version, files: walk('') }, null, 2)); return directory;
|
||||
}
|
||||
async function setup(installer, label, expected = 0) {
|
||||
const log = path.join(root, label + '-setup.log');
|
||||
let code = 0, output = '';
|
||||
try { const result = await run(installer.path, ['/VERYSILENT', '/SUPPRESSMSGBOXES', '/NORESTART', '/LOG=' + log, '/DIR=' + installRoot, '/DATA_ROOT=' + dataRoot, '/PORT=' + port], { windowsHide: true, timeout: 300000, maxBuffer: 1024 * 1024, env }); output = result.stdout + result.stderr; }
|
||||
catch (error) { code = typeof error.code === 'number' ? error.code : -1; output = String(error.stdout ?? '') + String(error.stderr ?? ''); }
|
||||
fs.writeFileSync(path.join(root, label + '-process.txt'), output);
|
||||
if (expected === 0) assert.equal(code, 0, 'Actual installer failed; inspect ' + log); else assert.notEqual(code, 0, 'Actual installer unexpectedly accepted the rejected condition.');
|
||||
const logText = fs.readFileSync(log, 'utf8'); assert.match(logText, /AWBPROGRESS /, 'Actual installer must receive original stage output.');
|
||||
const records = fs.readdirSync(path.join(installRoot, '.installation')).filter(name => /^progress-.*\.json$/.test(name)).map(name => JSON.parse(fs.readFileSync(path.join(installRoot, '.installation', name)))).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
||||
assert.equal(records[0].state, expected === 0 ? 'completed' : 'failed');
|
||||
if (expected === 0) { assert.ok(records[0].history.some(row => row.stage === 'verify-candidate' && row.scope === 'full-manifest' && row.counts.completedFiles === row.counts.totalFiles)); assert.match(logText, /AWBFILES \d+\/\d+/); }
|
||||
return { code, log, sha256: hash(fs.readFileSync(log)) };
|
||||
}
|
||||
async function launchElectron() {
|
||||
const current = JSON.parse(fs.readFileSync(path.join(installRoot, 'current.json'))), directory = path.join(installRoot, 'versions', current.active.directory);
|
||||
const client = await _electron.launch({ executablePath: path.join(directory, 'desktop/AgentWorkbench.exe'), args: ['--app-root', directory, '--data-root', dataRoot, '--port', String(port), '--hidden-window'], cwd: directory, env, timeout: 60000 });
|
||||
const page = await client.firstWindow(); await page.locator('.connection.connected').waitFor({ timeout: 30000 }); return client;
|
||||
}
|
||||
function ownedManager(candidateRoot) { return new InstallationManager({ installRoot, dataRoot, candidateRoot, port }); }
|
||||
try {
|
||||
const a = payload('owned-a-' + version), b = payload('owned-b-' + version), fault = payload('owned-fault-' + version);
|
||||
stage = 'compile isolated actual installers';
|
||||
const firstSetup = buildWindowsInstaller({ root: appRoot, portableRoot: a, outputRoot: root, qaFixture: { appId } });
|
||||
const secondSetup = buildWindowsInstaller({ root: appRoot, portableRoot: b, outputRoot: root, qaFixture: { appId } });
|
||||
const faultSetup = buildWindowsInstaller({ root: appRoot, portableRoot: fault, outputRoot: root, qaFixture: { appId, fault: 'corrupt-installed-candidate' } });
|
||||
stage = 'actual clean user-level installation'; const installed = await setup(firstSetup, 'clean');
|
||||
const activeA = path.join(installRoot, 'versions', path.basename(a)); controller = new HostController({ appRoot: activeA, dataRoot, port, nodePath: process.execPath });
|
||||
original = (await controller.start()).identity; application = await launchElectron();
|
||||
assert.equal((await controller.status()).desktopWindows.count, 1); record(stage, { installer: firstSetup, ...installed, hostPid: original.pid, electronPid: await application.evaluate(() => process.pid) });
|
||||
stage = 'actual installer rejects an open attested desktop window'; const windowRejection = await setup(secondSetup, 'window-block', 1);
|
||||
assert.equal(ownedManager(null).current().active.directory, path.basename(a)); assert.equal((await controller.connect()).identity.pid, original.pid); record(stage, windowRejection);
|
||||
await application.close(); application = null; await until(async () => (await controller.status()).desktopWindows.count === 0);
|
||||
const key = 'owned-install-credential-' + randomUUID(); let authenticated = 0, hold = false;
|
||||
ownedModel = http.createServer(async (request, response) => {
|
||||
const chunks = []; for await (const bytes of request) chunks.push(bytes);
|
||||
assert.equal(request.headers.authorization, 'Bearer ' + key); authenticated++;
|
||||
if (hold) { heldRequest = true; await new Promise(resolve => { releaseRequest = resolve; request.on('close', resolve); }); }
|
||||
response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'OK' }, finish_reason: 'stop' }], usage: { prompt_tokens: 4, completion_tokens: 1, total_tokens: 5 } }));
|
||||
}); await new Promise(resolve => ownedModel.listen(0, '127.0.0.1', resolve));
|
||||
const connection = await controller.api('/model-center/connections', { method: 'POST', input: { name: 'Owned desktop install credential', model: 'owned', baseUrl: 'http://127.0.0.1:' + ownedModel.address().port + '/v1', apiKey: key, persistKey: true } });
|
||||
const connectionId = connection.id ?? connection.connection?.id ?? connection.center?.connections.at(-1).id;
|
||||
assert.ok(connectionId); await controller.api('/model-center/probe', { method: 'POST', input: { id: connectionId } }); assert.equal(authenticated, 1);
|
||||
stage = 'actual update rejects a pending owned model request'; hold = true;
|
||||
// Manifest verification is synchronous and can occupy this fixture's event
|
||||
// loop. Keep the explicitly held local response pending across that work;
|
||||
// this is the QA client's deadline, not a product/model timeout override.
|
||||
const pendingStarted = Date.now();
|
||||
const pending = controller.api('/model-center/probe', { method: 'POST', input: { id: connectionId }, timeoutMs: 300000 }); pending.catch(() => {});
|
||||
await until(() => heldRequest); await assert.rejects(ownedManager(null).preflight(), /任务|请求/);
|
||||
assert.equal((await controller.connect()).identity.pid, original.pid);
|
||||
assert.equal(authenticated, 2, 'Exactly the initial and explicitly held owned requests; no retry.');
|
||||
record(stage, { fixture: 'Actual owned HTTP response held until full manifest/preflight and original PID verification settle. No configured-model call or product timeout change.', elapsedMs: Date.now() - pendingStarted });
|
||||
releaseRequest?.(); hold = false; assert.equal((await pending).ok, true);
|
||||
stage = 'actual installer rejects a persistent original task approval';
|
||||
const projectRoot = path.join(root, 'approval-project'); fs.mkdirSync(projectRoot); fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned installer approval project\n');
|
||||
const project = await controller.api('/projects', { method: 'POST', input: { root: projectRoot } }), session = await controller.api('/sessions', { method: 'POST', input: { projectId: project.id } });
|
||||
await controller.api('/sessions/' + session.id + '/messages', { method: 'POST', input: { content: 'Read this owned project and prepare the offline demonstration proposal.' } });
|
||||
await until(async () => (await controller.api('/sessions/' + session.id)).approvals.some(value => value.status === 'pending'));
|
||||
const approval = (await controller.api('/sessions/' + session.id)).approvals.find(value => value.status === 'pending');
|
||||
const approvalRejection = await setup(secondSetup, 'approval-block', 1); assert.equal((await controller.api('/sessions/' + session.id)).approvals.find(value => value.status === 'pending').id, approval.id);
|
||||
await controller.api('/sessions/' + session.id + '/stop', { method: 'POST', input: {} }); record(stage, { ...approvalRejection, approvalId: approval.id });
|
||||
stage = 'actual manifest fault installer preserves original pointer, PID and all data'; const snapshot = snapshotPersistentData(dataRoot), pointer = fs.readFileSync(path.join(installRoot, 'current.json'));
|
||||
const faultResult = await setup(faultSetup, 'candidate-fault', 1);
|
||||
assert.deepEqual(fs.readFileSync(path.join(installRoot, 'current.json')), pointer); assert.deepEqual(snapshotPersistentData(dataRoot), snapshot); assert.equal((await controller.connect()).identity.pid, original.pid); assert.ok(fs.existsSync(path.join(installRoot, 'AgentWorkbench.exe'))); record(stage, faultResult);
|
||||
stage = 'actual pointer-published fault rolls original host and pointer back';
|
||||
const originalBootstrap = fs.readFileSync(path.join(installRoot, 'AgentWorkbench.exe'));
|
||||
const faultCandidate = path.join(installRoot, 'versions', 'owned-pointer-fault'); fs.cpSync(b, faultCandidate, { recursive: true });
|
||||
await assert.rejects(ownedManager(faultCandidate).activate({ afterPointer: () => { throw Error('owned-post-pointer-fault'); } }), /owned-post-pointer-fault/);
|
||||
assert.deepEqual(fs.readFileSync(path.join(installRoot, 'current.json')), pointer); assert.deepEqual(snapshotPersistentData(dataRoot), snapshot); assert.equal((await controller.connect()).identity.pid, original.pid); record(stage, { fixture: 'Explicit in-process afterPointer error with real retained host; original pointer bytes, database rows and artifact bytes checked.' });
|
||||
assert.deepEqual(fs.readFileSync(path.join(installRoot, 'AgentWorkbench.exe')), originalBootstrap);
|
||||
stage = 'actual update retains original PID, instance, DPAPI ciphertext and every persistent byte';
|
||||
const updated = await setup(secondSetup, 'update'); const activeB = path.join(installRoot, 'versions', path.basename(b));
|
||||
controller = new HostController({ appRoot: activeB, dataRoot, port, nodePath: process.execPath }); const after = (await controller.connect()).identity;
|
||||
assert.equal(after.pid, original.pid); assert.equal(after.instanceId, original.instanceId); assert.deepEqual(snapshotPersistentData(dataRoot), snapshot); record(stage, { ...updated, hostPid: after.pid, instanceId: after.instanceId, data: snapshot });
|
||||
assert.notDeepEqual(fs.readFileSync(path.join(installRoot, 'AgentWorkbench.exe')), originalBootstrap);
|
||||
assert.deepEqual(fs.readFileSync(path.join(installRoot, 'AgentWorkbench.exe')), fs.readFileSync(path.join(activeB, 'AgentWorkbench.exe')));
|
||||
stage = 'installed native bootstrap verifies and opens the registered release';
|
||||
const nativeLauncher = spawn(path.join(installRoot, 'AgentWorkbench.exe'), ['--hidden-window'], { windowsHide: true, stdio: 'ignore', env });
|
||||
await new Promise((resolve, reject) => { nativeLauncher.once('error', reject); nativeLauncher.once('exit', code => code === 0 ? resolve() : reject(Error('Owned native bootstrap failed (' + code + ')'))); });
|
||||
await until(async () => (await controller.status()).desktopWindows.count === 1);
|
||||
const nativeWindow = (await controller.status()).desktopWindows.entries[0], nativeProcess = observeWindowsProcess(nativeWindow.pid); assert.ok(nativeProcess); assert.equal((await controller.connect()).identity.pid, original.pid);
|
||||
// A controlled WM_CLOSE to the owned main window is delegated to Electron by
|
||||
// a separate automation connection below; no unknown process is terminated.
|
||||
const child = spawn(path.join(activeB, 'desktop/AgentWorkbench.exe'), ['--app-root', activeB, '--data-root', dataRoot, '--port', String(port), '--hidden-window'], { cwd: activeB, env, windowsHide: true, stdio: 'ignore' });
|
||||
await new Promise(resolve => child.once('exit', resolve)); assert.equal((await controller.status()).desktopWindows.count, 1);
|
||||
record(stage, { electronPid: nativeWindow.pid, hostPid: original.pid, repeatLaunchReusedWindow: true });
|
||||
// The native process has no debugger. Its owned renderer closes itself through
|
||||
// the product's browser window action using a localhost, attested stop below.
|
||||
stage = 'native desktop remains registered before explicit owned cleanup';
|
||||
// Closing only this exactly observed test Electron tree after the launch check
|
||||
// models a desktop crash. Host/persistent results remain; stale registration is
|
||||
// removed through the normal identity check, without signalling a reused PID.
|
||||
const observed = observeWindowsProcess(nativeWindow.pid); assert.equal(observed.createdAt, nativeWindow.createdAt);
|
||||
await run('taskkill.exe', ['/PID', String(observed.pid), '/T', '/F'], { windowsHide: true, timeout: 15000, env });
|
||||
await until(async () => (await controller.status()).desktopWindows.count === 0); assert.equal((await controller.connect()).identity.pid, original.pid);
|
||||
record(stage, { fixture: 'Explicit owned Electron crash after exact actual PID/creation check; independent host retained.' });
|
||||
stage = 'real host restart restores the owned saved credential and actual HTTP authentication';
|
||||
const beforeRestart = snapshotPersistentData(dataRoot); await controller.stop(); const restarted = (await controller.start()).identity;
|
||||
assert.notEqual(restarted.pid, original.pid); assert.deepEqual(snapshotPersistentData(dataRoot), beforeRestart);
|
||||
await controller.api('/model-center/probe', { method: 'POST', input: { id: connectionId } }); assert.equal(authenticated, 3); record(stage, { originalPid: original.pid, restartedPid: restarted.pid, authenticatedRequests: authenticated });
|
||||
if (['1.24.0', '1.24.1', '1.24.2', '1.24.3', '1.24.4', '1.24.5', '1.24.6', '1.24.7', '1.24.8', '1.24.9', '1.25.0'].includes(version)) {
|
||||
stage = 'actual installed current results and saved HTML user paths';
|
||||
const pointer = JSON.parse(fs.readFileSync(path.join(installRoot, 'current.json')));
|
||||
const installedRoot = fs.realpathSync.native(path.join(installRoot, 'versions', pointer.active.directory)), proofs = [];
|
||||
const suites = [
|
||||
['verify-file-results-ui.mjs', [], 16], ['verify-local-files.mjs', [], 9],
|
||||
...(['1.24.9', '1.25.0'].includes(version) ? [['verify-requirements-ui.mjs', [], 7], ['verify-requirements-desktop.mjs', [], 3], ['verify-turn-progress-ui.mjs', [], 9], ['verify-turn-progress-ui.mjs', ['--desktop'], 9]] : []),
|
||||
...(version === '1.25.0' ? [['verify-task-agreement-ui.mjs', [], 6], ['verify-task-agreement-ui.mjs', ['--desktop'], 6], ['verify-task-follow-up-ui.mjs', [], 5], ['verify-task-follow-up-ui.mjs', ['--desktop'], 5]] : []),
|
||||
];
|
||||
for (const [script, args, expected] of suites) {
|
||||
const output = await run(process.execPath, [path.join(appRoot, 'scripts', script), ...args], { cwd: appRoot,
|
||||
env: { ...env, AGENT_WORKBENCH_QA_APP_ROOT: installedRoot }, windowsHide: true, timeout: 180000, maxBuffer: 1024 * 1024 });
|
||||
const log = path.join(root, 'installed-' + script + (args.includes('--desktop') ? '-desktop' : '') + '.log'); fs.writeFileSync(log, output.stdout + output.stderr);
|
||||
const summary = output.stdout.trim().split(/\r?\n/).filter(line => line.startsWith('{')).map(line => { try { return JSON.parse(line); } catch { return null; } }).findLast(row => row?.success !== undefined);
|
||||
const proofRoot = summary.directory ?? summary.root; assert.ok(proofRoot && path.resolve(proofRoot).startsWith(path.join(appRoot, '.qa') + path.sep));
|
||||
const proof = JSON.parse(fs.readFileSync(path.join(proofRoot, 'result.json'), 'utf8'));
|
||||
assert.equal(proof.success, true); assert.equal(proof.version, version); assert.equal(fs.realpathSync.native(proof.appRoot), installedRoot);
|
||||
assert.equal((proof.checks ?? proof.passed).length, expected); assert.deepEqual(proof.errors, []);
|
||||
assert.equal(proof.realModelRequests, 0); proofs.push({ script, proof, log, sha256: hash(fs.readFileSync(log)) });
|
||||
}
|
||||
installedUI = { success: true, version, installedRoot, packagedSource, proofs, realModelRequests: 0 };
|
||||
fs.writeFileSync('.qa/existing-results-installed-ui.json', JSON.stringify(installedUI, null, 2)); record(stage, installedUI);
|
||||
}
|
||||
stage = 'actual uninstall closes only the idle owned host and preserves user data';
|
||||
const uninstallSnapshot = snapshotPersistentData(dataRoot), uninstallLog = path.join(root, 'uninstall.log');
|
||||
await run(path.join(installRoot, 'unins000.exe'), ['/VERYSILENT', '/SUPPRESSMSGBOXES', '/NORESTART', '/LOG=' + uninstallLog], { windowsHide: true, timeout: 180000, env });
|
||||
assert.equal(observeWindowsProcess(restarted.pid), null); assert.deepEqual(snapshotPersistentData(dataRoot), uninstallSnapshot); assert.equal(fs.existsSync(path.join(installRoot, 'AgentWorkbench.exe')), false); assert.equal(fs.existsSync(path.join(installRoot, 'current.json')), false);
|
||||
record(stage, { log: uninstallLog, originalDataRoot: dataRoot, data: uninstallSnapshot });
|
||||
report = { success: true, version, root, installRoot, dataRoot, port, appId, packagedSource, results, installedUI, realModelRequests: 0, scope: 'Actual isolated user-level Windows installers and shipped Electron/Node runtimes; own credentials and HTTP fixture only. QA appId, corruption, pointer-fault and crash fixtures explicitly recorded.' };
|
||||
} catch (error) { report = { success: false, version, root, installRoot, dataRoot, port, stage, error: error.stack, results, realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
releaseRequest?.(); await application?.close().catch(() => {});
|
||||
if (controller) { try { const status = await controller.status(); if (!status.desktopWindows.count) await controller.stop(); } catch {} }
|
||||
if (ownedModel) { ownedModel.closeAllConnections(); await new Promise(resolve => ownedModel.close(resolve)); }
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(report, null, 2)); fs.writeFileSync('.qa/desktop-install-location.json', JSON.stringify({ root, result: path.join(root, 'result.json') })); console.log('EVIDENCE ' + root);
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import net from 'node:net';
|
||||
import http from 'node:http';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { _electron } from 'playwright';
|
||||
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
||||
import { observeWindowsProcess } from '../src/core/desktop/process-identity.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-desktop-window.mjs [--native-picker] — isolated actual Electron runtime; default uses explicitly identified native-dialog/shell fixtures, --native-picker waits for actual Windows UI selection. No existing credentials or configured model.'); process.exit(0); }
|
||||
if (process.argv.slice(2).some(arg => arg !== '--native-picker')) throw new Error('Use --help.');
|
||||
const native = process.argv.includes('--native-picker'), appRoot = process.cwd(), root = path.resolve('.qa', 'desktop-window-' + randomUUID());
|
||||
const dataRoot = path.join(root, 'data'), projectRoot = path.join(root, 'owned-project'); fs.mkdirSync(projectRoot, { recursive: true });
|
||||
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Owned desktop project\n\nOriginal source for desktop lifecycle acceptance.\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'package.json'), JSON.stringify({ name: 'owned-desktop-project', type: 'module', scripts: { test: 'node --test case.test.mjs' } }));
|
||||
fs.writeFileSync(path.join(projectRoot, 'case.test.mjs'), "import test from 'node:test';import assert from 'node:assert/strict';import fs from 'node:fs';test('original desktop source retained',()=>assert.match(fs.readFileSync('README.md','utf8'),/Original source/));\n");
|
||||
const server = net.createServer(); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const port = server.address().port; await new Promise(resolve => server.close(resolve));
|
||||
const controller = new HostController({ appRoot, dataRoot, port, nodePath: process.execPath, timeoutMs: 60000 });
|
||||
let application, page, original, externalServer, stage = 'start', expectedNavigation = false; const results = [], errors = [], requests = [];
|
||||
const record = (name, detail = {}) => { results.push({ name, passed: true, ...detail }); console.log(name + ': passed'); };
|
||||
const until = async (action, timeout = 30000) => { const deadline = Date.now() + timeout; while (!await action()) { if (Date.now() > deadline) throw new Error('Timeout: ' + stage); await new Promise(resolve => setTimeout(resolve, 100)); } };
|
||||
const save = result => { fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/desktop-window-' + (native ? 'native' : 'latest') + '.json', JSON.stringify(result, null, 2)); };
|
||||
const completedTools = id => {
|
||||
const db = new DatabaseSync(path.join(dataRoot, 'workbench.sqlite'), { readOnly: true });
|
||||
try { return db.prepare("SELECT payload FROM records WHERE kind='tool' AND session_id=? ORDER BY id").all(id).map(row => JSON.parse(row.payload)).filter(value => value.state === 'completed'); } finally { db.close(); }
|
||||
};
|
||||
async function launch() {
|
||||
const executable = desktopExecutableIdentity(appRoot).path, packaged = executable.toLowerCase().endsWith('agentworkbench.exe');
|
||||
const current = await _electron.launch({ executablePath: executable,
|
||||
args: [...(packaged ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', dataRoot, '--node', process.execPath, '--port', String(port), ...(native ? [] : ['--hidden-window'])],
|
||||
cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
||||
const window = await current.firstWindow();
|
||||
window.on('pageerror', error => errors.push(error.message)); window.on('requestfailed', request => { if (!expectedNavigation) requests.push({ url: request.url(), failure: request.failure()?.errorText }); });
|
||||
await window.locator('.connection.connected').waitFor({ timeout: 30000 });
|
||||
return { application: current, page: window };
|
||||
}
|
||||
async function snapshot(name, width) {
|
||||
const window = await application.browserWindow(page); await window.evaluate((value, width) => { value.showInactive(); value.setContentSize(width, 900); }, width);
|
||||
await until(() => page.evaluate(width => innerWidth === width, width));
|
||||
await page.screenshot({ path: path.join(root, name + '.png') }); fs.writeFileSync(path.join(root, name + '.html'), await page.content());
|
||||
const overflow = await page.evaluate(() => document.documentElement.scrollWidth > innerWidth);
|
||||
assert.equal(overflow, false); return { name, width, png: path.join(root, name + '.png') };
|
||||
}
|
||||
try {
|
||||
original = (await controller.start()).identity; const project = await controller.api('/projects', { method: 'POST', input: { root: projectRoot } });
|
||||
const session = await controller.api('/sessions', { method: 'POST', input: { projectId: project.id } });
|
||||
({ application, page } = await launch());
|
||||
const pid = await application.evaluate(() => process.pid), launcherPid = application.process().pid, observed = observeWindowsProcess(pid), status = await controller.status();
|
||||
assert.equal(status.identity.pid, original.pid); assert.equal(status.desktopWindows.count, 1); assert.equal(status.desktopWindows.entries[0].pid, pid); assert.equal(status.desktopWindows.entries[0].createdAt, observed.createdAt); record('actual Electron process registered and original host reused', { electronPid: pid, launcherPid, hostPid: original.pid });
|
||||
await page.getByRole('combobox', { name: '当前项目', exact: true }).selectOption(project.id);
|
||||
if (native) {
|
||||
stage = 'actual native directory picker'; const chosenRoot = path.join(root, 'native-picked-project'); fs.mkdirSync(chosenRoot); fs.writeFileSync(path.join(chosenRoot, 'README.md'), '# Actual native picker project\n');
|
||||
await page.getByRole('button', { name: '桌面宿主', exact: true }).click();
|
||||
const pending = page.getByRole('button', { name: '选择项目文件夹', exact: true }).click();
|
||||
console.log(JSON.stringify({ nativePickerReady: true, expectedPath: chosenRoot, electronPid: pid, hostPid: original.pid, root }));
|
||||
await until(async () => (await controller.api('/bootstrap')).projects.some(value => value.root.toLowerCase() === chosenRoot.toLowerCase()), 1200000); await pending;
|
||||
await until(async () => (await page.getByRole('combobox', { name: '当前项目', exact: true }).textContent()).includes('native-picked-project'));
|
||||
record(stage, { directory: chosenRoot, dialog: 'Actual Windows openDirectory dialog; no dialog/shell replacement in this run.' });
|
||||
await snapshot('native-picker-result', 1440);
|
||||
} else {
|
||||
stage = 'sandboxed renderer and fixed preload';
|
||||
const surface = await page.evaluate(() => ({ methods: Object.keys(window.desktopWorkbench).sort(), require: typeof window.require, process: typeof window.process, ipc: typeof window.ipcRenderer, frozen: Object.isFrozen(window.desktopWorkbench) }));
|
||||
// This verifier runs the current source and its current preload. Keep an
|
||||
// explicit allowlist so later versions retain these reviewed capabilities
|
||||
// while any unexpected privileged method still fails acceptance.
|
||||
assert.deepEqual(surface.methods, ['browsingLayout', 'openFilePreview', 'pickProject', 'projectFolder', 'protocol', 'revealFile', 'reviewTask', 'status', 'stopHost'].sort());
|
||||
assert.equal(surface.require, 'undefined'); assert.equal(surface.process, 'undefined'); assert.equal(surface.ipc, 'undefined'); assert.equal(surface.frozen, true);
|
||||
const preferences = await (await application.browserWindow(page)).evaluate(value => value.webContents.getLastWebPreferences());
|
||||
assert.equal(preferences.sandbox, true); assert.equal(preferences.contextIsolation, true); assert.equal(preferences.nodeIntegration, false); record(stage, { surface });
|
||||
stage = 'external navigation, requests and popups rejected'; let outsideRequests = 0;
|
||||
externalServer = http.createServer((_req, res) => { outsideRequests++; res.end('owned external fixture'); }); await new Promise(resolve => externalServer.listen(0, '127.0.0.1', resolve));
|
||||
const externalUrl = 'http://127.0.0.1:' + externalServer.address().port + '/outside'; expectedNavigation = true;
|
||||
await page.evaluate(url => { window.open(url); location.href = url; }, externalUrl); await new Promise(resolve => setTimeout(resolve, 300));
|
||||
assert.equal(new URL(page.url()).origin, controller.url); assert.equal(application.windows().length, 1); assert.equal(outsideRequests, 0);
|
||||
const navigation = await (await application.browserWindow(page)).evaluate(value => ({ url: value.webContents.getURL(), loading: value.webContents.isLoading() }));
|
||||
assert.equal(navigation.loading, false); assert.equal(new URL(navigation.url).origin, controller.url);
|
||||
// Chromium reports requestedNavigation to Playwright before Electron vetoes it.
|
||||
// A real window refresh clears that automation-only pending document state.
|
||||
const refreshed = page.waitForEvent('domcontentloaded'); await (await application.browserWindow(page)).evaluate(value => value.webContents.reload()); await refreshed;
|
||||
await page.locator('.connection.connected').waitFor(); expectedNavigation = false; record(stage, { navigation, originalWindowRefreshed: true });
|
||||
stage = 'offline original task and approval retained across refresh and window close';
|
||||
await page.getByRole('textbox', { name: '任务内容', exact: true }).fill('Read this owned project and prepare the offline demonstration proposal.'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).waitFor({ timeout: 20000 });
|
||||
const before = await controller.api('/sessions/' + session.id), tools = completedTools(session.id), hash = createHash('sha256').update(JSON.stringify(tools)).digest('hex'); assert.ok(tools.length >= 3);
|
||||
await page.reload(); await page.getByRole('button', { name: '确认执行', exact: true }).waitFor({ timeout: 20000 }); assert.equal((await controller.api('/sessions/' + session.id)).approvals.at(-1).id, before.approvals.at(-1).id);
|
||||
const closed = application.waitForEvent('close'); await (await application.browserWindow(page)).evaluate(value => value.close()); await closed; application = null;
|
||||
assert.equal((await controller.connect()).identity.instanceId, original.instanceId); assert.equal(observeWindowsProcess(pid), null); assert.equal((await controller.status()).desktopWindows.count, 0);
|
||||
({ application, page } = await launch()); await page.getByRole('button', { name: '确认执行', exact: true }).waitFor({ timeout: 20000 });
|
||||
const reopened = await controller.api('/sessions/' + session.id); assert.equal(reopened.execution.task.id, before.execution.task.id); assert.equal(reopened.execution.runs.at(-1).id, before.execution.runs.at(-1).id); assert.equal(reopened.eventSeq, before.eventSeq); assert.equal(reopened.approvals.at(-1).id, before.approvals.at(-1).id);
|
||||
assert.equal(createHash('sha256').update(JSON.stringify(completedTools(session.id))).digest('hex'), hash); record(stage, { taskId: before.execution.task.id, runId: before.execution.runs.at(-1).id, approvalId: before.approvals.at(-1).id, eventSeq: before.eventSeq, completedTools: tools.length });
|
||||
stage = 'actual command approval, proposal and project check'; await page.getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
await until(async () => (await controller.api('/sessions/' + session.id)).status === 'needs_review');
|
||||
const done = await controller.api('/sessions/' + session.id); const change = done.changes.at(-1); assert.equal(change.status, 'pending');
|
||||
await controller.api('/sessions/' + session.id + '/changes/' + change.id + '/apply', { method: 'POST', input: {} });
|
||||
const planned = await controller.api('/sessions/' + session.id + '/check-plans', { method: 'POST', input: {} });
|
||||
const started = await controller.api('/sessions/' + session.id + '/check-jobs', { method: 'POST', input: { planId: planned.execution.plans.at(-1).id } }), checkId = started.execution.checks.at(-1).id;
|
||||
await until(async () => !['queued', 'running'].includes((await controller.api('/sessions/' + session.id + '/check-jobs/' + checkId)).status));
|
||||
const check = await controller.api('/sessions/' + session.id + '/check-jobs/' + checkId); assert.equal(check.status, 'passed'); record(stage, { changeId: change.id, checkId, checkStatus: check.status });
|
||||
stage = 'current project, task and file desktop entry scope';
|
||||
await application.evaluate(({ dialog, shell }, projectRoot) => { globalThis.desktopFixtureCalls = []; dialog.showOpenDialog = async () => ({ canceled: false, filePaths: [projectRoot] }); dialog.showMessageBox = async () => ({ response: 0 }); shell.openPath = async value => { globalThis.desktopFixtureCalls.push({ folder: value }); return ''; }; shell.showItemInFolder = value => globalThis.desktopFixtureCalls.push({ reveal: value }); }, projectRoot);
|
||||
await page.getByRole('button', { name: 'REA 工件研究', exact: true }).waitFor();
|
||||
await page.getByRole('button', { name: 'README.md', exact: true }).click(); await page.locator('.file-heading').getByText('README.md', { exact: true }).waitFor();
|
||||
const file = await controller.api('/projects/' + project.id + '/file?path=README.md'), scope = { projectId: project.id, sessionId: session.id, taskId: done.execution.task.id };
|
||||
const invalid = await page.evaluate(async input => { const rejected = []; for (const value of [{ ...input, path: '../outside' }, { ...input, taskId: '00000000-0000-0000-0000-000000000000' }, { ...input, sha256: '0'.repeat(64) }, { ...input, command: 'foreign' }]) { try { await window.desktopWorkbench.revealFile(value); rejected.push(false); } catch { rejected.push(true); } } return rejected; }, { ...scope, path: file.path, sha256: file.sha256 });
|
||||
assert.deepEqual(invalid, [true, true, true, true]);
|
||||
await page.getByRole('button', { name: '桌面宿主', exact: true }).click(); await page.getByRole('button', { name: '打开项目目录', exact: true }).click(); await page.getByRole('button', { name: '定位当前文件', exact: true }).click();
|
||||
await until(() => application.evaluate(() => globalThis.desktopFixtureCalls.length === 2));
|
||||
const fixtureCalls = await application.evaluate(() => globalThis.desktopFixtureCalls); assert.deepEqual(fixtureCalls, [{ folder: projectRoot }, { reveal: path.join(projectRoot, 'README.md') }]);
|
||||
const screenshots = [await snapshot('desktop-host-1440', 1440), await snapshot('desktop-host-390', 390)];
|
||||
await page.getByRole('button', { name: '审查当前任务', exact: true }).click(); await page.getByLabel('任务记录', { exact: true }).waitFor();
|
||||
record(stage, { invalidRejected: invalid.length, fixtureCalls, screenshots, nativeOperations: 'Dialog/shell adapters are declared fixtures in this automated run; actual Windows picker separately verified.' });
|
||||
stage = 'native stop cancelled, then original host explicitly stopped';
|
||||
await page.getByRole('button', { name: '关闭检视区', exact: true }).click();
|
||||
await page.getByRole('button', { name: '桌面宿主', exact: true }).click(); await page.getByRole('button', { name: '停止并退出', exact: true }).click();
|
||||
await until(() => page.getByRole('button', { name: '停止并退出', exact: true }).isEnabled());
|
||||
assert.equal((await controller.connect()).identity.instanceId, original.instanceId);
|
||||
await application.evaluate(({ dialog }) => { dialog.showMessageBox = async () => ({ response: 1 }); });
|
||||
const exited = application.waitForEvent('close'); expectedNavigation = true; await page.getByRole('button', { name: '停止并退出', exact: true }).click(); await exited; application = null;
|
||||
assert.equal(observeWindowsProcess(original.pid), null); assert.equal(await controller.health(), null); expectedNavigation = false; record(stage, { hostPid: original.pid, exited: true, confirmation: 'Declared native confirmation fixtures: cancelled first, confirmed second.' });
|
||||
}
|
||||
assert.deepEqual(errors, []); assert.deepEqual(requests, []);
|
||||
save({ success: true, root, appRoot, dataRoot, port, results, errors, requests, native, inference: 'Offline demo/local engineering checks only; no configured model or existing credentials.' });
|
||||
} catch (error) {
|
||||
if (page && !page.isClosed()) { try { fs.writeFileSync(path.join(root, 'failure.html'), await page.content()); await page.screenshot({ path: path.join(root, 'failure.png') }); } catch {} }
|
||||
save({ success: false, root, appRoot, dataRoot, stage, results, errors, requests, message: error.message, stack: error.stack, native }); throw error;
|
||||
} finally {
|
||||
if (application) await application.close();
|
||||
if (await controller.health()) await controller.stop();
|
||||
if (externalServer) { externalServer.closeAllConnections(); await new Promise(resolve => externalServer.close(resolve)); }
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4319';
|
||||
const root = fs.mkdtempSync(path.resolve('.qa', 'diagnostics-ui-'));
|
||||
const projectRoot = path.join(root, 'project'); fs.mkdirSync(projectRoot);
|
||||
fs.writeFileSync(path.join(projectRoot, 'README.md'), '# Diagnostics UI fixture\n');
|
||||
fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.equal(bootstrap.config.mode, 'demo', 'Diagnostics UI validation requires an independent demo service.');
|
||||
assert.equal(bootstrap.config.hasKey, false, 'Diagnostics UI validation must not replace a configured model connection.');
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
||||
};
|
||||
let mode = 'failure', requestCount = 0, sessionId = '', page, diagnosticReads = 0, diagnosticProjectionUsed = false;
|
||||
const passed = [], pageErrors = [], networkErrors = [];
|
||||
const model = http.createServer(async (request, response) => {
|
||||
try {
|
||||
const chunks = []; for await (const chunk of request) chunks.push(chunk);
|
||||
if (request.url === '/v1/models') { response.writeHead(200, { 'Content-Type': 'application/json' }); return response.end(JSON.stringify({ data: [{ id: 'ui-diagnostics-fixture' }] })); }
|
||||
const input = JSON.parse(Buffer.concat(chunks).toString('utf8')); requestCount++;
|
||||
assert.equal(request.headers.authorization, 'Bearer ui-fixture-memory-key');
|
||||
const answer = content => { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content }, finish_reason: 'stop' }] })); };
|
||||
const tool = (name, args) => { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: '', tool_calls: [{ id: 'ui-' + randomUUID(), type: 'function', function: { name, arguments: JSON.stringify(args) } }] }, finish_reason: 'tool_calls' }] })); };
|
||||
const failure = (status, headers = {}) => { response.writeHead(status, { 'Content-Type': 'application/json', ...headers }); response.end(JSON.stringify({ error: { message: 'Controlled model-service failure' } })); };
|
||||
if (!input.tools) return answer('OK');
|
||||
const lastUser = input.messages.findLastIndex(message => message.role === 'user');
|
||||
const results = input.messages.slice(lastUser + 1).filter(message => message.role === 'tool');
|
||||
if (mode === 'failure') return results.length ? failure(503) : tool('read_file', { path: 'README.md' });
|
||||
if (mode === 'recover') { assert.equal(input.messages.at(-1).role, 'tool'); return answer('**模型请求已恢复**,已完成的读取结果保留。'); }
|
||||
if (mode === 'question') {
|
||||
assert.match(input.messages[0].content, /model_connection/); assert.match(input.messages[0].content, /503/);
|
||||
// Large owned snapshots may be projected into mandatory latest-tool
|
||||
// feedback. The fixture must consume that saved result like the model,
|
||||
// rather than replaying an inspection whose raw group was archived.
|
||||
const feedback = input.messages.filter(message => message.role === 'system').map(message => {
|
||||
try { return JSON.parse(message.content.split('\n').at(-1)); } catch { return {}; }
|
||||
}).find(value => value.latest_tools)?.data.entries.find(entry => entry.tool === 'inspect_session');
|
||||
const snapshot = results.length ? JSON.parse(results.at(-1).content) : feedback?.result;
|
||||
if (!snapshot) { diagnosticReads++; return tool('inspect_session', {}); }
|
||||
assert.equal(snapshot.session.id, sessionId); assert.equal(snapshot.lastFailure.source, 'model_connection');
|
||||
if (feedback) { diagnosticProjectionUsed = true; assert.equal(feedback.recorded_state, 'completed'); assert.match(snapshot.lastFailure.message, /HTTP 503/); }
|
||||
else assert.equal(snapshot.lastFailure.httpStatus, 503);
|
||||
return answer('已读取工作台执行记录。上次错误发生在请求模型服务时:**HTTP 503**。项目读取已经成功,文件未改动。');
|
||||
}
|
||||
if (mode === 'proposal') {
|
||||
if (!results.length) return tool('propose_file', { path: 'REVIEWED.md', content: '# Reviewed through the UI\n', reason: 'UI review fixture' });
|
||||
await new Promise(resolve => setTimeout(resolve, 700)); return answer('改动已准备好,请查看差异后应用。');
|
||||
}
|
||||
if (mode === 'command') {
|
||||
if (!results.length) return tool('run_command', { command: "@'\nconsole.log('中文输入与输出正确');\n'@ | node --input-type=module\nexit $LASTEXITCODE", reason: '验证中文通过项目终端进入 Node。' });
|
||||
const result = JSON.parse(results.at(-1).content); assert.equal(result.status, 'completed'); assert.match(result.stdout, /中文输入与输出正确/);
|
||||
return answer('**UTF-8 验证通过**:中文输入与输出正确。');
|
||||
}
|
||||
if (mode === 'authentication') return failure(401);
|
||||
if (mode === 'backoff') return failure(503, { 'Retry-After': '3' });
|
||||
if (mode === 'transient') return requestCount === 1 ? failure(503) : answer('临时连接错误已自动恢复。');
|
||||
throw new Error('Unknown fixture mode');
|
||||
} catch (error) { if (!response.headersSent) response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
|
||||
});
|
||||
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
||||
const browser = await chromium.launch({ executablePath: path.join(process.env['ProgramFiles(x86)'] || 'C:\\Program Files (x86)', 'Microsoft', 'Edge', 'Application', 'msedge.exe'), headless: true });
|
||||
const mark = text => { passed.push(text); console.log('PASS ' + text); };
|
||||
const state = async () => api('/sessions/' + sessionId);
|
||||
const waitState = async status => {
|
||||
await page.waitForFunction(expected => document.querySelector('.task-toolbar .status')?.classList.contains('status-' + expected), status);
|
||||
};
|
||||
const checkViewport = async () => {
|
||||
const bounds = await page.evaluate(() => {
|
||||
const header = document.querySelector('.work-header').getBoundingClientRect(), composer = document.querySelector('.composer-wrap').getBoundingClientRect();
|
||||
return { width: innerWidth, height: innerHeight, scrollWidth: document.documentElement.scrollWidth, scrollHeight: document.documentElement.scrollHeight, headerTop: header.top, composerBottom: composer.bottom };
|
||||
});
|
||||
assert.ok(bounds.scrollWidth <= bounds.width + 1 && bounds.scrollHeight <= bounds.height + 1, JSON.stringify(bounds));
|
||||
assert.ok(bounds.headerTop >= -1 && bounds.composerBottom <= bounds.height + 1, JSON.stringify(bounds));
|
||||
};
|
||||
const screenshot = async name => { await checkViewport(); await page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' }); };
|
||||
const task = async content => {
|
||||
const newTask = page.getByRole('button', { name: '新建任务', exact: true });
|
||||
if (!await newTask.isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await newTask.click();
|
||||
await page.getByLabel('任务内容').fill(content); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await page.waitForFunction(() => !!localStorage.getItem('awb-session')); sessionId = await page.evaluate(() => localStorage.getItem('awb-session'));
|
||||
};
|
||||
const settings = async () => {
|
||||
await page.getByRole('button', { name: '选择与管理模型', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '模型中心', exact: true }).locator('.mc-connection').filter({ has: page.getByRole('heading', { name: '原模型连接', exact: true }) }).getByRole('button', { name: '编辑', exact: true }).click();
|
||||
return page.getByRole('dialog', { name: '原连接与演示设置', exact: true });
|
||||
};
|
||||
try {
|
||||
page = await browser.newPage({ viewport: { width: 1440, height: 960 }, deviceScaleFactor: 1 }); page.setDefaultTimeout(15000);
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => { if (!request.url().includes('/api/events')) networkErrors.push(request.failure()?.errorText); });
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.locator('.connection.connected').waitFor();
|
||||
await page.getByRole('button', { name: '打开项目', exact: true }).click();
|
||||
const projectDialog = page.getByRole('dialog', { name: '打开项目' });
|
||||
await projectDialog.getByLabel('项目文件夹路径').fill(projectRoot); await projectDialog.getByRole('button', { name: '打开项目', exact: true }).click();
|
||||
await projectDialog.waitFor({ state: 'hidden' });
|
||||
const connection = await settings(); await connection.getByRole('button', { name: '兼容模型服务' }).click();
|
||||
await connection.getByLabel('接口根地址', { exact: true }).fill('http://127.0.0.1:' + model.address().port + '/v1');
|
||||
await connection.getByLabel('API Key(按服务要求填写)').fill('ui-fixture-memory-key');
|
||||
await connection.getByLabel('模型名称').fill('ui-diagnostics-fixture');
|
||||
await connection.getByRole('button', { name: '验证模型响应', exact: true }).click();
|
||||
await connection.getByText(/模型响应验证通过/).waitFor(); assert.equal((await api('/diagnostics')).connection.mode, 'demo');
|
||||
await connection.getByRole('button', { name: '保存并切换原连接', exact: true }).click(); await connection.waitFor({ state: 'hidden' });
|
||||
mark('model response validation checks the tentative connection before saving');
|
||||
|
||||
requestCount = 0; await task('制造读取成功后的模型服务错误'); await waitState('failed');
|
||||
const failed = await state(); assert.equal(failed.failure.source, 'model_connection'); assert.equal(failed.failure.httpStatus, 503); assert.equal(failed.failure.attempts, 3);
|
||||
assert.equal(failed.events.filter(event => event.tool === 'read_file').length, 1);
|
||||
await page.locator('.failure-card').waitFor(); await screenshot('formal-model-error');
|
||||
await page.locator('.failure-card').getByRole('button', { name: '查看运行诊断' }).click();
|
||||
const diagnostic = page.getByRole('dialog', { name: '运行诊断' }); await diagnostic.getByText('Windows PowerShell 5.1', { exact: true }).waitFor();
|
||||
await diagnostic.getByText(/HTTP 503/).first().waitFor(); await screenshot('formal-diagnostics');
|
||||
assert.ok(!(await diagnostic.innerText()).includes('ui-fixture-memory-key'));
|
||||
await diagnostic.getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
mark('a model failure after successful tools shows source, attempts and local runtime diagnosis');
|
||||
|
||||
mode = 'recover'; await page.locator('.failure-card').getByRole('button', { name: '重试模型请求', exact: true }).click(); await waitState('completed');
|
||||
const recovered = await state(); assert.equal(recovered.events.filter(event => event.tool === 'read_file').length, 1);
|
||||
assert.equal(recovered.messages.filter(message => message.role === 'user').length, 1);
|
||||
mark('explicit retry preserves the completed tool result without replaying the task');
|
||||
|
||||
mode = 'question'; await page.getByLabel('任务内容').fill('刚才工作台报错怎么回事?请查执行记录。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await page.getByText(/已读取工作台执行记录/).waitFor(); await waitState('completed');
|
||||
const answered = await state(); assert.ok(answered.events.some(event => event.tool === 'inspect_session' && event.phase === 'completed'));
|
||||
assert.equal(diagnosticReads, 1, 'Read the actual saved diagnosis once, including when its raw group is projected.');
|
||||
assert.equal(answered.events.filter(event => event.tool === 'inspect_session').length, 1);
|
||||
assert.equal(answered.events.filter(event => event.tool === 'search_files').length, 0);
|
||||
assert.ok(!await page.locator('.message-assistant .message-text').last().innerText().then(text => text.includes('**')));
|
||||
await screenshot('formal-diagnostic-answer'); mark('an error question receives the actual session diagnosis and formatted answer');
|
||||
|
||||
mode = 'proposal'; await task('准备一个可审查文件'); await page.getByRole('button', { name: '审核 1 项', exact: true }).waitFor();
|
||||
await page.getByRole('button', { name: '审核 1 项', exact: true }).click();
|
||||
const apply = page.getByRole('button', { name: '应用这项改动', exact: true }); assert.ok(await apply.isDisabled());
|
||||
assert.equal(fs.existsSync(path.join(projectRoot, 'REVIEWED.md')), false);
|
||||
await waitState('needs_review'); assert.equal(await page.getByRole('button', { name: '继续任务', exact: true }).count(), 0);
|
||||
await screenshot('formal-review'); await apply.click(); await waitState('completed');
|
||||
assert.equal(fs.readFileSync(path.join(projectRoot, 'REVIEWED.md'), 'utf8'), '# Reviewed through the UI\n');
|
||||
mark('review state exposes the review action and prevents application during a running request');
|
||||
|
||||
mode = 'command'; await task('验证中文终端输入输出'); await waitState('awaiting_approval');
|
||||
await page.locator('.approval-shell').getByText(/PowerShell 5.1/).waitFor(); await screenshot('formal-command-approval');
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitState('completed');
|
||||
assert.match((await state()).events.find(event => event.tool === 'run_command').output, /中文输入与输出正确/);
|
||||
mark('approved Unicode commands execute in the declared shell and retain Chinese output');
|
||||
|
||||
mode = 'authentication'; requestCount = 0; await task('验证身份失败的处理'); await waitState('failed'); assert.equal(requestCount, 1);
|
||||
assert.match(await page.locator('.failure-card').innerText(), /HTTP 401/);
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await screenshot('formal-mobile-error');
|
||||
await page.locator('.failure-card').getByRole('button', { name: '查看运行诊断' }).click();
|
||||
await page.getByRole('dialog', { name: '运行诊断' }).getByText('Windows PowerShell 5.1', { exact: true }).waitFor();
|
||||
const box = await page.getByRole('dialog', { name: '运行诊断' }).boundingBox(); assert.ok(box.x >= 0 && box.x + box.width <= 391);
|
||||
await screenshot('formal-mobile-diagnostics'); await page.getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
await page.setViewportSize({ width: 1440, height: 960 }); mark('authentication errors do not retry and error/diagnostic controls fit a 390 px viewport');
|
||||
|
||||
mode = 'backoff'; requestCount = 0; await task('停止自动重试'); await page.getByText('正在重试模型连接', { exact: true }).waitFor();
|
||||
await page.getByRole('button', { name: '停止', exact: true }).click(); await waitState('paused'); assert.equal(requestCount, 1);
|
||||
assert.equal((await state()).events.find(event => event.kind === 'model').phase, 'interrupted');
|
||||
assert.equal((await api('/diagnostics')).connection.lastRequest.status, 'interrupted');
|
||||
await page.locator('.task-toolbar').getByRole('button', { name: '运行诊断', exact: true }).click();
|
||||
const stoppedDiagnostic = page.getByRole('dialog', { name: '运行诊断' });
|
||||
await stoppedDiagnostic.getByText(/最近请求:已停止/).waitFor();
|
||||
await screenshot('formal-stopped-diagnostics');
|
||||
await stoppedDiagnostic.getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
mark('stop cancels pending retry and clears running model state');
|
||||
|
||||
mode = 'transient'; requestCount = 0; await task('验证自动恢复'); await waitState('completed'); assert.equal(requestCount, 2);
|
||||
await screenshot('formal-desktop'); mark('a transient service failure recovers within the bounded retry policy');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(networkErrors, []);
|
||||
const result = { success: true, version: bootstrap.version, url, date: new Date().toISOString(), passed, pageErrors, requestFailures: networkErrors, diagnosticReads, diagnosticProjectionUsed, modelValidation: 'controlled local HTTP fixture; actual configured model is verified separately' };
|
||||
fs.writeFileSync('.qa/diagnostics-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
if (page) await page.screenshot({ path: '.qa/screenshots/formal-ui-failure.png', scale: 'css' }).catch(() => {});
|
||||
console.error(error); throw error;
|
||||
} finally {
|
||||
await browser.close();
|
||||
await api('/config', { ...bootstrap.config, mode: 'demo', apiKey: '' }).catch(() => {});
|
||||
model.closeAllConnections(); await new Promise(resolve => model.close(resolve));
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { _electron, expect } from 'playwright/test';
|
||||
import { fixture, until } from '../tests/helpers.mjs';
|
||||
import { previewProject, freePreviewPort } from '../tests/fixtures/preview-project.mjs';
|
||||
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
||||
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { verifyBrowsingMobile } from './verify-browsing-mobile.mjs';
|
||||
|
||||
if(process.argv.includes('--help')) { console.log('node scripts/verify-embedded-browser.mjs [--app-root <absolute package>] [--native-review] — actual Electron + owned React/Vite + 390px mobile original observation; no real model or production state.'); process.exit(0); }
|
||||
const options=process.argv.slice(2).filter(value=>value!=='--native-review');
|
||||
assert.ok(options.length===0||options.length===2&&options[0]==='--app-root'&&path.isAbsolute(options[1]));
|
||||
const appRoot=fs.realpathSync.native(options[1]??process.cwd()),root=path.resolve('.qa','embedded-browser-'+randomUUID());fs.mkdirSync(root,{recursive:true});
|
||||
const nodePath=options[1]?path.join(appRoot,'runtime/node.exe'):process.execPath;assert.ok(fs.existsSync(nodePath));
|
||||
const hash=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const files=(folder,relative='')=>fs.readdirSync(path.join(folder,relative),{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name)).flatMap(entry=>entry.isDirectory()?files(folder,path.join(relative,entry.name)):[path.join(relative,entry.name)]);
|
||||
const inputNames=['dist-client','dist-server','dist-desktop',...['src','tests','scripts'].filter(folder=>fs.existsSync(path.join(appRoot,folder)))].flatMap(folder=>files(path.join(appRoot,folder)).map(name=>path.join(folder,name))).concat(['package.json','package-lock.json','MANIFEST.json'].filter(name=>fs.existsSync(path.join(appRoot,name))));
|
||||
const inputSnapshot=()=>inputNames.map(name=>({path:name.replaceAll('\\','/'),bytes:fs.statSync(path.join(appRoot,name)).size,sha256:hash(fs.readFileSync(path.join(appRoot,name)))}));
|
||||
const inputs=inputSnapshot();let sourceCommit=null,verifierSourceState='archive-without-git';try{sourceCommit=execFileSync('git',['rev-parse','HEAD'],{encoding:'utf8',windowsHide:true}).trim();verifierSourceState=execFileSync('git',['status','--porcelain','--','src','scripts','tests','package.json','package-lock.json'],{encoding:'utf8',windowsHide:true}).trim()?'modified-source-with-exact-input-manifest':'clean';}catch{}
|
||||
fs.writeFileSync(path.join(root,'inputs.json'),JSON.stringify({sourceCommit,verifierSourceState,appRoot,inputs},null,2));
|
||||
const cleanups=[],f=fixture({after:callback=>cleanups.push(callback)}),p=await previewProject(f);
|
||||
fs.writeFileSync(path.join(p.root,'index.html'),'<!doctype html><html lang="zh-CN"><head><meta charset="UTF-8"><title>所属React真实预览</title></head><body><div id="root"></div><script type="module" src="/src.jsx"></script></body></html>');
|
||||
const source=`import React,{useState} from 'react';import{createRoot}from'react-dom/client';function App(){const[name,setName]=useState(''),[saved,setSaved]=useState('尚未保存');return <main style={{padding:32,fontFamily:'sans-serif'}}><h1>真实React工程预览</h1><label>项目名称<input aria-label="项目名称" value={name} onChange={e=>setName(e.target.value)}/></label><button onClick={()=>setSaved('已保存:'+name)}>保存计划</button><p role="status">{saved}</p><a href="/next.html">下一页</a><button onClick={()=>window.open('https://example.com/')}>尝试新窗口</button><input aria-label="密码" type="password"/><a href="/download.txt" download>原下载链接</a></main>}createRoot(document.getElementById('root')).render(<App/>);`;
|
||||
fs.writeFileSync(path.join(p.root,'src.jsx'),source);fs.writeFileSync(path.join(p.root,'next.html'),'<!doctype html><meta charset="UTF-8"><h1>所属第二页</h1><a href="/">返回</a>');
|
||||
fs.writeFileSync(path.join(p.root,'download.txt'),'原下载字节\r\nowned-download\r\n');
|
||||
p.session.title='内置浏览器真实验收';f.store.save(p.session);f.store.close();
|
||||
const port=await freePreviewPort(),controller=new HostController({appRoot,dataRoot:f.data,port,nodePath,timeoutMs:60000});
|
||||
const results=[],errors=[];let application,page,nativeId,stage='host startup',previewJob,version=null;
|
||||
const record=(name,detail={})=>{results.push({name,passed:true,...detail});console.log(name+': passed');};
|
||||
async function nativeReview(label){if(!process.argv.includes('--native-review'))return;const pending=path.join(root,'native-review-'+label+'.json'),release=path.join(root,'native-continue-'+label);fs.writeFileSync(pending,JSON.stringify({label,pid:await application.evaluate(()=>process.pid),root}));console.log('NATIVE_REVIEW '+pending);while(!fs.existsSync(release))await new Promise(resolve=>setTimeout(resolve,300));}
|
||||
const api=async(action,input)=>{const response=await fetch(controller.url+'/api/sessions/'+p.session.id+'/'+action,{method:input===undefined?'GET':'POST',headers:{'X-Workbench-Token':controller.token,'Content-Type':'application/json'},body:input===undefined?undefined:JSON.stringify(input),signal:AbortSignal.timeout(45000)});const value=await response.json();if(!response.ok)throw Object.assign(new Error('HTTP '+response.status+': '+value.error),{status:response.status});return value;};
|
||||
const state=()=>controller.api('/sessions/'+p.session.id);
|
||||
async function capture(name){
|
||||
const native=await application.browserWindow(page),png=await native.evaluate(async win=>(await win.capturePage()).toPNG().toString('base64'));
|
||||
const filename=path.join(root,name+'.png');fs.writeFileSync(filename,Buffer.from(png,'base64'));fs.writeFileSync(path.join(root,name+'.html'),await page.content());
|
||||
assert.equal(await page.evaluate(()=>document.documentElement.scrollWidth>innerWidth),false);return filename;
|
||||
}
|
||||
async function site(code){return application.evaluate(async({webContents},{id,code})=>{const contents=webContents.fromId(id);if(!contents)throw new Error('Original native contents ended');return contents.executeJavaScript(code);},{id:nativeId,code});}
|
||||
async function nativeVisible(){return application.evaluate(({BrowserWindow},id)=>BrowserWindow.getAllWindows()[0].contentView.children.some(view=>view.webContents?.id===id&&view.getVisible()),nativeId);}
|
||||
async function observe(){await api('browsing/tabs/'+(await currentTab()).id+'/observe',{});return currentObservation();}
|
||||
async function currentTab(){return(await api('browsing')).tabs.filter(tab=>tab.status==='open').at(-1);}
|
||||
async function currentObservation(){const snapshot=await api('browsing'),tab=snapshot.tabs.filter(tab=>tab.status==='open').at(-1);return snapshot.observations.find(value=>value.id===tab?.observationId);}
|
||||
async function act(input){const tab=await currentTab(),observation=await currentObservation();return api('browsing/tabs/'+tab.id+'/act',{...input,observationId:observation.id,revision:observation.revision,approved:true});}
|
||||
function save(extra){const receipt={schemaVersion:1,kind:'actual-electron-owned-preview',root,appRoot,version,nodePath,nodeSha256:hash(fs.readFileSync(nodePath)),stage,results,checks:results,errors,realModelRequests:0,sourceCommit,verifierSourceState,inputManifestSha256:hash(fs.readFileSync(path.join(root,'inputs.json'))),screenshots:'BrowserWindow images show the main renderer only. Independent native WebContents requires native OS screenshots or the original observation PNG.',...extra};fs.writeFileSync(path.join(root,'result.json'),JSON.stringify(receipt,null,2));fs.writeFileSync('.qa/embedded-browser-latest.json',JSON.stringify(receipt,null,2));}
|
||||
try{
|
||||
const host=await controller.start(),executable=desktopExecutableIdentity(appRoot).path,packaged=executable.toLowerCase().endsWith('agentworkbench.exe');version=(await controller.health()).version;
|
||||
application=await _electron.launch({executablePath:executable,args:[...(packaged?[]:[path.join(appRoot,'dist-desktop/main.cjs')]),'--app-root',appRoot,'--data-root',f.data,'--node',nodePath,'--port',String(port),'--hidden-window'],cwd:appRoot,env:projectEnvironment(),timeout:60000});
|
||||
page=await application.firstWindow();page.on('pageerror',error=>errors.push(error.message));await page.locator('.connection.connected').waitFor({timeout:30000});
|
||||
const native=await application.browserWindow(page);await native.evaluate(win=>{win.showInactive();win.setContentSize(1440,1000);});
|
||||
await page.getByRole('combobox',{name:'当前项目',exact:true}).selectOption(p.project.id);await page.getByRole('button',{name:'查看任务记录',exact:true}).click();
|
||||
stage='owned Vite UI plan';const service=page.getByRole('region',{name:'项目开发预览'});await service.getByRole('spinbutton',{name:'预览端口'}).fill(String(p.port));await service.getByRole('button',{name:'创建启动计划',exact:true}).click();
|
||||
await service.getByRole('button',{name:'核对并批准预览',exact:true}).click();await expect(page.getByRole('dialog',{name:'确认项目预览'})).toContainText('Vite 配置');await capture('desktop-preview-approval');
|
||||
await page.getByRole('button',{name:'确认启动预览',exact:true}).click();await until(async()=>{previewJob=(await state()).execution.previewJobs.at(-1);return previewJob&&!['queued','starting'].includes(previewJob.status);},40000);assert.equal(previewJob.status,'ready',JSON.stringify(previewJob));
|
||||
record('actual owned React/Vite UI approval, OS listener identity and HTTP readiness',{hostPid:host.identity.pid,previewPid:previewJob.process.pid});
|
||||
await service.getByRole('button',{name:'打开浏览区',exact:true}).click();await page.getByRole('button',{name:'打开目标',exact:true}).click();await expect(page.getByLabel('浏览目标地址')).toHaveValue(previewJob.url);await page.getByRole('button',{name:'确认授权并打开',exact:true}).click();
|
||||
await expect(page.getByRole('dialog')).toHaveCount(0,{timeout:30000});stage='native page';await until(async()=>!!(await currentTab()),30000);
|
||||
nativeId=await application.evaluate(({webContents},origin)=>webContents.getAllWebContents().find(value=>value.getURL().startsWith(origin))?.id,new URL(previewJob.url).origin);assert.ok(nativeId);
|
||||
await until(()=>site("document.querySelector('h1')?.textContent==='真实React工程预览'"),20000);await until(nativeVisible,10000);
|
||||
assert.deepEqual(await site("({node:typeof require,process:typeof process,bridge:typeof window.desktopWorkbench})"),{node:'undefined',process:'undefined',bridge:'undefined'});
|
||||
assert.equal(await site(`fetch(${JSON.stringify(controller.url+'/api/bootstrap')}).then(()=>false,()=>true)`),true);
|
||||
record('same actual native page, isolated Node/preload and blocked workbench API');
|
||||
stage='resize and zoom';const separator=page.getByRole('separator',{name:'调整检视区宽度'}),separatorBox=await separator.boundingBox(),beforeWidth=await page.locator('.inspector').evaluate(node=>node.getBoundingClientRect().width);
|
||||
await page.mouse.move(separatorBox.x+3,separatorBox.y+80);await page.mouse.down();await until(async()=>!await nativeVisible());await page.mouse.move(separatorBox.x-110,separatorBox.y+80);await page.mouse.up();await until(nativeVisible);
|
||||
assert.ok(await page.locator('.inspector').evaluate(node=>node.getBoundingClientRect().width)>beforeWidth+90);
|
||||
await separator.focus();await page.keyboard.press('ArrowLeft');await expect(separator).toHaveAttribute('aria-valuenow',String(Math.round(beforeWidth+145)));
|
||||
await native.evaluate(win=>win.webContents.setZoomFactor(1.25));await until(async()=>await nativeVisible());
|
||||
await until(async()=>{const box=await page.locator('.native-browser-slot').boundingBox(),bounds=await application.evaluate(({BrowserWindow},id)=>BrowserWindow.getAllWindows()[0].contentView.children.find(view=>view.webContents?.id===id)?.getBounds(),nativeId);return bounds&&Math.abs(bounds.x-Math.round(box.x*1.25))<=2&&Math.abs(bounds.width-Math.floor(box.width*1.25))<=2;});
|
||||
await native.evaluate(win=>win.webContents.setZoomFactor(1));await until(nativeVisible);record('drag and keyboard width adjustment, native occlusion during drag and 125% zoom geometry');
|
||||
await capture('desktop-native-browser');await nativeReview('desktop');
|
||||
stage='real React input action';let observation=await observe(),input=observation.controls.find(value=>value.label==='项目名称');assert.ok(input);
|
||||
await act({action:'fill',ref:input.ref,value:'专业工作台验收'});observation=await currentObservation();let button=observation.controls.find(value=>value.label==='保存计划');await act({action:'click',ref:button.ref});
|
||||
assert.equal(await site("document.querySelector('[role=status]').textContent"),'已保存:专业工作台验收');record('fixed original-control fill and click operate real React controlled state');
|
||||
stage='stale page';observation=await observe();await site("document.querySelector('h1').textContent='人工更改的页面'");
|
||||
await assert.rejects(api('browsing/tabs/'+(await currentTab()).id+'/act',{action:'click',ref:button.ref,observationId:observation.id,revision:observation.revision,approved:true}),/409/);
|
||||
assert.match((await api('browsing')).actions.at(-1).error,/变化/);
|
||||
record('actual DOM mutation invalidates previous observation without submitting');
|
||||
stage='HMR';fs.writeFileSync(path.join(p.root,'src.jsx'),source.replace('真实React工程预览','热更新后的React预览'));await until(()=>site("document.querySelector('h1')?.textContent==='热更新后的React预览'"),20000);await observe();record('actual source update reaches same native WebContents through Vite live reload');
|
||||
stage='phone original observation';await verifyBrowsingMobile({controller,session:await state(),root,record});
|
||||
stage='modal visibility';await page.getByRole('button',{name:'打开目标',exact:true}).click();await until(async()=>!await nativeVisible(),8000);await capture('native-hidden-for-modal');await page.getByRole('dialog').getByRole('button',{name:'取消',exact:true}).click();await until(nativeVisible,8000);record('native page hides for main-window authorization modal and restores');
|
||||
stage='popup';let before=await application.evaluate(({webContents})=>webContents.getAllWebContents().length);await site("[...document.querySelectorAll('button')].find(value=>value.textContent==='尝试新窗口').click()");assert.equal(await application.evaluate(({webContents})=>webContents.getAllWebContents().length),before);record('unapproved popup is denied in actual Electron');
|
||||
stage='navigation';observation=await observe();const link=observation.controls.find(value=>value.label==='下一页');await act({action:'click',ref:link.ref});await until(()=>site("document.querySelector('h1')?.textContent==='所属第二页'"));await act({action:'back'});await until(()=>site("document.querySelector('h1')?.textContent==='热更新后的React预览'"));await act({action:'forward'});assert.equal(await site("document.querySelector('h1').textContent"),'所属第二页');await act({action:'back'});record('actual native navigation, back and forward preserve original scoped receipts');
|
||||
stage='download';await observe();await act({action:'download',url:previewJob.url+'download.txt'});const downloaded=(await api('browsing')).actions.findLast(value=>value.downloadArtifactId);assert.ok(downloaded?.downloadSha256);record('explicit exact-URL download keeps original binary artifact and SHA',{sha256:downloaded.downloadSha256});
|
||||
stage='profile isolation';await site("document.cookie='awb_ephemeral=private;SameSite=Lax'");const firstTab=await currentTab(),firstNativeId=nativeId;
|
||||
await api('browsing/tabs',{url:previewJob.url,previewJobId:previewJob.id,interact:true,persistent:true,approved:true});let second=await currentTab();nativeId=await application.evaluate(({webContents},{origin,first})=>webContents.getAllWebContents().find(value=>value.id!==first&&value.getURL().startsWith(origin))?.id,{origin:new URL(previewJob.url).origin,first:firstNativeId});
|
||||
assert.equal(await site("document.cookie.includes('awb_ephemeral')"),false);await site("document.cookie='awb_persistent=stored;Max-Age=3600;SameSite=Lax'");await api('browsing/tabs/'+second.id+'/close',{approved:true});
|
||||
await api('browsing/tabs',{url:previewJob.url,previewJobId:previewJob.id,profileId:second.profileId,interact:true,approved:true});second=await currentTab();nativeId=await application.evaluate(({webContents},{origin,first})=>webContents.getAllWebContents().find(value=>value.id!==first&&value.getURL().startsWith(origin))?.id,{origin:new URL(previewJob.url).origin,first:firstNativeId});assert.equal(await site("document.cookie.includes('awb_persistent=stored')"),true);
|
||||
await api('browsing/tabs/'+second.id+'/clear',{approved:true});assert.equal(await application.evaluate(async({session},id)=>(await session.fromPartition('persist:awb-browsing-'+id).cookies.get({})).length,second.profileId),0);nativeId=firstNativeId;assert.equal(await site("document.cookie.includes('awb_ephemeral=private')"),true);record('actual independent temporary/persistent sessions, explicit reuse and selective cookie/storage/cache revocation');
|
||||
stage='mobile';await native.evaluate(win=>win.setContentSize(390,844));await until(()=>page.evaluate(()=>innerWidth===390));await page.getByRole('button',{name:'查看任务记录',exact:true}).click();await page.getByRole('button',{name:'浏览',exact:true}).click();await until(nativeVisible,10000);await capture('mobile-native-browser');await nativeReview('mobile');
|
||||
await page.getByRole('button',{name:'打开目标',exact:true}).click();await until(async()=>!await nativeVisible());await capture('mobile-authorize-modal');await page.getByRole('dialog').getByRole('button',{name:'取消',exact:true}).click();await page.getByRole('button',{name:'返回任务',exact:true}).click();await until(async()=>!await nativeVisible());record('390px native geometry, modal occlusion and return-to-task with no horizontal overflow');
|
||||
stage='close view';await native.evaluate(win=>win.setContentSize(1440,1000));await page.getByRole('button',{name:'浏览',exact:true}).click();await until(nativeVisible);await page.getByRole('button',{name:'关闭浏览标签页',exact:true}).click();await until(async()=>!(await currentTab()));assert.equal((await state()).execution.previewJobs.at(-1).status,'ready');record('closing actual view leaves owned development service running');
|
||||
stage='host lifetime';await application.close();application=null;assert.equal((await controller.health()).pid,host.identity.pid);assert.equal((await state()).execution.previewJobs.at(-1).status,'ready');await api('preview-jobs/'+previewJob.id+'/stop',{});assert.equal((await state()).execution.previewJobs.at(-1).status,'stopped');record('window close retains host and preview; explicit service stop confirms original process tree');
|
||||
assert.deepEqual(errors,[]);assert.deepEqual(inputSnapshot(),inputs,'Verification inputs changed during the actual browser flow');stage='complete';save({passed:true,success:true});console.log('EVIDENCE '+root);console.log(JSON.stringify({passed:true,root,groups:results.length}));
|
||||
}catch(error){if(application&&page)await capture('original-failure').catch(()=>{});if(await controller.health()){fs.writeFileSync(path.join(root,'original-session.json'),JSON.stringify(await state(),null,2));fs.writeFileSync(path.join(root,'original-browsing.json'),JSON.stringify(await api('browsing'),null,2));}save({passed:false,error:error.stack});console.error(JSON.stringify({passed:false,stage,root,error:error.message}));process.exitCode=1;}
|
||||
finally{if(application)await application.close().catch(()=>{});if(await controller.health())await controller.stop();for(const cleanup of cleanups.reverse())await cleanup();}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { originalEngineRun,verifyEngineModelInput } from '../src/shared/contracts/engine-execution.mjs';
|
||||
import { requestMetrics } from '../src/shared/contracts/model-request.mjs';
|
||||
|
||||
// Offline verification of the A repair. It never creates a Runtime or Provider.
|
||||
const root=path.resolve('docs/live-context-a-evidence'),sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
|
||||
const read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
|
||||
const manifest=read('manifest.json'),entries=new Map(manifest.entries.map(entry=>[entry.path,entry]));
|
||||
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.match(manifest.sourceCommit,/^[a-f0-9]{40}$/);
|
||||
let bytes=0;
|
||||
for(const entry of entries.values()){
|
||||
assert.ok(fs.lstatSync(inside(entry.path)).isFile());const data=fs.readFileSync(inside(entry.path));
|
||||
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;
|
||||
}
|
||||
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
|
||||
for(const input of manifest.sourceInputs){
|
||||
assert.match(input.path,/^(?:src|scripts|tests|benchmarks)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);
|
||||
assert.ok(!input.path.split('/').includes('..'));
|
||||
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
|
||||
assert.equal(sha(blob),input.gitBlobSha256,input.path);
|
||||
const raw=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
|
||||
assert.equal(sha(raw),input.sha256,input.path);assert.equal(raw.length,input.bytes,input.path);
|
||||
assert.equal(raw.toString('utf8').replace(/\r\n/g,'\n'),blob.toString('utf8').replace(/\r\n/g,'\n'),input.path);
|
||||
}
|
||||
let originalRequests=0,bounded=0,artifactRecords=0;
|
||||
for(const sample of manifest.samples){
|
||||
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value]));
|
||||
const core={};for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
|
||||
artifactRecords+=core.artifact.size;
|
||||
const blobs=new Map(sample.artifacts.map(value=>[value.id,value]));
|
||||
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,'Original owner: '+id);return value;},
|
||||
artifactStore:{read(artifact){const entry=blobs.get(artifact.id),data=fs.readFileSync(inside(entry.file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
|
||||
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
|
||||
if(!sample.passed){
|
||||
assert.equal(result.success,false);assert.ok(result.engineRuns.some(job=>job.inputRejections?.length));
|
||||
for(const job of core.engine_run.values())for(const rejected of job.inputRejections??[]){
|
||||
const artifact=store.owned('artifact',rejected.artifactId,job.sessionId),receipt=JSON.parse(store.artifactStore.read(artifact));
|
||||
assert.equal(artifact.sha256,rejected.sha256);assert.equal(receipt.dispatched,false);assert.equal(receipt.budget.contextLimit,16000);
|
||||
assert.ok(receipt.budget.requiredInputTokens>receipt.budget.inputBudget);
|
||||
}
|
||||
continue; // Earlier development projections are not reinterpreted as v1.
|
||||
}
|
||||
assert.equal(result.success,true);assert.equal(result.metrics.contextLimit,16000);assert.equal(result.metrics.tests,9);assert.equal(result.parentCheck.results[0].tests.passed,9);
|
||||
assert.equal(result.writable.length,3);assert.equal(result.realModelRequests,0);assert.match(result.commit,/^[a-f0-9]{40}$/);
|
||||
assert.ok(result.checks.some(check=>check.status==='failed'&&check.results[0].tests.failed===9));
|
||||
assert.ok([...core.tool.values()].some(tool=>tool.tool==='prepare_change_set'));
|
||||
const sampleBodies=[];
|
||||
for(const job of core.engine_run.values()){
|
||||
const session=sessions.get(job.sessionId);originalEngineRun(store,session,job);
|
||||
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);
|
||||
for(const request of job.requests){
|
||||
const input=verifyEngineModelInput(store,session,job,request),metrics=requestMetrics(input.body);originalRequests++;
|
||||
assert.ok(metrics.estimatedInputTokens+input.native.maxTokens+input.view.safetyTokens<=16000);sampleBodies.push(input.body);
|
||||
if(input.view.mode==='bounded'){bounded++;assert.ok(input.view.beforeInputTokens>input.view.inputBudget);}
|
||||
}
|
||||
assert.equal(job.inputRejections?.length??0,0);
|
||||
}
|
||||
const text=JSON.stringify(sampleBodies);assert.ok(text.includes('logArtifactId'));assert.ok(text.includes('failed'));
|
||||
if(sample.name==='responses'){
|
||||
const attempts=read(sample.attempts);assert.equal(attempts.length,result.metrics.requests);
|
||||
assert.ok(attempts.every(attempt=>attempt.protocol==='responses'&&attempt.stream.terminal&&attempt.responseArtifactId));
|
||||
}
|
||||
}
|
||||
assert.ok(bounded>0);assert.equal(manifest.samples.filter(sample=>sample.passed).length,2);
|
||||
console.log(JSON.stringify({passed:true,manifestSha256:sha(fs.readFileSync(inside('manifest.json'))),files:entries.size,bytes,artifactRecords,sourceCommit:manifest.sourceCommit,
|
||||
originalRequests,bounded,realModelRequests:0,scope:'Offline original bytes, source, native view/provenance, failed development receipts and two owned-model engineering chains; real-model/Purge benefit not measured.'}));
|
||||
@@ -0,0 +1,52 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium,expect } from 'playwright/test';
|
||||
import { fixture,launchWorkbench } from '../tests/helpers.mjs';
|
||||
import { controlledWorker } from '../tests/fixtures/controlled-worker.mjs';
|
||||
|
||||
// A declared UI state fixture. Native execution and capacity are independently
|
||||
// checked by verify-controlled-projects --context-16k, not by this sample.
|
||||
const directory=path.resolve('.qa','engine-context-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),cases=[],screenshots=[],errors=[];
|
||||
let browser,host,report;
|
||||
try{
|
||||
const context=controlledWorker(f),group=await context.manager.plan(f.session,context.definition);
|
||||
await (await context.manager.prepare(f.session,group.id,group.fingerprint)).promise;
|
||||
assert.equal(group.status,'ready',group.summary);
|
||||
const child=group.children[0],session=f.store.session(child.sessionId),run=f.store.core.run.get(session.runId);
|
||||
run.engine='deepseek-workbench';run.budgetPause={schema:1,reason:'engine_input_budget',contextLimit:16000,outputReservation:2048,safetyTokens:1600,
|
||||
inputBudget:12352,requiredInputTokens:12437,originalInputTokens:13110,runRemainingTokens:198488,taskRemainingTokens:198488,
|
||||
breakdown:{requestOverheadTokens:40,toolDefinitionTokens:2704,fixedInstructionTokens:7818,retainedHistoryAndViewTokens:1875}};
|
||||
session.status='paused';f.store.finishRun(session,run,'engine_input_budget');f.store.save(session);
|
||||
await context.runtime.stopAll();f.store.close();
|
||||
host=await launchWorkbench(t,f.data);assert.notEqual(new URL(host.url).port,'4318');
|
||||
browser=await chromium.launch({channel:'msedge',headless:true});
|
||||
for(const width of [1440,390]){
|
||||
const page=await browser.newPage({viewport:{width,height:1000}});page.on('pageerror',error=>errors.push(error.message));
|
||||
await page.goto(host.url);await page.waitForLoadState('networkidle');
|
||||
const project=page.getByLabel('当前项目',{exact:true});
|
||||
if(!await project.isVisible())await page.getByRole('button',{name:'展开项目栏',exact:true}).click();
|
||||
await project.selectOption(child.projectId);
|
||||
const panel=page.getByLabel('任务预算诊断',{exact:true});await expect(panel).toBeVisible();
|
||||
await expect(panel).toContainText('原生引擎输入或剩余预算不足');await expect(panel).toContainText('12,437');
|
||||
await expect(panel).toContainText('完整目标与原生提示');await expect(panel).toContainText('198,488');await expect(panel).not.toContainText('不限');
|
||||
await expect(panel.getByRole('button',{name:'核对预算并续接原任务',exact:true})).toHaveCount(0);
|
||||
await page.reload();await expect(page.getByLabel('任务预算诊断',{exact:true})).toContainText('13,110');
|
||||
const screenshot=path.join(directory,'native-budget-'+width+'.png');await panel.scrollIntoViewIfNeeded();await page.screenshot({path:screenshot,fullPage:true});screenshots.push(screenshot);
|
||||
fs.writeFileSync(path.join(directory,'native-budget-'+width+'.html'),await page.content());
|
||||
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+1),'No viewport overflow');
|
||||
await panel.getByRole('button',{name:'打开隔离协作',exact:true}).click();
|
||||
const dialog=page.getByRole('dialog',{name:'隔离协作',exact:true});await expect(dialog).toContainText('当前是受限子任务');
|
||||
await dialog.getByRole('button',{name:'返回父目标',exact:true}).click();await expect(project).toHaveValue(group.projectId);
|
||||
const original=await host.api('/sessions/'+child.sessionId);assert.equal(original.execution.task.usage.requests,0);
|
||||
assert.equal(original.execution.run.budgetPause.reason,'engine_input_budget');assert.equal(original.execution.progress.totalTools,0);
|
||||
cases.push(width+': native diagnostic persists, names exact capacity, and returns to the parent without dispatch or unsupported resume');await page.close();
|
||||
}
|
||||
assert.deepEqual(errors,[]);report={passed:true,cases,screenshots,errors,realModelRequests:0,scope:'Declared paused UI fixture; native execution is separate.'};
|
||||
}catch(error){report={passed:false,cases,screenshots,errors,error:error.stack,realModelRequests:0};process.exitCode=1;}
|
||||
finally{
|
||||
await browser?.close();await host?.close();for(const dispose of disposers.reverse())await dispose();
|
||||
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({...report,directory}));
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { releaseEvidencePrefix } from './qa-evidence.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { chromium } from 'playwright';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
import { createEngineFixture } from './engine-fixtures.mjs';
|
||||
import { resolveDeepSeekInstallation } from '../src/core/engines/deepseek-installation.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-engines-ui.mjs — isolated browser flows, declared stop fixture and actual installed SDK'); process.exit(0); }
|
||||
const root = path.resolve('.qa', 'engines-ui-' + randomUUID()), data = path.join(root, 'data'), projectRoot = path.join(root, 'project');
|
||||
fs.mkdirSync(projectRoot, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const projectBytes = Buffer.from('# Owned engine UI validation project\n'); fs.writeFileSync(path.join(projectRoot, 'README.md'), projectBytes);
|
||||
const store = new Store(data), project = store.addProject(projectRoot), session = store.createSession(project.id), other = store.createSession(project.id);
|
||||
session.title = '引擎连接与原记录验收'; other.title = '引擎归属对照'; store.save(session); store.save(other); store.close();
|
||||
const fixtureRoot = createEngineFixture(path.join(root, '本地 SDK #')), native = resolveDeepSeekInstallation();
|
||||
const disposers = [], t = { after: fn => disposers.push(fn) };
|
||||
let host = await launchWorkbench(t, data, { AGENT_WORKBENCH_DEEPSEEK_ROOT: fixtureRoot }), selectedRoute = '/sessions/' + session.id, deliberateNavigation = false;
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1050 } });
|
||||
const passed = [], pageErrors = [], requestFailures = [], expectedRequestFailures = [], screenshots = [], starts = [];
|
||||
const sha = value => createHash('sha256').update(value).digest('hex'), state = () => host.api(selectedRoute);
|
||||
const panel = () => page.getByRole('dialog', { name: '本地代理引擎', exact: true });
|
||||
const view = () => page.getByRole('dialog', { name: '引擎原检查记录', exact: true });
|
||||
const card = job => panel().locator('[data-engine-job="' + job.id + '"]');
|
||||
const textContains = async (locator, text) => { await locator.waitFor({ state: 'visible' }); await until(async () => (await locator.innerText()).includes(text)); };
|
||||
const enabled = async locator => { await locator.waitFor({ state: 'visible' }); await until(() => locator.isEnabled()); };
|
||||
const mark = text => { passed.push(text); console.log('PASS ' + text); };
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => { const failure = { url: request.url(), error: request.failure()?.errorText };
|
||||
(deliberateNavigation && failure.error?.includes('ERR_ABORTED') ? expectedRequestFailures : requestFailures).push(failure); });
|
||||
page.on('request', request => { if (request.method() === 'POST' && request.url().endsWith('/engines/jobs')) starts.push(request.postDataJSON()); });
|
||||
async function screenshot(name) { const directory = path.join(root, 'screenshots'); fs.mkdirSync(directory, { recursive: true }); const filename = path.join(directory, 'engines-' + name + '.png'); await page.screenshot({ path: filename, scale: 'css' }); screenshots.push(filename); }
|
||||
async function open(title = session.title) {
|
||||
deliberateNavigation = true; await page.goto(host.url); await page.waitForLoadState('networkidle'); deliberateNavigation = false;
|
||||
if (!await page.getByLabel('当前项目').isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await page.getByLabel('当前项目').selectOption(project.id); await page.locator('.session-list button').filter({ hasText: title }).click();
|
||||
await page.getByRole('button', { name: '本地代理引擎', exact: true }).click(); await panel().waitFor({ state: 'visible' });
|
||||
await textContains(panel(), '已找到 0.2.1-rc.2');
|
||||
}
|
||||
async function detect(installationRoot) {
|
||||
await panel().getByLabel('DeepSeek安装目录').fill(installationRoot); await textContains(panel(), '目录已改变');
|
||||
assert.equal(await panel().getByRole('button', { name: '核验连接', exact: true }).isEnabled(), false);
|
||||
await panel().getByRole('button', { name: '检测安装', exact: true }).click();
|
||||
await enabled(panel().getByRole('button', { name: '核验连接', exact: true }));
|
||||
}
|
||||
async function waitJob(before) {
|
||||
await until(async () => (await state()).execution.engineJobs.length === before + 1);
|
||||
await until(async () => (await state()).execution.engineJobs.at(-1).reportArtifactId, 20000);
|
||||
const job = (await state()).execution.engineJobs.at(-1); await textContains(card(job), job.status === 'completed' ? '连接通过' : '已停止'); return job;
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
assert.notEqual(new URL(host.url).port, '4318'); assert.equal(host.bootstrap.config.hasKey, false);
|
||||
await open(); assert.equal((await state()).execution.task, null); await textContains(panel(), '本页连接检查不发送模型任务');
|
||||
mark('independent desktop engine entry detects the declared installation and clearly separates connection from project execution');
|
||||
await panel().getByLabel('DeepSeek安装目录').fill(path.join(root, '不存在的安装')); await panel().getByRole('button', { name: '检测安装', exact: true }).click();
|
||||
await textContains(panel(), '本地引擎关键输入不完整'); assert.equal(await panel().getByRole('button', { name: '核验连接', exact: true }).isEnabled(), false);
|
||||
assert.equal((await state()).execution.engineJobs.length, 0); await detect(fixtureRoot);
|
||||
mark('missing installations and edited directories cannot launch a check until detection succeeds');
|
||||
await panel().getByRole('button', { name: '核验连接', exact: true }).click(); const first = await waitJob(0);
|
||||
assert.equal(first.taskId, (await state()).execution.task.id); assert.equal(first.reportSummary.modelRequests, 0); assert.equal(first.reportSummary.rootExited, true);
|
||||
await textContains(card(first).getByLabel('引擎实际检查依据'), first.installation.identitySha256.slice(0, 12));
|
||||
assert.ok(starts[0].identitySha256); assert.equal(starts[0].identitySha256, first.installation.identitySha256);
|
||||
mark('UI launch seals the selected installation into the current Task/Run and renders actual startup and process-close facts');
|
||||
await card(first).getByRole('button', { name: '查看原记录', exact: true }).click(); await view().waitFor({ state: 'visible' });
|
||||
const firstPage = await host.api(selectedRoute + '/engines/jobs/' + first.id + '/report?limit=4000');
|
||||
await until(async () => (await view().getByLabel('引擎原记录内容').innerText()) === firstPage.content);
|
||||
assert.equal(firstPage.truncated, true); await view().getByRole('button', { name: '下一页', exact: true }).click();
|
||||
const secondPage = await host.api(selectedRoute + '/engines/jobs/' + first.id + '/report?offset=4000&limit=4000');
|
||||
await until(async () => (await view().getByLabel('引擎原记录内容').innerText()) === secondPage.content);
|
||||
await view().getByRole('button', { name: '回到首页', exact: true }).click(); await textContains(view(), first.reportSha256);
|
||||
const pendingDownload = page.waitForEvent('download'); await view().getByRole('button', { name: '下载完整记录', exact: true }).click();
|
||||
const download = await pendingDownload, filename = path.join(root, 'downloaded-report.json'); await download.saveAs(filename);
|
||||
assert.equal(sha(fs.readFileSync(filename)), first.reportSha256); assert.equal(JSON.parse(fs.readFileSync(filename, 'utf8')).report.responses.length, 6);
|
||||
await page.keyboard.press('Escape'); await view().waitFor({ state: 'hidden' }); assert.equal(await panel().isVisible(), true);
|
||||
mark('original record paging, full download SHA256 and nested-dialog keyboard close preserve the exact evidence');
|
||||
await card(first).getByRole('button', { name: '重新核验', exact: true }).click();
|
||||
const retry = page.getByRole('dialog', { name: '重新核验引擎连接', exact: true }); await retry.waitFor({ state: 'visible' });
|
||||
assert.equal((await state()).execution.engineJobs.length, 1); await retry.getByRole('button', { name: '开始新的核验', exact: true }).click();
|
||||
const second = await waitJob(1); assert.notEqual(second.pid, first.pid); assert.notEqual(second.id, first.id);
|
||||
assert.equal((await state()).execution.engineJobs.find(job => job.id === first.id).reportSha256, first.reportSha256);
|
||||
mark('an explicit new check creates a different process and record while retaining the original check unchanged');
|
||||
fs.writeFileSync(path.join(fixtureRoot, 'fixture-options.json'), JSON.stringify({ mode: 'hang-init' }));
|
||||
await panel().getByRole('button', { name: '核验连接', exact: true }).click();
|
||||
await until(async () => (await state()).execution.engineJobs.at(-1).status === 'initializing');
|
||||
await enabled(panel().getByRole('button', { name: '停止检查', exact: true })); await panel().getByRole('button', { name: '停止检查', exact: true }).click();
|
||||
const stopped = await waitJob(2); assert.equal(stopped.status, 'cancelled'); assert.equal(stopped.reportSummary.rootExited, true);
|
||||
assert.equal(fs.readFileSync(path.join(fixtureRoot, 'wire.jsonl'), 'utf8').includes('notifications/cancelled'), false);
|
||||
fs.writeFileSync(path.join(fixtureRoot, 'fixture-options.json'), '{}');
|
||||
await card(stopped).getByRole('button', { name: '重新核验', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '重新核验引擎连接', exact: true }).getByRole('button', { name: '开始新的核验', exact: true }).click();
|
||||
const resumed = await waitJob(3); assert.equal(resumed.status, 'completed');
|
||||
mark('declared pending-initialize fixture stops its owned process and explicit retry preserves the stopped result');
|
||||
assert.equal(native.status, 'available', native.message); await detect(native.root);
|
||||
await panel().getByRole('button', { name: '核验连接', exact: true }).click(); const nativeJob = await waitJob(4);
|
||||
assert.equal(nativeJob.status, 'completed'); assert.equal(nativeJob.installation.root, native.root); assert.equal(nativeJob.reportSummary.observedResponses, 6);
|
||||
assert.equal(nativeJob.reportSummary.modelRequests, 0); assert.equal(nativeJob.reportSummary.rootExited, true); await screenshot('native-desktop');
|
||||
mark('the same product UI verifies the actual installed DeepSeek SDK without a prompt or configured-model call');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await textContains(card(nativeJob), '连接通过');
|
||||
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true);
|
||||
await card(nativeJob).getByRole('button', { name: '查看原记录', exact: true }).click(); await view().waitFor({ state: 'visible' });
|
||||
await textContains(view(), nativeJob.reportSha256); assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true);
|
||||
await screenshot('record-mobile'); await page.keyboard.press('Escape'); await view().waitFor({ state: 'hidden' });
|
||||
await panel().getByLabel('DeepSeek安装目录').scrollIntoViewIfNeeded(); await screenshot('native-mobile');
|
||||
mark('390px installation paths, source chain, original record, paging and keyboard close remain readable without page overflow');
|
||||
await page.setViewportSize({ width: 1440, height: 1050 });
|
||||
let held = false, release;
|
||||
const delay = new Promise(resolve => { release = resolve; });
|
||||
const pattern = '**/api/engines/deepseek/availability*'; await page.route(pattern, async route => {
|
||||
held = true; await delay; await route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: 'owned-late-engine-error-must-stay-in-original-view' }) });
|
||||
});
|
||||
await panel().getByRole('button', { name: '检测安装', exact: true }).click(); await until(() => held);
|
||||
await panel().getByRole('button', { name: '关闭窗口', exact: true }).click(); await panel().waitFor({ state: 'hidden' });
|
||||
if (!await page.locator('.session-list').isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await page.locator('.session-list button').filter({ hasText: other.title }).click(); selectedRoute = '/sessions/' + other.id;
|
||||
const lateResponse = page.waitForResponse(response => response.url().includes('/engines/deepseek/availability') && response.status() === 500);
|
||||
release(); await lateResponse; await page.unroute(pattern);
|
||||
await page.getByRole('button', { name: '本地代理引擎', exact: true }).click(); await panel().waitFor({ state: 'visible' });
|
||||
await textContains(panel(), '已找到 0.2.1-rc.2'); await textContains(panel(), '先验证这个安装能否连接');
|
||||
assert.equal((await state()).execution.engineJobs.length, 0); assert.equal(await page.getByText('owned-late-engine-error-must-stay-in-original-view').count(), 0);
|
||||
mark('late detection failure from a closed panel cannot appear in a different task or reveal its prior engine records');
|
||||
const wireBefore = fs.readFileSync(path.join(fixtureRoot, 'wire.jsonl')), saved = await host.api('/sessions/' + session.id);
|
||||
deliberateNavigation = true; await host.close(); host = await launchWorkbench(t, data, { AGENT_WORKBENCH_DEEPSEEK_ROOT: fixtureRoot });
|
||||
selectedRoute = '/sessions/' + session.id; await open();
|
||||
assert.deepEqual((await state()).execution.engineJobs, saved.execution.engineJobs); assert.deepEqual(fs.readFileSync(path.join(fixtureRoot, 'wire.jsonl')), wireBefore);
|
||||
await card(nativeJob).getByRole('button', { name: '查看原记录', exact: true }).click(); await view().waitFor({ state: 'visible' }); await textContains(view(), nativeJob.reportSha256);
|
||||
assert.deepEqual(fs.readFileSync(path.join(projectRoot, 'README.md')), projectBytes); assert.equal((await state()).execution.task.usage.requests, 0);
|
||||
mark('actual host restart preserves all native and declared-fixture records with no protocol replay or project modification');
|
||||
assert.equal(pageErrors.length, 0); assert.equal(requestFailures.length, 0);
|
||||
result = { success: true, version: host.bootstrap.version, testedAt: new Date().toISOString(), root, browser: browser.version(), passed, pageErrors,
|
||||
requestFailures, expectedRequestFailures, screenshots, starts, nativeJob,
|
||||
actualSdk: { version: native.version, identitySha256: nativeJob.installation.identitySha256, nativeMethods: 6, modelRequests: 0, rootExited: true },
|
||||
stopScope: 'declared private SDK pending-initialize fixture', scope: 'Independent service and data. Original 4318 Provider, tasks and credentials were not accessed.' };
|
||||
} catch (error) {
|
||||
await screenshot('initial-failure').catch(() => {}); fs.writeFileSync(path.join(root, 'failure-dom.html'), await page.content().catch(() => ''));
|
||||
result = { success: false, version: host.bootstrap.version, testedAt: new Date().toISOString(), root, passed, pageErrors, requestFailures, expectedRequestFailures, screenshots, error: error.stack }; process.exitCode = 1;
|
||||
} finally { deliberateNavigation = true; await browser.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(result.version === '1.9.0' ? 'docs/engines-ui-validation.json' : `docs/${releaseEvidencePrefix(result.version)}-engines-ui-validation.json`, JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, groups: passed.length, root, pageErrors, requestFailures, error: result.error }));
|
||||
@@ -0,0 +1,52 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { EngineManager } from '../src/core/engine-manager.mjs';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { APP_VERSION } from '../src/server/diagnostics.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-engines.mjs [--root <installed-DeepSeek-Harness-root>] — native SDK connection, no prompt or configured model'); process.exit(0);
|
||||
}
|
||||
const rootIndex = process.argv.indexOf('--root'), requestedRoot = rootIndex >= 0 ? process.argv[rootIndex + 1] : undefined;
|
||||
if (rootIndex >= 0 && !requestedRoot) throw new Error('--root needs an installation directory');
|
||||
const directory = path.resolve('.qa', 'native-engine-' + randomUUID()), projectRoot = path.join(directory, 'project');
|
||||
fs.mkdirSync(projectRoot, { recursive: true });
|
||||
const projectBytes = Buffer.from('# Owned native SDK connection validation\n'); fs.writeFileSync(path.join(projectRoot, 'README.md'), projectBytes);
|
||||
const store = new Store(path.join(directory, 'data')), manager = new EngineManager(store), project = store.addProject(projectRoot), session = store.createSession(project.id);
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
let result;
|
||||
try {
|
||||
const installation = manager.availability(requestedRoot);
|
||||
assert.equal(installation.status, 'available', installation.message);
|
||||
const started = manager.start(session, { root: installation.root, identitySha256: installation.identitySha256 }), job = await started.promise;
|
||||
assert.equal(job.status, 'completed', job.summary); assert.equal(job.verification?.verified, true);
|
||||
const reportText = manager.originalReport(session, job), original = JSON.parse(reportText);
|
||||
assert.equal(sha(reportText), job.reportSha256);
|
||||
assert.deepEqual(original.owner, { sessionId: session.id, projectId: project.id, taskId: session.taskId, runId: session.runId, jobId: job.id });
|
||||
assert.equal(original.report.readiness.signal, 'appReady'); assert.equal(original.report.readiness.pid, job.pid);
|
||||
assert.equal(original.report.before.identitySha256, original.report.after.identitySha256);
|
||||
assert.equal(original.report.before.identitySha256, installation.identitySha256);
|
||||
assert.deepEqual(original.report.responses.map(value => value.method), ['initialize', 'session/working-directory/get', 'session/wait', 'session/resume', 'session/cancel', 'shutdown']);
|
||||
assert.equal(original.report.process.pid, original.report.cleanup.pid); assert.equal(original.report.cleanup.rootExited, true); assert.equal(original.report.cleanup.exit.code, 0);
|
||||
assert.equal(original.report.modelRequests, 0); assert.equal(original.report.providerRequests, 0);
|
||||
assert.equal(store.core.task.get(session.taskId).usage.requests, 0); assert.equal(session.changes.length, 0);
|
||||
assert.equal(original.report.workspace.unchanged, true); assert.deepEqual(fs.readFileSync(path.join(projectRoot, 'README.md')), projectBytes);
|
||||
fs.writeFileSync(path.join(directory, 'original-report.json'), reportText);
|
||||
result = { success: true, version: APP_VERSION, testedAt: new Date().toISOString(), directory,
|
||||
evidenceKind: 'actual-native-DeepSeek-Harness-SDK; no protocol fixture', scope: job.scope,
|
||||
owner: original.owner, installation: { root: installation.root, version: installation.version, profile: installation.profile,
|
||||
identitySha256: installation.identitySha256, inputs: installation.inputs, node: installation.node },
|
||||
process: original.report.process, readiness: original.report.readiness, responses: original.report.responses,
|
||||
cleanup: original.report.cleanup, workspace: original.report.workspace, reportSha256: job.reportSha256,
|
||||
modelRequests: 0, providerRequests: 0, projectChanges: 0,
|
||||
executionBoundary: 'Connection validation only; session/prompt was not sent. Original user service and credentials were not used.' };
|
||||
} catch (error) {
|
||||
result = { success: false, version: APP_VERSION, testedAt: new Date().toISOString(), directory,
|
||||
error: error.stack, jobs: manager.jobs(session).map(job => manager.summary(session, job)) }; process.exitCode = 1;
|
||||
} finally { await manager.stopAll(); store.close(); }
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(path.resolve('.qa', 'native-engine-validation.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, version: result.version, directory, sdkVersion: result.installation?.version,
|
||||
nativeMethods: result.responses?.map(value => value.method), modelRequests: result.modelRequests, rootExited: result.cleanup?.rootExited, error: result.error }));
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createRequire} from 'node:module';
|
||||
import {randomUUID,createHash} from 'node:crypto';
|
||||
import {fixture} from '../tests/helpers.mjs';
|
||||
import {deliveryGit} from '../tests/fixtures/delivery-project.mjs';
|
||||
import {WorkspaceManager} from '../src/core/workspaces/git-workspaces.mjs';
|
||||
import {EnvironmentManager} from '../src/core/environment/environment-manager.mjs';
|
||||
import {CheckRunner} from '../src/core/check-runner.mjs';
|
||||
|
||||
const directory=path.resolve('.qa','environment-projects-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),reports=[],requests=[];
|
||||
const hash=value=>createHash('sha256').update(value).digest('hex'),require=createRequire(import.meta.url);
|
||||
const cacache=require(path.join(path.dirname(process.execPath),'node_modules/npm/node_modules/cacache'));
|
||||
const cache=path.join(process.env.LOCALAPPDATA,'npm-cache','_cacache');
|
||||
const manifestRoot=path.resolve('benchmarks/react-delivery-v1/baseline');fs.cpSync(manifestRoot,f.projectRoot,{recursive:true,filter:file=>!file.split(path.sep).includes('node_modules')});
|
||||
const lockFile=path.join(f.projectRoot,'package-lock.json'),originalLock=fs.readFileSync(lockFile),lock=JSON.parse(originalLock),tarballs=new Map();
|
||||
const server=http.createServer(async(request,response)=>{
|
||||
const entry=tarballs.get(request.url);requests.push({path:request.url,integrity:entry?.integrity??null});
|
||||
if(!entry){response.writeHead(404);response.end('No owned tarball');return;}
|
||||
try{const bytes=await cacache.get.byDigest(cache,entry.integrity);response.writeHead(200,{'Content-Type':'application/octet-stream'});response.end(bytes);}
|
||||
catch{response.writeHead(503);response.end('Required integrity not in local QA download cache');}
|
||||
});
|
||||
await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const registry='http://127.0.0.1:'+server.address().port+'/';
|
||||
disposers.push(()=>{server.closeAllConnections();return new Promise(resolve=>server.close(resolve));});
|
||||
for(const entry of Object.values(lock.packages))if(entry.resolved){const url=new URL(entry.resolved);assert.equal(url.origin,'https://registry.npmjs.org');assert.equal(url.username,'');
|
||||
const target='/tarballs/'+hash(entry.resolved)+'.tgz';tarballs.set(target,{integrity:entry.integrity});entry.resolved=registry+target.slice(1);}
|
||||
fs.writeFileSync(lockFile,JSON.stringify(lock,null,2));
|
||||
deliveryGit(f,['init','--initial-branch=main']);deliveryGit(f,['-c','core.autocrlf=false','add','.']);deliveryGit(f,['-c','user.name=Workbench QA','-c','user.email=qa@workbench.local','commit','-m','Own React source and integrity-bound local tarball feed']);
|
||||
const parent=f.store.beginRun(f.session,'验证真实React工程的依赖准备及原版/候选检查',false,f.store.config);f.session.status='paused';f.store.finishRun(f.session,parent);
|
||||
const workspaces=new WorkspaceManager(f.store),environment=new EnvironmentManager(f.store,{allowHttpLoopback:true}),checks=new CheckRunner(f.store);
|
||||
disposers.push(()=>workspaces.stopAll(),()=>environment.stopAll(),()=>checks.stopAll());
|
||||
async function child(label){const record=await workspaces.prepare(f.session,label);await(await workspaces.start(f.session,record.id,record.fingerprint)).promise;assert.equal(record.status,'ready',record.summary);
|
||||
const project=f.store.project(record.childProjectId),session=f.store.session(record.childSessionId),run=f.store.beginRun(session,label,false,f.store.config);session.status='paused';f.store.finishRun(session,run);return {project,session};}
|
||||
async function prepare(p){const plan=await environment.plan(p.session,{registry}),job=await(await environment.start(p.session,plan.id,plan.fingerprint,{approved:true})).promise;
|
||||
reports.push({stage:'prepare',projectId:p.project.id,plan,job});assert.equal(job.status,'prepared',job.summary+'\n'+job.logExcerpt);console.log('PASS actual React npm ci: '+job.id);return job;}
|
||||
function dependencyFiles(root,relative='node_modules'){const directory=path.join(root,relative);if(!fs.existsSync(directory))return [];
|
||||
return fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name)).flatMap(entry=>{const name=path.posix.join(relative,entry.name),target=path.join(root,name);
|
||||
return entry.isSymbolicLink()?[{path:name,link:fs.readlinkSync(target)}]:entry.isDirectory()?[{path:name,directory:true},...dependencyFiles(root,name)]:[{path:name,sha256:hash(fs.readFileSync(target))}];});}
|
||||
async function check(p,names){const before=dependencyFiles(p.project.root),plan=await checks.plan(p.session,names),job=await(await checks.start(p.session,plan.id)).promise,after=dependencyFiles(p.project.root);
|
||||
const left=new Map(before.map(row=>[row.path,JSON.stringify(row)])),right=new Map(after.map(row=>[row.path,JSON.stringify(row)]));
|
||||
const changed=[...new Set([...left.keys(),...right.keys()])].filter(name=>left.get(name)!==right.get(name));
|
||||
reports.push({stage:'checks',projectId:p.project.id,plan,job,dependencySnapshot:{before,after,changed}});if(changed.length)console.log('Observed dependency changes: '+JSON.stringify(changed));return job;}
|
||||
let result;
|
||||
try{
|
||||
const original=await child('React原版环境'),candidate=await child('React候选环境'),left=await prepare(original),right=await prepare(candidate);
|
||||
assert.equal(left.after.dependencyInputs,right.after.dependencyInputs);assert.equal(left.after.installed.sha256,right.after.installed.sha256);
|
||||
const baseline=await check(original,['test','typecheck','build']);assert.equal(baseline.status,'failed',baseline.summary);assert.ok(baseline.results.some(value=>value.tests?.failed>0));
|
||||
const app=fs.readFileSync(path.join(candidate.project.root,'src/App.tsx'),'utf8'),fixes={
|
||||
'src/query.ts':"import type { Task } from './types.ts';\nexport function selectTasks(tasks: Task[], filter: string, query: string): Task[] { const text=query.trim().toLocaleLowerCase(); return tasks.filter(task => (filter==='done'?task.done:filter==='pending'?!task.done:true)&&task.title.toLocaleLowerCase().includes(text)); }\n",
|
||||
'src/summary.ts':"import type { Task } from './types.ts';\nexport function summarizeTasks(tasks: Task[]) { return {total:tasks.length,completed:tasks.filter(task=>task.done).length,pendingHours:tasks.filter(task=>!task.done).reduce((sum,task)=>sum+task.hours,0)}; }\n",
|
||||
'src/transition.ts':"import type { Task } from './types.ts';\nexport function toggleTask(tasks: Task[], id: string): Task[] { return tasks.map(task=>task.id===id?{...task,done:!task.done}:task); }\n",
|
||||
'src/App.tsx':app.replace("toggleTask(current, 'missing')",'toggleTask(current, task.id)')};
|
||||
for(const[name,value]of Object.entries(fixes))fs.writeFileSync(path.join(candidate.project.root,name),value);
|
||||
const candidateJob=await check(candidate,['test','typecheck','build']);assert.equal(candidateJob.status,'passed',candidateJob.summary);assert.equal(candidateJob.results[0].tests.passed,9);
|
||||
assert.ok(candidateJob.environment.preparation.jobId);assert.equal(fs.existsSync(path.join(f.projectRoot,'node_modules')),false);assert.equal(hash(fs.readFileSync(lockFile)),hash(JSON.stringify(lock,null,2)));
|
||||
const pyRoot=path.join(f.root,'python-owned');fs.mkdirSync(pyRoot);fs.mkdirSync(path.join(pyRoot,'tests'));
|
||||
fs.writeFileSync(path.join(pyRoot,'pyproject.toml'),'[tool.pytest.ini_options]\ntestpaths=["tests"]\n');
|
||||
fs.writeFileSync(path.join(pyRoot,'calc.py'),'def add(a,b):\n return a+b\n');fs.writeFileSync(path.join(pyRoot,'tests/test_calc.py'),'from calc import add\ndef test_positive():\n assert add(2,3)==5\ndef test_negative():\n assert add(-2,3)==1\n');
|
||||
const project=f.store.addProject(pyRoot),session=f.store.createSession(project.id),run=f.store.beginRun(session,'实际Python环境及检查',false,f.store.config);session.status='paused';f.store.finishRun(session,run);
|
||||
const py=new EnvironmentManager(f.store,{pythonExecutable:process.env.AGENT_WORKBENCH_TEST_PYTHON}),diagnostic=await py.diagnose(session);assert.equal(diagnostic.python.pytest.version,'8.4.2');
|
||||
const runner=new CheckRunner(f.store,{pythonExecutable:process.env.AGENT_WORKBENCH_TEST_PYTHON}),plan=await runner.plan(session,['python:test','python:compile']),job=await(await runner.start(session,plan.id)).promise;
|
||||
reports.push({stage:'python',diagnostic,plan,job});assert.equal(job.status,'passed',job.summary);assert.equal(job.results[0].tests.passed,2);
|
||||
result={success:true,directory,reports:reports.length,reactOriginal:baseline.status,reactCandidate:candidateJob.status,reactTests:9,pythonTests:2,requests:requests.length,
|
||||
originalLockSha256:hash(originalLock),ownedLockSha256:hash(fs.readFileSync(lockFile)),downloadSource:'local HTTP feed of existing npm tarball bytes by locked SRI; no config/credentials or remote network',realModelRequests:0};
|
||||
}catch(error){result={success:false,error:error.stack,directory,reports:reports.length,requests:requests.length};process.exitCode=1;}
|
||||
finally{fs.writeFileSync(path.join(directory,'reports.json'),JSON.stringify({reports,requests},null,2));fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));console.log(JSON.stringify(result));for(const dispose of disposers.reverse())await dispose();}
|
||||
@@ -0,0 +1,71 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {randomUUID,createHash} from 'node:crypto';
|
||||
import {chromium,expect} from 'playwright/test';
|
||||
import {fixture,launchWorkbench,until} from '../tests/helpers.mjs';
|
||||
import {environmentProject} from '../tests/fixtures/environment-project.mjs';
|
||||
import {EnvironmentManager} from '../src/core/environment/environment-manager.mjs';
|
||||
|
||||
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]));
|
||||
const appRoot=args[1]??process.cwd(),directory=path.resolve('.qa','environment-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),cases={},checks=[],screenshots=[],pageErrors=[],failures=[],events=[];
|
||||
let browser,page,host,result;
|
||||
const mark=value=>{checks.push(value);console.log('PASS '+value);};
|
||||
for(const name of ['desktop','mobile','failure','stop','stale','recovery','late']) {
|
||||
let own=f;
|
||||
if(Object.keys(cases).length){const projectRoot=path.join(f.root,'project-'+name);fs.mkdirSync(projectRoot);const project=f.store.addProject(projectRoot);own={...f,projectRoot,project,session:f.store.createSession(project.id)};}
|
||||
const postinstall=name==='failure'?`node -e "console.error('OWNED_INSTALL_FAILURE');process.exit(23)"`:name==='stop'?`node -e "console.log('OWNED_WAITING');setTimeout(()=>{},60000)"`:undefined;
|
||||
const project=await environmentProject(own,{dependency:false,...(postinstall?{postinstall}:{})});project.session.title='环境验收 '+name;f.store.save(project.session);cases[name]=project;
|
||||
if(name==='recovery'){
|
||||
const manager=new EnvironmentManager(f.store),plan=await manager.plan(project.session,{offline:true});
|
||||
f.store.putRecord('environment_job',project.session,{id:randomUUID(),sessionId:project.session.id,projectId:project.project.id,taskId:project.session.taskId,runId:project.session.runId,
|
||||
planId:plan.id,planFingerprint:plan.fingerprint,status:'queued',process:null,requiresVerification:false},'environment.queued');
|
||||
}
|
||||
}
|
||||
f.store.close();
|
||||
const panel=()=>page.getByRole('region',{name:'工程环境与依赖',exact:true});
|
||||
async function choose(name){const p=cases[name];if(page.viewportSize().width<=620||!await page.locator('.project-sidebar').isVisible())await page.getByRole('button',{name:'展开项目栏',exact:true}).click();
|
||||
await page.getByLabel('当前项目',{exact:true}).selectOption(p.project.id);await expect(page.locator('.work-title h1')).toHaveText(p.session.title);
|
||||
if(page.viewportSize().width<=620&&await page.getByRole('button',{name:'收起项目栏',exact:true}).isVisible())await page.getByRole('button',{name:'收起项目栏',exact:true}).click();
|
||||
await page.getByRole('button',{name:'查看任务记录',exact:true}).click();await expect(panel()).toBeVisible();return p;}
|
||||
async function capture(name){await panel().scrollIntoViewIfNeeded();const bounds=await page.evaluate(()=>({width:innerWidth,scroll:document.documentElement.scrollWidth}));assert.ok(bounds.scroll<=bounds.width+1,'horizontal overflow '+name);
|
||||
const file=path.join(directory,name+'.png');await page.screenshot({path:file,scale:'css'});screenshots.push({path:file,sha256:createHash('sha256').update(fs.readFileSync(file)).digest('hex')});}
|
||||
async function plan({allow=false}={}){await panel().getByRole('button',{name:/诊断工程环境/}).click();await expect(panel().getByText('Node / npm',{exact:true})).toBeVisible();
|
||||
await panel().getByText('下载与脚本选项',{exact:true}).click();await panel().getByLabel('只使用本工作区的下载缓存(离线)').check();if(allow)await panel().getByLabel('允许依赖生命周期脚本').check();
|
||||
await panel().getByRole('button',{name:'创建依赖准备计划',exact:true}).click();await expect(panel().getByRole('button',{name:'核对并批准依赖准备',exact:true})).toBeVisible();}
|
||||
async function approve(){await panel().getByRole('button',{name:'核对并批准依赖准备',exact:true}).click();const dialog=page.getByRole('dialog',{name:'确认依赖准备',exact:true});await expect(dialog).toBeVisible();
|
||||
await expect(dialog.getByText('独立工作区',{exact:true})).toBeVisible();await expect(dialog.locator('pre').first()).toContainText('--offline=true');return dialog;}
|
||||
async function settled(p){let session;await until(async()=>{session=await host.api('/sessions/'+p.session.id);return session.execution.environmentJobs?.at(-1)?.finishedAt;},30000);return session;}
|
||||
try{
|
||||
host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath:fs.existsSync(path.join(appRoot,'runtime','node.exe'))?path.join(appRoot,'runtime','node.exe'):process.execPath});
|
||||
assert.notEqual(new URL(host.url).port,'4318');assert.equal(host.bootstrap.config.hasKey,false);
|
||||
browser=await chromium.launch({executablePath:'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe',headless:true});page=await browser.newPage({viewport:{width:1440,height:1000}});
|
||||
page.on('pageerror',error=>pageErrors.push(error.message));page.on('requestfailed',request=>{if(request.resourceType()!=='eventsource')failures.push(request.url()+': '+request.failure()?.errorText);});
|
||||
page.on('response',response=>{if(response.url().includes('/environment-'))events.push({url:response.url(),status:response.status()});});
|
||||
await page.goto(host.url);let p=await choose('desktop');await plan();let dialog=await approve();await capture('desktop-plan');
|
||||
await dialog.getByRole('button',{name:'确认执行依赖准备',exact:true}).click();let session=await settled(p);assert.equal(session.execution.environmentJobs.at(-1).status,'prepared');
|
||||
assert.equal(session.execution.environmentDiagnostics.at(-1).node.status,'ready');await expect(panel().getByText('依赖已就绪',{exact:true})).toBeVisible();
|
||||
await expect(panel().getByText('依赖准备完成',{exact:true})).toBeVisible();await capture('desktop-prepared');
|
||||
await panel().getByRole('button',{name:'查看依赖日志',exact:true}).click();await expect(page.getByRole('dialog').last()).toContainText('stdout');await page.getByRole('dialog').last().getByRole('button',{name:'关闭窗口',exact:true}).click();
|
||||
await page.reload();await page.getByRole('button',{name:'查看任务记录',exact:true}).click();await expect(panel().getByText('依赖准备完成',{exact:true})).toBeVisible();
|
||||
assert.equal(fs.existsSync(path.join(p.sourceRoot,'node_modules')),false);mark('desktop diagnosis, fixed plan, explicit approval, actual npm, original log and refresh');
|
||||
await page.setViewportSize({width:390,height:844});p=await choose('mobile');await plan();dialog=await approve();await capture('mobile-plan');await dialog.getByRole('button',{name:'确认执行依赖准备',exact:true}).click();session=await settled(p);
|
||||
assert.equal(session.execution.environmentJobs.at(-1).status,'prepared');await capture('mobile-prepared');mark('390px same real flow and bounded plan modal');
|
||||
await page.setViewportSize({width:1440,height:1000});p=await choose('failure');await plan({allow:true});dialog=await approve();await expect(dialog.getByText('允许执行根工程和依赖包安装脚本',{exact:true})).toBeVisible();
|
||||
await dialog.getByRole('button',{name:'确认执行依赖准备',exact:true}).click();session=await settled(p);assert.equal(session.execution.environmentJobs.at(-1).status,'failed');await expect(panel().getByText('准备失败',{exact:true})).toBeVisible();await capture('failure-original-log');
|
||||
assert.match(session.execution.environmentJobs.at(-1).logExcerpt,/OWNED_INSTALL_FAILURE/);mark('explicit script approval and original nonzero install failure');
|
||||
p=await choose('stop');await plan({allow:true});dialog=await approve();await dialog.getByRole('button',{name:'确认执行依赖准备',exact:true}).click();await expect(panel().locator('pre').filter({hasText:'OWNED_WAITING'})).toBeVisible();
|
||||
await panel().getByRole('button',{name:'停止依赖准备',exact:true}).click();session=await settled(p);assert.equal(session.execution.environmentJobs.at(-1).status,'cancelled',session.execution.environmentJobs.at(-1).summary);await capture('stopped-owned-process');mark('stop kills only owned npm tree and keeps cancelled logs');
|
||||
p=await choose('stale');await plan();dialog=await approve();fs.appendFileSync(path.join(p.root,'package-lock.json'),'\n');await dialog.getByRole('button',{name:'确认执行依赖准备',exact:true}).click();
|
||||
await expect(page.getByRole('alert')).toContainText('变化');assert.equal((await host.api('/sessions/'+p.session.id)).execution.environmentJobs?.length??0,0);await capture('stale-input-rejected');await dialog.getByRole('button',{name:'取消',exact:true}).click();mark('changed lock rejected before process dispatch');
|
||||
p=await choose('recovery');await expect(panel().getByText('结果待核对',{exact:true})).toBeVisible();await panel().getByRole('button',{name:'核对中断现场',exact:true}).click();await page.getByRole('dialog',{name:'核对中断的依赖准备',exact:true}).getByRole('button',{name:'确认只读核对',exact:true}).click();
|
||||
await expect(panel().getByText('已核对原进程结束并保留现场,原未知结果保持。再次执行需要新计划。',{exact:true})).toBeVisible();await capture('recovery-reconciled');mark('recovered unknown remains original unknown after explicit reconciliation');
|
||||
p=await choose('late');await panel().getByRole('button',{name:'诊断工程环境',exact:true}).click();await expect(panel().getByText('Node / npm',{exact:true})).toBeVisible();
|
||||
let release,observed,finish;const gate=new Promise(resolve=>{release=resolve;}),seen=new Promise(resolve=>{observed=resolve;}),finished=new Promise(resolve=>{finish=resolve;});
|
||||
await page.route('**/environment-plans',async route=>{const response=await route.fetch();observed();await gate;try{await route.fulfill({response});}finally{finish();}});
|
||||
await panel().getByRole('button',{name:'创建依赖准备计划',exact:true}).click();await seen;await choose('desktop');release();await finished;await page.unroute('**/environment-plans');await expect(page.locator('.work-title h1')).toHaveText(cases.desktop.session.title);mark('late plan response cannot replace another project or task');
|
||||
assert.deepEqual(pageErrors,[]);assert.deepEqual(failures,[]);const receipts={};for(const[name,p]of Object.entries(cases))receipts[name]=await host.api('/sessions/'+p.session.id);
|
||||
fs.writeFileSync(path.join(directory,'receipts.json'),JSON.stringify(receipts,null,2));result={success:true,appRoot,version:host.bootstrap.version,checks,screenshots,pageErrors,failures,events,directory,realModelRequests:0};
|
||||
}catch(error){if(page){fs.writeFileSync(path.join(directory,'failure-dom.html'),await page.content());await page.screenshot({path:path.join(directory,'failure.png')}).catch(()=>{});}result={success:false,error:error.stack,checks,screenshots,pageErrors,failures,events,directory};process.exitCode=1;}
|
||||
finally{fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));console.log(JSON.stringify(result));await browser?.close();for(const dispose of disposers.reverse())await dispose();}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import { fixture, compatibleFixture, call, completion, assertToolPairs } from '../tests/helpers.mjs';
|
||||
import { specification } from '../tests/fixtures/engineering-specification.mjs';
|
||||
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { ownerBytes, applyWorkerProposals, approveWorkerChecks } from '../tests/fixtures/workers-project.mjs';
|
||||
import { Runtime } from '../src/server/runtime.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
||||
import { evaluationSummary } from '../src/shared/contracts/evaluations.mjs';
|
||||
|
||||
if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-projects.mjs — actual fixed HTTP evaluation of Node8/React9/Python10 and a source-bound Workbench summary module; original tests, isolated candidate, review, checks, Edge, local Git. Set AGENT_WORKBENCH_TEST_PYTHON. No real inference claim.');process.exit(0);}
|
||||
assert.equal(process.argv.length,2);
|
||||
const directory=path.resolve('.qa','engineering-evals-projects-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const version=JSON.parse(fs.readFileSync('package.json','utf8')).version,hash=value=>createHash('sha256').update(value).digest('hex');
|
||||
const sourceFixture=JSON.parse(fs.readFileSync(new URL('../tests/fixtures/evaluation-summary-source.json',import.meta.url),'utf8'));
|
||||
const sourceCommit=sourceFixture.commit;assert.equal(hash(sourceFixture.source),sourceFixture.moduleSha256,'Original committed module fixture bytes');
|
||||
const python=process.env.AGENT_WORKBENCH_TEST_PYTHON;assert.ok(python&&path.isAbsolute(python)&&fs.existsSync(python),'Set the isolated QA Python runtime.');
|
||||
const assertions=[],results=[];const mark=value=>{assertions.push(value);console.log('PASS '+value);};
|
||||
function selfSpecification(){
|
||||
const source=sourceFixture.source;
|
||||
const baseline=source.slice(source.indexOf('export function evaluationSummary(batch)'));
|
||||
assert.ok(baseline.startsWith('export function evaluationSummary(batch)'));
|
||||
const before="candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : 'pending'";
|
||||
const after="candidate: ['failed','stopped','unknown'].includes(sample.status) ? sample.status : ['candidate_running','candidate_complete','awaiting_review','paused','collecting','adopting','sealed'].includes(sample.status) ? 'unknown' : 'pending'";
|
||||
assert.ok(baseline.includes(before),'Self pilot must use the original committed summary implementation, without an invented defect.');
|
||||
const tests="import test from 'node:test';import assert from 'node:assert/strict';import {evaluationSummary} from '../src/summary.mjs';\nconst summary=(status,report=null)=>evaluationSummary({evidenceMode:'local-fixture',samples:[{status,report}]});\n"+
|
||||
"test('pending sample keeps its original denominator',()=>{const value=summary('pending');assert.equal(value.samples,1);assert.equal(value.candidate.pending,1);});\n"+
|
||||
"test('fixed fixture never proves inference',()=>assert.equal(summary('pending').inferenceValidated,false));\n"+
|
||||
"test('original failure and candidate pass remain paired',()=>{const value=summary('sealed',{summary:{original:'failed',candidate:'passed'}});assert.equal(value.original.failed,1);assert.equal(value.candidate.passed,1);});\n"+
|
||||
"test('stopped attempt remains stopped',()=>assert.equal(summary('stopped').candidate.stopped,1));\n"+
|
||||
"test('failed attempt remains failed',()=>assert.equal(summary('failed').candidate.failed,1));\n"+
|
||||
"test('finished candidate without report is unknown, never unexecuted',()=>assert.equal(summary('candidate_complete').candidate.unknown,1));\n"+
|
||||
"test('paused candidate without report is unknown, never unexecuted',()=>assert.equal(summary('paused').candidate.unknown,1));\n"+
|
||||
"test('reviewed pending evidence is unknown, never unexecuted',()=>assert.equal(summary('awaiting_review').candidate.unknown,1));\n"+
|
||||
"test('running candidate without report is unknown, never unexecuted',()=>assert.equal(summary('candidate_running').candidate.unknown,1));\n";
|
||||
return {tests:9,checks:['test','build'],source:{commit:sourceCommit,path:'src/shared/contracts/evaluations.mjs',moduleSha256:hash(source),extractedFunctionSha256:hash(baseline),scope:'Exact committed function extracted into an isolated module project; this is not an entire-repository autonomous upgrade.'},
|
||||
initial:{'src/summary.mjs':baseline,'test/summary.test.mjs':tests,'package.json':JSON.stringify({name:'workbench-summary-pilot',private:true,type:'module',scripts:{test:'node --test test/*.test.mjs',build:'node scripts/build.mjs'}}),'scripts/build.mjs':"import fs from 'node:fs';fs.mkdirSync('dist',{recursive:true});fs.copyFileSync('src/summary.mjs','dist/summary.mjs');console.log('source-bound summary built');\n"},fixes:{'src/summary.mjs':baseline.replace(before,after)}};
|
||||
}
|
||||
async function sample(kind){
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),saved=path.join(directory,kind);fs.mkdirSync(saved);let runtime,result;
|
||||
try{
|
||||
if(kind!=='self')fs.cpSync(path.resolve(kind==='react'?'benchmarks/react-delivery-v1/baseline':'benchmarks/workers-delivery-v1/'+kind),f.projectRoot,{recursive:true,filter:file=>!file.split(path.sep).includes('node_modules')});
|
||||
const spec=kind==='self'?selfSpecification():specification(kind,f.projectRoot);
|
||||
for(const [file,content]of Object.entries(spec.initial??{}))write(f,file,content);
|
||||
write(f,'.gitignore','node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
|
||||
write(f,'owner-staged.txt','Owner baseline\n');write(f,'owner-unstaged.txt','Owner baseline\n');
|
||||
git(f,['init','--initial-branch=main']);git(f,['config','core.autocrlf','false']);git(f,['config','user.name','Workbench QA']);git(f,['config','user.email','qa@workbench.invalid']);git(f,['add','--','.']);git(f,['commit','--no-gpg-sign','-m','Seal fixed '+kind+' evaluation source']);
|
||||
write(f,'owner-staged.txt','Owner staged change\n');git(f,['add','--','owner-staged.txt']);write(f,'owner-staged.txt','Owner working copy after staging\n');write(f,'owner-unstaged.txt','Owner unstaged change\n');write(f,'owner-untracked.txt','Owner untracked file\n');
|
||||
const original=ownerBytes(f),writable=Object.keys(spec.fixes).sort();
|
||||
const protectedFiles=Object.fromEntries(git(f,['ls-files','-z']).split('\0').filter(file=>file&&!writable.includes(file)).map(file=>[file,hash(fs.readFileSync(path.join(f.projectRoot,file)))]));
|
||||
const verify=()=>{for(const[file,sha]of Object.entries(protectedFiles))assert.equal(hash(fs.readFileSync(path.join(f.projectRoot,file))),sha,'Protected '+file);assert.deepEqual(ownerBytes(f).files,original.files);};
|
||||
let phase='new',candidateSession;
|
||||
const model=await compatibleFixture(t,(_route,input)=>{
|
||||
assertToolPairs(input.messages);let calls=[];
|
||||
const owner=candidateSession&&f.store.session(candidateSession);
|
||||
if(phase==='new'){phase='indexed';calls=[call(randomUUID(),'index_project',{})];}
|
||||
else if(phase==='indexed'){phase='read';calls=writable.map(file=>call(randomUUID(),'read_file',{path:file}));}
|
||||
else if(phase==='read'){phase='proposed';calls=Object.entries(spec.fixes).map(([file,content])=>call(randomUUID(),'propose_file',{path:file,content,reason:'本地固定HTTP候选;保留原测试与运行声明'}));}
|
||||
else if(!owner.changes.some(change=>change.status==='pending')){
|
||||
const plan=[...f.store.core.check_plan.values()].find(plan=>plan.taskId===owner.taskId),job=[...f.store.core.check_job.values()].find(job=>job.taskId===owner.taskId);
|
||||
if(!plan)calls=[call(randomUUID(),'plan_checks',{names:spec.checks})];else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
|
||||
}
|
||||
return completion(calls,calls.length?'':'固定HTTP返回结束,宿主按实际成果核验');
|
||||
});
|
||||
f.store.saveConfig({mode:'compatible',baseUrl:model.baseUrl,model:'owned-fixed-engineering-http',maxSteps:48});
|
||||
const provider=new Provider();provider.setKey('self-owned-engineering-fixture');runtime=new Runtime(f.store,provider);runtime.checkRunner().pythonExecutable=python;
|
||||
const manager=runtime.evaluationManager(),batch=await manager.plan(f.session,{title:kind+' 同源工程评测',evidenceMode:'local-fixture',samples:[{label:kind+' 原版与候选',goal:kind==='self'?'修复工作台评测汇总:已经执行但尚未封存报告的候选应显示未知,不能显示未执行。':'修复实际工程边界并保持原行为;原测试与构建必须通过。',criteria:spec.tests+'项固定原测试及完整检查通过;不改测试、脚本、运行声明,保留父原暂存和未提交内容。',paths:writable,checks:spec.checks}]});
|
||||
const item=batch.samples[0];await manager.prepare(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(item.status,'ready',batch.message);
|
||||
const group=f.store.core.collaboration.get(item.groupId);candidateSession=group.children.find(child=>child.evaluation.role==='candidate').sessionId;
|
||||
const dependencyActions=[];
|
||||
if(kind==='react')for(const project of [f.project,...group.children.map(child=>f.store.project(child.projectId))]){
|
||||
const cli=path.join(path.dirname(process.execPath),'node_modules/npm/bin/npm-cli.js'),action=spawnSync(process.execPath,[cli,'ci','--offline','--no-audit','--no-fund'],{cwd:project.root,windowsHide:true,timeout:120000,maxBuffer:8*1024*1024});
|
||||
const file=path.join(saved,'npm-ci-'+project.id+'.log');fs.writeFileSync(file,Buffer.concat([action.stdout??Buffer.alloc(0),action.stderr??Buffer.alloc(0)]));assert.equal(action.status,0,fs.readFileSync(file,'utf8'));dependencyActions.push({projectId:project.id,operation:'QA explicitly installed dependencies from the fixed original lockfile',log:file,sha256:hash(fs.readFileSync(file))});
|
||||
}
|
||||
await manager.baseline(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(item.status,'baseline_complete',batch.message);assert.equal(model.requests.length,0);verify();
|
||||
const baseline=f.store.core.check_job.get(item.baselineJobId);assert.equal(baseline.status,'failed',baseline.summary);assert.equal(baseline.results.find(result=>result.kind==='test').tests.total,spec.tests);
|
||||
mark(kind+': fixed same-source commands run the original failing '+spec.tests+' tests with no original model request; original tests/config and owner bytes stay exact');
|
||||
await manager.start(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(item.status,'awaiting_review',batch.message);await applyWorkerProposals(f,runtime,group);
|
||||
await approveWorkerChecks(f,runtime,group,manager.start(f.session,batch.id,batch.fingerprint,item.id));assert.equal(item.status,'candidate_complete',batch.message);verify();
|
||||
const candidate=f.store.session(candidateSession),plans=[...f.store.core.context_plan.values()].filter(plan=>plan.sessionId===candidateSession);assert.ok(plans.length>0);for(const plan of plans)verifyPlanEvidence(f.store,candidate,plan);assertToolPairs(candidate.providerMessages);
|
||||
await manager.seal(f.session,batch.id,batch.fingerprint,item.id).promise;const report=manager.readReport(f.session,batch,item);assert.deepEqual(report.summary,{original:'failed',candidate:'passed'});assert.equal(report.candidate.attempts.at(-1).results.find(result=>result.tests).tests.passed,spec.tests);
|
||||
assert.equal(report.inferenceValidated,false);assert.equal(report.evidenceMode,'local-fixture');assert.ok(report.candidate.approvals.some(approval=>approval.status==='approved'));
|
||||
mark(kind+': actual HTTP coding tools and approved original checks pass '+spec.tests+' tests; full original failure/pass report and sealed request provenance are verified');
|
||||
await manager.adopt(f.session,batch.id,batch.fingerprint,item.id).promise;assert.equal(batch.status,'awaiting_delivery',batch.message);
|
||||
const set=f.store.core.change_set.get(group.merge.changeSetId);await (await runtime.changeSetManager().start(f.session,set.id,set.fingerprint)).promise;assert.equal(set.status,'completed',set.summary);
|
||||
const parentPlan=await runtime.checkRunner().plan(f.session,spec.checks),parentCheck=await (await runtime.checkRunner().start(f.session,parentPlan.id)).promise;assert.equal(parentCheck.status,'passed',parentCheck.summary);
|
||||
const browserJobs=[];
|
||||
if(kind==='react')for(const viewport of [{width:1280,height:900},{width:390,height:844}]){
|
||||
const steps=[{action:'assert_count',selector:'.task-list article',count:3},{action:'select',selector:'select[aria-label="任务状态"]',value:'pending'},{action:'assert_count',selector:'.task-list article',count:2},{action:'fill',selector:'input[aria-label="任务搜索"]',value:' 编写 '},{action:'assert_count',selector:'.task-list article',count:1},{action:'fill',selector:'input[aria-label="任务搜索"]',value:''},{action:'select',selector:'select[aria-label="任务状态"]',value:'all'},{action:'click',selector:'.task-list article[data-task="api"] button'},{action:'assert_text',selector:'.task-list article[data-task="api"] .tag',text:'已完成'},{action:'assert_text',selector:'.task-list article[data-task="tests"] .tag',text:'待完成'},{action:'assert_text',selector:'.summary > div:nth-child(2) strong',text:'2'},{action:'assert_text',selector:'.summary > div:nth-child(3) strong',text:viewport.width>600?'5小时':'5\n小时'}];
|
||||
const plan=await runtime.browserManager().plan(f.session,{viewport,steps}),job=await runtime.browserManager().start(f.session,plan.id).promise;assert.equal(job.status,'passed',job.summary);assert.equal(job.cleanup.exited,true);browserJobs.push(job);
|
||||
for(const image of job.screenshots)fs.writeFileSync(path.join(saved,'page-'+viewport.width+'-'+image.artifactId+'.png'),f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId)));
|
||||
mark('react: actual Edge '+viewport.width+'px interaction passes 12 fixed assertions after parent review');
|
||||
}
|
||||
const deliveryPlan=await runtime.gitDelivery().plan(f.session,{message:'Reviewed fixed '+kind+' evaluation candidate'}),delivery=await (await runtime.gitDelivery().start(f.session,deliveryPlan.id)).promise;assert.equal(delivery.status,'completed',delivery.summary);
|
||||
await manager.finalize(f.session,batch.id,batch.fingerprint);assert.equal(batch.adoption.status,'completed');assert.deepEqual(git(f,['show','--format=','--name-only',delivery.commit]).trim().split(/\r?\n/).sort(),writable);verify();assert.equal(git(f,['show',':owner-staged.txt']),'Owner staged change\n');
|
||||
mark(kind+': candidate adoption, fresh parent checks and exact-scope local Git complete; original staged/unstaged/untracked bytes remain untouched');
|
||||
fs.writeFileSync(path.join(saved,'report.json'),JSON.stringify(report,null,2));fs.writeFileSync(path.join(saved,'fixed-http-requests.json'),JSON.stringify(model.requests.map(request=>({url:request.url,input:request.input,sha256:hash(JSON.stringify(request.input))})),null,2));
|
||||
result={success:true,version,kind,source:spec.source??null,batch,summary:evaluationSummary(batch),report,baseline,parentCheck,dependencyActions,browserJobs,commit:delivery.commit,protectedFiles,fixtureRequests:model.requests.length,realModelRequests:0};return result;
|
||||
}catch(error){result={success:false,kind,error:error.stack,checks:[...f.store.core.check_job.values()],evaluations:[...f.store.core.evaluation.values()]};throw error;}
|
||||
finally{await runtime?.stopAll();f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');fs.cpSync(f.root,path.join(saved,'original-fixture'),{recursive:true,filter:file=>!file.split(path.sep).includes('node_modules')});fs.writeFileSync(path.join(saved,'result.json'),JSON.stringify(result,null,2));for(const dispose of disposers.reverse())await dispose();}
|
||||
}
|
||||
let report;
|
||||
try{for(const kind of ['node','react','python','self'])results.push(await sample(kind));report={success:true,version,directory,assertions,results,sourceCommit,realModelRequests:0,scope:'Fixed own HTTP responses validate actual engineering execution and controlled module improvement; they do not measure autonomous inference or whole-repository self-upgrade.'};}
|
||||
catch(error){report={success:false,version,directory,assertions,results,sourceCommit,error:error.stack,realModelRequests:0};process.exitCode=1;}
|
||||
finally{fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));fs.writeFileSync('.qa/engineering-evals-projects-location.json',JSON.stringify({directory,result:path.join(directory,'result.json')}));console.log('EVIDENCE '+directory);console.log(JSON.stringify({success:report.success,assertions:assertions.length,error:report.error??null}));}
|
||||
@@ -0,0 +1,66 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import {randomUUID,createHash} from 'node:crypto';
|
||||
import {chromium,expect} from 'playwright/test';
|
||||
import {fixture,launchWorkbench,until,call,completion,assertToolPairs} from '../tests/helpers.mjs';
|
||||
import {createDeliveryProject,deliveryGit} from '../tests/fixtures/delivery-project.mjs';
|
||||
import {ownerBytes} from '../tests/fixtures/workers-project.mjs';
|
||||
|
||||
if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-release.mjs [--app-root <absolute-package>] — actual compiled host/bundled Node, fixed HTTP candidate, original failing checks, review, desktop/390px comparison/export, parent checks/Git, stop and source rejection. Independent data; no real inference.');process.exit(0);}
|
||||
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]));
|
||||
const appRoot=fs.realpathSync.native(args[1]??process.cwd()),nodePath=args.length?path.join(appRoot,'runtime/node.exe'):process.execPath;
|
||||
const expectedVersion=JSON.parse(fs.readFileSync(path.join(appRoot,fs.existsSync(path.join(appRoot,'package.json'))?'package.json':'MANIFEST.json'),'utf8')).version;
|
||||
const directory=path.resolve('.qa','engineering-evals-release-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t);createDeliveryProject(f,{broken:true,ownerChanges:true});const original=ownerBytes(f);f.store.close();
|
||||
const hash=bytes=>createHash('sha256').update(bytes).digest('hex'),requests=[],held=new Set(),sockets=new Set(),assertions=[],screenshots=[],pageErrors=[];
|
||||
let host,childRoute,hold=false,browser,report;
|
||||
const model=http.createServer(async(req,res)=>{try{
|
||||
const chunks=[];for await(const chunk of req)chunks.push(chunk);const body=JSON.parse(Buffer.concat(chunks));requests.push({route:req.url,body});assert.equal(req.url,'/v1/chat/completions');assertToolPairs(body.messages);
|
||||
if(hold){res.writeHead(200,{'Content-Type':'application/json'});res.flushHeaders();held.add(res);res.on('close',()=>held.delete(res));return;}
|
||||
const current=await host.api(childRoute),task=current.taskId,changes=current.changes.filter(row=>row.taskId===task),plan=current.execution.plans.at(-1),job=current.execution.checks.at(-1);let calls=[];
|
||||
if(!changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'随包固定HTTP候选,保留原测试'})];
|
||||
else if(!changes.some(row=>row.status==='pending')){if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})];else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];}
|
||||
res.writeHead(200,{'Content-Type':'application/json'});res.end(JSON.stringify(completion(calls,calls.length?'':'固定响应完成,仍需核对实际结果')));
|
||||
}catch(error){res.writeHead(500,{'Content-Type':'application/json'});res.end(JSON.stringify({error:{message:error.message}}));}});
|
||||
model.on('connection',socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));});await new Promise(resolve=>model.listen(0,'127.0.0.1',resolve));
|
||||
const mark=value=>{assertions.push(value);console.log('PASS '+value);},route='/sessions/'+f.session.id;
|
||||
const definition=title=>({title,evidenceMode:'local-fixture',samples:[{label:'原加法边界',goal:'修复加法,保留原调用与测试。',criteria:'原3测试和构建通过,原测试/脚本保持不变。',paths:['src/add.mjs'],checks:['test','build']}],budget:{maxTokens:200000,maxSteps:32,maxToolCalls:64,maxDurationMs:600000}});
|
||||
const get=async id=>(await host.api(route+'/evaluations')).batches.find(row=>row.id===id);
|
||||
async function operation(batch,name){await host.api(route+'/evaluations/'+batch.id+'/'+name,{fingerprint:batch.fingerprint,sampleId:batch.samples[0].id});await until(async()=>!['preparing','baseline_running','candidate_running','collecting','adopting'].includes((await get(batch.id)).status),120000);return get(batch.id);}
|
||||
async function apply(target,set){await host.api(target+'/change-sets/'+set.id+'/apply',{fingerprint:set.fingerprint});let value;await until(async()=>{value=await host.api(target+'/change-sets/'+set.id);return !['prepared','applying'].includes(value.status);});assert.equal(value.status,'completed',value.summary);}
|
||||
try{
|
||||
host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath});assert.equal(host.bootstrap.version,expectedVersion,'Compiled service identity must match its selected package.');assert.equal(fs.realpathSync.native(host.bootstrap.runtimeRoot),appRoot);assert.notEqual(new URL(host.url).port,'4318');
|
||||
await host.api('/config',{mode:'compatible',model:'owned-evaluation-release',baseUrl:'http://127.0.0.1:'+model.address().port+'/v1',apiKey:'self-owned-evaluation-release',maxSteps:32});
|
||||
let batch=(await host.api(route+'/evaluations',definition('随包工程评测'))).batch;batch=await operation(batch,'prepare');assert.equal(batch.samples[0].status,'ready',batch.message);
|
||||
const group=(await host.api(route+'/collaborations')).groups.find(row=>row.id===batch.samples[0].groupId);childRoute='/sessions/'+group.children.find(row=>row.evaluation.role==='candidate').sessionId;
|
||||
batch=await operation(batch,'baseline');assert.equal(batch.samples[0].status,'baseline_complete',batch.message);assert.equal(requests.length,0);assert.deepEqual(ownerBytes(f),original);
|
||||
mark('selected compiled host prepares same-source workspaces and retains original three-test failure with zero original model requests and unchanged owner staging');
|
||||
batch=await operation(batch,'start');assert.equal(batch.samples[0].status,'awaiting_review',batch.message);let current=await host.api(childRoute);
|
||||
const set=(await host.api(childRoute+'/change-sets',{changeIds:current.changes.filter(row=>row.status==='pending').map(row=>row.id)})).execution.changeSets.at(-1);await apply(childRoute,set);
|
||||
await host.api(route+'/evaluations/'+batch.id+'/start',{fingerprint:batch.fingerprint,sampleId:batch.samples[0].id});
|
||||
await until(async()=>{current=await host.api(childRoute);for(const approval of current.approvals.filter(row=>row.status==='pending'))await host.api(childRoute+'/approvals/'+approval.id,{allow:true});batch=await get(batch.id);return !['candidate_running'].includes(batch.status);},120000);
|
||||
assert.equal(batch.samples[0].status,'candidate_complete',batch.message);assert.equal(batch.summary.candidate.unknown,1);assert.equal(batch.summary.candidate.pending,0);
|
||||
batch=await operation(batch,'seal');const comparison=await host.api(route+'/evaluations/'+batch.id+'/report?sampleId='+batch.samples[0].id);assert.deepEqual(comparison.summary,{original:'failed',candidate:'passed'});assert.equal(comparison.original.attempts[0].results[0].tests.failed,3);assert.equal(comparison.candidate.attempts.at(-1).results[0].tests.passed,3);assert.equal(comparison.inferenceValidated,false);assert.deepEqual(ownerBytes(f),original);
|
||||
fs.writeFileSync(path.join(directory,'original-comparison.json'),JSON.stringify(comparison,null,2));
|
||||
mark('real owned HTTP tools, applied candidate review and approved original checks pass; unsealed attempted outcomes stay unknown and sealed report retains original failure');
|
||||
browser=await chromium.launch({channel:'msedge',headless:true});const page=await browser.newPage({viewport:{width:1440,height:1000},acceptDownloads:true});page.on('pageerror',error=>pageErrors.push(error.message));
|
||||
await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id});
|
||||
await page.goto(host.url);await page.waitForLoadState('networkidle');await page.getByRole('button',{name:'工程评测',exact:true}).click();const panel=page.getByLabel('工程评测控制台',{exact:true});await expect(panel).toContainText('随包工程评测');
|
||||
const refresh=panel.getByRole('button',{name:'刷新',exact:true});await refresh.focus();assert.ok(await refresh.evaluate(node=>node.matches(':focus')));await page.keyboard.press('Tab');
|
||||
await panel.getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();const dialog=page.getByRole('dialog',{name:'原版与候选对照',exact:true});await expect(dialog).toContainText('测试 0/3 · 失败 3');await expect(dialog).toContainText('测试 3/3 · 失败 0');await expect(dialog).toContainText('本地固定响应');await expect(dialog).toContainText('费用未报告');
|
||||
for(const viewport of [{width:1440,height:1000},{width:390,height:844}]){await page.setViewportSize(viewport);if(viewport.width===390)await dialog.getByText('测试 3/3 · 失败 0',{exact:true}).scrollIntoViewIfNeeded();assert.ok(await page.locator('body').evaluate(node=>node.scrollWidth<=innerWidth));const file=path.join(directory,'comparison-'+viewport.width+'.png');await page.screenshot({path:file,fullPage:true});screenshots.push({file,sha256:hash(fs.readFileSync(file))});}
|
||||
const downloaded=page.waitForEvent('download');await dialog.getByRole('button',{name:'导出原报告',exact:true}).click();const file=path.join(directory,'downloaded-report.json');await(await downloaded).saveAs(file);assert.deepEqual(JSON.parse(fs.readFileSync(file)).summary,comparison.summary);await page.keyboard.press('Escape');await page.reload();await page.waitForLoadState('networkidle');await page.getByRole('button',{name:'工程评测',exact:true}).click();await expect(panel).toContainText('随包工程评测');
|
||||
mark('compiled desktop/390px UI renders original and candidate checks, fixed-response label, unknown fee, keyboard focus and downloadable report; refresh preserves the same batch');
|
||||
batch=await operation(batch,'adopt');assert.equal(batch.status,'awaiting_delivery',batch.message);const adopted=(await host.api(route+'/collaborations')).groups.find(row=>row.id===group.id);await apply(route,await host.api(route+'/change-sets/'+adopted.merge.changeSetId));
|
||||
const plan=(await host.api(route+'/check-plans',{names:['test','build']})).execution.plans.at(-1),job=(await host.api(route+'/check-jobs',{planId:plan.id})).execution.checks.at(-1);let checked;await until(async()=>{checked=await host.api(route+'/check-jobs/'+job.id);return !['queued','running'].includes(checked.status);});assert.equal(checked.status,'passed',checked.summary);
|
||||
const deliveryPlan=(await host.api(route+'/delivery-plans',{message:'Adopt compiled evaluation candidate'})).execution.deliveryPlans.at(-1),deliveryJob=(await host.api(route+'/delivery-jobs',{planId:deliveryPlan.id})).execution.deliveries.at(-1);let delivered;await until(async()=>{delivered=await host.api(route+'/delivery-jobs/'+deliveryJob.id);return ['completed','failed','cancelled','unknown'].includes(delivered.status);},120000);assert.equal(delivered.status,'completed',delivered.summary);batch=await operation(batch,'finalize');assert.equal(batch.adoption.status,'completed');assert.deepEqual(ownerBytes(f).files,original.files);assert.equal(deliveryGit(f,['diff-tree','--no-commit-id','--name-only','-r','HEAD']).trim(),'src/add.mjs');
|
||||
mark('fresh parent review, actual original parent checks and exact-scope local Git complete adoption while preserving owner staged/unstaged/untracked bytes');
|
||||
hold=true;let stopped=(await host.api(route+'/evaluations',definition('随包停止原请求'))).batch;stopped=await operation(stopped,'prepare');const stoppedGroup=(await host.api(route+'/collaborations')).groups.find(row=>row.id===stopped.samples[0].groupId);childRoute='/sessions/'+stoppedGroup.children.find(row=>row.evaluation.role==='candidate').sessionId;stopped=await operation(stopped,'baseline');const before=requests.length;
|
||||
await host.api(route+'/evaluations/'+stopped.id+'/start',{fingerprint:stopped.fingerprint,sampleId:stopped.samples[0].id});await until(()=>requests.length===before+1);await host.api(route+'/evaluations/'+stopped.id+'/stop',{fingerprint:stopped.fingerprint,sampleId:stopped.samples[0].id});await until(()=>held.size===0);stopped=await operation(stopped,'seal');assert.equal(stopped.samples[0].report.summary.candidate,'stopped');assert.equal(requests.length,before+1);
|
||||
const stale=(await host.api(route+'/evaluations',definition('随包来源变化'))).batch;fs.appendFileSync(path.join(f.projectRoot,'README.md'),'\nOwner source changed\n');const rejected=await operation(stale,'prepare');assert.equal(rejected.samples[0].status,'failed');assert.match(rejected.message,/来源工程已变化/);
|
||||
mark('bundled candidate stop closes only its original owned HTTP request and keeps stopped outcome; changed source rejects before new workspaces without replay');
|
||||
assert.deepEqual(pageErrors,[]);report={success:true,version:host.bootstrap.version,appRoot,nodePath,containedRuntime:!!args.length,directory,assertions,screenshots,pageErrors,requests,comparison,parentCheck:checked,delivery:delivered,stopped,rejected,realModelRequests:0};
|
||||
}catch(error){report={success:false,error:error.stack,directory,appRoot,assertions,screenshots,pageErrors,requests,realModelRequests:0};throw error;}
|
||||
finally{await browser?.close();await host?.close();for(const response of held)response.destroy();for(const socket of sockets)socket.destroy();await new Promise(resolve=>model.close(resolve));fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));for(const dispose of disposers.reverse())await dispose();console.log('EVIDENCE '+directory);}
|
||||
@@ -0,0 +1,79 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, call } from '../tests/helpers.mjs';
|
||||
import { workerHttp } from '../tests/fixtures/workers-http.mjs';
|
||||
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
||||
|
||||
if(process.argv.includes('--help')){console.log('node scripts/verify-evaluations-ui.mjs — independent local HTTP host, fixed-response candidate, real Git/test/build, desktop/390px queue, review, comparison, adoption, stop and source errors. No real model inference.');process.exit(0);}
|
||||
assert.equal(process.argv.length,2);
|
||||
const directory=path.resolve('.qa','engineering-evals-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),hash=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
let hold=false;
|
||||
const host=await workerHttp(f,{broken:true,respond:async({project,signal})=>{
|
||||
if(hold)return new Promise((resolve,reject)=>{if(signal.aborted)reject(signal.reason);else signal.addEventListener('abort',()=>reject(signal.reason),{once:true});});
|
||||
const owner=[...f.store.sessions.values()].find(session=>session.projectId===project.id),taskId=owner.taskId;
|
||||
const plan=[...f.store.core.check_plan.values()].find(plan=>plan.taskId===taskId),job=[...f.store.core.check_job.values()].find(job=>job.taskId===taskId);
|
||||
let calls=[];
|
||||
if(!owner.changes.length)calls=[call(randomUUID(),'propose_file',{path:'src/add.mjs',content:'export const add = (a,b) => a + b;\n',reason:'本地固定响应修复加法'})];
|
||||
else if(owner.changes.some(change=>change.status==='pending'))calls=[];
|
||||
else if(!plan)calls=[call(randomUUID(),'plan_checks',{names:['test','build']})];
|
||||
else if(!job)calls=[call(randomUUID(),'run_checks',{plan_id:plan.id})];
|
||||
return {message:{role:'assistant',content:calls.length?'':'固定响应已返回,仍需实际工程证据',...(calls.length?{tool_calls:calls}:{})},usage:{total_tokens:168,prompt_tokens:123,completion_tokens:45}};
|
||||
}});
|
||||
const browser=await chromium.launch({channel:'msedge',headless:true}),page=await browser.newPage({viewport:{width:1440,height:1000},acceptDownloads:true});
|
||||
const assertions=[],screenshots=[],errors=[],exports=[];page.on('pageerror',error=>errors.push(error.message));
|
||||
const panel=()=>page.getByLabel('工程评测控制台',{exact:true});
|
||||
const mark=value=>{assertions.push(value);console.log('PASS '+value);};
|
||||
const shot=async name=>{const file=path.join(directory,name+'.png');await page.screenshot({path:file,fullPage:true});screenshots.push({file,sha256:hash(fs.readFileSync(file))});assert.ok(await page.locator('body').evaluate(node=>node.scrollWidth<=innerWidth),'body overflow '+name);assert.ok(await panel().evaluate(node=>node.scrollWidth<=node.clientWidth+1),'panel overflow '+name);};
|
||||
const open=async()=>{await page.getByRole('button',{name:'工程评测',exact:true}).click();await expect(panel().getByRole('button',{name:'刷新',exact:true})).toBeEnabled();};
|
||||
const confirm=async(name,button)=>{await panel().getByRole('button',{name,exact:true}).click();await page.getByRole('dialog',{name:'确认评测操作',exact:true}).getByRole('button',{name:button,exact:true}).click();};
|
||||
const refresh=async()=>{await panel().getByRole('button',{name:'刷新',exact:true}).click();};
|
||||
const latest=()=>[...f.store.core.evaluation.values()].at(-1);
|
||||
const parent=async()=>{await page.getByRole('button',{name:'隔离协作',exact:true}).click();await page.getByLabel('隔离协作控制台',{exact:true}).getByRole('button',{name:'返回父目标',exact:true}).click();await expect.poll(()=>page.evaluate(()=>localStorage.getItem('awb-session'))).toBe(f.session.id);await open();};
|
||||
async function fill(title){await panel().getByLabel('评测名称',{exact:true}).fill(title);await panel().getByLabel('评测来源',{exact:true}).selectOption('local-fixture');await panel().getByLabel('样本名称',{exact:true}).fill('正负数和整数加法边界');await panel().getByLabel('具体需求',{exact:true}).fill('修复加法运算,保留原测试及已有调用方式');await panel().getByLabel('验收条件',{exact:true}).fill('原3项加法测试与构建通过,原测试文件和运行声明保持不变');await panel().getByLabel('允许修改的源码文件',{exact:true}).fill('src/add.mjs');}
|
||||
async function apply(prepared=false){if(await page.getByRole('button',{name:'查看改动',exact:true}).isVisible())await page.getByRole('button',{name:'查看改动',exact:true}).click();else await page.locator('.inspector-tabs').getByRole('button',{name:/^改动/}).click();await page.getByRole('button',{name:prepared?'审查变更集':'组合审查',exact:true}).click();if(!prepared)await page.getByRole('dialog',{name:'选择要一起审查的提案',exact:true}).getByRole('button',{name:'封存审查范围',exact:true}).click();const dialog=page.getByRole('dialog',{name:'多文件变更集审查',exact:true});await dialog.getByRole('button',{name:'确认应用全部 1 项',exact:true}).click();await expect(dialog).toContainText('全部应用完成',{timeout:20000});await page.keyboard.press('Escape');}
|
||||
let result;
|
||||
try{
|
||||
await page.addInitScript(({projectId,sessionId})=>{localStorage.setItem('awb-project',projectId);localStorage.setItem('awb-session',sessionId);},{projectId:f.project.id,sessionId:f.session.id});
|
||||
await page.goto(host.url);await page.waitForLoadState('networkidle');await open();await shot('desktop-empty');
|
||||
await expect(panel().getByRole('button',{name:'封存评测计划',exact:true})).toBeDisabled();await fill('固定工程改进 · 中文完整评测');
|
||||
await panel().getByLabel('允许修改的源码文件',{exact:true}).fill('test/add.test.mjs');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect(panel().getByRole('alert')).toContainText('测试');await expect(panel().getByLabel('具体需求',{exact:true})).toHaveValue('修复加法运算,保留原测试及已有调用方式');
|
||||
await page.setViewportSize({width:390,height:844});await shot('mobile-form-scope-error');await page.setViewportSize({width:1440,height:1000});
|
||||
await panel().getByLabel('允许修改的源码文件',{exact:true}).fill('src/add.mjs');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect(panel()).toContainText('计划已封存');
|
||||
const batch=latest(),sample=batch.samples[0],original=ownerBytes(f);assert.equal(batch.evidenceMode,'local-fixture');assert.equal(host.requests.length,0);
|
||||
await page.reload();await page.waitForLoadState('networkidle');await open();await expect(panel()).toContainText(batch.title);
|
||||
mark('desktop/390px form rejects candidate test edits, keeps entered requirements, persists fixed model/source/budget and reloads the original plan');
|
||||
await panel().getByRole('button',{name:'准备同源工作区',exact:true}).click();await page.keyboard.press('Escape');assert.equal(f.store.core.worker.size,0);
|
||||
await confirm('准备同源工作区','确认准备');await expect.poll(()=>sample.status,{timeout:30000}).toBe('ready');await expect(panel()).toContainText('node --test test/*.test.mjs');assert.deepEqual(ownerBytes(f),original);await shot('desktop-ready');
|
||||
await confirm('核对并运行原版','确认运行原版');await expect.poll(()=>sample.status,{timeout:30000}).toBe('baseline_complete');assert.equal(host.requests.length,0);assert.equal(f.store.core.check_job.get(sample.baselineJobId).results[0].tests.failed,3);
|
||||
mark('explicit preparation seals two same-source checkouts and actual commands; original real three-test failure is retained with zero model requests');
|
||||
await confirm('开始候选改进','确认开始候选');await expect.poll(()=>sample.status,{timeout:30000}).toBe('awaiting_review');
|
||||
await panel().getByRole('button',{name:'审查候选改动',exact:true}).click();await apply();await parent();
|
||||
await confirm('继续候选任务','确认开始候选');const group=f.store.core.collaboration.get(sample.groupId),child=group.children.find(child=>child.evaluation.role==='candidate');
|
||||
await expect.poll(()=>f.store.session(child.sessionId).status,{timeout:15000}).toBe('awaiting_approval');await expect(panel().getByRole('button',{name:'核对并审批检查',exact:true})).toBeVisible();
|
||||
await panel().getByRole('button',{name:'核对并审批检查',exact:true}).click();await page.locator('.approval-card').getByRole('button',{name:'确认执行',exact:true}).click();await parent();
|
||||
await expect.poll(()=>sample.status,{timeout:30000}).toBe('candidate_complete');assert.deepEqual(ownerBytes(f),original);await shot('desktop-candidate-complete');
|
||||
mark('candidate navigates to its original restricted task for real change review and check approval, runs the original test/build and preserves the parent source');
|
||||
await confirm('封存当前结果','确认封存结果');await expect.poll(()=>batch.status,{timeout:15000}).toBe('completed');await panel().getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();
|
||||
const report=page.getByRole('dialog',{name:'原版与候选对照',exact:true});await expect(report).toContainText('测试 0/3 · 失败 3');await expect(report).toContainText('测试 3/3 · 失败 0');await expect(report).toContainText('不证明模型自主完成率');await shot('desktop-comparison');
|
||||
const download=page.waitForEvent('download');await report.getByRole('button',{name:'导出原报告',exact:true}).click();const file=path.join(directory,'downloaded-report.json');await (await download).saveAs(file);const downloaded=JSON.parse(fs.readFileSync(file,'utf8'));assert.deepEqual(downloaded.summary,{original:'failed',candidate:'passed'});exports.push({file,sha256:hash(fs.readFileSync(file))});
|
||||
await page.setViewportSize({width:390,height:844});await shot('mobile-comparison');await report.getByText('测试 3/3 · 失败 0',{exact:true}).scrollIntoViewIfNeeded();await expect(report.getByText('测试 3/3 · 失败 0',{exact:true})).toBeVisible();await shot('mobile-candidate-evidence');await page.keyboard.press('Escape');await page.reload();await page.waitForLoadState('networkidle');await open();await panel().getByRole('button',{name:'查看原版 / 候选对照',exact:true}).click();await expect(page.getByRole('dialog',{name:'原版与候选对照',exact:true})).toContainText('测试 0/3 · 失败 3');await page.keyboard.press('Escape');await page.setViewportSize({width:1440,height:1000});
|
||||
mark('parallel comparison preserves original failure, candidate pass, real approval/application counts and absent cost; export and refresh retain the original report and fixture label');
|
||||
await confirm('采纳此候选,准备父工程审查','确认准备采纳');await expect.poll(()=>batch.status,{timeout:30000}).toBe('awaiting_delivery');await panel().getByRole('button',{name:'审查父工程变更集',exact:true}).click();await apply(true);
|
||||
const plan=await host.runtime.checkRunner().plan(f.session,['test','build']);assert.equal((await (await host.runtime.checkRunner().start(f.session,plan.id)).promise).status,'passed');
|
||||
const delivery=await host.runtime.gitDelivery().plan(f.session,{message:'Adopt fixed engineering evaluation candidate'});assert.equal((await (await host.runtime.gitDelivery().start(f.session,delivery.id)).promise).status,'completed');await open();await panel().getByRole('button',{name:'核对父工程检查与 Git',exact:true}).click();await expect.poll(()=>batch.adoption.status).toBe('completed');assert.deepEqual(ownerBytes(f).files,original.files);await shot('desktop-delivered');
|
||||
mark('adoption forms a fresh parent review batch; completion requires applied source, fresh real parent checks and local Git delivery while owner modifications remain exact');
|
||||
hold=true;await panel().getByRole('button',{name:'新建下一批评测',exact:true}).click();await fill('停止中的候选,保留原失败分母');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect.poll(()=>f.store.core.evaluation.size).toBe(2);await expect(panel()).toContainText('停止中的候选,保留原失败分母');const stopped=latest(),stoppedSample=stopped.samples[0];
|
||||
await confirm('准备同源工作区','确认准备');await expect.poll(()=>stoppedSample.status,{timeout:30000}).toBe('ready');await confirm('核对并运行原版','确认运行原版');await expect.poll(()=>stoppedSample.status,{timeout:30000}).toBe('baseline_complete');const requests=host.requests.length;
|
||||
await confirm('开始候选改进','确认开始候选');await expect.poll(()=>host.requests.length).toBe(requests+1);await panel().getByRole('button',{name:'停止本批执行',exact:true}).click();await expect.poll(()=>host.runtime.hasActivity(),{timeout:10000}).toBe(false);await refresh();assert.notEqual(stoppedSample.status,'candidate_complete');
|
||||
await confirm('封存当前结果','确认封存结果');await expect.poll(()=>!!stoppedSample.report).toBe(true);assert.equal(stoppedSample.report.summary.candidate,'stopped');assert.equal(host.requests.length,requests+1);await page.setViewportSize({width:390,height:844});await shot('mobile-stopped');
|
||||
mark('stop waits for the owned candidate request, preserves stopped outcome and never turns cancellation into a passed sample or a replay');
|
||||
await panel().getByRole('button',{name:'新建下一批评测',exact:true}).click();await fill('来源变化时保持失效原因');await panel().getByRole('button',{name:'封存评测计划',exact:true}).click();await expect.poll(()=>f.store.core.evaluation.size).toBe(3);await expect(panel()).toContainText('来源变化时保持失效原因');const stale=latest();fs.appendFileSync(path.join(f.projectRoot,'README.md'),'\nOwner changed after plan\n');await confirm('准备同源工作区','确认准备');await expect.poll(()=>stale.samples[0].status,{timeout:10000}).toBe('failed');await expect(panel()).toContainText('来源工程已变化');await shot('mobile-source-error');
|
||||
mark('changed owner source is rejected before workspace creation and the error stays attached to the original batch instead of a generic chat reply');
|
||||
for(const batch of f.store.core.evaluation.values())for(const sample of batch.samples){if(sample.report){const file=path.join(directory,'report-'+sample.id+'.json');fs.writeFileSync(file,JSON.stringify(host.runtime.evaluationManager().readReport(f.session,batch,sample),null,2));exports.push({file,sha256:hash(fs.readFileSync(file))});}}
|
||||
assert.deepEqual(errors,[]);result={passed:true,version:JSON.parse(fs.readFileSync('package.json','utf8')).version,evidenceKind:'owned fixed response, actual isolated Git/checks and desktop/390px UI',realModelRequests:0,fixtureRequests:host.requests.length,assertions,screenshots,exports,errors};
|
||||
}catch(error){result={passed:false,error:error.stack,assertions,screenshots,exports,errors};await shot('failure').catch(()=>{});throw error;}
|
||||
finally{await browser.close();await host.runtime.stopAll();f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');fs.cpSync(f.root,path.join(directory,'original-fixture'),{recursive:true});fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));if(result.passed)fs.writeFileSync('docs/engineering-evals-ui-validation.json',JSON.stringify(result,null,2));for(const dispose of disposers.reverse())await dispose();console.log('EVIDENCE '+directory);}
|
||||
@@ -0,0 +1,145 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
|
||||
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4319';
|
||||
if (!/^http:\/\/127\.0\.0\.1:(?!4318(?:\/|$))\d+\/?$/.test(url)) throw new Error('Use an independent local QA service.');
|
||||
const root = path.resolve('.qa', 'execution-ui-' + randomUUID()), projectRoot = path.join(root, 'project'), otherRoot = path.join(root, 'other-project');
|
||||
fs.mkdirSync(path.join(projectRoot, 'test'), { recursive: true }); fs.mkdirSync(otherRoot, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
fs.writeFileSync(path.join(projectRoot, 'package.json'), JSON.stringify({ name: 'execution-ui-project', type: 'module', scripts: { test: 'node --test test/*.test.mjs', build: 'node --check src.mjs' } }));
|
||||
fs.writeFileSync(path.join(projectRoot, 'src.mjs'), 'export const add = (a, b) => a + b;\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'test/cases.test.mjs'), "import test from 'node:test'; import assert from 'node:assert/strict'; import { add } from '../src.mjs'; test('addition', () => assert.equal(add(2, 3), 5)); test('negative values', () => assert.equal(add(-1, 3), 2));");
|
||||
fs.writeFileSync(path.join(otherRoot, 'README.md'), '# Secondary project\n');
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
||||
};
|
||||
let mode = 'checks', expectedDisconnect = false, eventsPaused = false, page, selectedId = '';
|
||||
const eventConnections = new Set();
|
||||
const proxy = http.createServer((request, response) => {
|
||||
const eventStream = request.url.startsWith('/api/events?');
|
||||
if (eventStream && eventsPaused) { response.writeHead(503); response.end(); return; }
|
||||
const headers = { ...request.headers, host: new URL(url).host };
|
||||
if (headers.origin) headers.origin = url;
|
||||
const upstream = http.request(url + request.url, { method: request.method, headers }, result => { response.writeHead(result.statusCode, result.headers); result.pipe(response); });
|
||||
const connection = { response, upstream }; if (eventStream) eventConnections.add(connection);
|
||||
response.once('close', () => { eventConnections.delete(connection); upstream.destroy(); });
|
||||
upstream.once('error', () => { if (!response.destroyed) { response.writeHead(502); response.end(); } });
|
||||
request.pipe(upstream);
|
||||
});
|
||||
await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve));
|
||||
const browserUrl = 'http://127.0.0.1:' + proxy.address().port;
|
||||
const passed = [], errors = [], requestFailures = [];
|
||||
const mark = message => { passed.push(message); console.log('PASS ' + message); };
|
||||
const model = http.createServer(async (request, response) => {
|
||||
try {
|
||||
const parts = []; for await (const part of request) parts.push(part);
|
||||
const input = JSON.parse(Buffer.concat(parts).toString());
|
||||
const lastUser = input.messages.findLastIndex(message => message.role === 'user');
|
||||
const results = input.messages.slice(lastUser + 1).filter(message => message.role === 'tool').map(message => JSON.parse(message.content));
|
||||
const plan = results.find(value => value.id && value.checks && value.fingerprint);
|
||||
let call;
|
||||
if (mode === 'checks') call = !results.length ? ['inspect_project', {}] : results.length === 1 ? ['plan_checks', {}] : results.length === 2 ? ['run_checks', { plan_id: plan.id }] : null;
|
||||
if (mode === 'budget') call = ['list_files', { path: '' }];
|
||||
if (mode === 'proposal') call = !results.length ? ['propose_file', { path: 'src.mjs', content: 'export const add = (a, b) => a + b + 0;\n', reason: 'Verify reviewed task continuation' }] : null;
|
||||
response.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: call ? '' : mode === 'answer' ? '依据已保存记录完成本次回应。' : '已记录实际工程检查和日志。', ...(call ? { tool_calls: [{ id: randomUUID(), type: 'function', function: { name: call[0], arguments: JSON.stringify(call[1]) } }] } : {}) }, finish_reason: call ? 'tool_calls' : 'stop' }], usage: mode === 'budget' ? { prompt_tokens: 1000, completion_tokens: 100 } : { prompt_tokens: 40, completion_tokens: 20 } }));
|
||||
} catch (error) { response.writeHead(500, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ error: { message: error.message } })); }
|
||||
});
|
||||
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
|
||||
const state = () => api('/sessions/' + selectedId);
|
||||
const waitState = async status => { await expect.poll(async () => (await state()).status, { timeout: 20000 }).toBe(status); };
|
||||
const screenshot = name => page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' });
|
||||
const taskPanel = () => page.getByLabel('任务记录', { exact: true });
|
||||
async function newTask(content) {
|
||||
if (!await page.getByRole('button', { name: '新建任务', exact: true }).isVisible()) await page.getByRole('button', { name: '展开项目栏' }).click();
|
||||
await page.getByRole('button', { name: '新建任务', exact: true }).click(); await page.getByLabel('任务内容').fill(content); await page.getByRole('button', { name: '发送任务' }).click();
|
||||
await expect.poll(async () => (await api('/sessions')).find(value => value.title === content)?.id, { timeout: 10000 }).toBeTruthy();
|
||||
selectedId = (await api('/sessions')).find(value => value.title === content).id;
|
||||
}
|
||||
try {
|
||||
const project = await api('/projects', { root: projectRoot }), other = await api('/projects', { root: otherRoot });
|
||||
await api('/config', { ...bootstrap.config, mode: 'compatible', model: 'execution-ui-fixture', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', apiKey: 'execution-ui-fixture-key' });
|
||||
page = await context.newPage(); page.on('pageerror', error => errors.push(error.message)); page.on('requestfailed', request => { if (!expectedDisconnect) requestFailures.push(request.url() + ': ' + request.failure()?.errorText); });
|
||||
await page.goto(browserUrl); await page.waitForLoadState('networkidle');
|
||||
await page.getByLabel('当前项目').selectOption(project.id);
|
||||
await newTask('验证真实工程检查'); await waitState('awaiting_approval');
|
||||
await expect(page.locator('.approval-card')).toContainText('npm run test'); await expect(page.locator('.approval-card')).toContainText('npm run build');
|
||||
assert.equal((await state()).execution.checks.length, 0);
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitState('completed');
|
||||
await page.getByRole('button', { name: '查看任务记录' }).click(); await expect(taskPanel()).toContainText('验证真实工程检查');
|
||||
await expect(taskPanel()).toContainText('execution-ui-fixture'); await expect(taskPanel()).toContainText('240 tokens'); await expect(taskPanel()).toContainText('测试 2/2 通过');
|
||||
assert.equal((await state()).execution.checks.at(-1).status, 'passed');
|
||||
await screenshot('execution-desktop'); mark('model-selected checks wait for approval and display actual counted results');
|
||||
|
||||
await taskPanel().getByRole('button', { name: '查看检查日志' }).first().click();
|
||||
const artifact = page.getByRole('dialog', { name: '执行结果工件' }); await expect(artifact).toContainText('tests 2'); await expect(artifact).toContainText('SHA256');
|
||||
await screenshot('execution-artifact'); await artifact.getByRole('button', { name: '关闭窗口' }).click(); mark('verified artifact preview shows the real check log and hash');
|
||||
|
||||
fs.appendFileSync(path.join(projectRoot, 'src.mjs'), '\n// Source changed after verification.\n');
|
||||
await taskPanel().getByRole('button', { name: '重新识别检查计划' }).click(); await expect(taskPanel().locator('.project-checks > .record-heading .status')).toHaveText('需重新检查');
|
||||
await taskPanel().getByRole('button', { name: '执行这些检查' }).click(); const confirmation = page.getByRole('dialog', { name: '执行工程检查' });
|
||||
await expect(confirmation).toContainText('node --test'); await expect(confirmation).toContainText('node --check');
|
||||
await confirmation.getByRole('button', { name: '确认执行检查' }).click();
|
||||
await expect.poll(async () => (await state()).execution.checks.at(-1).status, { timeout: 15000 }).toBe('passed');
|
||||
await expect(taskPanel().locator('.project-checks > .record-heading .status')).toHaveText('检查通过'); mark('changed source invalidates old checks and manual confirmation runs the new plan');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: '查看任务记录' }).click();
|
||||
await expect(taskPanel()).toBeVisible(); await taskPanel().getByRole('button', { name: '查看检查日志' }).first().click();
|
||||
await expect(artifact).toBeVisible(); await screenshot('execution-mobile-artifact'); await artifact.getByRole('button', { name: '关闭窗口' }).click();
|
||||
await screenshot('execution-mobile-task'); await page.getByRole('button', { name: '关闭检视区' }).click();
|
||||
const composerBox = await page.getByLabel('任务内容').boundingBox(), headerBox = await page.locator('.work-header').boundingBox();
|
||||
assert.ok(composerBox.y >= 0 && composerBox.y + composerBox.height <= 844); assert.ok(headerBox.y >= 0); assert.ok((await page.locator('body').evaluate(node => node.scrollWidth)) <= 390);
|
||||
mark('390px task drawer, log preview, close controls and fixed composer remain operable');
|
||||
|
||||
await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
expectedDisconnect = true; eventsPaused = true; for (const connection of eventConnections) { connection.response.destroy(); connection.upstream.destroy(); } await expect(page.locator('.connection')).not.toHaveClass(/connected/);
|
||||
mode = 'answer'; const offlineSession = await api('/sessions', { projectId: other.id }); await api('/sessions/' + offlineSession.id + '/messages', { content: '断线期间的其他项目任务' });
|
||||
await expect.poll(async () => (await api('/sessions/' + offlineSession.id)).status).toBe('completed');
|
||||
eventsPaused = false; await expect(page.locator('.connection')).toHaveClass(/connected/, { timeout: 15000 });
|
||||
await page.getByRole('button', { name: '展开项目栏' }).click(); await page.getByLabel('当前项目').selectOption(other.id);
|
||||
await expect(page.locator('.session-list')).toContainText('断线期间的其他项目任务'); await page.getByLabel('当前项目').selectOption(project.id); expectedDisconnect = false;
|
||||
mark('reconnection synchronizes missed session changes in other projects');
|
||||
|
||||
const openLegacy = async () => {
|
||||
await page.getByRole('button', { name: '选择与管理模型', exact: true }).click();
|
||||
await page.getByRole('dialog', { name: '模型中心', exact: true }).locator('.mc-connection').filter({ has: page.getByRole('heading', { name: '原模型连接', exact: true }) }).getByRole('button', { name: '编辑', exact: true }).click();
|
||||
};
|
||||
await openLegacy(); const settings = page.getByRole('dialog', { name: '原连接与演示设置', exact: true });
|
||||
await settings.getByText('上下文与任务预算', { exact: true }).click(); await settings.getByLabel('单次运行token预算').fill('1000'); await settings.getByLabel('运行时间预算').fill('2'); await settings.getByLabel('任务累计token预算').fill('50000');
|
||||
await settings.getByRole('button', { name: '保存并切换原连接' }).click(); mode = 'budget'; await newTask('保留预算暂停的任务目标'); await waitState('paused');
|
||||
const original = await state(); assert.equal(original.execution.run.stopReason, 'token_limit'); assert.equal(original.approvals.length, 0);
|
||||
await openLegacy(); await settings.getByText('上下文与任务预算', { exact: true }).click(); await settings.getByLabel('单次运行token预算').fill('16000'); await settings.getByRole('button', { name: '保存并切换原连接' }).click();
|
||||
mode = 'answer'; await page.getByRole('button', { name: '继续任务', exact: true }).click(); await waitState('completed');
|
||||
const resumed = await state(); assert.equal(resumed.execution.task.id, original.execution.task.id); assert.equal(resumed.execution.runs.length, 2);
|
||||
await page.getByRole('button', { name: '查看任务记录' }).click(); await taskPanel().getByLabel('选择运行记录').selectOption(original.execution.run.id);
|
||||
await expect(taskPanel()).toContainText('到达本次 token 预算'); await expect(taskPanel()).toContainText('1,000 tokens'); await screenshot('execution-budget-history');
|
||||
mark('budget controls pause before tools and preserve the goal across recorded runs');
|
||||
|
||||
mode = 'proposal'; await newTask('应用后继续同一个工程任务'); await waitState('needs_review'); const reviewed = await state();
|
||||
await page.locator('.inspector-tabs').getByRole('button', { name: /改动/ }).click(); await page.getByRole('button', { name: '应用这项改动', exact: true }).click();
|
||||
await waitState('completed'); mode = 'checks'; await page.getByRole('button', { name: '继续原任务', exact: true }).click(); await waitState('awaiting_approval');
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).click(); await waitState('completed');
|
||||
const delivered = await state(); assert.equal(delivered.execution.task.id, reviewed.execution.task.id); assert.equal(delivered.execution.task.goal, '应用后继续同一个工程任务'); assert.equal(delivered.execution.checks.at(-1).status, 'passed');
|
||||
mark('applied proposals continue the original task into actual engineering verification');
|
||||
|
||||
assert.deepEqual(errors, []); assert.deepEqual(requestFailures, []);
|
||||
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), browser: browser.version(), passed, pageErrors: errors, requestFailures, projectRoot, model: 'controlled local HTTP fixture' };
|
||||
fs.writeFileSync('.qa/execution-ui-validation.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
if (page) { await screenshot('execution-ui-failure').catch(() => {}); fs.writeFileSync('.qa/execution-failure-dom.html', await page.content().catch(() => '')); }
|
||||
fs.writeFileSync('.qa/execution-ui-validation.json', JSON.stringify({ success: false, error: error.message, passed, pageErrors: errors, requestFailures }, null, 2)); throw error;
|
||||
} finally {
|
||||
await context.setOffline(false).catch(() => {}); await browser.close();
|
||||
if (selectedId) await api('/sessions/' + selectedId + '/stop', {}).catch(() => {});
|
||||
await api('/config', { ...bootstrap.config, mode: 'demo', apiKey: '' }).catch(() => {});
|
||||
model.closeAllConnections(); await new Promise(resolve => model.close(resolve));
|
||||
proxy.closeAllConnections(); await new Promise(resolve => proxy.close(resolve));
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID, X509Certificate } from 'node:crypto';
|
||||
import { chromium, _electron, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench } from '../tests/helpers.mjs';
|
||||
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
import { FileResults } from '../src/core/tasks/file-results.mjs';
|
||||
import { readText } from '../src/server/files.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-file-results-ui.mjs — isolated Edge/Electron existing-file receipts, changed bytes, explicit adoption, no-change delivery and late-response scope. Optional AGENT_WORKBENCH_QA_APP_ROOT. No model requests.'); process.exit(0); }
|
||||
if (process.argv.length > 2) throw new Error('Use --help.');
|
||||
const appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.'), directory = path.resolve('.qa/file-results-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const versionFile = path.join(appRoot, fs.existsSync(path.join(appRoot, 'package.json')) ? 'package.json' : 'MANIFEST.json');
|
||||
const version = JSON.parse(fs.readFileSync(versionFile, 'utf8')).version;
|
||||
const checks = [], screenshots = [], errors = [];
|
||||
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
||||
let stage = 'setup', result;
|
||||
const shot = async (page, name) => { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file, scale: 'css' }); screenshots.push(file); };
|
||||
const overflow = page => page.evaluate(() => ({ width: innerWidth, document: document.documentElement.scrollWidth })).then(value => assert.ok(value.document <= value.width + 1, JSON.stringify(value)));
|
||||
|
||||
async function mobileFlow(browser, host, f, htmlPath) {
|
||||
const initial = await host.api('/companion'); assert.ok(initial.addresses.length);
|
||||
const address = initial.addresses[0].address, reservation = http.createServer(); await new Promise(resolve => reservation.listen(0, address, resolve));
|
||||
const port = reservation.address().port; await new Promise(resolve => reservation.close(resolve));
|
||||
const gateway = await host.api('/companion/start', { address, port }), target = new URL(gateway.origin);
|
||||
const cert = await fetch(host.url + '/api/companion/certificate', { headers: { 'X-Workbench-Token': host.bootstrap.token } }); assert.equal(cert.status, 200);
|
||||
const ca = new X509Certificate(Buffer.from(await cert.arrayBuffer())).toString(); assert.equal(new X509Certificate(ca).ca, true);
|
||||
const pair = await host.api('/companion/pair', { projectIds: [f.project.id], permissions: { observe: true, approve: false, stop: false }, expiresInHours: 24 });
|
||||
let cookie = '';
|
||||
const relay = http.createServer((request, response) => {
|
||||
const forwarded = https.request(new URL(request.url, target), { method: request.method, ca, servername: '', headers: { Host: target.host, Origin: target.origin, Connection: 'close', ...(cookie ? { Cookie: cookie } : {}), ...(request.headers['content-type'] ? { 'Content-Type': request.headers['content-type'] } : {}) } }, incoming => {
|
||||
if (incoming.headers['set-cookie']?.length) cookie = incoming.headers['set-cookie'][0].split(';')[0];
|
||||
const headers = { ...incoming.headers }; delete headers['set-cookie']; response.writeHead(incoming.statusCode, headers); incoming.pipe(response);
|
||||
});
|
||||
forwarded.on('error', () => { if (!response.headersSent) response.writeHead(503); response.end(); }); request.pipe(forwarded);
|
||||
});
|
||||
await new Promise(resolve => relay.listen(0, '127.0.0.1', resolve));
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true }), mobile = await context.newPage();
|
||||
mobile.on('pageerror', error => errors.push('mobile: ' + error.message));
|
||||
try {
|
||||
await mobile.goto('http://127.0.0.1:' + relay.address().port + '/#pair=' + new URLSearchParams(new URL(pair.url).hash.slice(1)).get('pair'));
|
||||
await mobile.getByLabel('设备名称', { exact: true }).fill('当前成果观察夹具'); await mobile.getByRole('button', { name: '连接这台电脑', exact: true }).click();
|
||||
await mobile.locator('.mobile-task-row').filter({ hasText: f.session.title }).click();
|
||||
const panel = mobile.getByLabel('当前文件成果', { exact: true }); await expect(panel).toContainText('当前文件已核对');
|
||||
const row = (await host.api('/sessions/' + f.session.id + '/file-results')).results[0]; await expect(panel).toContainText(row.files[0].sha256);
|
||||
await expect(panel).toContainText(row.taskId.slice(0, 8)); await expect(panel).toContainText(row.runId.slice(0, 8)); await overflow(mobile); await shot(mobile, 'mobile-current-results');
|
||||
await expect(mobile.getByRole('button', { name: '按当前成果结束任务', exact: true })).toHaveCount(0);
|
||||
mark('mobile 390px: actual paired TLS gateway exposes current Task/Run and file SHA through observation only; no OS trust changes');
|
||||
fs.appendFileSync(htmlPath, '<!-- 手机观察期间用户编辑 -->\n'); await mobile.reload();
|
||||
await expect(mobile.getByRole('heading', { name: '项目任务', exact: true })).toBeVisible();
|
||||
await mobile.locator('.mobile-task-row').filter({ hasText: f.session.title }).click();
|
||||
await expect(mobile.getByLabel('当前文件成果', { exact: true })).toContainText('文件或要求已变化'); await overflow(mobile); await shot(mobile, 'mobile-stale-results');
|
||||
mark('mobile 390px: authenticated refresh reports stale bytes and retains original SHA without granting file/finish permissions');
|
||||
} catch (error) { await shot(mobile, 'mobile-failure').catch(() => {}); fs.writeFileSync(path.join(directory, 'mobile-failure.html'), await mobile.content().catch(() => 'unavailable')); throw error; }
|
||||
finally { await context.close(); relay.closeAllConnections(); await new Promise(resolve => relay.close(resolve)); await host.api('/companion/close', {}); }
|
||||
}
|
||||
|
||||
async function surface(kind) {
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t);
|
||||
const htmlPath = path.join(f.projectRoot, 'result.html');
|
||||
fs.writeFileSync(htmlPath, '<!doctype html><html lang="zh"><meta charset="UTF-8"><title>已有成果</title><h1>当前已有成果</h1><p>用户保存的内容</p></html>\n');
|
||||
const begin = (session, goal, title, forceNewTask = false) => {
|
||||
const run = f.store.beginRun(session, goal, false, f.store.config, { forceNewTask }); session.status = 'paused'; session.title = title; f.store.finishRun(session, run, 'delivery_incomplete'); return run;
|
||||
};
|
||||
const goal = '创建 result.html 文件,已有内容满足时沿用,无需测试';
|
||||
const originalRun = begin(f.session, goal, '核对已有HTML');
|
||||
const sourceSession = f.store.createSession(f.project.id);
|
||||
begin(sourceSession, goal, '显式采用旧成果');
|
||||
const source = new FileResults(f.store).record(sourceSession, { files: [{ path: 'result.html', sha256: readText(f.project, 'result.html').sha256 }], reason: '原任务已保存HTML成果,作为后续显式采用的来源。' }, { source: 'user' });
|
||||
begin(sourceSession, goal, '显式采用旧成果', true);
|
||||
const blocked = f.store.createSession(f.project.id); begin(blocked, '修改 result.html 文件标题,无需测试', '修改要求不能绕过');
|
||||
f.store.close();
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot });
|
||||
assert.notEqual(new URL(host.url).port, '4318'); assert.equal(host.bootstrap.config.hasKey, false);
|
||||
assert.equal(host.bootstrap.version, version);
|
||||
let browser, application, page;
|
||||
try {
|
||||
if (kind === 'edge') { browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); await page.goto(host.url); }
|
||||
else {
|
||||
const executable = desktopExecutableIdentity(appRoot).path, packaged = executable.toLowerCase().endsWith('agentworkbench.exe');
|
||||
application = await _electron.launch({ executablePath: executable, args: [...(packaged ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', f.data, '--node', process.execPath, '--port', new URL(host.url).port, '--hidden-window'], cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
||||
page = await application.firstWindow(); await (await application.browserWindow(page)).evaluate(window => { window.showInactive(); window.setContentSize(1440, 1000); });
|
||||
await expect.poll(() => page.evaluate(() => innerWidth)).toBe(1440);
|
||||
}
|
||||
page.on('pageerror', error => errors.push(kind + ': ' + error.message));
|
||||
await page.locator('.connection.connected').waitFor();
|
||||
const panel = () => page.getByLabel('当前文件成果', { exact: true });
|
||||
const select = async session => {
|
||||
if (!await page.getByLabel('当前项目', { exact: true }).isVisible()) await page.getByRole('button', { name: '展开项目栏', exact: true }).click();
|
||||
await page.getByLabel('当前项目', { exact: true }).selectOption(f.project.id);
|
||||
await page.locator('.session-list button').filter({ hasText: session.title }).click();
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await expect(panel()).toBeVisible();
|
||||
};
|
||||
const form = async () => { const details = panel().locator('.file-result-form'); if (!await details.evaluate(node => node.open)) await details.locator('summary').click(); };
|
||||
const choose = async () => { await form(); await panel().getByLabel('成果文件', { exact: true }).selectOption('result.html'); await expect(panel().getByLabel('成果文件内容')).toContainText('当前已有成果'); };
|
||||
const route = '/sessions/' + f.session.id;
|
||||
stage = kind + ': stale-save recovery'; await select(f.session); await choose();
|
||||
const reason = '核对已有HTML的标题与完整内容,沿用用户已有成果。'; await panel().getByLabel('成果采用原因').fill(reason);
|
||||
fs.appendFileSync(htmlPath, '<!-- 保存前由用户修改 -->\n');
|
||||
await panel().getByRole('button', { name: '核对并采用此文件', exact: true }).click();
|
||||
await expect(panel().getByRole('status')).toContainText('变化'); await expect(panel().getByLabel('成果采用原因')).toHaveValue(reason);
|
||||
assert.equal((await host.api(route + '/file-results')).results.length, 0);
|
||||
await panel().getByRole('button', { name: '重新读取当前文件', exact: true }).click(); await expect(panel().getByLabel('成果文件内容')).toContainText('保存前由用户修改');
|
||||
await panel().getByRole('button', { name: '核对并采用此文件', exact: true }).click();
|
||||
await expect(panel().locator('.file-result-row')).toHaveCount(1); await expect(panel()).toContainText('当前文件已核对');
|
||||
mark(kind + ': changed SHA refuses stale save, retains input, rereads actual bytes and records owned receipt'); await shot(page, kind + '-current-results');
|
||||
await panel().getByRole('button', { name: '查看原成果凭据', exact: true }).click();
|
||||
const modal = page.getByRole('dialog', { name: '执行结果工件', exact: true }); await expect(modal).toContainText('file-result'); await expect(modal).toContainText('SHA256'); await page.keyboard.press('Escape');
|
||||
await panel().getByRole('button', { name: '按当前成果结束任务', exact: true }).click();
|
||||
await expect(panel().getByRole('status')).toContainText('任务已按当前成果结束');
|
||||
const completed = await host.api(route); assert.equal(completed.status, 'completed'); assert.equal(completed.execution.task.delivery.status, 'existing_verified');
|
||||
assert.equal(completed.changes.length, 0); assert.equal(completed.execution.checks.length, 0); assert.equal(completed.execution.deliveries.length, 0); assert.equal(completed.execution.task.usage.requests, 0);
|
||||
assert.deepEqual(completed.execution.run, originalRun);
|
||||
mark(kind + ': current receipt ends no-test task, preserves old failed run and creates no file change/check/commit/model request');
|
||||
await page.reload(); await page.locator('.connection.connected').waitFor(); await select(f.session); await expect(panel()).toContainText('结束时采用已有成果');
|
||||
fs.appendFileSync(htmlPath, '<!-- 交付后用户继续编辑 -->\n'); await panel().getByRole('button', { name: '刷新成果状态', exact: true }).click(); await expect(panel()).toContainText('需要重新核对');
|
||||
await panel().getByRole('button', { name: '重新打开此文件核对', exact: true }).click(); await form(); await expect(panel().getByLabel('成果采用原因')).toHaveValue(reason);
|
||||
await expect(panel().getByLabel('成果文件内容')).toContainText('交付后用户继续编辑');
|
||||
mark(kind + ': refresh retains receipt and later file edits produce specific stale recovery without discarding reason');
|
||||
stage = kind + ': narrow current results';
|
||||
if (application) await (await application.browserWindow(page)).evaluate(window => { window.showInactive(); window.setContentSize(390, 844); }); else await page.setViewportSize({ width: 390, height: 844 });
|
||||
await expect.poll(() => page.evaluate(() => innerWidth)).toBe(390);
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click(); await expect(panel()).toBeVisible(); await panel().scrollIntoViewIfNeeded(); await overflow(page); await shot(page, kind + '-390-results');
|
||||
mark(kind + ': 390px task entry displays readable paths/hashes and actions without page overflow');
|
||||
if (application) await (await application.browserWindow(page)).evaluate(window => window.setContentSize(1440, 1000)); else await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
await expect.poll(() => page.evaluate(() => innerWidth)).toBe(1440);
|
||||
stage = kind + ': explicit adoption'; await select(sourceSession); await choose(); await panel().getByLabel('成果采用原因').fill('按当前要求再次核对完整HTML,显式采用原任务成果。');
|
||||
await expect(panel().getByRole('button', { name: '核对并采用此文件', exact: true })).toBeDisabled();
|
||||
await panel().getByLabel('成果来源').selectOption(source.result_id); await panel().getByRole('button', { name: '核对并采用此文件', exact: true }).click();
|
||||
await expect(panel().locator('.file-result-row')).toHaveCount(1); await expect(panel()).toContainText('采用自 Task');
|
||||
const adopted = (await host.api('/sessions/' + sourceSession.id + '/file-results')).results[0]; assert.equal(adopted.files[0].origin.resultId, source.result_id); assert.notEqual(adopted.files[0].origin.sha256, adopted.files[0].sha256);
|
||||
mark(kind + ': new task explicitly selects original source, preserving old/current SHA and Task/Run'); await shot(page, kind + '-adopted');
|
||||
stage = kind + ': change guard'; await select(blocked); await choose(); await panel().getByLabel('成果采用原因').fill('只核对现有文件,不能替代本次要求的标题修改。');
|
||||
await panel().getByRole('button', { name: '核对并采用此文件', exact: true }).click(); await expect(panel().locator('.file-result-row')).toHaveCount(1);
|
||||
await panel().getByRole('button', { name: '按当前成果结束任务', exact: true }).click(); await expect(panel().getByRole('status')).toContainText('包含修改');
|
||||
assert.equal((await host.api('/sessions/' + blocked.id)).status, 'paused');
|
||||
mark(kind + ': unchanged adoption cannot satisfy explicit modification, error retains current task and form');
|
||||
stage = kind + ': late response'; await select(f.session); await choose();
|
||||
const delayedReason = '延迟回执仍须归属原任务,不能写入后来选择的任务。'; await panel().getByLabel('成果采用原因').fill(delayedReason);
|
||||
let entered, release, finished, didEnter = false; const held = new Promise(resolve => { entered = resolve; }), released = new Promise(resolve => { release = resolve; }), responded = new Promise(resolve => { finished = resolve; });
|
||||
const routeErrors = [];
|
||||
const matcher = '**/api/sessions/' + f.session.id + '/file-results';
|
||||
await page.route(matcher, async route => {
|
||||
if (route.request().method() !== 'POST') return route.continue();
|
||||
try { const response = await route.fetch(); didEnter = true; entered(); await released; await route.fulfill({ response }); }
|
||||
catch (error) { routeErrors.push(error.message); entered(); }
|
||||
finally { finished(); }
|
||||
});
|
||||
try {
|
||||
await panel().getByRole('button', { name: '核对并采用此文件', exact: true }).click(); await held;
|
||||
await select(blocked); release(); await responded; await page.unroute(matcher); assert.deepEqual(routeErrors, []);
|
||||
await expect(panel()).not.toContainText(delayedReason); assert.equal((await host.api('/sessions/' + blocked.id)).status, 'paused');
|
||||
assert.equal((await host.api(route + '/file-results')).results[0].reason, delayedReason);
|
||||
mark(kind + ': delayed successful save remains with original Task after UI selection changes');
|
||||
} finally { release(); if (didEnter) await responded; await page.unroute(matcher); }
|
||||
if (kind === 'edge') { stage = 'mobile current files'; await mobileFlow(browser, host, f, htmlPath); }
|
||||
assert.equal((await host.api('/bootstrap')).config.hasKey, false);
|
||||
} catch (error) { if (page) { await shot(page, kind + '-failure').catch(() => {}); fs.writeFileSync(path.join(directory, kind + '-failure.html'), await page.content().catch(() => 'unavailable')); } throw error; }
|
||||
finally { await browser?.close(); await application?.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
}
|
||||
try { await surface('edge'); await surface('electron'); assert.deepEqual(errors, []); result = { success: true, version, appRoot, directory, checks, screenshots, errors, realModelRequests: 0, scope: 'Actual isolated Edge and Electron UI; 390px is desktop narrow validation, not a new iPhone test.' }; }
|
||||
catch (error) { result = { success: false, stage, version, appRoot, directory, checks, screenshots, errors, error: error.stack }; process.exitCode = 1; }
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/file-results-ui-latest.json', JSON.stringify(result, null, 2)); console.log('EVIDENCE ' + directory); console.log(JSON.stringify(result));
|
||||
@@ -0,0 +1,143 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
|
||||
const url = process.env.AGENT_WORKBENCH_URL;
|
||||
if (!url || !/^http:\/\/127\.0\.0\.1:(?!4318(?:\/|$))\d+\/?$/.test(url)) throw new Error('Set AGENT_WORKBENCH_URL to an independent local QA service.');
|
||||
const root = path.resolve('.qa', 'index-ui-' + randomUUID()), primaryRoot = path.join(root, 'index-ui-project'), otherRoot = path.join(root, 'index-ui-other');
|
||||
fs.mkdirSync(path.join(primaryRoot, 'src'), { recursive: true }); fs.mkdirSync(otherRoot, { recursive: true }); fs.mkdirSync('.qa/screenshots', { recursive: true });
|
||||
const sourcePath = path.join(primaryRoot, 'src', 'invoice.ts');
|
||||
const banner = Array.from({ length: 180 }, (_, line) => '// 保留的项目说明 ' + (line + 1) + ' · 原文🙂').join('\r\n') + '\r\n';
|
||||
const original = '\uFEFF' + banner + 'export function total(amount: number, discount: number) { return amount - discount; }\r\nexport const currency = "CNY";\r\n';
|
||||
fs.writeFileSync(sourcePath, original);
|
||||
fs.writeFileSync(path.join(primaryRoot, 'src', 'Card.tsx'), 'import { total } from "./invoice.js";\nexport const Card = () => <p>{total(10, 2)}</p>;\n');
|
||||
fs.writeFileSync(path.join(primaryRoot, 'legacy.py'), '# def FakeComment():\n"""def FakeString():"""\ndef repair():\n pass\n');
|
||||
fs.writeFileSync(path.join(primaryRoot, 'broken.js'), 'export function unfinished(');
|
||||
fs.writeFileSync(path.join(primaryRoot, 'package.json'), JSON.stringify({ name: 'index-ui-project', type: 'module' }));
|
||||
fs.writeFileSync(path.join(otherRoot, 'other.ts'), 'export function otherOnly() { return true; }\n');
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.equal(bootstrap.version, JSON.parse(fs.readFileSync('package.json', 'utf8')).version); assert.equal(bootstrap.config.mode, 'demo'); assert.equal(bootstrap.config.hasKey, false);
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token }, ...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
||||
};
|
||||
const primary = await api('/projects', { root: primaryRoot }), other = await api('/projects', { root: otherRoot });
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } }), page = await context.newPage();
|
||||
const passed = [], pageErrors = [], requestFailures = []; let heldRelease;
|
||||
page.on('pageerror', cause => pageErrors.push(cause.message));
|
||||
page.on('requestfailed', request => { if (request.resourceType() !== 'eventsource') requestFailures.push(request.url() + ': ' + request.failure()?.errorText); });
|
||||
const panel = () => page.getByLabel('工程索引', { exact: true });
|
||||
const indexTab = () => page.locator('.inspector-tabs').getByRole('button', { name: '索引', exact: true });
|
||||
const screenshot = name => page.screenshot({ path: '.qa/screenshots/' + name + '.png', scale: 'css' });
|
||||
async function find(query, kind = 'symbols') {
|
||||
await panel().getByLabel('索引查询').fill(query);
|
||||
if (await panel().getByLabel('索引结果类型').inputValue() !== kind) await panel().getByLabel('索引结果类型').selectOption(kind);
|
||||
else await panel().getByRole('button', { name: '查找', exact: true }).click();
|
||||
await expect(panel().getByRole('button', { name: '查找', exact: true })).toBeEnabled();
|
||||
await expect(panel().getByRole('status')).toContainText('“' + query + '”');
|
||||
}
|
||||
async function checkPosition(line) {
|
||||
await expect(page.locator('.code-line-target')).toHaveAttribute('data-line', String(line));
|
||||
await expect.poll(() => page.evaluate(() => {
|
||||
const line = document.querySelector('.code-line-target').getBoundingClientRect(), area = document.querySelector('.code-view').getBoundingClientRect();
|
||||
return line.top >= area.top && line.bottom <= area.bottom;
|
||||
})).toBe(true);
|
||||
const layout = await page.evaluate(() => ({ scrollY, header: document.querySelector('.work-header').getBoundingClientRect().top,
|
||||
composerBottom: document.querySelector('.composer-wrap').getBoundingClientRect().bottom, height: innerHeight }));
|
||||
assert.equal(layout.scrollY, 0); assert.ok(layout.header >= 0); assert.ok(layout.composerBottom <= layout.height + 1);
|
||||
}
|
||||
try {
|
||||
await page.goto(url); await page.waitForLoadState('networkidle');
|
||||
await page.getByLabel('当前项目', { exact: true }).selectOption(primary.id);
|
||||
fs.writeFileSync(path.join(root, 'initial-dom.html'), await page.content());
|
||||
assert.ok((await page.locator('.inspector-tabs button').allTextContents()).some(value => value === '索引'));
|
||||
await indexTab().click(); await expect(panel()).toBeVisible();
|
||||
await expect(panel().getByRole('button', { name: '查找', exact: true })).toBeEnabled();
|
||||
await expect(panel()).toContainText('当前项目 · index-ui-project'); await expect(panel()).toContainText('部分结果');
|
||||
assert.equal((await api('/sessions')).filter(value => value.projectId === primary.id).length, 0);
|
||||
passed.push('工程索引可独立于任务使用,实际覆盖范围与损坏文件显示明确');
|
||||
|
||||
await find('total'); await expect(panel().locator('.index-result-location')).toContainText(':181');
|
||||
await screenshot('m2-index-desktop');
|
||||
await panel().getByRole('button', { name: '打开 src/invoice.ts 第 181 行 total', exact: true }).click();
|
||||
await checkPosition(181); await screenshot('m2-index-desktop-line');
|
||||
passed.push('真实AST查询打开原文件第181行并仅滚动代码容器,输入区和顶部保持可见');
|
||||
|
||||
await indexTab().click(); await expect(panel().getByLabel('索引查询')).toHaveValue('total');
|
||||
fs.writeFileSync(sourcePath, '\uFEFF// 新说明1\r\n// 新说明2\r\n// 新说明3\r\n' + original.slice(1));
|
||||
await panel().getByRole('button', { name: '打开 src/invoice.ts 第 181 行 total', exact: true }).click();
|
||||
await expect(page.getByText('文件在索引后已变化,已打开当前内容。请回到索引刷新后重新定位。', { exact: true })).toBeVisible();
|
||||
await expect(page.locator('.code-line-target')).toHaveCount(0); await expect(page.locator('.code-view')).toContainText('新说明3');
|
||||
await indexTab().click(); await panel().getByRole('button', { name: '刷新工程索引' }).click();
|
||||
await expect(panel().getByRole('button', { name: '查找', exact: true })).toBeEnabled();
|
||||
await panel().getByRole('button', { name: '打开 src/invoice.ts 第 184 行 total', exact: true }).click(); await checkPosition(184);
|
||||
passed.push('查询词保留;过期SHA不高亮旧位置,增量刷新后正确定位第184行');
|
||||
|
||||
await indexTab().click(); let held = false;
|
||||
const gate = new Promise(resolve => { heldRelease = resolve; });
|
||||
const routePattern = new RegExp('/api/projects/' + primary.id + '/code-query\\?');
|
||||
await page.route(routePattern, async route => { const response = await route.fetch(); held = true; await gate; await route.fulfill({ response }); });
|
||||
await panel().getByRole('button', { name: '刷新工程索引' }).click(); await expect.poll(() => held).toBe(true);
|
||||
await page.getByLabel('当前项目', { exact: true }).selectOption(other.id);
|
||||
await expect(panel()).toContainText('当前项目 · index-ui-other'); await expect(panel()).toContainText('otherOnly');
|
||||
heldRelease(); await page.waitForTimeout(300); await page.unroute(routePattern);
|
||||
await expect(panel()).not.toContainText('src/invoice.ts'); await expect(panel()).toContainText('otherOnly');
|
||||
passed.push('切换项目时延迟返回的旧索引不覆盖新项目结果');
|
||||
|
||||
await page.getByLabel('当前项目', { exact: true }).selectOption(primary.id);
|
||||
await expect(panel().getByRole('button', { name: '查找', exact: true })).toBeEnabled(); await find('repair');
|
||||
await expect(panel()).toContainText('Python 候选 · 需读源码核对'); await expect(panel().locator('.index-result-location')).toContainText('legacy.py');
|
||||
await find('FakeString'); await expect(panel().getByText('当前范围没有匹配', { exact: true })).toBeVisible();
|
||||
await find('invoice.js', 'imports'); await expect(panel()).toContainText('文件候选 → src/invoice.ts');
|
||||
await panel().getByText('查看索引范围与依据', { exact: true }).click(); await expect(panel()).toContainText('不证明实际运行时模块解析');
|
||||
passed.push('Python候选、字符串误报排除和局部导入文件候选保留真实证据标签');
|
||||
|
||||
const current = await api('/projects/' + primary.id + '/file?path=src%2Finvoice.ts'), before = fs.readFileSync(sourcePath);
|
||||
const session = await api('/sessions', { projectId: primary.id });
|
||||
const proposed = await api('/sessions/' + session.id + '/patch-proposals', { path: 'src/invoice.ts', base_sha256: current.sha256, reason: '精确调整金额与币种', edits: [
|
||||
{ old_text: 'return amount - discount;', new_text: 'return Math.max(0, amount - discount);' }, { old_text: 'currency = "CNY"', new_text: 'currency = "USD"' }] });
|
||||
assert.deepEqual(fs.readFileSync(sourcePath), before);
|
||||
await page.evaluate(id => localStorage.setItem('awb-session', id), session.id); await page.reload(); await page.waitForLoadState('networkidle');
|
||||
await page.locator('.inspector-tabs').getByRole('button', { name: /^改动/ }).click();
|
||||
await expect(page.locator('.patch-detail')).toHaveText('精确补丁 · 2 处替换 · 保留 CRLF 换行 · 保留 BOM');
|
||||
await expect.poll(() => page.evaluate(() => {
|
||||
const row = document.querySelector('.diff-removed').getBoundingClientRect(), area = document.querySelector('.diff-view').getBoundingClientRect();
|
||||
return row.top > area.top && row.bottom <= area.bottom;
|
||||
})).toBe(true);
|
||||
await page.evaluate(() => { document.querySelector('.diff-view').scrollTop = 0; });
|
||||
await page.getByRole('button', { name: '定位首处改动', exact: true }).click();
|
||||
await expect.poll(() => page.evaluate(() => document.querySelector('.diff-view').scrollTop > 0)).toBe(true);
|
||||
await screenshot('m2-index-patch-desktop'); await page.getByRole('button', { name: '应用这项改动', exact: true }).click();
|
||||
await expect(page.getByText('这项改动已写入项目。', { exact: true })).toBeVisible();
|
||||
const expected = before.toString('utf8').replace('return amount - discount;', 'return Math.max(0, amount - discount);').replace('currency = "CNY"', 'currency = "USD"');
|
||||
assert.deepEqual(fs.readFileSync(sourcePath), Buffer.from(expected)); assert.equal((await api('/sessions/' + proposed.id)).changes[0].status, 'applied');
|
||||
passed.push('两处唯一锚点成为待审核提案,真实界面应用后BOM/CRLF和未选择原字节保留');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await expect(page.getByRole('button', { name: '查看文件', exact: true })).toBeVisible();
|
||||
await page.getByRole('button', { name: '查看文件', exact: true }).click(); await indexTab().click();
|
||||
await panel().getByRole('button', { name: '刷新工程索引' }).click(); await expect(panel().getByRole('button', { name: '查找', exact: true })).toBeEnabled();
|
||||
await find('total'); await panel().getByText('查看索引范围与依据', { exact: true }).click();
|
||||
const bounds = await page.evaluate(() => ({ page: document.documentElement.scrollWidth <= innerWidth,
|
||||
panel: document.querySelector('.code-index-panel').scrollWidth <= document.querySelector('.code-index-panel').clientWidth + 1,
|
||||
tabs: document.querySelector('.inspector-tabs').scrollWidth <= document.querySelector('.inspector-tabs').clientWidth + 1 }));
|
||||
assert.deepEqual(bounds, { page: true, panel: true, tabs: true }); await screenshot('m2-index-mobile');
|
||||
await panel().getByRole('button', { name: '打开 src/invoice.ts 第 184 行 total', exact: true }).click(); await checkPosition(184); await screenshot('m2-index-mobile-line');
|
||||
passed.push('390px索引查询、范围和完整指纹可读且无横向溢出,实际文件行定位通过');
|
||||
|
||||
await indexTab().click(); const errorPattern = new RegExp('/api/projects/' + primary.id + '/code-query\\?'); let failNext = true;
|
||||
await page.route(errorPattern, route => { if (failNext) { failNext = false; return route.fulfill({ status: 409, contentType: 'application/json', body: JSON.stringify({ error: '建立索引期间文件已变化,请刷新工程索引。' }) }); } return route.continue(); });
|
||||
await panel().getByRole('button', { name: '刷新工程索引' }).click(); await expect(panel().getByRole('alert')).toContainText('索引未刷新');
|
||||
await expect(panel().locator('.index-result').first()).toBeDisabled(); await panel().getByRole('button', { name: '重试索引' }).click();
|
||||
await expect(panel().getByRole('alert')).toHaveCount(0); await expect(panel().locator('.index-result').first()).toBeEnabled(); await page.unroute(errorPattern);
|
||||
passed.push('索引刷新失败保留归属和解释,旧结果禁用并可重试恢复');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
const result = { checkedAt: new Date().toISOString(), version: bootstrap.version, passed, pageErrors, requestFailures, root, primaryProjectId: primary.id, otherProjectId: other.id };
|
||||
fs.writeFileSync('.qa/index-ui-result.json', JSON.stringify(result, null, 2) + '\n'); console.log(JSON.stringify(result, null, 2));
|
||||
} catch (cause) {
|
||||
fs.writeFileSync('.qa/index-ui-failure.json', JSON.stringify({ passed, pageErrors, requestFailures, message: cause.message, root }, null, 2));
|
||||
await screenshot('m2-index-failure').catch(() => {}); throw cause;
|
||||
} finally { heldRelease?.(); await browser.close(); }
|
||||
@@ -0,0 +1,101 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
|
||||
const previous = JSON.parse(fs.readFileSync('docs/live-react-engineering-validation.json', 'utf8'));
|
||||
const url = 'http://127.0.0.1:4318', directory = path.resolve('.qa', 'live-browser-' + randomUUID()); fs.mkdirSync(directory);
|
||||
const sha = value => createHash('sha256').update(Buffer.isBuffer(value) || typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
|
||||
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
||||
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
let token, sessionId, taskId, current, result; const decisions = [], continuations = [], startedAt = Date.now();
|
||||
async function api(route, input) {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { Connection: 'close', 'Content-Type': 'application/json', ...(token ? { 'X-Workbench-Token': token } : {}) },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
||||
const value = await response.json(); if (!response.ok) throw new Error(value.error ?? 'HTTP ' + response.status); return value;
|
||||
}
|
||||
const root = previous.workspaceRoot, git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', root, ...args], { env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }).trim();
|
||||
const sourceFiles = () => {
|
||||
const walk = (relative = '') => fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
||||
if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'out'].includes(entry.name)) return [];
|
||||
assert.equal(entry.isSymbolicLink(), false); const file = path.join(relative, entry.name);
|
||||
return entry.isDirectory() ? walk(file) : [{ path: file, sha256: sha(fs.readFileSync(path.join(root, file))) }];
|
||||
}); return walk();
|
||||
};
|
||||
async function original(job) {
|
||||
let text = '', offset = 0, page;
|
||||
do { page = await api('/sessions/' + sessionId + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=16000'); text += page.content; offset += page.content.length; } while (page.truncated);
|
||||
assert.equal(sha(text), job.resultSha256); const value = JSON.parse(text); assert.equal(value.taskId, taskId);
|
||||
fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text);
|
||||
for (const image of job.screenshots) {
|
||||
const response = await fetch(url + '/api/sessions/' + sessionId + '/browser-jobs/' + job.id + '/images/' + image.artifactId, { headers: { 'X-Workbench-Token': token }, signal: AbortSignal.timeout(15000) });
|
||||
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), image.sha256);
|
||||
fs.writeFileSync(path.join(directory, job.id + '-' + image.label + '.png'), bytes);
|
||||
} return value;
|
||||
}
|
||||
try {
|
||||
assert.equal(previous.success, true);
|
||||
const before = await api('/bootstrap'); token = before.token; assert.equal(before.version, '1.10.1'); assert.equal(before.config.mode, 'compatible'); assert.equal(before.config.hasKey, true);
|
||||
const project = before.projects.find(value => value.id === previous.projectId); assert.ok(project); assert.equal(project.root, root);
|
||||
assert.ok(!before.sessions.some(value => value.projectId === project.id && ['running', 'awaiting_approval', 'stopping'].includes(value.status)));
|
||||
const baseline = { source: sourceFiles(), head: git(['rev-parse', 'HEAD']), index: sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))),
|
||||
originalSession: sha(canonical(await api('/sessions/' + previous.sessionId + '/export'))), configuration: sha(canonical(before.config)) };
|
||||
// exportedAt is a transport timestamp, not a persisted session field.
|
||||
const old = await api('/sessions/' + previous.sessionId + '/export'); delete old.exportedAt; baseline.originalSession = sha(canonical(old));
|
||||
const created = await api('/sessions', { projectId: project.id }); sessionId = created.id;
|
||||
const prompt = '请使用工作台内置plan_browser、run_browser、read_browser_result验证当前已经完成的React工程实际页面。工程代码已经通过测试并提交,本次只检查页面,不修改文件、不执行终端、不准备Git提交。'
|
||||
+ '先读取实际DOM/元素,再由你根据实际页面选择CSS交互和断言。桌面1280×900:初始3项任务,状态筛选pending为2项;搜索带空格的“ 接口 ”只显示编写接口;将该任务标为完成后待办筛选下变空,统计应变为完成2项、待办工时5小时。'
|
||||
+ '再以390×844独立浏览器核对手机布局可见且没有横向溢出。每份计划最多20步,需要实际断言与原截图,observed不算通过。根据原错误准备新计划,最后基于原结果向我报告,不把截图或推测当成功。';
|
||||
await api('/sessions/' + sessionId + '/messages', { content: prompt });
|
||||
const decided = new Set();
|
||||
while (Date.now() - startedAt < 210000) {
|
||||
current = await api('/sessions/' + sessionId); taskId ??= current.execution?.task?.id; assert.equal(current.execution.task.id, taskId);
|
||||
if (current.changes.some(value => ['pending', 'conflict', 'applied'].includes(value.status))) throw new Error('The page-only task proposed source changes; approval was not given.');
|
||||
if (current.status === 'awaiting_approval') {
|
||||
const approval = current.approvals.find(value => value.status === 'pending' && !decided.has(value.id));
|
||||
if (approval) {
|
||||
assert.ok(approval.browserPlanId, 'Only the explicitly scoped page execution may be approved.');
|
||||
const plan = current.execution.browserPlans.find(value => value.id === approval.browserPlanId);
|
||||
assert.equal(plan.taskId, taskId); assert.equal(plan.projectId, project.id); assert.equal(plan.build.directory, 'dist'); assert.equal(plan.path, '/');
|
||||
assert.ok([1280, 390].includes(plan.viewport.width)); assert.ok(plan.steps.length <= 20);
|
||||
decided.add(approval.id); decisions.push({ id: approval.id, taskId, planId: plan.id, fingerprint: plan.fingerprint, viewport: plan.viewport, steps: plan.steps.length });
|
||||
await api('/sessions/' + sessionId + '/approvals/' + approval.id, { allow: true });
|
||||
}
|
||||
} else if (['failed', 'cancelled'].includes(current.status)) throw new Error(current.failure?.message ?? current.status);
|
||||
else if (['completed', 'paused'].includes(current.status)) {
|
||||
const passed = current.execution.browserJobs.filter(job => job.taskId === taskId && job.status === 'passed');
|
||||
if (current.status === 'completed' && passed.some(job => current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 1280)
|
||||
&& passed.some(job => current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 390)) break;
|
||||
if (continuations.length >= 2 || current.status !== 'paused' || current.execution.run.stopReason !== 'step_limit') throw new Error('Actual model did not complete both requested page checks: ' + current.status + '/' + current.execution.run.stopReason);
|
||||
continuations.push({ previousRunId: current.execution.run.id, reason: current.execution.run.stopReason, at: new Date().toISOString() });
|
||||
await api('/sessions/' + sessionId + '/resume', {});
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 75));
|
||||
}
|
||||
assert.equal(current.status, 'completed', 'Actual model task exceeded the acceptance time bound.');
|
||||
const jobs = current.execution.browserJobs.filter(job => job.taskId === taskId), records = [];
|
||||
for (const job of jobs) { assert.equal(job.cleanup?.exited, true); records.push({ job, original: await original(job), viewport: current.execution.browserPlans.find(plan => plan.id === job.planId).viewport }); }
|
||||
const desktop = records.find(value => value.viewport.width === 1280 && value.job.status === 'passed' && ['fill', 'select', 'click'].every(action => value.job.steps.some(step => step.action === action)));
|
||||
assert.ok(desktop, 'Desktop acceptance needs real search/filter/click, not only screenshots.');
|
||||
const text = desktop.original.pages.at(-1).text; assert.match(text, /完成任务\s*2/); assert.match(text, /待办工时\s*5\s*小时/);
|
||||
const mobile = records.find(value => value.viewport.width === 390 && value.job.status === 'passed'); assert.ok(mobile); assert.ok(mobile.original.pages.at(-1).viewport.scrollWidth <= 390);
|
||||
assert.deepEqual(sourceFiles(), baseline.source); assert.equal(git(['rev-parse', 'HEAD']), baseline.head); assert.equal(sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))), baseline.index);
|
||||
const afterOld = await api('/sessions/' + previous.sessionId + '/export'); delete afterOld.exportedAt; assert.equal(sha(canonical(afterOld)), baseline.originalSession);
|
||||
assert.equal(sha(canonical((await api('/bootstrap')).config)), baseline.configuration);
|
||||
const exported = await api('/sessions/' + sessionId + '/export'); fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n');
|
||||
result = { success: true, version: '1.10.1', testedAt: new Date().toISOString(), kind: 'configured-model-inner-browser-acceptance', model: before.config.model,
|
||||
directory, projectId: project.id, sessionId, taskId, root, sourceCommit: baseline.head, sourcePreserved: true, indexPreserved: true, originalEngineeringTaskPreserved: true,
|
||||
modelStatus: current.status, usage: current.execution.task.usage, elapsedSeconds: (Date.now() - startedAt) / 1000, decisions, continuations,
|
||||
jobs: records.map(value => ({ id: value.job.id, status: value.job.status, viewport: value.viewport, assertions: value.job.assertions, steps: value.job.steps, resultSha256: value.job.resultSha256, screenshots: value.job.screenshots, cleanup: value.job.cleanup })),
|
||||
scope: 'Configured model chooses CSS plans and reads original fields through the self harness. The verifier confirms only sealed local page plans and validates original DOM/PNG, source/index and previous Task. This read-only page sample is separate from code-repair success rate and Purge benefit.',
|
||||
credentialsHandling: 'The original in-memory Provider performs requests; no model key is read, copied, printed or saved.' };
|
||||
} catch (error) {
|
||||
if (sessionId) { await api('/sessions/' + sessionId + '/stop', {}).catch(() => {}); current = await api('/sessions/' + sessionId).catch(() => current); }
|
||||
result = { success: false, version: '1.10.1', testedAt: new Date().toISOString(), directory, sessionId, taskId, modelStatus: current?.status, usage: current?.execution?.task?.usage, decisions, continuations, error: error.stack }; process.exitCode = 1;
|
||||
if (sessionId) { const exported = await api('/sessions/' + sessionId + '/export').catch(() => null); if (exported) fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n'); }
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('docs/browser-engine-live-model-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, sessionId, taskId, modelStatus: result.modelStatus, usage: result.usage, elapsedSeconds: result.elapsedSeconds, jobs: result.jobs?.map(value => ({ id: value.id, status: value.status, viewport: value.viewport, assertions: value.assertions })), error: result.error, directory }));
|
||||
@@ -0,0 +1,134 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
const root = path.resolve('docs/live-context-o-evidence');
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const file = name => { const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target; };
|
||||
const read = name => JSON.parse(fs.readFileSync(file(name), 'utf8'));
|
||||
const manifest = read('manifest.json');
|
||||
assert.equal(manifest.kind, 'owned-live-context-o-evidence');
|
||||
const names = new Set(); let bytes = 0;
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(!names.has(row.path)); names.add(row.path);
|
||||
const content = fs.readFileSync(file(row.path)); assert.equal(content.length, row.bytes); assert.equal(sha(content), row.sha256, row.path); bytes += content.length;
|
||||
}
|
||||
const owned = reference => { assert.ok(names.has(reference)); return read(reference); };
|
||||
const real = [owned(manifest.references.preflight), owned(manifest.references.originalNode), owned(manifest.references.fixedNode)];
|
||||
let requests = 0, reportedTokens = 0, unknownRequests = 0, estimatedUnknownTokens = 0;
|
||||
for (const [index, result] of real.entries()) {
|
||||
const count = result.requests ?? result.usage.requests;
|
||||
assert.ok(count <= (index === 0 ? 1 : 8));
|
||||
const total = result.usage.total_tokens ?? result.usage.totalTokens;
|
||||
assert.ok(total <= (index === 0 ? 16000 : 90000));
|
||||
const unknown = result.usage.unknownRequests ?? 0;
|
||||
const estimatedUnknown = result.usage.estimatedUnknownTokens ?? 0;
|
||||
assert.ok(total + estimatedUnknown <= (index === 0 ? 16000 : 90000));
|
||||
requests += count; reportedTokens += total; unknownRequests += unknown;
|
||||
estimatedUnknownTokens += estimatedUnknown;
|
||||
assert.equal(result.modelWeightsAuthenticated, false);
|
||||
const folder = index === 0 ? 'preflight-resumed' : index === 1 ? 'node' : 'node-after-fix';
|
||||
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, count);
|
||||
let actualTotal = 0, actualUnknown = 0;
|
||||
for (const attempt of attempts) {
|
||||
assert.equal(attempt.connectionId, '93d0cf6d-4f32-4918-9da8-6f125aa8a6e5');
|
||||
assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
|
||||
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions');
|
||||
assert.equal(attempt.requestUrl, 'https://api.apikey.fan/v1/chat/completions');
|
||||
const request = result.captures.find(c => c.attemptId === attempt.id && c.kind === 'request');
|
||||
assert.ok(request); const requestBytes = fs.readFileSync(file('run/' + request.path));
|
||||
assert.equal(sha(requestBytes), request.sha256); assert.equal(request.sha256, attempt.requestSha256);
|
||||
assert.equal(JSON.parse(requestBytes).model, attempt.model);
|
||||
const response = result.captures.find(c => c.attemptId === attempt.id && c.kind === 'response');
|
||||
assert.ok(response); const responseBytes = fs.readFileSync(file('run/' + response.path));
|
||||
assert.equal(sha(responseBytes), response.sha256); assert.equal(response.sha256, attempt.responseSha256);
|
||||
if (attempt.status !== 'completed') {
|
||||
assert.equal(result.success, false); assert.equal(index, 2); assert.equal(attempt.status, 'failed');
|
||||
assert.equal(attempt.httpStatus, 502); assert.equal(attempt.usage, null); assert.equal(attempt.usageStatus, 'unknown'); actualUnknown++;
|
||||
assert.equal(attempt.modelIdentity.status, 'incomplete'); assert.equal(attempt.modelIdentity.complete, false);
|
||||
assert.equal(attempt.diagnostic.origin, 'upstream'); assert.equal(attempt.diagnostic.retryAfterMs, 60000);
|
||||
assert.equal(attempt.diagnostic.upstream.headers['content-type'], 'text/html; charset=UTF-8');
|
||||
assert.match(responseBytes.toString('utf8'), /502: Bad gateway/);
|
||||
assert.equal(attempt.diagnostic.upstream.artifactSha256, response.sha256);
|
||||
assert.equal(attempt.diagnostic.upstream.capture.complete, true);
|
||||
continue;
|
||||
}
|
||||
const events = responseBytes.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim()).filter(line => line && line !== '[DONE]').map(line => JSON.parse(line));
|
||||
const models = [...new Set(events.map(event => event.model).filter(Boolean))]; assert.deepEqual(models, ['gpt-6.1-sol']);
|
||||
const usage = events.findLast(event => event.usage)?.usage; assert.ok(usage);
|
||||
assert.equal(usage.prompt_tokens, attempt.usage.prompt_tokens); assert.equal(usage.completion_tokens, attempt.usage.completion_tokens);
|
||||
actualTotal += usage.total_tokens ?? usage.prompt_tokens + usage.completion_tokens;
|
||||
assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true);
|
||||
}
|
||||
assert.equal(actualTotal, total);
|
||||
assert.equal(actualUnknown, unknown);
|
||||
if (index > 0) {
|
||||
const restoration = owned('run/' + folder + '/restoration.json'); assert.equal(restoration.passed, true);
|
||||
assert.equal(restoration.originalCredentialRevision, 1); assert.equal(restoration.finalCredentialRevision, 1);
|
||||
if (index === 2) {
|
||||
assert.equal(restoration.originalPublicProjectsExact, true);
|
||||
assert.equal(restoration.originalAttemptsExact, owned('run/node-after-fix/original-attempts.json').length);
|
||||
assert.equal(restoration.rawConfigExceptRevisionExact, true); assert.equal(restoration.registryExceptDeclaredRevisionTimestampOrderExact, true);
|
||||
assert.equal(restoration.otherMetadataRowsExact, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.equal(real[0].passed, true); assert.equal(real[0].actualTools, 0);
|
||||
assert.equal(real[1].success, false); assert.equal(real[1].stopReason, 'context_limit');
|
||||
assert.equal(real[1].usage.requests, 2); assert.equal(real[1].usage.totalTokens, 11269);
|
||||
assert.equal(real[2].success, false); assert.equal(real[2].state, 'failed'); assert.equal(real[2].usage.requests, 2);
|
||||
assert.equal(real[2].usage.totalTokens, 4838); assert.equal(real[2].usage.unknownRequests, 1); assert.equal(real[2].usage.estimatedUnknownTokens, 11095);
|
||||
assert.deepEqual(real[2].approvals, []); assert.deepEqual(real[2].applications, []); assert.equal(real[2].modelImplementationSuppliedBySupervisor, false);
|
||||
for (const name of ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']) {
|
||||
assert.deepEqual(fs.readFileSync(file('run/node/source-before/' + name)), fs.readFileSync(file('run/node/source-after/' + name)));
|
||||
assert.deepEqual(fs.readFileSync(file('run/node-after-fix/source-before/' + name)), fs.readFileSync(file('run/node-after-fix/source-after/' + name)));
|
||||
}
|
||||
const formal = owned(manifest.references.formal), freeze = owned(manifest.references.freeze), release = owned(manifest.references.release);
|
||||
assert.equal(formal.success, true); assert.equal(formal.pid, 36552); assert.equal(formal.hasKey, true);
|
||||
assert.equal(formal.version, '1.24.5'); assert.equal(formal.sourceCommit, manifest.sourceCommit); assert.equal(freeze.sourceCommit, manifest.sourceCommit);
|
||||
assert.equal(release.documentation.sourceCommit, manifest.sourceCommit);
|
||||
assert.equal(formal.retention.originalRows, 'exact'); assert.equal(formal.retention.originalExports, 'exact');
|
||||
assert.equal(formal.retention.originalArtifactBytes, 'exact'); assert.equal(formal.retention.originalModelCenter, 'exact');
|
||||
const browserReview = owned('run/release/browser-review-result.json'), nativeReview = owned('run/release/native-review-result.json');
|
||||
assert.equal(browserReview.success, true); assert.equal(browserReview.views.length, 2); assert.equal(browserReview.actualImagesReviewed, true);
|
||||
assert.deepEqual(browserReview.errors, []); assert.deepEqual(browserReview.rejectedWrites, []);
|
||||
assert.equal(nativeReview.success, true); assert.equal(nativeReview.version, manifest.version); assert.equal(nativeReview.checked.length, 3);
|
||||
assert.equal(nativeReview.noTaskStart, true); assert.equal(nativeReview.realModelRequests, 0);
|
||||
const archives = owned('release/archives-validation.json'), portable = owned('release/release-validation.json');
|
||||
assert.equal(archives.success, true); assert.equal(archives.version, manifest.version); assert.equal(archives.archives.length, 3);
|
||||
for (const archive of archives.archives) { assert.equal(archive.crcPassed, true); assert.equal(archive.manifestAndDirectoryMatch, true); }
|
||||
assert.equal(portable.success, true); assert.equal(portable.appRoot, release.portableRoot);
|
||||
const rebuild = owned('release/independent-source-rebuild/result.json');
|
||||
assert.equal(rebuild.success, true); assert.equal(rebuild.sourceCommit, manifest.sourceCommit); assert.equal(rebuild.inputCount, 552);
|
||||
assert.deepEqual(rebuild.mismatches, []);
|
||||
for (const log of rebuild.logs) assert.equal(sha(fs.readFileSync(file('release/independent-source-rebuild/' + log.name + '.log'))), log.sha256);
|
||||
const installation = owned('release/isolated-install/result.json');
|
||||
assert.equal(installation.success, true); assert.equal(installation.version, manifest.version); assert.equal(installation.results.length, 12);
|
||||
for (const { proof, script } of installation.installedUI.proofs) {
|
||||
assert.equal(proof.success, true); assert.equal(proof.version, manifest.version); assert.equal(proof.realModelRequests, 0);
|
||||
assert.equal(proof.checks.length, script === 'verify-file-results-ui.mjs' ? 16 : 9); assert.deepEqual(proof.errors, []);
|
||||
owned('ui/installed/' + path.basename(proof.directory ?? proof.root) + '/result.json');
|
||||
}
|
||||
let sourceUiGroups = 0;
|
||||
for (const [folder, count] of [['model-selection-k-3bd2d5b6-b0c2-4592-89a7-5221f614cc91', 7],
|
||||
['model-identity-ui-71450f91-9e8f-48d7-b7b9-ea6c69bb1b28', 3], ['task-plan-ui-5b6fee37-1b8d-46e4-984f-f4202e6a66e3', 8],
|
||||
['messages-ui-m-c3f7030c-d9c0-4eba-a608-b5c4c9429180', 7]]) {
|
||||
const proof = owned('ui/source/' + folder + '/result.json');
|
||||
assert.equal(proof.success, true); assert.equal(proof.version ?? proof.testedVersion, manifest.version);
|
||||
assert.equal((proof.checks ?? proof.passed).length, count); sourceUiGroups += count;
|
||||
}
|
||||
for (const label of ['source', 'portable', 'installed']) {
|
||||
const result = owned('run/compiled-context-' + label + '/result.json');
|
||||
assert.equal(result.success, true); assert.equal(result.requests, 3); assert.equal(result.externalModelRequests, 0);
|
||||
assert.equal(result.contextLimit, 16000); assert.equal(result.outputReservation, 2048); assert.equal(result.safetyTokens, 1600);
|
||||
assert.equal(result.rawToolEvidenceVerified, true); assert.equal(result.stopReason, 'verification_incomplete');
|
||||
}
|
||||
if (real[2].success) {
|
||||
assert.ok(real[2].commit); assert.equal(real[2].tests.total, 9); assert.equal(real[2].tests.passed, 9); assert.equal(real[2].tests.failed, 0);
|
||||
assert.equal(real[2].identitiesValid, true);
|
||||
}
|
||||
console.log(JSON.stringify({ success: true, manifestSha256: sha(fs.readFileSync(file('manifest.json'))), files: manifest.files.length, bytes,
|
||||
requests, reportedTokens, unknownRequests, estimatedUnknownTokens, originalNodeDelivered: false, fixedNodeDelivered: real[2].success,
|
||||
sourceUiGroups, isolatedInstallerCases: installation.results.length, installedFileUiCases: 25,
|
||||
originalPurgeBenefitTested: false, sourceCommit: manifest.sourceCommit, candidate: manifest.candidate,
|
||||
scope: 'Offline original-byte, outgoing/requested/reported identifiers, usage, bounded samples and declared restoration checks. No model requests, credential access, task replay or weight authentication.' }));
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { RequestPlanner, historyGroups, verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-live-context-retention.mjs OUTPUT [retained|original] — offline replay of sealed E source/context, no model calls. OUTPUT must be a new directory.');
|
||||
process.exit(0);
|
||||
}
|
||||
assert.ok(process.argv[2], 'Choose a new output directory.');
|
||||
const destination = path.resolve(process.argv[2]);
|
||||
assert.ok(!fs.existsSync(destination), 'Keep prior evidence; choose a new directory.');
|
||||
fs.mkdirSync(destination, { recursive: true });
|
||||
const root = path.resolve('docs/live-original-purge-e-evidence');
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
||||
const manifest = read('manifest.json');
|
||||
for (const entry of manifest.entries) {
|
||||
const raw = fs.readFileSync(path.join(root, entry.path));
|
||||
assert.equal(raw.length, entry.bytes); assert.equal(sha(raw), entry.sha256);
|
||||
}
|
||||
const sample = manifest.samples.find(sample => sample.name === 'react');
|
||||
const exported = read('owned-exports/' + sample.sessionId + '.json');
|
||||
const original = exported.session;
|
||||
const rows = read('receipts/owned-records.json');
|
||||
const originalCore = {};
|
||||
for (const row of rows) (originalCore[row.kind] ??= new Map()).set(row.id, JSON.parse(row.payload));
|
||||
const artifacts = new Map(manifest.artifacts.map(artifact => [artifact.id, artifact]));
|
||||
function rawArtifact(id) { return fs.readFileSync(path.join(root, artifacts.get(id).file), 'utf8'); }
|
||||
const plans = [...originalCore.context_plan.values()].filter(plan => plan.sessionId === sample.sessionId).sort((a, b) => a.requestIndex - b.requestIndex);
|
||||
const reports = [];
|
||||
for (const prior of plans.filter(plan => plan.requestIndex >= 7)) {
|
||||
const store = new Store(path.join(destination, 'request-' + prior.requestIndex));
|
||||
try {
|
||||
for (const [kind, map] of Object.entries(originalCore)) store.core[kind] = structuredClone(map);
|
||||
const readNew = store.artifactStore.read.bind(store.artifactStore);
|
||||
store.artifactStore.read = artifact => artifacts.has(artifact.id) ? rawArtifact(artifact.id) : readNew(artifact);
|
||||
const session = structuredClone(original);
|
||||
store.sessions.set(session.id, session);
|
||||
store.save = () => {}; // Scratch planner records only; never persist original sessions.
|
||||
const project = store.project.bind(store);
|
||||
store.project = id => id === exported.project.id ? exported.project : project(id);
|
||||
session.providerMessages = session.providerMessages.slice(0, prior.originalMessages);
|
||||
session.changes = session.changes.filter(change => change.createdAt <= prior.createdAt);
|
||||
const run = store.core.run.get(prior.runId), task = store.core.task.get(prior.taskId);
|
||||
// Restore this request's exact remaining credits, not the final run's usage.
|
||||
for (const [record, remaining] of [[run, prior.budget.runRemainingTokens], [task, prior.budget.taskRemainingTokens]]) {
|
||||
record.usage = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, estimatedUnknownTokens: 0 };
|
||||
record.budget.maxTokens = remaining ?? 0;
|
||||
}
|
||||
run.consumption = []; // Context selection only; no fabricated upstream usage.
|
||||
const request = JSON.parse(rawArtifact(prior.requestArtifactId));
|
||||
const wire = request.variants?.[0].body ?? request;
|
||||
const config = { ...run.model, modelRouting: run.model.routing, streaming: wire.stream, nativeParameters: run.model.nativeParameters ?? {} };
|
||||
const planner = new RequestPlanner(store);
|
||||
const prepared = planner.prepare(session, run, config, { system: request.messages[0].content, tools: wire.tools });
|
||||
assert.equal(prepared.record.budget.inputBudget, prior.budget.inputBudget);
|
||||
assert.ok(prepared.record.estimatedInputTokens <= prepared.record.budget.inputBudget);
|
||||
verifyPlanEvidence(store, session, prepared.record);
|
||||
const sources = [];
|
||||
for (const group of historyGroups(session.providerMessages)) for (const message of group.messages.filter(message => message.role === 'tool')) {
|
||||
const call = group.messages[0].tool_calls.find(call => call.id === message.tool_call_id);
|
||||
const value = JSON.parse(message.content);
|
||||
if (call.function.name === 'read_file' && !value.truncated && value.start_line === 1 && value.end_line === value.total_lines) {
|
||||
sources.push({ path: value.path, content: value.content });
|
||||
}
|
||||
}
|
||||
function strings(value, out = []) {
|
||||
if (typeof value === 'string') {
|
||||
out.push(value);
|
||||
for (const line of [value, ...value.split('\n').filter(line => /^[\[{]/.test(line))]) { try { strings(JSON.parse(line), out); } catch {} }
|
||||
} else if (value && typeof value === 'object') for (const item of Object.values(value)) strings(item, out);
|
||||
return out;
|
||||
}
|
||||
const sent = strings(prepared.messages);
|
||||
const report = { request: prior.requestIndex, originalInputTokens: prior.estimatedInputTokens,
|
||||
inputTokens: prepared.record.estimatedInputTokens, inputBudget: prior.budget.inputBudget,
|
||||
softTarget: prior.budget.targetInputTokens,
|
||||
sources: [...new Map(sources.map(source => [source.path, source])).values()].map(source => ({ path: source.path,
|
||||
characters: source.content.length, retained: sent.some(text => text.includes(source.content)) })) };
|
||||
reports.push(report);
|
||||
fs.writeFileSync(path.join(destination, 'request-' + prior.requestIndex + '.json'), JSON.stringify({ report, plan: prepared.record, body: prepared.body }, null, 2), { flag: 'wx' });
|
||||
} finally { store.close(); }
|
||||
}
|
||||
const result = { schemaVersion: 1, manifestSha256: sha(fs.readFileSync(path.join(root, 'manifest.json'))),
|
||||
scope: 'Offline context-selection counterfactual using original E system/tools/history/source bytes and remaining credits. No model requests or task execution. Historical outcomes unchanged.',
|
||||
realModelRequests: 0, reports };
|
||||
if (process.argv[3] === 'retained') assert.ok(reports.every(report => report.sources.length === 6 && report.sources.every(source => source.retained)));
|
||||
if (process.argv[3] === 'original') {
|
||||
assert.ok(reports.every(report => report.inputTokens === report.originalInputTokens));
|
||||
assert.deepEqual(reports.map(report => report.sources.filter(source => source.retained).length), [6, 5, 5, 4, 2, 2]);
|
||||
}
|
||||
fs.writeFileSync(path.join(destination, 'result.json'), JSON.stringify(result, null, 2) + '\n', { flag: 'wx' });
|
||||
console.log(JSON.stringify(result));
|
||||
@@ -0,0 +1,132 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
const root = path.resolve('docs/live-delivery-p-evidence');
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const file = name => {
|
||||
assert.equal(typeof name, 'string'); assert.ok(!/[\\\x00\r\n]/.test(name) && !name.split('/').includes('..'));
|
||||
const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target;
|
||||
};
|
||||
const read = name => JSON.parse(fs.readFileSync(file(name), 'utf8'));
|
||||
const manifest = read('manifest.json'); assert.equal(manifest.kind, 'owned-live-delivery-p-evidence');
|
||||
const names = new Set(); let bytes = 0;
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(!names.has(row.path)); names.add(row.path);
|
||||
const content = fs.readFileSync(file(row.path)); assert.equal(content.length, row.bytes); assert.equal(sha(content), row.sha256, row.path); bytes += content.length;
|
||||
}
|
||||
const list = (folder = root) => fs.readdirSync(folder, { withFileTypes: true }).flatMap(entry => {
|
||||
assert.equal(entry.isSymbolicLink(), false); const target = path.join(folder, entry.name);
|
||||
return entry.isDirectory() ? list(target) : [path.relative(root, target).replaceAll('\\', '/')];
|
||||
});
|
||||
assert.deepEqual(list().sort(), ['manifest.json', ...names].sort());
|
||||
const owned = name => { assert.ok(names.has(name)); return read(name); };
|
||||
const initial = owned(manifest.references.initial), preflight = owned(manifest.references.preflight), node = owned(manifest.references.node);
|
||||
const final = owned(manifest.references.final), network = owned(manifest.references.network);
|
||||
const connectionId = '5b4daed9-b2f7-4de0-b949-573eba70c7b3';
|
||||
for (const [folder, outcome] of [['preflight', preflight], ['node', node]]) {
|
||||
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, 1);
|
||||
const attempt = attempts[0]; assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
|
||||
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions'); assert.equal(attempt.streaming, false);
|
||||
assert.equal(attempt.requestUrl, 'https://jojocode.com/v1/chat/completions'); assert.equal(attempt.credentialRevision, 1);
|
||||
assert.equal(attempt.modelIdentity.policy, 'require-match');
|
||||
const request = outcome.captures.find(value => value.kind === 'request'); assert.ok(request);
|
||||
assert.equal(request.attemptId, attempt.id); assert.ok(names.has('run/' + request.path));
|
||||
const requestBytes = fs.readFileSync(file('run/' + request.path));
|
||||
assert.equal(sha(requestBytes), request.sha256); assert.equal(request.sha256, attempt.requestSha256); assert.equal(requestBytes.length, attempt.requestBytes);
|
||||
const body = JSON.parse(requestBytes); assert.equal(body.model, 'gpt-6.1-sol'); assert.notEqual(body.stream, true);
|
||||
const restoration = owned('run/' + folder + '/restoration.json'); assert.equal(restoration.passed, true);
|
||||
if (folder === 'preflight') {
|
||||
assert.equal(preflight.passed, true); assert.equal(body.tools?.length ?? 0, 0); assert.equal(body.max_tokens, 256);
|
||||
assert.equal(preflight.actualTools, 0); assert.equal(attempt.status, 'completed'); assert.equal(attempt.usageStatus, 'reported');
|
||||
assert.equal(attempt.modelIdentity.status, 'matched'); assert.equal(attempt.modelIdentity.complete, true);
|
||||
const response = preflight.captures.find(value => value.kind === 'response'); assert.ok(response);
|
||||
const responseBytes = fs.readFileSync(file('run/' + response.path)); assert.equal(sha(responseBytes), response.sha256);
|
||||
assert.equal(response.sha256, attempt.responseSha256); assert.equal(response.sha256, preflight.identity.responseSource.sha256);
|
||||
const original = JSON.parse(responseBytes); assert.equal(original.model, 'gpt-6.1-sol'); assert.deepEqual(original.usage, attempt.usage);
|
||||
assert.equal(original.usage.total_tokens, 112); assert.deepEqual(preflight.usage, original.usage);
|
||||
assert.equal(original.choices[0].message.tool_calls?.length ?? 0, 0);
|
||||
assert.ok(original.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce));
|
||||
assert.equal(restoration.originalConfigRegistryAndPoolExact, true); assert.equal(restoration.inspectorClosed, true);
|
||||
} else {
|
||||
assert.equal(body.max_tokens, 2048); assert.ok(body.tools.length > 0); assert.equal(node.success, false);
|
||||
assert.equal(attempt.status, 'failed'); assert.equal(attempt.httpStatus, null); assert.equal(attempt.usage, null); assert.equal(attempt.usageStatus, 'unknown');
|
||||
assert.equal(attempt.responseArtifactId, undefined); assert.equal(node.captures.length, 1);
|
||||
assert.equal(attempt.modelIdentity.complete, false); assert.equal(attempt.diagnostic.code, 'UND_ERR_CONNECT_TIMEOUT');
|
||||
assert.equal(attempt.diagnostic.network.stage, 'connect'); assert.equal(attempt.diagnostic.network.replaySafety, 'before_model_send');
|
||||
assert.equal(attempt.diagnostic.network.responseReceived, false); assert.equal(attempt.diagnostic.retryDecision, 'attempt_limit');
|
||||
assert.equal(attempt.diagnostic.network.transport.connectTimeoutMs, 10000);
|
||||
assert.equal(node.usage.requests, 1); assert.equal(node.usage.totalTokens, 0); assert.equal(node.usage.unknownRequests, 1); assert.equal(node.usage.estimatedUnknownTokens, 10141);
|
||||
const session = owned('run/node/session.json'), plan = session.execution.contextPlans[0];
|
||||
assert.equal(plan.tokenReservation.reservedTokens, node.usage.estimatedUnknownTokens);
|
||||
assert.equal(plan.budget.contextLimit, 16000); assert.equal(plan.budget.outputReservation, 2048); assert.equal(plan.budget.safetyTokens, 1600);
|
||||
assert.equal(session.execution.run.budget.maxSteps, 8); assert.equal(session.execution.run.budget.maxTokens, 90000);
|
||||
assert.equal(session.execution.run.toolCalls, 0); assert.deepEqual(session.execution.checks, []); assert.deepEqual(session.changes, []);
|
||||
assert.deepEqual(node.approvals, []); assert.deepEqual(node.applications, []); assert.equal(node.modelImplementationSuppliedBySupervisor, false);
|
||||
assert.equal(restoration.budgetsAndSelectionRestored, true); assert.equal(restoration.poolsUnchanged, true);
|
||||
assert.equal(restoration.originalCredentialRevision, 1); assert.equal(restoration.finalCredentialRevision, 1);
|
||||
assert.equal(restoration.registryExceptDeclaredRevisionTimestampOrderAndIdentityDefaultsExact, true);
|
||||
assert.deepEqual(restoration.identityDefaultNormalization, { modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' });
|
||||
}
|
||||
}
|
||||
assert.equal(network.realModelRequests, 0); assert.equal(network.httpRequests, 0); assert.equal(network.keyValuesReadOrSent, false);
|
||||
assert.equal(network.tlsCertificateValidationEnabled, true); assert.equal(network.productConfigurationChanged, false);
|
||||
assert.equal(network.results.length, 4);
|
||||
const trials = network.results.flatMap(value => value.trials); assert.equal(trials.length, 12); assert.equal(trials.filter(value => value.success).length, 11);
|
||||
const failure = trials.find(value => !value.success); assert.equal(failure.mode, 'ipv6-only-diagnostic'); assert.equal(failure.code, 'QA_TLS_TIMEOUT');
|
||||
assert.ok(trials.filter(value => value.success).every(value => value.certificateAuthorized === true));
|
||||
assert.equal(final.passed, true); assert.equal(final.pid, 36552); assert.equal(final.version, '1.24.5');
|
||||
assert.equal(final.originalProjects, 59); assert.equal(final.originalSessions, 75); assert.equal(final.originalArtifacts, 3605); assert.equal(final.originalAttempts, 207);
|
||||
assert.equal(final.finalProjects, 61); assert.equal(final.finalSessions, 77); assert.equal(final.addedAttempts, 2);
|
||||
assert.equal(final.originalBudgetsSelectionAndPoolRestored, true); assert.equal(final.originalSessionExportsRowsAndArtifactsExact, true);
|
||||
assert.equal(final.exactSinceNodeRestoration, true); assert.equal(final.inspectorClosed, true); assert.equal(final.keyReadOrResaved, false);
|
||||
const snapshot = owned('run/final/snapshot.json');
|
||||
for (const row of initial.baseline.exports) assert.deepEqual(snapshot.exports.find(value => value.id === row.id), row);
|
||||
for (const table of ['sessions', 'records', 'events', 'outbox']) {
|
||||
const rows = new Map(snapshot.database.tables[table].rows.map(value => [value.id, value]));
|
||||
for (const row of initial.baseline.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row);
|
||||
}
|
||||
const artifacts = new Map(snapshot.persistent.artifacts.map(value => [value.path, value]));
|
||||
for (const row of initial.baseline.persistent.artifacts) assert.deepEqual(artifacts.get(row.path), row);
|
||||
assert.deepEqual(snapshot.modelCenter.selected, initial.center.selected); assert.deepEqual(snapshot.modelCenter.pools, initial.center.pools);
|
||||
assert.deepEqual(final.connectionRevision, [3, 5]); assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision, [61, 65]);
|
||||
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
||||
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
const beforeMeta = owned('run/node/original-public-metadata.json'), afterMeta = owned('run/node/final-public-metadata.json');
|
||||
assert.deepEqual({ ...afterMeta.config, configRevision: beforeMeta.config.configRevision }, beforeMeta.config);
|
||||
const registry = structuredClone(afterMeta.registry); registry.revision = beforeMeta.registry.revision;
|
||||
const currentConnection = registry.connections.find(value => value.id === connectionId), oldConnection = beforeMeta.registry.connections.find(value => value.id === connectionId);
|
||||
assert.equal(currentConnection.revision, oldConnection.revision + 2);
|
||||
currentConnection.revision = oldConnection.revision; currentConnection.updatedAt = oldConnection.updatedAt;
|
||||
const expected = structuredClone(beforeMeta.registry); Object.assign(expected.connections.find(value => value.id === connectionId), {
|
||||
modelIdentityPolicy: 'record', modelIdentityAliases: [], modelIdentityAliasSource: '' });
|
||||
for (const value of [expected, registry]) value.connections.sort((a, b) => a.id.localeCompare(b.id));
|
||||
assert.deepEqual(canonical(registry), canonical(expected));
|
||||
for (const name of ['src/amount.mjs', 'src/summary.mjs', 'src/report.mjs']) assert.deepEqual(fs.readFileSync(file('run/node/source-before/' + name)), fs.readFileSync(file('run/node/source-after/' + name)));
|
||||
const git = owned('run/node/git-final.json'), baseline = owned('run/node/source-identity.json');
|
||||
assert.equal(git.head, baseline.head); assert.equal(git.stagedAfter, baseline.stagedBefore); assert.equal(git.status, baseline.status); assert.equal(git.diff, '');
|
||||
assert.equal(manifest.releaseSourceCommit, owned('provenance/release-1.24.5.json').sourceCommit);
|
||||
assert.equal(final.productSourceChanged, false); assert.equal(final.nodeDelivered, false); assert.equal(final.purgeBenefitTested, false);
|
||||
let gitIndex;
|
||||
if (process.argv.includes('--index')) {
|
||||
const evidenceRoot = path.relative(process.cwd(), root).replaceAll('\\', '/');
|
||||
const manifestBytes = fs.readFileSync(file('manifest.json'));
|
||||
const entries = [...manifest.files, { path: 'manifest.json', bytes: manifestBytes.length, sha256: sha(manifestBytes) }];
|
||||
const output = execFileSync('git', ['cat-file', '--batch'], { input: entries.map(row => ':' + evidenceRoot + '/' + row.path).join('\n') + '\n',
|
||||
maxBuffer: 100 * 1024 * 1024, windowsHide: true });
|
||||
let offset = 0, indexBytes = 0;
|
||||
for (const entry of entries) {
|
||||
const end = output.indexOf(10, offset), header = output.subarray(offset, end).toString('utf8');
|
||||
const match = header.match(/^[a-f0-9]{40} blob (\d+)$/); assert.ok(match, header);
|
||||
const size = Number(match[1]); offset = end + 1;
|
||||
const content = output.subarray(offset, offset + size); offset += size;
|
||||
assert.equal(output[offset++], 10); assert.equal(size, entry.bytes, entry.path); assert.equal(sha(content), entry.sha256, entry.path); indexBytes += size;
|
||||
}
|
||||
assert.equal(offset, output.length); gitIndex = { passed: true, files: entries.length, bytes: indexBytes, exactOriginalBytes: true };
|
||||
}
|
||||
console.log(JSON.stringify({ success: true, files: manifest.files.length, bytes, manifestSha256: sha(fs.readFileSync(file('manifest.json'))),
|
||||
requests: 2, reportedTokens: 112, unknownRequests: 1, estimatedUnknownTokens: 10141, nodeDelivered: false,
|
||||
tlsHandshakes: trials.length, tlsPassed: 11, tlsTimedOut: 1, originalDataAndDefaultsVerified: true, productVersion: '1.24.5',
|
||||
originalPurgeBenefitTested: false, ...(gitIndex ? { gitIndex } : {}),
|
||||
scope: 'Offline original bytes, frozen configuration, accounting and declared original-state restoration; no network, model or credential access.' }));
|
||||
@@ -0,0 +1,187 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
const root = path.resolve('docs/live-delivery-r-evidence');
|
||||
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
||||
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
||||
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
const digest = value => sha(JSON.stringify(canonical(value)));
|
||||
const file = name => {
|
||||
assert.equal(typeof name, 'string'); assert.ok(!/[\\\x00\r\n]/.test(name) && !name.split('/').includes('..'));
|
||||
const target = path.resolve(root, name); assert.ok(target.startsWith(root + path.sep)); return target;
|
||||
};
|
||||
const read = name => JSON.parse(fs.readFileSync(file(name)));
|
||||
const manifest = read('manifest.json'); assert.equal(manifest.kind, 'owned-live-delivery-r-evidence');
|
||||
const names = new Set(); let totalBytes = 0;
|
||||
for (const row of manifest.files) {
|
||||
assert.ok(!names.has(row.path)); names.add(row.path);
|
||||
const bytes = fs.readFileSync(file(row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256, row.path); totalBytes += bytes.length;
|
||||
}
|
||||
const list = (directory = root) => fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
|
||||
assert.equal(entry.isSymbolicLink(), false); const target = path.join(directory, entry.name);
|
||||
return entry.isDirectory() ? list(target) : [path.relative(root, target).replaceAll('\\','/')];
|
||||
});
|
||||
assert.deepEqual(list().sort(), ['manifest.json', ...names].sort());
|
||||
const owned = name => { assert.ok(names.has(name)); return read(name); };
|
||||
const initial = owned(manifest.references.initial), preflight = owned(manifest.references.preflight), node = owned(manifest.references.node);
|
||||
const final = owned(manifest.references.final), network = owned(manifest.references.network), proof = owned(manifest.references.ownership);
|
||||
const reading = owned(manifest.references.reading), budget = owned(manifest.references.budget), artifacts = owned(manifest.references.artifacts).artifacts;
|
||||
const frozen = owned('contract/manifest.json'), connectionId = '5b4daed9-b2f7-4de0-b949-573eba70c7b3';
|
||||
assert.equal(frozen.modelRequestsAtFreeze, 0); assert.equal(frozen.maxNodeRequests, 8); assert.equal(frozen.maxNodeTokens, 90000);
|
||||
assert.equal(frozen.contextLimit, 16000); assert.equal(frozen.maxOutputTokens, 2048);
|
||||
assert.equal(sha(fs.readFileSync(file('run/current.json'))), frozen.originalBaselineSha256);
|
||||
assert.equal(sha(fs.readFileSync(file('provenance/frozen-plan.md'))), frozen.planSha256);
|
||||
const gitBytes = (commit, relative) => execFileSync('git', ['show', commit + ':' + relative], { maxBuffer: 2 * 1024 * 1024, windowsHide: true });
|
||||
for (const row of frozen.files) {
|
||||
const bytes = fs.readFileSync(file('contract/' + row.path)); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
|
||||
assert.deepEqual(bytes, gitBytes(manifest.experimentPlanCommit, 'docs/live-delivery-r-contract/' + row.path));
|
||||
}
|
||||
assert.deepEqual(fs.readFileSync(file('contract/manifest.json')), gitBytes(manifest.experimentPlanCommit, 'docs/live-delivery-r-contract/manifest.json'));
|
||||
for (const row of frozen.dependencySources) assert.equal(sha(gitBytes(manifest.experimentBaseCommit, row.path)), row.sha256);
|
||||
for (const row of budget.sourceInputs) {
|
||||
const bytes = fs.readFileSync(file('run/' + row.capturedPath)); assert.equal(sha(bytes), row.sha256);
|
||||
assert.deepEqual(bytes, gitBytes(manifest.releaseSourceCommit, row.path));
|
||||
}
|
||||
assert.equal(manifest.releaseSourceCommit, owned('provenance/release-1.24.6.json').sourceCommit);
|
||||
assert.equal(network.connectionId, connectionId); assert.equal(network.connectionRevision, 5);
|
||||
assert.equal(network.status, 'passed'); assert.equal(network.stages.dns.status, 'passed');
|
||||
assert.deepEqual(network.trials.map(row => row.mode), ['auto','ipv4','ipv6']);
|
||||
assert.ok(network.trials.every(row => row.status === 'passed' && row.certificateAuthorized === true));
|
||||
assert.equal(network.httpRequests, 0); assert.equal(network.modelRequests, 0); assert.equal(network.credentialAccess, false);
|
||||
assert.equal(network.route.kind, 'direct-endpoint-only'); assert.equal(network.route.modelApiRouteVerified, false);
|
||||
assert.equal(owned('run/network/restoration.json').originalStateExactExceptNewDiagnosticHistory, true);
|
||||
let totalRequests = 0, reportedTokens = 0;
|
||||
const allAttempts = [];
|
||||
for (const [folder, outcome, expectedCount] of [['preflight',preflight,1],['node',node,6]]) {
|
||||
const attempts = owned('run/' + folder + '/attempts.json'); assert.equal(attempts.length, expectedCount);
|
||||
allAttempts.push(...attempts); assert.equal(outcome.captures.length, expectedCount * 2);
|
||||
for (const attempt of attempts) {
|
||||
assert.equal(attempt.connectionId, connectionId); assert.equal(attempt.connectionRevision, folder === 'preflight' ? 5 : 6);
|
||||
assert.equal(attempt.credentialRevision, 1); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
|
||||
assert.equal(attempt.model, 'gpt-6.1-sol'); assert.equal(attempt.protocol, 'chat-completions'); assert.equal(attempt.streaming, false);
|
||||
assert.equal(attempt.requestUrl, 'https://jojocode.com/v1/chat/completions'); assert.equal(attempt.status, 'completed');
|
||||
assert.equal(attempt.modelIdentity.policy, 'require-match'); assert.equal(attempt.modelIdentity.status, 'matched');
|
||||
assert.equal(attempt.modelIdentity.complete, true); assert.deepEqual(attempt.modelIdentity.reportedModels, ['gpt-6.1-sol']);
|
||||
assert.equal(attempt.usageStatus, 'reported'); assert.equal(attempt.responseRedacted, false);
|
||||
const captures = Object.fromEntries(outcome.captures.filter(row => row.attemptId === attempt.id).map(row => [row.kind,row]));
|
||||
const requestBytes = fs.readFileSync(file('run/' + captures.request.path)), responseBytes = fs.readFileSync(file('run/' + captures.response.path));
|
||||
assert.equal(sha(requestBytes), attempt.requestSha256); assert.equal(sha(requestBytes), captures.request.sha256);
|
||||
assert.equal(sha(responseBytes), attempt.responseSha256); assert.equal(sha(responseBytes), captures.response.sha256);
|
||||
assert.equal(attempt.modelIdentity.responseSource.sha256, attempt.responseSha256); assert.equal(requestBytes.length, attempt.requestBytes);
|
||||
const body = JSON.parse(requestBytes), raw = JSON.parse(responseBytes); assert.equal(body.model, 'gpt-6.1-sol'); assert.notEqual(body.stream,true);
|
||||
assert.equal(raw.model, 'gpt-6.1-sol'); assert.deepEqual(raw.usage, attempt.usage);
|
||||
assert.ok(raw.usage.completion_tokens <= body.max_tokens); reportedTokens += raw.usage.total_tokens; totalRequests++;
|
||||
if (folder === 'preflight') {
|
||||
assert.equal(body.tools?.length ?? 0, 0); assert.equal(body.max_tokens, 256); assert.equal(preflight.actualTools, 0); assert.equal(preflight.passed,true);
|
||||
assert.equal(raw.choices[0].message.tool_calls?.length ?? 0, 0); assert.ok(raw.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce));
|
||||
assert.deepEqual(raw.usage, preflight.usage); assert.equal(raw.usage.total_tokens, 4492);
|
||||
} else {
|
||||
assert.equal(body.max_tokens,2048); assert.ok(body.tools.length > 0);
|
||||
assert.deepEqual(body.messages.filter(row => row.role === 'user').map(row => row.content), [fs.readFileSync(file('run/node/goal.txt'),'utf8')]);
|
||||
}
|
||||
}
|
||||
assert.equal(owned('run/' + folder + '/restoration.json').passed,true);
|
||||
}
|
||||
assert.equal(totalRequests,7); assert.equal(reportedTokens,77063); assert.equal(node.success,false); assert.equal(node.stopReason,'token_limit');
|
||||
assert.equal(node.usage.requests,6); assert.equal(node.usage.totalTokens,72571); assert.equal(node.usage.unknownRequests,0);
|
||||
assert.equal(node.usage.estimatedUnknownTokens,0); assert.deepEqual(node.applications,[]); assert.equal(node.modelImplementationSuppliedBySupervisor,false);
|
||||
const session = owned('run/node/session.json'); assert.deepEqual(session.changes,[]); assert.equal(session.execution.run.toolCalls,12);
|
||||
assert.equal(session.execution.checks.length,1); const checks=session.execution.checks[0]; assert.equal(checks.status,'failed');
|
||||
assert.deepEqual(checks.results.find(row => row.name==='test').tests,{total:9,passed:0,failed:9,cancelled:0,skipped:0});
|
||||
assert.equal(checks.results.find(row => row.name==='build').status,'passed'); assert.equal(node.approvals.length,1); assert.equal(node.approvals[0].allow,true);
|
||||
assert.equal(node.approvals[0].command,'npm run test\nnpm run build');
|
||||
assert.deepEqual(session.execution.run.budgetPause,budget.pause); assert.equal(budget.pause.remainingTokens,17429);
|
||||
assert.equal(budget.pause.tokenReservation.reservedTokens,29623); assert.equal(budget.pause.requiredInputTokens,12352);
|
||||
assert.ok(budget.pause.requiredInputTokens<=budget.pause.inputBudget); assert.ok(budget.pause.tokenReservation.reservedTokens>budget.pause.remainingTokens);
|
||||
assert.equal(budget.pause.runBudget.maxTokens,90000); assert.equal(budget.pause.runBudget.maxSteps,8);
|
||||
assert.equal(budget.pause.contextLimit,16000); assert.equal(budget.pause.safetyTokens,1600); assert.equal(budget.pause.outputReservation,2048);
|
||||
for (const plan of session.execution.contextPlans) {
|
||||
assert.ok(plan.estimatedInputTokens<=plan.budget.inputBudget);
|
||||
assert.ok(plan.tokenReservation.reservedTokens<=Math.min(plan.budget.runRemainingTokens,plan.budget.taskRemainingTokens));
|
||||
assert.equal(plan.requestAttempts,1);
|
||||
}
|
||||
const rawFourth=JSON.parse(fs.readFileSync(file('run/'+node.captures.filter(row=>row.kind==='response')[3].path)));
|
||||
assert.equal(rawFourth.usage.prompt_tokens,27164); assert.equal(rawFourth.usage.prompt_tokens_details.cached_tokens,24192);
|
||||
const testSource=fs.readFileSync(file('run/node/source-before/test/expense.test.mjs'),'utf8').replaceAll('\r\n','\n');
|
||||
const filePages=reading.pages.filter(row=>row.path==='test/expense.test.mjs');
|
||||
assert.deepEqual(filePages.map(row=>[row.offset,row.next_offset]),[[0,473],[473,875],[875,1271]]);
|
||||
for(const row of filePages){assert.equal(row.content,testSource.slice(row.offset,row.next_offset));assert.equal(row.total_characters,2584);}
|
||||
assert.equal(budget.fileCoverage.missingCharacters,1313); assert.equal(budget.pageSizing[2].readMessageBytes,1024);
|
||||
const wrapper=reading.pages.find(row=>row.kind==='tool-result'&&row.artifact_id===filePages[0].artifactId);
|
||||
assert.ok(wrapper); assert.equal(wrapper.next_offset,495); assert.equal(wrapper.total_characters,891);
|
||||
const artifactMap=new Map(artifacts.map(row=>[row.id,row])); assert.equal(artifacts.length,61);
|
||||
for(const artifact of artifacts){
|
||||
assert.ok(final.ownedSessions.includes(artifact.sessionId)); assert.ok(final.ownedProjects.includes(artifact.projectId));
|
||||
const bytes=fs.readFileSync(file('run/'+artifact.capturedPath)); assert.equal(sha(bytes),artifact.sha256); assert.equal(bytes.length,artifact.bytes);
|
||||
}
|
||||
for(const result of checks.results){const artifact=artifactMap.get(result.logArtifactId);assert.ok(artifact);assert.equal(artifact.kind,'check-log');}
|
||||
const snapshot=owned('run/final/snapshot.json');
|
||||
for(const item of initial.baseline.exports)assert.deepEqual(snapshot.exports.find(row=>row.id===item.id),item);
|
||||
for(const table of ['sessions','records','events','outbox']){
|
||||
const before=initial.baseline.database.tables[table],after=snapshot.database.tables[table];assert.equal(after.schemaSha256,before.schemaSha256);
|
||||
const rows=new Map(after.rows.map(row=>[row.id,row]));for(const row of before.rows)assert.deepEqual(rows.get(row.id),row);
|
||||
assert.equal(proof.additions[table].length,after.count-before.count);
|
||||
for(const row of proof.additions[table]){
|
||||
assert.ok(final.ownedSessions.includes(row.sessionId));if(row.projectId)assert.ok(final.ownedProjects.includes(row.projectId));
|
||||
assert.equal(rows.get(row.id).sha256,row.sha256);
|
||||
}
|
||||
}
|
||||
assert.equal(proof.additions.records.filter(row=>row.kind==='artifact').length,artifacts.length);
|
||||
for(const artifact of artifacts)assert.ok(proof.additions.records.some(row=>row.kind==='artifact'&&row.id===artifact.id));
|
||||
const originalProjectRows=new Map(initial.baseline.database.tables.projects.rows.map(row=>[row.id,row]));
|
||||
const finalProjectRows=new Map(snapshot.database.tables.projects.rows.map(row=>[row.id,row]));
|
||||
for(const row of proof.projects){
|
||||
assert.equal(row.actualSha256,finalProjectRows.get(row.id).sha256);
|
||||
if(row.originalSha256){assert.equal(row.originalSha256,originalProjectRows.get(row.id).sha256);assert.equal(row.normalizedSha256,row.originalSha256);}
|
||||
else assert.ok(final.ownedProjects.includes(row.id));
|
||||
}
|
||||
assert.equal(proof.projects.filter(row=>row.originalSha256).length,61);
|
||||
assert.equal(proof.projects.filter(row=>row.owned).length,2); assert.equal(proof.projectInsertionOffset,2);
|
||||
const oldArtifactMap=new Map(initial.baseline.persistent.artifacts.map(row=>[row.path,row]));
|
||||
const nextArtifactMap=new Map(snapshot.persistent.artifacts.map(row=>[row.path,row]));
|
||||
for(const [name,row]of oldArtifactMap)assert.deepEqual(nextArtifactMap.get(name),row);
|
||||
const newFiles=snapshot.persistent.artifacts.filter(row=>!oldArtifactMap.has(row.path));assert.equal(newFiles.length,50);
|
||||
assert.ok(newFiles.every(row=>artifacts.some(artifact=>artifact.sha256===row.sha256&&artifact.bytes===row.bytes)));
|
||||
assert.deepEqual(snapshot.modelCenter.selected,initial.center.selected);assert.deepEqual(snapshot.modelCenter.pools,initial.center.pools);
|
||||
const beforeMeta=owned('run/node/original-public-metadata.json'),afterMeta=owned('run/node/final-public-metadata.json');
|
||||
const finalMetadataRows=new Map(snapshot.database.tables.metadata.rows.map(row=>[row.id,row]));
|
||||
const oldAttempts=owned('run/preflight/original-attempts.json');assert.equal(oldAttempts.length,209);
|
||||
assert.deepEqual(owned('run/node/original-attempts.json'),[...oldAttempts,...owned('run/preflight/attempts.json')]);
|
||||
assert.equal(finalMetadataRows.get('modelAttempts').sha256,digest({key:'modelAttempts',value:JSON.stringify([...oldAttempts,...allAttempts])}));
|
||||
for(const [key,value]of [['config',afterMeta.config],['modelRegistry',afterMeta.registry]])assert.equal(finalMetadataRows.get(key).sha256,digest({key,value:JSON.stringify(value)}));
|
||||
const rawNetwork={...network};delete rawNetwork.stale;delete rawNetwork.currentConnectionRevision;
|
||||
assert.equal(finalMetadataRows.get('modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify([rawNetwork])}));
|
||||
for(const row of initial.baseline.database.tables.metadata.rows)if(!['config','modelRegistry','modelAttempts'].includes(row.id))assert.deepEqual(finalMetadataRows.get(row.id),row);
|
||||
assert.deepEqual([...finalMetadataRows.keys()].filter(key=>!initial.baseline.database.tables.metadata.rows.some(row=>row.id===key)),['modelNetworkDiagnostics']);
|
||||
assert.deepEqual({...afterMeta.config,configRevision:beforeMeta.config.configRevision},beforeMeta.config);
|
||||
const registry=structuredClone(afterMeta.registry);registry.revision=beforeMeta.registry.revision;
|
||||
const connection=registry.connections.find(row=>row.id===connectionId),oldConnection=beforeMeta.registry.connections.find(row=>row.id===connectionId);
|
||||
assert.equal(connection.revision,oldConnection.revision+2);connection.revision=oldConnection.revision;connection.updatedAt=oldConnection.updatedAt;
|
||||
const expected=structuredClone(beforeMeta.registry);for(const value of [expected,registry])value.connections.sort((a,b)=>a.id.localeCompare(b.id));
|
||||
assert.deepEqual(canonical(registry),canonical(expected));assert.deepEqual(owned('run/node/restoration.json').identityDefaultNormalization,{});
|
||||
for(const name of ['src/amount.mjs','src/summary.mjs','src/report.mjs'])assert.deepEqual(fs.readFileSync(file('run/node/source-before/'+name)),fs.readFileSync(file('run/node/source-after/'+name)));
|
||||
const git=owned('run/node/git-final.json'),baseline=owned('run/node/source-identity.json');
|
||||
assert.equal(git.head,baseline.head);assert.equal(git.status,baseline.status);assert.equal(git.stagedAfter,baseline.stagedBefore);assert.equal(git.diff,'');
|
||||
assert.equal(final.passed,true);assert.equal(final.version,'1.24.6');assert.equal(final.pid,36552);assert.equal(final.inspectorClosed,true);
|
||||
assert.equal(final.originalProjects,61);assert.equal(final.originalSessions,77);assert.equal(final.originalArtifacts,3610);assert.equal(final.originalAttempts,209);
|
||||
assert.equal(final.finalProjects,63);assert.equal(final.finalSessions,79);assert.equal(final.finalArtifacts,3660);assert.equal(final.addedAttempts,7);
|
||||
assert.deepEqual(final.connectionRevision,[5,7]);assert.equal(final.credentialRevision,1);assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision,[65,69]);
|
||||
assert.equal(final.originalBudgetsSelectionAndPoolRestored,true);assert.equal(final.reportedTokens,reportedTokens);assert.equal(final.unknownRequests,0);
|
||||
assert.equal(final.nodeDelivered,false);assert.equal(final.purgeBenefitTested,false);assert.equal(final.productSourceChanged,false);assert.equal(final.keyReadOrResaved,false);
|
||||
let gitIndex;
|
||||
if(process.argv.includes('--index')){
|
||||
const prefix=path.relative(process.cwd(),root).replaceAll('\\','/'),manifestBytes=fs.readFileSync(file('manifest.json'));
|
||||
const entries=[...manifest.files,{path:'manifest.json',bytes:manifestBytes.length,sha256:sha(manifestBytes)}];
|
||||
const output=execFileSync('git',['cat-file','--batch'],{input:entries.map(row=>':'+prefix+'/'+row.path).join('\n')+'\n',maxBuffer:256*1024*1024,windowsHide:true});
|
||||
let offset=0,indexBytes=0;
|
||||
for(const entry of entries){const end=output.indexOf(10,offset),header=output.subarray(offset,end).toString('utf8'),match=header.match(/^[a-f0-9]{40} blob (\d+)$/);assert.ok(match,header);
|
||||
const size=Number(match[1]);offset=end+1;const bytes=output.subarray(offset,offset+size);offset+=size;assert.equal(output[offset++],10);
|
||||
assert.equal(size,entry.bytes,entry.path);assert.equal(sha(bytes),entry.sha256,entry.path);indexBytes+=size;}
|
||||
assert.equal(offset,output.length);gitIndex={passed:true,files:entries.length,bytes:indexBytes,exactOriginalBytes:true};
|
||||
}
|
||||
console.log(JSON.stringify({success:true,files:manifest.files.length,bytes:totalBytes,manifestSha256:sha(fs.readFileSync(file('manifest.json'))),
|
||||
modelRequests:totalRequests,reportedTokens,unknownRequests:0,preflightPassed:true,nodeDelivered:false,originalPurgeBenefitTested:false,
|
||||
ownedArtifactRecords:artifacts.length,originalDataAndDefaultsVerified:true,readingPaginationAndBudgetPauseVerified:true,productVersion:'1.24.6',
|
||||
...(gitIndex?{gitIndex}:{}),scope:'Offline original bytes, frozen plan/source, actual reports, paging, ownership and declared restoration. No network, host mutation, model or credential access.'}));
|
||||
@@ -0,0 +1,196 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
const root=path.resolve('docs/live-delivery-t-evidence');
|
||||
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const canonical=value=>Array.isArray(value)?value.map(canonical):value&&typeof value==='object'
|
||||
?Object.fromEntries(Object.keys(value).sort().map(key=>[key,canonical(value[key])])):value;
|
||||
const digest=value=>sha(JSON.stringify(canonical(value)));
|
||||
const file=name=>{assert.equal(typeof name,'string');assert.ok(!/[\\\x00\r\n]/.test(name)&&!name.split('/').includes('..'));
|
||||
const target=path.resolve(root,name);assert.ok(target.startsWith(root+path.sep));return target;};
|
||||
const bytes=name=>fs.readFileSync(file(name)),read=name=>JSON.parse(bytes(name));
|
||||
const manifest=read('manifest.json');assert.equal(manifest.kind,'owned-live-delivery-t-evidence');
|
||||
const names=new Set();let totalBytes=0;
|
||||
for(const row of manifest.files){assert.ok(!names.has(row.path));names.add(row.path);const value=bytes(row.path);
|
||||
assert.equal(value.length,row.bytes);assert.equal(sha(value),row.sha256,row.path);totalBytes+=value.length;}
|
||||
const list=(directory=root)=>fs.readdirSync(directory,{withFileTypes:true}).flatMap(entry=>{assert.equal(entry.isSymbolicLink(),false);
|
||||
const target=path.join(directory,entry.name);return entry.isDirectory()?list(target):[path.relative(root,target).replaceAll('\\','/')];});
|
||||
assert.deepEqual(list().sort(),['manifest.json',...names].sort());
|
||||
const owned=name=>{assert.ok(names.has(name),name);return read(name);};
|
||||
const initial=owned(manifest.references.initial),preflight=owned(manifest.references.preflight),node=owned(manifest.references.node);
|
||||
const final=owned(manifest.references.final),network=owned(manifest.references.network),proof=owned(manifest.references.ownership);
|
||||
const context=owned(manifest.references.context),replay=owned(manifest.references.candidates),artifacts=owned(manifest.references.artifacts).artifacts;
|
||||
const frozen=owned('contract/manifest.json'),connectionId='5b4daed9-b2f7-4de0-b949-573eba70c7b3';
|
||||
assert.equal(frozen.modelRequestsAtFreeze,0);assert.equal(frozen.maxNetworkJobs,1);assert.equal(frozen.maxPreflightRequests,1);
|
||||
assert.equal(frozen.maxNodeRequests,8);assert.equal(frozen.maxNodeTokens,90000);assert.equal(frozen.maxNodeDurationMs,600000);
|
||||
assert.equal(frozen.contextLimit,16000);assert.equal(frozen.maxOutputTokens,2048);assert.equal(frozen.safetyReserveTokens,1600);
|
||||
assert.equal(sha(bytes('run/current.json')),frozen.originalBaselineSha256);
|
||||
assert.equal(sha(bytes('provenance/frozen-plan.md')),frozen.planSha256);
|
||||
const gitBytes=(commit,relative)=>execFileSync('git',['show',commit+':'+relative],{maxBuffer:2*1024*1024,windowsHide:true});
|
||||
for(const row of frozen.files){assert.equal(bytes('contract/'+row.path).length,row.bytes);assert.equal(sha(bytes('contract/'+row.path)),row.sha256);
|
||||
assert.deepEqual(bytes('contract/'+row.path),gitBytes(manifest.experimentPlanCommit,'docs/live-delivery-t-contract/'+row.path));}
|
||||
assert.deepEqual(bytes('contract/manifest.json'),gitBytes(manifest.experimentPlanCommit,'docs/live-delivery-t-contract/manifest.json'));
|
||||
for(const row of frozen.dependencySources)assert.equal(sha(gitBytes(manifest.experimentBaseCommit,row.path)),row.sha256);
|
||||
for(const row of context.sourceInputs){assert.equal(sha(bytes('run/'+row.capturedPath)),row.sha256);assert.equal(bytes('run/'+row.capturedPath).length,row.bytes);
|
||||
assert.deepEqual(bytes('run/'+row.capturedPath),gitBytes(manifest.releaseSourceCommit,row.path));}
|
||||
assert.equal(context.sourceInputs.length,9);assert.equal(context.sourceCommit,manifest.releaseSourceCommit);
|
||||
assert.equal(manifest.releaseSourceCommit,owned('provenance/release-1.24.7.json').sourceCommit);
|
||||
assert.equal(sha(bytes('provenance/release-1.24.7.json')),frozen.releaseIdentitySha256);
|
||||
assert.deepEqual(bytes('run/node/goal.txt'),gitBytes('fbedb738','docs/live-delivery-r-evidence/run/node/goal.txt'));
|
||||
const stripDerived=row=>{const {stale,currentConnectionRevision,...raw}=row;return raw;};
|
||||
assert.equal(network.connectionId,connectionId);assert.equal(network.connectionRevision,7);assert.equal(network.status,'passed');
|
||||
assert.equal(network.stages.dns.status,'passed');assert.deepEqual(network.trials.map(row=>row.mode),['auto','ipv4','ipv6']);
|
||||
assert.ok(network.trials.every(row=>row.status==='passed'&&row.certificateAuthorized===true));
|
||||
assert.equal(network.httpRequests,0);assert.equal(network.modelRequests,0);assert.equal(network.credentialAccess,false);
|
||||
assert.equal(network.route.kind,'direct-endpoint-only');assert.equal(network.route.modelApiRouteVerified,false);
|
||||
assert.equal(owned('run/network/restoration.json').originalStateExactExceptOneAppendedDiagnostic,true);
|
||||
assert.equal(owned('run/network/driver-error.json').jobId,network.id);
|
||||
assert.equal(sha(bytes('run/network/audit-first-driver.mjs.txt')),owned('run/network/audit-first-error.json').driverSha256);
|
||||
const allAttempts=[];let totalRequests=0,reportedTokens=0;
|
||||
const assertPairs=messages=>{const pending=new Set();for(const message of messages){
|
||||
if(message.role==='assistant'){assert.equal(pending.size,0);for(const call of message.tool_calls??[]){assert.ok(!pending.has(call.id));pending.add(call.id);}}
|
||||
else if(message.role==='tool'){assert.ok(pending.has(message.tool_call_id));pending.delete(message.tool_call_id);}
|
||||
else assert.equal(pending.size,0);
|
||||
}assert.equal(pending.size,0);};
|
||||
for(const [folder,outcome,expectedCount]of[['preflight',preflight,1],['node',node,8]]){
|
||||
const attempts=owned('run/'+folder+'/attempts.json');assert.equal(attempts.length,expectedCount);allAttempts.push(...attempts);
|
||||
assert.equal(outcome.captures.length,expectedCount*2);
|
||||
for(const attempt of attempts){
|
||||
assert.equal(attempt.connectionId,connectionId);assert.equal(attempt.connectionRevision,folder==='preflight'?7:8);
|
||||
assert.equal(attempt.credentialRevision,1);assert.equal(attempt.poolId,null);assert.equal(attempt.ordinal,1);
|
||||
assert.equal(attempt.model,'gpt-6.1-sol');assert.equal(attempt.protocol,'chat-completions');assert.equal(attempt.streaming,false);
|
||||
assert.equal(attempt.requestUrl,'https://jojocode.com/v1/chat/completions');assert.equal(attempt.status,'completed');
|
||||
assert.equal(attempt.modelIdentity.policy,'require-match');assert.equal(attempt.modelIdentity.status,'matched');
|
||||
assert.equal(attempt.modelIdentity.complete,true);assert.deepEqual(attempt.modelIdentity.reportedModels,['gpt-6.1-sol']);
|
||||
assert.equal(attempt.usageStatus,'reported');assert.equal(attempt.responseRedacted,false);
|
||||
const captures=Object.fromEntries(outcome.captures.filter(row=>row.attemptId===attempt.id).map(row=>[row.kind,row]));
|
||||
const request=bytes('run/'+captures.request.path),response=bytes('run/'+captures.response.path);
|
||||
assert.equal(sha(request),attempt.requestSha256);assert.equal(sha(request),captures.request.sha256);
|
||||
assert.equal(sha(response),attempt.responseSha256);assert.equal(sha(response),captures.response.sha256);
|
||||
assert.equal(attempt.modelIdentity.responseSource.sha256,attempt.responseSha256);assert.equal(request.length,attempt.requestBytes);
|
||||
const body=JSON.parse(request),raw=JSON.parse(response);assertPairs(body.messages);
|
||||
assert.equal(body.model,'gpt-6.1-sol');assert.notEqual(body.stream,true);assert.equal(raw.model,'gpt-6.1-sol');assert.deepEqual(raw.usage,attempt.usage);
|
||||
assert.ok(raw.usage.completion_tokens<=body.max_tokens);reportedTokens+=raw.usage.total_tokens;totalRequests++;
|
||||
if(folder==='preflight'){
|
||||
assert.equal(body.tools?.length??0,0);assert.equal(body.max_tokens,256);assert.equal(preflight.actualTools,0);assert.equal(preflight.passed,true);
|
||||
assert.equal(raw.choices[0].message.tool_calls?.length??0,0);assert.ok(raw.choices[0].message.content.includes(owned('run/preflight/contract.json').nonce));
|
||||
assert.deepEqual(raw.usage,preflight.usage);assert.equal(raw.usage.total_tokens,112);
|
||||
}else{assert.equal(body.max_tokens,2048);assert.ok(body.tools.length>0);
|
||||
assert.deepEqual(body.messages.filter(row=>row.role==='user').map(row=>row.content),[bytes('run/node/goal.txt').toString('utf8')]);}
|
||||
}assert.equal(owned('run/'+folder+'/restoration.json').passed,true);
|
||||
}
|
||||
assert.equal(totalRequests,9);assert.equal(reportedTokens,76174);assert.equal(node.success,false);assert.equal(node.stopReason,'step_limit');
|
||||
assert.equal(node.usage.requests,8);assert.equal(node.usage.totalTokens,76062);assert.equal(node.usage.unknownRequests,0);assert.equal(node.usage.estimatedUnknownTokens,0);
|
||||
assert.deepEqual(node.applications,[]);assert.equal(node.modelImplementationSuppliedBySupervisor,false);
|
||||
const session=owned('run/node/session.json');assert.deepEqual(session.changes,[]);assert.equal(session.execution.run.toolCalls,16);
|
||||
assert.deepEqual(session.execution.run.budget,{maxSteps:8,maxDurationMs:600000,maxTokens:90000,maxTaskTokens:90000});
|
||||
assert.equal(session.execution.checks.length,1);const checks=session.execution.checks[0];assert.equal(checks.status,'failed');
|
||||
assert.deepEqual(checks.results.find(row=>row.name==='test').tests,{total:9,passed:0,failed:9,cancelled:0,skipped:0});
|
||||
assert.equal(checks.results.find(row=>row.name==='build').status,'passed');assert.equal(node.approvals.length,1);assert.equal(node.approvals[0].allow,true);
|
||||
assert.equal(node.approvals[0].command,'npm run test\nnpm run build');
|
||||
assert.equal(session.execution.contextPlans.length,8);
|
||||
for(const plan of session.execution.contextPlans){
|
||||
assert.ok(plan.estimatedInputTokens<=plan.budget.inputBudget);
|
||||
assert.ok(plan.tokenReservation.reservedTokens<=Math.min(plan.budget.runRemainingTokens,plan.budget.taskRemainingTokens));
|
||||
assert.equal(plan.budget.contextLimit,16000);assert.equal(plan.budget.safetyTokens,1600);assert.equal(plan.budget.outputReservation,2048);assert.equal(plan.requestAttempts,1);
|
||||
}
|
||||
const testPath='test/expense.test.mjs',testSource=bytes('run/node/source-before/'+testPath).toString('utf8');assert.equal(testSource.length,2584);
|
||||
const filePages=context.sourceResults.filter(row=>row.path===testPath);
|
||||
assert.deepEqual(filePages.map(row=>[row.offset,row.next_offset]),[[0,473],[473,2089],[0,472],[2089,2557],[2557,2584]]);
|
||||
for(const row of context.sourceResults){const source=bytes('run/node/source-before/'+row.path).toString('utf8');
|
||||
assert.equal(row.content,source.slice(row.offset,row.next_offset));assert.equal(row.sha256,sha(source));}
|
||||
assert.deepEqual(context.returnedTestRanges,[[0,2584]]);assert.equal(context.returnedTestCharacters,2584);
|
||||
const union=ranges=>ranges.slice().sort((a,b)=>a[0]-b[0]).reduce((result,[a,b])=>{const previous=result.at(-1);
|
||||
if(previous&&a<=previous[1])previous[1]=Math.max(previous[1],b);else result.push([a,b]);return result;},[]);
|
||||
const actualCoverage=[];
|
||||
for(let index=0;index<8;index++){
|
||||
const capture=node.captures.filter(row=>row.kind==='request')[index],body=JSON.parse(bytes('run/'+capture.path)),segments=[];
|
||||
const add=(value,content,offset=0)=>{if(value?.path!==testPath||typeof content!=='string')return;
|
||||
assert.equal(value.sha256,sha(testSource));const start=value.offset+offset;assert.equal(content,testSource.slice(start,start+content.length));segments.push([start,start+content.length]);};
|
||||
for(const message of body.messages){
|
||||
if(message.role==='tool'){const value=JSON.parse(message.content);add(value,value.content);}
|
||||
if(message.role!=='system')continue;
|
||||
const start=message.content.indexOf('\n{');if(start<0)continue;let value;try{value=JSON.parse(message.content.slice(start+1));}catch{continue;}
|
||||
if(value.data?.kind==='task-working-set')for(const entry of value.data.entries)add(entry.observation,entry.content,entry.content_offset??0);
|
||||
if(value.data?.kind==='latest-tool-feedback')for(const entry of value.data.entries)if(entry.tool==='read_file')add(entry.result,entry.result?.content_excerpt);
|
||||
}
|
||||
const ranges=union(segments),count=ranges.reduce((sum,[a,b])=>sum+b-a,0);actualCoverage.push(count);
|
||||
const expected=context.perRequest[index];assert.equal(expected.requestSha256,capture.sha256);assert.deepEqual(expected.files[0].ranges,ranges);assert.equal(expected.files[0].characters,count);
|
||||
assert.equal(expected.originalCheckJobIdMentionedAnywhere,bytes('run/'+capture.path).includes(checks.id));
|
||||
}
|
||||
assert.deepEqual(actualCoverage,[0,0,0,400,1616,1060,27,421]);
|
||||
assert.ok(context.perRequest.every(row=>!row.files[0].fullSourcePresentInStructuredFileViews));
|
||||
assert.deepEqual(context.perRequest.slice(5).map(row=>row.originalCheckJobIdMentionedAnywhere),[false,false,false]);
|
||||
assert.equal(replay.passed,true);assert.equal(replay.totalExternalModelRequests,0);assert.equal(replay.hostMutations,0);
|
||||
assert.deepEqual(replay.result[2].testCandidateRanges,[[473,2089],[2557,2584]]);assert.deepEqual(replay.result[2].missingTestRangesBeforeBudgetSelection,[[0,473],[2089,2557]]);
|
||||
assert.equal(replay.result[2].checkCandidatePresent,true);
|
||||
const ownedToolRows=owned('run/analysis/owned-tool-rows.json');assert.equal(ownedToolRows.length,16);
|
||||
for(const row of ownedToolRows)assert.equal(digest(row),proof.additions.records.find(item=>item.id===row.id)?.sha256);
|
||||
const artifactMap=new Map(artifacts.map(row=>[row.id,row]));assert.equal(artifacts.length,74);
|
||||
for(const artifact of artifacts){assert.ok(final.ownedSessions.includes(artifact.sessionId));assert.ok(final.ownedProjects.includes(artifact.projectId));
|
||||
assert.equal(sha(bytes('run/'+artifact.capturedPath)),artifact.sha256);assert.equal(bytes('run/'+artifact.capturedPath).length,artifact.bytes);}
|
||||
for(const row of context.sourceResults){const artifact=artifactMap.get(row.artifactId);assert.ok(artifact);assert.equal(artifact.sha256,row.artifactSha256);
|
||||
const value=owned('run/'+artifact.capturedPath);assert.equal(value.content,row.content);assert.equal(value.offset,row.offset);}
|
||||
for(const result of checks.results){assert.equal(artifactMap.get(result.logArtifactId)?.kind,'check-log');}
|
||||
const snapshot=owned('run/final/snapshot.json');
|
||||
for(const item of initial.baseline.exports)assert.deepEqual(snapshot.exports.find(row=>row.id===item.id),item);
|
||||
for(const table of ['sessions','records','events','outbox']){
|
||||
const before=initial.baseline.database.tables[table],after=snapshot.database.tables[table];assert.equal(after.schemaSha256,before.schemaSha256);
|
||||
const rows=new Map(after.rows.map(row=>[row.id,row]));for(const row of before.rows)assert.deepEqual(rows.get(row.id),row);
|
||||
assert.equal(proof.additions[table].length,after.count-before.count);
|
||||
for(const row of proof.additions[table]){assert.ok(final.ownedSessions.includes(row.sessionId));if(row.projectId)assert.ok(final.ownedProjects.includes(row.projectId));assert.equal(rows.get(row.id).sha256,row.sha256);}
|
||||
}
|
||||
assert.equal(proof.additions.records.filter(row=>row.kind==='artifact').length,artifacts.length);
|
||||
for(const artifact of artifacts)assert.ok(proof.additions.records.some(row=>row.kind==='artifact'&&row.id===artifact.id));
|
||||
const originalProjectRows=new Map(initial.baseline.database.tables.projects.rows.map(row=>[row.id,row]));
|
||||
const finalProjectRows=new Map(snapshot.database.tables.projects.rows.map(row=>[row.id,row]));
|
||||
for(const row of proof.projects){assert.equal(row.actualSha256,finalProjectRows.get(row.id).sha256);
|
||||
if(row.originalSha256){assert.equal(row.originalSha256,originalProjectRows.get(row.id).sha256);assert.equal(row.normalizedSha256,row.originalSha256);}else assert.ok(final.ownedProjects.includes(row.id));}
|
||||
assert.equal(proof.projects.filter(row=>row.originalSha256).length,63);assert.equal(proof.projects.filter(row=>row.owned).length,2);assert.equal(proof.projectInsertionOffset,2);
|
||||
const oldArtifactMap=new Map(initial.baseline.persistent.artifacts.map(row=>[row.path,row])),nextArtifactMap=new Map(snapshot.persistent.artifacts.map(row=>[row.path,row]));
|
||||
for(const [name,row]of oldArtifactMap)assert.deepEqual(nextArtifactMap.get(name),row);
|
||||
const addedFiles=snapshot.persistent.artifacts.filter(row=>!oldArtifactMap.has(row.path));assert.equal(addedFiles.length,60);
|
||||
assert.ok(addedFiles.every(row=>artifacts.some(artifact=>artifact.sha256===row.sha256&&artifact.bytes===row.bytes)));
|
||||
assert.deepEqual(snapshot.modelCenter.selected,initial.center.selected);assert.deepEqual(snapshot.modelCenter.pools,initial.center.pools);
|
||||
const beforeMeta=owned('run/node/original-public-metadata.json'),afterMeta=owned('run/node/final-public-metadata.json');
|
||||
const metadataRows=new Map(snapshot.database.tables.metadata.rows.map(row=>[row.id,row]));
|
||||
const oldAttempts=owned('run/preflight/original-attempts.json');assert.equal(oldAttempts.length,216);
|
||||
assert.deepEqual(owned('run/node/original-attempts.json'),[...oldAttempts,...owned('run/preflight/attempts.json')]);
|
||||
assert.equal(metadataRows.get('modelAttempts').sha256,digest({key:'modelAttempts',value:JSON.stringify([...oldAttempts,...allAttempts])}));
|
||||
for(const [key,value]of[['config',afterMeta.config],['modelRegistry',afterMeta.registry]])assert.equal(metadataRows.get(key).sha256,digest({key,value:JSON.stringify(value)}));
|
||||
const oldNetwork=owned('run/network/history-before.json').jobs.map(stripDerived).reverse(),nextNetwork=owned('run/network/history-after.json').jobs.map(stripDerived).reverse();
|
||||
assert.equal(oldNetwork.length,1);assert.deepEqual(nextNetwork,[...oldNetwork,stripDerived(network)]);
|
||||
assert.equal(initial.baseline.database.tables.metadata.rows.find(row=>row.id==='modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify(oldNetwork)}));
|
||||
assert.equal(metadataRows.get('modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify(nextNetwork)}));
|
||||
for(const row of initial.baseline.database.tables.metadata.rows)if(!['config','modelRegistry','modelAttempts','modelNetworkDiagnostics'].includes(row.id))assert.deepEqual(metadataRows.get(row.id),row);
|
||||
assert.deepEqual([...metadataRows.keys()].filter(key=>!initial.baseline.database.tables.metadata.rows.some(row=>row.id===key)),[]);
|
||||
assert.deepEqual({...afterMeta.config,configRevision:beforeMeta.config.configRevision},beforeMeta.config);
|
||||
const registry=structuredClone(afterMeta.registry);registry.revision=beforeMeta.registry.revision;
|
||||
const connection=registry.connections.find(row=>row.id===connectionId),oldConnection=beforeMeta.registry.connections.find(row=>row.id===connectionId);
|
||||
assert.equal(connection.revision,oldConnection.revision+2);connection.revision=oldConnection.revision;connection.updatedAt=oldConnection.updatedAt;
|
||||
const expectedRegistry=structuredClone(beforeMeta.registry);for(const value of[expectedRegistry,registry])value.connections.sort((a,b)=>a.id.localeCompare(b.id));
|
||||
assert.deepEqual(canonical(registry),canonical(expectedRegistry));assert.deepEqual(owned('run/node/restoration.json').identityDefaultNormalization,{});
|
||||
for(const name of ['src/amount.mjs','src/summary.mjs','src/report.mjs'])assert.deepEqual(bytes('run/node/source-before/'+name),bytes('run/node/source-after/'+name));
|
||||
const git=owned('run/node/git-final.json'),baseline=owned('run/node/source-identity.json');
|
||||
assert.equal(git.head,baseline.head);assert.equal(git.status,baseline.status);assert.equal(git.stagedAfter,baseline.stagedBefore);assert.equal(git.diff,'');
|
||||
assert.equal(final.passed,true);assert.equal(final.version,'1.24.7');assert.equal(final.pid,36552);assert.equal(final.inspectorClosed,true);
|
||||
assert.equal(final.originalProjects,63);assert.equal(final.originalSessions,79);assert.equal(final.originalArtifacts,3660);assert.equal(final.originalAttempts,216);
|
||||
assert.equal(final.finalProjects,65);assert.equal(final.finalSessions,81);assert.equal(final.finalArtifacts,3720);assert.equal(final.addedAttempts,9);
|
||||
assert.deepEqual(final.connectionRevision,[7,9]);assert.equal(final.credentialRevision,1);assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision,[69,73]);
|
||||
assert.equal(final.originalBudgetsSelectionAndPoolRestored,true);assert.equal(final.reportedTokens,reportedTokens);assert.equal(final.unknownRequests,0);
|
||||
assert.equal(final.nodeDelivered,false);assert.equal(final.purgeBenefitTested,false);assert.equal(final.productSourceChanged,false);assert.equal(final.keyReadOrResaved,false);
|
||||
let gitIndex;
|
||||
if(process.argv.includes('--index')){
|
||||
const prefix=path.relative(process.cwd(),root).replaceAll('\\','/'),manifestBytes=bytes('manifest.json');
|
||||
const entries=[...manifest.files,{path:'manifest.json',bytes:manifestBytes.length,sha256:sha(manifestBytes)}];
|
||||
const output=execFileSync('git',['cat-file','--batch'],{input:entries.map(row=>':'+prefix+'/'+row.path).join('\n')+'\n',maxBuffer:256*1024*1024,windowsHide:true});
|
||||
let offset=0,indexBytes=0;for(const entry of entries){const end=output.indexOf(10,offset),header=output.subarray(offset,end).toString('utf8'),match=header.match(/^[a-f0-9]{40} blob (\d+)$/);assert.ok(match,header);
|
||||
const size=Number(match[1]);offset=end+1;const value=output.subarray(offset,offset+size);offset+=size;assert.equal(output[offset++],10);assert.equal(size,entry.bytes,entry.path);assert.equal(sha(value),entry.sha256,entry.path);indexBytes+=size;}
|
||||
assert.equal(offset,output.length);gitIndex={passed:true,files:entries.length,bytes:indexBytes,exactOriginalBytes:true};
|
||||
}
|
||||
console.log(JSON.stringify({success:true,files:manifest.files.length,bytes:totalBytes,manifestSha256:sha(bytes('manifest.json')),
|
||||
modelRequests:totalRequests,reportedTokens,unknownRequests:0,preflightPassed:true,nodeDelivered:false,originalPurgeBenefitTested:false,
|
||||
ownedArtifactRecords:artifacts.length,originalDataAndDefaultsVerified:true,actualRequestRetentionAndCandidateReplayVerified:true,productVersion:'1.24.7',
|
||||
...(gitIndex?{gitIndex}:{}),scope:'Offline original bytes, frozen plan/source, original usage, file coverage, owned records and declared restoration. No network, host mutation, model or credential access.'}));
|
||||
@@ -0,0 +1,55 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
const root=path.resolve('docs/live-delivery-v-evidence');
|
||||
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const canonical=value=>Array.isArray(value)?value.map(canonical):value&&typeof value==='object'?Object.fromEntries(Object.keys(value).sort().map(key=>[key,canonical(value[key])])):value;
|
||||
const digest=value=>sha(JSON.stringify(canonical(value)));
|
||||
const file=name=>{assert.equal(typeof name,'string');assert.ok(!/[\\\x00\r\n]/.test(name)&&!name.split('/').includes('..'));const target=path.resolve(root,name);assert.ok(target.startsWith(root+path.sep));return target;};
|
||||
const bytes=name=>fs.readFileSync(file(name)),read=name=>JSON.parse(bytes(name));
|
||||
const manifest=read('manifest.json');assert.equal(manifest.kind,'owned-live-delivery-v-evidence');
|
||||
const names=new Set();let totalBytes=0;
|
||||
for(const row of manifest.files){assert.ok(!names.has(row.path));names.add(row.path);const value=bytes(row.path);assert.equal(value.length,row.bytes);assert.equal(sha(value),row.sha256,row.path);totalBytes+=value.length;}
|
||||
const list=(directory=root)=>fs.readdirSync(directory,{withFileTypes:true}).flatMap(entry=>{assert.equal(entry.isSymbolicLink(),false);const target=path.join(directory,entry.name);return entry.isDirectory()?list(target):[path.relative(root,target).replaceAll('\\','/')];});
|
||||
assert.deepEqual(list().sort(),['manifest.json',...names].sort());
|
||||
assert.ok([...names].every(name=>!name.startsWith('run/preflight/')&&!name.startsWith('run/node/')));
|
||||
const owned=name=>{assert.ok(names.has(name));return read(name);};
|
||||
const initial=owned(manifest.references.initial),network=owned(manifest.references.network),audit=owned(manifest.references.networkAudit),final=owned(manifest.references.final),proof=owned(manifest.references.ownership),readback=owned(manifest.references.readback),frozen=owned('contract/manifest.json');
|
||||
const gitBytes=(commit,relative)=>execFileSync('git',['show',commit+':'+relative],{maxBuffer:4*1024*1024,windowsHide:true});
|
||||
assert.equal(sha(bytes('run/current.json')),frozen.originalBaselineSha256);assert.equal(sha(bytes('provenance/frozen-plan.md')),frozen.planSha256);
|
||||
assert.deepEqual(bytes('contract/manifest.json'),gitBytes(manifest.experimentPlanCommit,'docs/live-delivery-v-contract/manifest.json'));
|
||||
for(const row of frozen.files){assert.equal(sha(bytes('contract/'+row.path)),row.sha256);assert.deepEqual(bytes('contract/'+row.path),gitBytes(manifest.experimentPlanCommit,'docs/live-delivery-v-contract/'+row.path));}
|
||||
for(const row of frozen.inputs){assert.equal(sha(bytes('contract/'+row.capturedPath)),row.sha256);assert.equal(row.sha256,frozen.originalElevenFileHashes[row.path]);}
|
||||
for(const row of frozen.dependencySources)assert.equal(sha(gitBytes(manifest.experimentBaseCommit,row.path)),row.sha256);
|
||||
assert.equal(sha(bytes('provenance/release-1.24.8.json')),frozen.releaseIdentitySha256);assert.equal(owned('provenance/release-1.24.8.json').sourceCommit,manifest.releaseSourceCommit);
|
||||
assert.equal(frozen.modelRequestsAtFreeze,0);assert.equal(frozen.maxNetworkJobs,1);assert.equal(frozen.maxPreflightRequests,1);assert.equal(frozen.maxNodeRequests,8);assert.equal(frozen.maxNodeTokens,90000);assert.equal(frozen.maxNodeDurationMs,600000);assert.equal(frozen.contextLimit,16000);assert.equal(frozen.maxOutputTokens,2048);assert.equal(frozen.safetyReserveTokens,1600);assert.equal(frozen.goalMatchesTExactly,true);
|
||||
assert.equal(network.status,'partial');assert.equal(network.connectionRevision,9);assert.equal(network.stages.dns.status,'passed');assert.equal(network.httpRequests,0);assert.equal(network.modelRequests,0);assert.equal(network.credentialAccess,false);assert.equal(network.route.modelApiRouteVerified,false);
|
||||
assert.deepEqual(network.trials.map(row=>[row.mode,row.status]),[['auto','passed'],['ipv4','failed'],['ipv6','passed']]);assert.equal(network.trials[1].code,'TLS_HANDSHAKE_TIMEOUT');
|
||||
assert.equal(audit.passed,true);assert.equal(audit.readOnlyReconciliation,true);assert.equal(audit.firstDriverFailurePreserved,true);assert.equal(owned('run/network/driver-error.json').jobId,network.id);
|
||||
const previous=owned('run/network/history-before.json'),next=owned('run/network/history-after.json');
|
||||
const strip=({stale,currentConnectionRevision,...raw})=>raw;
|
||||
const oldRaw=previous.jobs.map(strip).reverse(),newRaw=next.jobs.map(strip).reverse();
|
||||
assert.equal(oldRaw.length,2);assert.deepEqual(newRaw,[...oldRaw,strip(network)]);assert.equal(next.active,0);
|
||||
const snapshot=owned('run/final/snapshot.json'),before=initial.baseline;
|
||||
for(const field of ['pid','version','runtimeRoot','configSha256','projectsSha256','sessionsSha256','eventCursor','modelCenterSha256'])assert.deepEqual(snapshot[field],before[field],field);
|
||||
assert.deepEqual(snapshot.exports,before.exports);assert.deepEqual(snapshot.modelCenter,before.modelCenter);
|
||||
assert.deepEqual(snapshot.persistent.artifacts,before.persistent.artifacts);assert.deepEqual(snapshot.persistent.files,before.persistent.files);
|
||||
for(const [name,table] of Object.entries(before.database.tables)){
|
||||
assert.equal(snapshot.database.tables[name].count,table.count);assert.equal(snapshot.database.tables[name].schemaSha256,table.schemaSha256);
|
||||
const without=rows=>rows.filter(row=>name!=='metadata'||row.id!=='modelNetworkDiagnostics');assert.deepEqual(without(snapshot.database.tables[name].rows),without(table.rows));
|
||||
}
|
||||
for(const [snapshotValue,raw]of[[before,oldRaw],[snapshot,newRaw]])assert.equal(snapshotValue.database.tables.metadata.rows.find(row=>row.id==='modelNetworkDiagnostics').sha256,digest({key:'modelNetworkDiagnostics',value:JSON.stringify(raw)}));
|
||||
const attempts=owned('run/initial-attempts.json');assert.equal(attempts.length,225);assert.equal(snapshot.database.tables.metadata.rows.find(row=>row.id==='modelAttempts').sha256,digest({key:'modelAttempts',value:JSON.stringify(attempts)}));
|
||||
for(const rows of Object.values(proof.additions))assert.deepEqual(rows,[]);assert.equal(proof.projectInsertionOffset,0);assert.ok(proof.projects.every(row=>row.actualSha256===row.originalSha256&&row.normalizedSha256===row.originalSha256));
|
||||
assert.equal(final.passed,true);assert.equal(final.pid,36552);assert.equal(final.version,'1.24.8');assert.equal(final.modelRequests,0);assert.equal(final.reportedTokens,0);assert.equal(final.unknownRequests,0);assert.equal(final.originalProjects,65);assert.equal(final.originalSessions,81);assert.equal(final.originalArtifacts,3720);assert.equal(final.originalAttempts,225);assert.equal(final.nodeDelivered,false);assert.equal(final.purgeBenefitTested,false);assert.equal(final.productSourceChanged,false);assert.equal(final.keyReadOrResaved,false);assert.deepEqual(final.connectionRevision,[9,9]);assert.deepEqual(final.legacyDisplayRevisionFollowsGlobalConfigurationRevision,[73,73]);assert.equal(final.credentialRevision,1);
|
||||
assert.equal(readback.originalIdentityExact,true);assert.equal(readback.inspectorClosed,true);assert.equal(readback.modelCenterExactSinceRestoration,true);
|
||||
let gitIndex;
|
||||
if(process.argv.includes('--index')){
|
||||
const prefix=path.relative(process.cwd(),root).replaceAll('\\','/'),manifestBytes=bytes('manifest.json'),entries=[...manifest.files,{path:'manifest.json',bytes:manifestBytes.length,sha256:sha(manifestBytes)}];
|
||||
const output=execFileSync('git',['cat-file','--batch'],{input:entries.map(row=>':'+prefix+'/'+row.path).join('\n')+'\n',maxBuffer:256*1024*1024,windowsHide:true});let offset=0,indexBytes=0;
|
||||
for(const entry of entries){const end=output.indexOf(10,offset),header=output.subarray(offset,end).toString('utf8'),match=header.match(/^[a-f0-9]{40} blob (\d+)$/);assert.ok(match,header);const size=Number(match[1]);offset=end+1;const value=output.subarray(offset,offset+size);offset+=size;assert.equal(output[offset++],10);assert.equal(size,entry.bytes);assert.equal(sha(value),entry.sha256,entry.path);indexBytes+=size;}
|
||||
assert.equal(offset,output.length);gitIndex={passed:true,files:entries.length,bytes:indexBytes,exactOriginalBytes:true};
|
||||
}
|
||||
console.log(JSON.stringify({success:true,files:manifest.files.length,bytes:totalBytes,manifestSha256:sha(bytes('manifest.json')),diagnosticStatus:'partial',stoppedBeforePreflight:true,modelRequests:0,nodeDelivered:false,purgeBenefitTested:false,allOriginalStateExceptSingleDiagnosticAppendExact:true,...(gitIndex?{gitIndex}:{}),scope:'Offline bytes, frozen contract/source and exact state digests. No network, model calls, credential access or host mutation.'}));
|
||||
@@ -0,0 +1,265 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import { APP_VERSION } from '../src/server/diagnostics.mjs';
|
||||
|
||||
// Operates only on a fresh synthetic project. The configured model remains in
|
||||
// the running host; bootstrap contains an API session token, never its model key.
|
||||
const workspace = process.cwd();
|
||||
const batch = process.argv.includes('--batch');
|
||||
const precision = batch || process.argv.includes('--precision');
|
||||
const bannerLines = 420;
|
||||
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
|
||||
const origin = new URL(url);
|
||||
assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:');
|
||||
const bootstrap = await (await fetch(origin.origin + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
||||
assert.equal(bootstrap.version, APP_VERSION); assert.equal(bootstrap.config.mode, 'compatible');
|
||||
assert.ok(bootstrap.config.model, 'Configure a model on the running Workbench first.');
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(15000) });
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw Object.assign(new Error(result.error ?? 'Workbench HTTP ' + response.status), { status: response.status });
|
||||
return result;
|
||||
};
|
||||
|
||||
const caseRoot = path.resolve('.qa', (batch ? 'live-batch-' : precision ? 'live-precision-' : 'live-delivery-') + randomUUID());
|
||||
const projectRoot = path.join(caseRoot, batch ? '多文件变更集交付工程' : precision ? '精确补丁交付工程' : '交付报告工程');
|
||||
const allowed = ['src/normalize.mjs', 'src/summary.mjs', 'src/render.mjs', 'src/cli.mjs'];
|
||||
const scripts = { test: 'node --test test/*.test.mjs', build: 'node scripts/build.mjs' };
|
||||
fs.mkdirSync(caseRoot, { recursive: true });
|
||||
fs.cpSync(path.join(workspace, 'benchmarks/node-delivery-v1/baseline'), projectRoot, { recursive: true });
|
||||
const sourcePrefixes = {}, sourceFormats = {};
|
||||
if (precision) for (const file of allowed) {
|
||||
const target = path.join(projectRoot, file), body = fs.readFileSync(target, 'utf8').replaceAll('\r\n', '\n');
|
||||
const bom = ['src/normalize.mjs', 'src/cli.mjs'].includes(file), ending = bom ? '\r\n' : '\n';
|
||||
const banner = Array.from({ length: bannerLines }, (_, line) => '// 保留工程说明 ' + (line + 1) + ' · ' + file + ' · 原文🙂').join(ending) + ending;
|
||||
const prefix = (bom ? '\uFEFF' : '') + banner;
|
||||
sourcePrefixes[file] = Buffer.from(prefix); sourceFormats[file] = { bom, lineEnding: bom ? 'crlf' : 'lf', firstSourceLine: bannerLines + 1 };
|
||||
fs.writeFileSync(target, prefix + body.replaceAll('\n', ending));
|
||||
}
|
||||
fs.writeFileSync(path.join(projectRoot, '.gitignore'), 'dist/\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), '# Owner notes\nBaseline owner content.\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Baseline staged owner content.\n');
|
||||
const git = args => execFileSync('git', ['-c', 'core.autocrlf=false', '-c', 'user.name=Agent Workbench QA', '-c', 'user.email=qa@workbench.local', '-C', projectRoot, ...args], { encoding: 'utf8', windowsHide: true });
|
||||
git(['init', '--initial-branch=main']); git(['config', 'core.autocrlf', 'false']); git(['add', '.']); git(['commit', '-m', 'test: seal delivery-report acceptance baseline']);
|
||||
const baselineCommit = git(['rev-parse', 'HEAD']).trim();
|
||||
fs.appendFileSync(path.join(projectRoot, 'OWNER-NOTES.md'), 'Existing owner edit: ' + randomUUID() + '\n');
|
||||
fs.writeFileSync(path.join(projectRoot, 'UNTRACKED-OWNER.txt'), 'Existing untracked owner file: ' + randomUUID() + '\n');
|
||||
fs.appendFileSync(path.join(projectRoot, 'OWNER-STAGED.txt'), 'Existing staged owner edit: ' + randomUUID() + '\n');
|
||||
git(['add', '--', 'OWNER-STAGED.txt']);
|
||||
const sha = file => createHash('sha256').update(fs.readFileSync(path.join(projectRoot, file))).digest('hex');
|
||||
const protectedPaths = ['README.md', 'package.json', 'test/delivery.test.mjs', 'scripts/build.mjs', 'fixtures/tasks.json', 'fixtures/invalid.json', '.gitignore', 'OWNER-NOTES.md', 'OWNER-STAGED.txt', 'UNTRACKED-OWNER.txt'];
|
||||
const protectedHashes = Object.fromEntries(protectedPaths.map(file => [file, sha(file)]));
|
||||
const beforeSources = Object.fromEntries(allowed.map(file => [file, sha(file)]));
|
||||
const ownerDiff = git(['diff', '--', 'OWNER-NOTES.md']);
|
||||
const ownerStagedDiff = git(['diff', '--cached', '--', 'OWNER-STAGED.txt']);
|
||||
const assertProtected = () => {
|
||||
for (const [file, hash] of Object.entries(protectedHashes)) assert.equal(sha(file), hash, 'Protected input changed: ' + file);
|
||||
assert.equal(git(['diff', '--', 'OWNER-NOTES.md']), ownerDiff, 'Existing owner edits changed.');
|
||||
assert.equal(git(['diff', '--cached', '--', 'OWNER-STAGED.txt']), ownerStagedDiff, 'Existing owner stage changed.');
|
||||
if (precision) for (const file of allowed) {
|
||||
const bytes = fs.readFileSync(path.join(projectRoot, file)); assert.deepEqual(bytes.subarray(0, sourcePrefixes[file].length), sourcePrefixes[file], 'Protected source banner changed: ' + file);
|
||||
const text = bytes.toString('utf8');
|
||||
if (sourceFormats[file].lineEnding === 'crlf') assert.ok(!/(^|[^\r])\n/.test(text), 'CRLF changed: ' + file);
|
||||
else assert.ok(!text.includes('\r'), 'LF changed: ' + file);
|
||||
}
|
||||
};
|
||||
const project = await api('/projects', { root: projectRoot });
|
||||
const session = await api('/sessions', { projectId: project.id });
|
||||
const started = Date.now(), applied = [], decisions = [], batches = [], reviewLimit = 4;
|
||||
let reviewedRounds = 0, sealingFollowups = 0, state, succeeded = false, lastProgress = '', originalTaskId;
|
||||
const saveEvidence = result => {
|
||||
fs.writeFileSync(path.join(caseRoot, 'result.json'), JSON.stringify(result, null, 2));
|
||||
fs.writeFileSync(path.join(workspace, batch ? '.qa/live-batch-validation.json' : precision ? '.qa/live-precision-validation.json' : '.qa/live-delivery-validation.json'), JSON.stringify(result, null, 2));
|
||||
};
|
||||
|
||||
try {
|
||||
await api('/sessions/' + session.id + '/messages', { content:
|
||||
'完成当前 Delivery Report 工程,严格实现 README.md 的七项要求。先用 inspect_project 识别工程,' +
|
||||
'再用 plan_checks 选择 test/build,并用 run_checks 实际运行封存输入,取得失败的测试和日志。' +
|
||||
'依据实际失败读取有关文件,使用 read_artifact 进一步核对失败日志。' + (precision
|
||||
? '先用 index_project 建立工程索引,再用 query_code 的 symbols/imports 结果定位源码,特别是CLI入口的实际导入行。每个源码模块前420行都是须原字节保留的说明;用 read_file 指定start_line至少421的必要行范围,取得最新SHA256。已有源码只能用 propose_patch 及该SHA256、唯一old_text锚点精确修改;保留BOM及原LF/CRLF,不使用 propose_file,不读整个说明块。'
|
||||
: '只能用 propose_file 修改 src/normalize.mjs、src/summary.mjs、src/render.mjs、src/cli.mjs;') +
|
||||
'允许修改的源码仅为 src/normalize.mjs、src/summary.mjs、src/render.mjs、src/cli.mjs。' +
|
||||
'测试、fixtures、构建脚本、package.json、README 和 OWNER 文件必须保留。四模块都需修复,不能修改验收来通过。' +
|
||||
(batch ? '四个源码提案全部准备后,必须用 prepare_change_set 一次封存这四份change_id,等待整批审查。第一次不要只封存部分文件;到工具轮次上限会继续原任务,已保存提案无需重做。' : '') +
|
||||
'本工程没有第三方依赖,不用安装。不要使用 run_command、不要自己提交Git;本轮终端仅通过已识别的npm检查计划运行。' +
|
||||
'提案等待审查应用,随后继续同一目标并实际重新运行 test/build,依据日志修复直至通过。检查通过后用 plan_delivery 准备本Task的四文件本地Git预览,等待用户独立确认;最后用中文报告实际测试数量、构建结果和改动。' });
|
||||
while (Date.now() - started < 720000) {
|
||||
state = await api('/sessions/' + session.id); originalTaskId ??= state.execution.task.id;
|
||||
assert.equal(state.execution.task.id, originalTaskId, 'Engineering delivery must retain the original Task.');
|
||||
assertProtected();
|
||||
const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(',') + '|' + (state.execution.changeSets ?? []).map(set => set.status).join(',');
|
||||
if (progress !== lastProgress) {
|
||||
console.log(JSON.stringify({ status: state.status, elapsedSeconds: Math.round((Date.now() - started) / 1000),
|
||||
tools: state.events.filter(event => event.kind === 'tool').slice(-5).map(event => ({ tool: event.tool, phase: event.phase })),
|
||||
checks: state.execution.checks.map(job => ({ status: job.status, results: job.results.map(result => ({ name: result.name, status: result.status, tests: result.tests })) })),
|
||||
appliedFiles: applied.map(item => item.path), changeSets: (state.execution.changeSets ?? []).map(set => ({ id: set.id, status: set.status, files: set.files.length })) }));
|
||||
lastProgress = progress;
|
||||
}
|
||||
for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope proposal: ' + proposal.path);
|
||||
if (precision) {
|
||||
assert.ok(!state.events.some(event => event.tool === 'propose_file'), 'Precision sample must use exact patch proposals only.');
|
||||
for (const event of state.events.filter(event => event.tool === 'read_file' && allowed.includes(event.input?.path))) {
|
||||
assert.ok(event.input.start_line >= bannerLines + 1, 'Source reads must target indexed implementation lines: ' + event.input.path);
|
||||
}
|
||||
}
|
||||
for (const approval of state.approvals.filter(item => item.status === 'pending')) {
|
||||
const plan = state.execution.plans?.find(item => item.id === approval.checkPlanId);
|
||||
const allow = !!plan && plan.taskId === originalTaskId && plan.checks.length > 0 && plan.checks.length <= 2
|
||||
&& plan.checks.every(check => scripts[check.name] === check.script && check.command === 'npm run ' + check.name && check.lifecycle.length === 0)
|
||||
&& approval.command === plan.checks.map(check => check.command).join('\n');
|
||||
decisions.push({ approvalId: approval.id, planId: plan?.id ?? null, command: approval.command, allow });
|
||||
await api('/sessions/' + session.id + '/approvals/' + approval.id, { allow });
|
||||
assert.ok(allow, 'Model requested a command outside the sealed test/build contract.');
|
||||
}
|
||||
if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 1000)); continue; }
|
||||
if (state.status === 'failed') throw new Error(state.failure?.message ?? 'The real model task failed.');
|
||||
const preparedSets = (state.execution.changeSets ?? []).filter(set => set.taskId === originalTaskId && set.status === 'prepared');
|
||||
if (state.status === 'paused') {
|
||||
assert.equal(state.execution.run.stopReason, 'step_limit', 'Budget/cancellation must be inspected before continuing.');
|
||||
assert.ok(state.execution.runs.length < 5, 'Bounded continuation count exceeded.');
|
||||
if (!preparedSets.length) { await api('/sessions/' + session.id + '/resume', {}); continue; }
|
||||
}
|
||||
const pending = state.changes.filter(change => change.status === 'pending');
|
||||
if (pending.length) {
|
||||
assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(result => result.tests?.failed > 0)), 'Capture a real baseline test failure before applying repairs.');
|
||||
for (const proposal of pending) {
|
||||
assert.equal(sha(proposal.path), proposal.baseSha256, 'Proposal baseline changed.');
|
||||
assert.notEqual(proposal.content.trim(), '', 'Empty source proposal.');
|
||||
if (precision) {
|
||||
assert.equal(proposal.editKind, 'exact-patch'); assert.ok(proposal.patch.edits > 0);
|
||||
assert.ok(proposal.patch.ranges.every(range => range.startLine >= bannerLines + 1), 'Patch must not select the preserved banner.');
|
||||
assert.deepEqual(Buffer.from(proposal.content).subarray(0, sourcePrefixes[proposal.path].length), sourcePrefixes[proposal.path]);
|
||||
assert.equal(proposal.patch.preservedBom, sourceFormats[proposal.path].bom); assert.equal(proposal.patch.lineEnding, sourceFormats[proposal.path].lineEnding);
|
||||
}
|
||||
}
|
||||
if (batch && !preparedSets.length) {
|
||||
assert.ok(++sealingFollowups <= 1, 'The actual model did not seal the requested batch after the bounded follow-up.');
|
||||
await api('/sessions/' + session.id + '/messages', { content: '沿用原任务与已保存的提案,不重新修改或重放工具。请用 prepare_change_set 一次封存以下待审查change_id,并等待用户审查:' + JSON.stringify(pending.map(proposal => proposal.id)) });
|
||||
continue;
|
||||
}
|
||||
assert.ok(++reviewedRounds <= reviewLimit, 'Too many repair/review rounds.');
|
||||
let changeSetId = null;
|
||||
if (preparedSets.length) {
|
||||
assert.equal(preparedSets.length, 1); const set = preparedSets[0]; changeSetId = set.id;
|
||||
assert.equal(set.source, 'agent'); assert.equal(set.taskId, originalTaskId);
|
||||
assert.deepEqual(set.files.map(file => file.changeId).sort(), pending.map(proposal => proposal.id).sort(), 'The reviewed batch must cover all pending proposals.');
|
||||
if (batch && reviewedRounds === 1) assert.deepEqual(set.files.map(file => file.path).sort(), [...allowed].sort(), 'The first batch must seal all four modules.');
|
||||
for (const file of set.files) {
|
||||
const proposal = pending.find(proposal => proposal.id === file.changeId);
|
||||
assert.equal(file.baseSha256, proposal.baseSha256); assert.equal(file.sha256, proposal.sha256); assert.equal(file.sourceRunId, proposal.runId);
|
||||
}
|
||||
await api('/sessions/' + session.id + '/change-sets/' + set.id + '/apply', { fingerprint: set.fingerprint });
|
||||
let completedSet;
|
||||
for (let attempt = 0; attempt < 120; attempt++) {
|
||||
completedSet = await api('/sessions/' + session.id + '/change-sets/' + set.id);
|
||||
if (!['applying', 'rolling_back'].includes(completedSet.status)) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
}
|
||||
assert.equal(completedSet.status, 'completed', completedSet.summary); assert.equal(completedSet.requiresVerification, false);
|
||||
assert.equal(completedSet.fingerprint, set.fingerprint); assert.ok(completedSet.files.every(file => file.writeAttempted && file.state === 'after'));
|
||||
batches.push(completedSet);
|
||||
} else {
|
||||
for (const proposal of pending) await api('/sessions/' + session.id + '/changes/' + proposal.id + '/apply', {});
|
||||
}
|
||||
for (const proposal of pending) {
|
||||
assert.equal(sha(proposal.path), proposal.sha256, 'Applied bytes do not match the reviewed proposal.');
|
||||
applied.push({ id: proposal.id, path: proposal.path, sha256: proposal.sha256, taskId: originalTaskId, changeSetId, ...(precision ? { editKind: proposal.editKind, patch: proposal.patch } : {}) });
|
||||
}
|
||||
assertProtected();
|
||||
await api('/sessions/' + session.id + '/continue-reviewed', {}); continue;
|
||||
}
|
||||
assert.equal(state.status, 'completed'); break;
|
||||
}
|
||||
assert.equal(state?.status, 'completed', 'Engineering delivery timed out.');
|
||||
assert.equal(state.execution.task.verification.status, 'passed', 'No authoritative passing engineering check.');
|
||||
const passedJob = state.execution.checks.filter(job => job.status === 'passed').at(-1);
|
||||
assert.ok(passedJob); assert.ok(passedJob.results.some(result => result.name === 'test' && result.exitCode === 0 && result.tests?.total === 14 && result.tests.passed === 14));
|
||||
assert.ok(passedJob.results.some(result => result.name === 'build' && result.exitCode === 0 && result.status === 'passed'));
|
||||
assert.deepEqual([...new Set(applied.map(item => item.path))].sort(), [...allowed].sort());
|
||||
for (const file of allowed) assert.notEqual(sha(file), beforeSources[file], 'Module was not repaired: ' + file);
|
||||
assertProtected();
|
||||
let precisionEvidence = null;
|
||||
if (precision) {
|
||||
const indexed = state.events.filter(event => event.tool === 'index_project' && event.phase === 'completed'), queries = state.events.filter(event => event.tool === 'query_code' && event.phase === 'completed');
|
||||
assert.ok(indexed.length > 0 && queries.length > 0, 'The model must use project index and symbol/import queries.');
|
||||
const sourceReads = state.events.filter(event => event.tool === 'read_file' && event.phase === 'completed' && allowed.includes(event.input?.path));
|
||||
for (const file of allowed) assert.ok(sourceReads.some(event => event.input.path === file), 'Each patched source must be read by its actual lines: ' + file);
|
||||
precisionEvidence = { bannerLines, formats: sourceFormats, preservedPrefixes: Object.fromEntries(allowed.map(file => [file, { bytes: sourcePrefixes[file].length, sha256: createHash('sha256').update(sourcePrefixes[file]).digest('hex') }])),
|
||||
indexCalls: indexed.length, queryCalls: queries.length, sourceReads: sourceReads.map(event => ({ path: event.input.path, startLine: event.input.start_line, endLine: event.input.end_line ?? null, runId: event.runId })),
|
||||
exactPatchesOnly: true, sourcePrefixesPreserved: true, formatPreserved: true };
|
||||
}
|
||||
const built = spawnSync(process.execPath, [path.join(projectRoot, 'dist/cli.mjs'), path.join(projectRoot, 'fixtures/tasks.json'), '--json'], { cwd: projectRoot, encoding: 'utf8', windowsHide: true });
|
||||
assert.equal(built.status, 0); assert.equal(built.stderr, '');
|
||||
assert.deepEqual(JSON.parse(built.stdout), { total: 4, counts: { todo: 0, doing: 1, done: 2, blocked: 1 }, completedPercent: 50, remainingMinutes: 50 });
|
||||
const diff = git(['diff', '--', ...allowed]); assert.ok(diff.length > 0); fs.writeFileSync(path.join(caseRoot, 'reviewed-changes.diff'), diff);
|
||||
let deliveryPlan = state.execution.deliveryPlans?.filter(plan => plan.status === 'planned' && plan.taskId === originalTaskId && plan.runId === state.runId).at(-1);
|
||||
if (!deliveryPlan) {
|
||||
state = await api('/sessions/' + session.id + '/delivery-plans', { message: 'feat: repair delivery report with 14 verified acceptance cases' });
|
||||
deliveryPlan = state.execution.deliveryPlans.at(-1);
|
||||
}
|
||||
assert.deepEqual(deliveryPlan.paths.slice().sort(), allowed.slice().sort());
|
||||
assert.equal(deliveryPlan.checkJobId, passedJob.id); assert.equal(deliveryPlan.sourceFingerprint, passedJob.sourceFingerprint);
|
||||
assert.equal(git(['rev-parse', 'HEAD']).trim(), baselineCommit); assertProtected();
|
||||
const queued = await api('/sessions/' + session.id + '/delivery-jobs', { planId: deliveryPlan.id });
|
||||
const deliveryJobId = queued.execution.deliveries.at(-1).id;
|
||||
let deliveryJob;
|
||||
for (let attempt = 0; attempt < 120; attempt++) {
|
||||
deliveryJob = await api('/sessions/' + session.id + '/delivery-jobs/' + deliveryJobId);
|
||||
if (!['queued', 'preparing', 'prepared', 'publishing', 'committed', 'reporting'].includes(deliveryJob.status)) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
assert.equal(deliveryJob.status, 'completed', deliveryJob.summary); assert.ok(deliveryJob.reportArtifactId); assert.ok(deliveryJob.reportMarkdownArtifactId);
|
||||
const deliveredCommit = git(['rev-parse', 'HEAD']).trim(); assert.equal(deliveredCommit, deliveryJob.commit);
|
||||
assert.notEqual(deliveredCommit, baselineCommit); assertProtected();
|
||||
assert.equal(git(['rev-list', '--count', 'HEAD']).trim(), '2');
|
||||
const readArtifact = async id => {
|
||||
let offset = 0, content = '', artifact;
|
||||
do { artifact = await api('/artifacts/' + id + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); content += artifact.content; offset += artifact.content.length; } while (artifact.truncated && artifact.content.length);
|
||||
assert.equal(createHash('sha256').update(content).digest('hex'), artifact.sha256);
|
||||
return content;
|
||||
};
|
||||
const deliveryJson = await readArtifact(deliveryJob.reportArtifactId), deliveryMarkdown = await readArtifact(deliveryJob.reportMarkdownArtifactId);
|
||||
const report = JSON.parse(deliveryJson); assert.equal(report.status, 'verified_committed'); assert.equal(report.task.id, originalTaskId);
|
||||
if (batch) assert.ok(batches.length > 0, 'The actual model must prepare a reviewed ChangeSet.');
|
||||
for (const set of batches) {
|
||||
const recorded = report.changeSets.find(value => value.id === set.id);
|
||||
assert.equal(recorded.status, 'completed'); assert.equal(recorded.fingerprint, set.fingerprint); assert.equal(recorded.snapshotVerified, true);
|
||||
assert.ok(report.changes.filter(change => set.files.some(file => file.changeId === change.id)).every(change => change.changeSetId === set.id));
|
||||
}
|
||||
assert.equal(report.verifiedCheckJobId, passedJob.id); assert.equal(report.deliveries.at(-1).commit, deliveredCommit);
|
||||
assert.equal(report.task.requirementsAcceptance, 'not_recorded'); assert.match(deliveryMarkdown, /14\/14/);
|
||||
fs.writeFileSync(path.join(caseRoot, 'delivery-report.json'), deliveryJson); fs.writeFileSync(path.join(caseRoot, 'delivery-report.md'), deliveryMarkdown);
|
||||
state = await api('/sessions/' + session.id);
|
||||
assert.deepEqual(git(['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), [...allowed].sort());
|
||||
const exported = await api('/sessions/' + session.id + '/export');
|
||||
fs.writeFileSync(path.join(caseRoot, 'session.json'), JSON.stringify(exported, null, 2));
|
||||
for (const job of state.execution.checks) for (const result of job.results) {
|
||||
if (!result.logArtifactId) continue;
|
||||
let offset = 0, text = '', log;
|
||||
do { log = await api('/artifacts/' + result.logArtifactId + '?session=' + session.id + '&offset=' + offset + '&limit=16000'); text += log.content; offset += log.content.length; } while (log.truncated && log.content.length);
|
||||
fs.writeFileSync(path.join(caseRoot, job.id + '-' + result.name + '.log'), text);
|
||||
}
|
||||
const result = { success: true, version: bootstrap.version, mode: batch ? 'reviewed-multi-file-exact-patch' : precision ? 'indexed-exact-patch' : 'whole-file', testedAt: new Date().toISOString(), model: bootstrap.config.model,
|
||||
projectRoot, projectId: project.id, sessionId: session.id, taskId: originalTaskId, baselineCommit, deliveredCommit, deliveryPlan, deliveryJob,
|
||||
elapsedSeconds: Math.round((Date.now() - started) / 1000), runs: state.execution.runs.length, reviewRounds: reviewedRounds, sealingFollowups, applied, batches,
|
||||
protectedHashes, ownerEditsPreserved: true, ownerStagePreserved: true, precisionEvidence, reportStatus: report.status, reportPath: path.join(caseRoot, 'delivery-report.md'), buildOutput: JSON.parse(built.stdout), usage: state.execution.task.usage,
|
||||
checks: state.execution.checks, decisions, diffPath: path.join(caseRoot, 'reviewed-changes.diff'), sessionPath: path.join(caseRoot, 'session.json'),
|
||||
credentialsHandling: 'Calls use the running host model configuration; no model key was read, copied or saved.',
|
||||
intervention: 'Verifier approved only sealed npm scripts, reviewed all source proposals before applying them through the single-file or fingerprint-bound batch API, and confirmed an owned Git preview. Implementation and batch preparation came from the configured model; the Workbench verified the files, checks, commit and reports.',
|
||||
scope: 'One synthetic Node engineering sample; does not establish all M2 engineering classes, worktrees or M6 delivery success rates.' };
|
||||
saveEvidence(result); succeeded = true;
|
||||
console.log(JSON.stringify({ success: true, model: result.model, testsPassed: 14, buildPassed: true, deliveredCommit, elapsedSeconds: result.elapsedSeconds, reviewRounds: reviewedRounds, projectRoot, sessionId: session.id }, null, 2));
|
||||
} catch (error) {
|
||||
saveEvidence({ success: false, version: bootstrap.version, testedAt: new Date().toISOString(), model: bootstrap.config.model, projectRoot,
|
||||
projectId: project.id, sessionId: session.id, taskId: originalTaskId, error: error.message, elapsedSeconds: Math.round((Date.now() - started) / 1000), applied, decisions, batches,
|
||||
checks: state?.execution?.checks ?? [] });
|
||||
throw error;
|
||||
} finally {
|
||||
if (!succeeded) await api('/sessions/' + session.id + '/stop', {}).catch(() => {});
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { Store } from '../src/server/store.mjs';
|
||||
import { APP_VERSION } from '../src/server/diagnostics.mjs';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { createPythonDeliveryProject, pythonProtected } from '../tests/fixtures/python-project.mjs';
|
||||
import { deliveryGit, writeDeliveryFile } from '../tests/fixtures/delivery-project.mjs';
|
||||
import { captureWorkspaceSource, assertWorkspaceSource } from '../tests/fixtures/workspace-project.mjs';
|
||||
import { verifyLiveReactClient } from './live-react-client.mjs';
|
||||
import { observedTaskModels } from './live-model-observation.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-live-engineering.mjs --case react|python [--url http://127.0.0.1:4318] — fresh owned Git source/workspace, configured model and bounded review/check/delivery acceptance'); process.exit(0);
|
||||
}
|
||||
const index = process.argv.indexOf('--case'), kind = index >= 0 ? process.argv[index + 1] : null;
|
||||
assert.ok(['react', 'python'].includes(kind), '--case react or --case python is required');
|
||||
const urlIndex = process.argv.indexOf('--url'), origin = new URL(urlIndex >= 0 ? process.argv[urlIndex + 1] : 'http://127.0.0.1:4318');
|
||||
assert.equal(origin.hostname, '127.0.0.1'); assert.equal(origin.protocol, 'http:');
|
||||
const bootstrap = await (await fetch(origin.origin + '/api/bootstrap', { headers: { Connection: 'close' } })).json();
|
||||
assert.equal(bootstrap.version, APP_VERSION); assert.equal(bootstrap.config.mode, 'compatible'); assert.ok(bootstrap.config.model);
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(origin.origin + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
||||
const value = await response.json(); if (!response.ok) throw Object.assign(new Error(value.error ?? 'Workbench HTTP ' + response.status), { status: response.status }); return value;
|
||||
};
|
||||
const root = path.resolve('.qa', 'live-' + kind + '-' + randomUUID()), sourceRoot = path.join(root, kind + '-source');
|
||||
fs.mkdirSync(sourceRoot, { recursive: true }); fs.writeFileSync(path.join(sourceRoot, 'README.md'), '# Owned ' + kind + ' engineering acceptance\n');
|
||||
const sourceStore = new Store(path.join(root, 'fixture-data')), sourceProject = sourceStore.addProject(sourceRoot), fixtureSession = sourceStore.createSession(sourceProject.id);
|
||||
const sourceF = { projectRoot: sourceRoot, store: sourceStore, session: fixtureSession };
|
||||
const allowed = kind === 'react' ? ['src/query.ts', 'src/summary.ts', 'src/transition.ts', 'src/App.tsx']
|
||||
: ['src/workflow/normalize.py', 'src/workflow/summary.py', 'src/workflow/render.py'];
|
||||
const checkNames = kind === 'react' ? ['test', 'typecheck', 'build'] : ['python:test', 'python:compile'];
|
||||
try {
|
||||
if (kind === 'python') createPythonDeliveryProject(sourceF);
|
||||
else {
|
||||
fs.cpSync(path.resolve('benchmarks/react-delivery-v1/baseline'), sourceRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
||||
writeDeliveryFile(sourceF, '.gitignore', 'node_modules/\ndist/\n.env\n.awb-*\n');
|
||||
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(sourceF, name, 'Owner baseline bytes\n');
|
||||
deliveryGit(sourceF, ['init', '--initial-branch=main']); deliveryGit(sourceF, ['config', 'core.autocrlf', 'false']);
|
||||
deliveryGit(sourceF, ['config', 'user.name', 'Workbench Engineering QA']); deliveryGit(sourceF, ['config', 'user.email', 'qa@workbench.invalid']);
|
||||
deliveryGit(sourceF, ['add', '--', '.']); deliveryGit(sourceF, ['commit', '--no-gpg-sign', '-m', 'Seal four-file React failure for actual-model acceptance']);
|
||||
writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner staged change\n'); deliveryGit(sourceF, ['add', '--', 'owner-staged.txt']);
|
||||
writeDeliveryFile(sourceF, 'owner-staged.txt', 'Owner working copy after staging\n'); writeDeliveryFile(sourceF, 'owner-unstaged.txt', 'Owner unstaged change\n');
|
||||
writeDeliveryFile(sourceF, 'owner-untracked.txt', 'Owner untracked change\n');
|
||||
}
|
||||
} finally { sourceStore.close(); }
|
||||
const originalSource = captureWorkspaceSource(sourceF), protectedNames = kind === 'react'
|
||||
? ['README.md', 'package.json', 'package-lock.json', 'tsconfig.json', 'vite.config.mjs', 'index.html', 'test/domain.test.mjs', 'src/types.ts', 'src/main.tsx', 'src/styles.css']
|
||||
: ['README.md', ...pythonProtected, 'package.json', 'test/add.test.mjs', 'scripts/build.mjs', 'src/add.mjs'];
|
||||
const parentProject = await api('/projects', { root: sourceRoot }), parentSession = await api('/sessions', { projectId: parentProject.id });
|
||||
const parentRoute = '/sessions/' + parentSession.id;
|
||||
let state, workspace, childProject, childSession, route, taskId, result, succeeded = false, completedDelivery = null, lastProgress = '';
|
||||
const decisions = [], applications = [], continuations = [], startedAt = Date.now(), maxModelResponses = 30, maxReportedTokens = 180000, maxDurationMs = 360000;
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
const wait = async (predicate, timeout = 30000) => {
|
||||
const deadline = Date.now() + timeout; while (!await predicate()) { if (Date.now() > deadline) throw new Error('Acceptance wait timed out'); await new Promise(resolve => setTimeout(resolve, 250)); }
|
||||
};
|
||||
let protectedBytes, childF, ownerStage;
|
||||
const assertProtected = () => {
|
||||
assertWorkspaceSource(assert, sourceF, originalSource, workspace.branch);
|
||||
for (const [name, bytes] of Object.entries(protectedBytes)) assert.deepEqual(fs.readFileSync(path.join(childProject.root, name)), bytes, 'Protected input changed: ' + name);
|
||||
assert.equal(deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']), ownerStage);
|
||||
};
|
||||
const readArtifact = async id => {
|
||||
let text = '', offset = 0, page;
|
||||
do { page = await api('/artifacts/' + id + '?session=' + childSession.id + '&offset=' + offset + '&limit=16000');
|
||||
assert.ok(page.content.length || !page.truncated); text += page.content; offset += page.content.length; } while (page.truncated);
|
||||
assert.equal(sha(text), page.sha256); return text;
|
||||
};
|
||||
try {
|
||||
const prepared = await api(parentRoute + '/workspaces', { label: kind + ' actual engineering' }); workspace = prepared;
|
||||
await api(parentRoute + '/workspaces/' + workspace.id + '/create', { fingerprint: workspace.fingerprint });
|
||||
await wait(async () => { workspace = await api(parentRoute + '/workspaces/' + workspace.id); return workspace.status === 'ready'; }, 30000);
|
||||
const opened = await api(parentRoute + '/workspaces/' + workspace.id + '/open', {}); childProject = opened.project; childSession = opened.session; route = '/sessions/' + childSession.id;
|
||||
childF = { projectRoot: childProject.root };
|
||||
if (kind === 'react') {
|
||||
const npmCli = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
||||
const installed = execFileSync(process.execPath, [npmCli, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: childProject.root, env: gitEnvironment(), windowsHide: true, encoding: 'utf8' });
|
||||
fs.writeFileSync(path.join(root, 'offline-install.log'), installed);
|
||||
}
|
||||
for (const name of ['owner-staged.txt', 'owner-unstaged.txt']) writeDeliveryFile(childF, name, 'Child owner baseline\n');
|
||||
deliveryGit(childF, ['add', '--', 'owner-staged.txt']); writeDeliveryFile(childF, 'owner-staged.txt', 'Child owner working copy after stage\n');
|
||||
writeDeliveryFile(childF, 'owner-untracked.txt', 'Child owner untracked change\n');
|
||||
protectedBytes = Object.fromEntries([...protectedNames, 'owner-staged.txt', 'owner-unstaged.txt', 'owner-untracked.txt'].map(name => [name, fs.readFileSync(path.join(childProject.root, name))]));
|
||||
ownerStage = deliveryGit(childF, ['diff', '--cached', '--', 'owner-staged.txt']); assertProtected();
|
||||
const requirements = kind === 'react'
|
||||
? '修复交付看板的query、summary、transition和App四个模块。状态筛选与trim后忽略大小写的搜索取交集;统计针对全部任务,完成数量及待办工时准确;toggle只切换指定id并保持其他对象和原数组,App绑定筛选/搜索和函数式状态更新,切换一条任务后界面与统计同步。必须通过9项测试、类型检查、构建,以及真正浏览器的筛选/搜索/单任务切换。'
|
||||
: '修复normalize、summary和render三个Python模块:normalize把数字或数字字符串转换为数值并把负数归零;summary返回count和数值合计total,空列表为0;render返回“共 N 项 · 合计 X”,整数合计没有多余小数。必须通过六项unittest及源码编译。';
|
||||
const goal = requirements + '先用inspect_project核对工程与工作区,再用plan_checks明确选择' + JSON.stringify(checkNames)
|
||||
+ '并用run_checks实际执行,保留基线失败与日志。根据真实失败读取有关源码;只能修改' + JSON.stringify(allowed)
|
||||
+ ',测试、配置、README、类型声明、样式、OWNER文件和已有暂存保持。不能改测试来通过,不使用run_command、不能直接提交Git。'
|
||||
+ '用propose_patch或propose_file准备全部允许模块的提案,再用prepare_change_set一次封存完整三/四文件批次,等待用户审查应用;未审查的提案不是已写入。应用后继续同一Task,用相同检查范围重新执行并根据日志修复。全部检查通过后用plan_delivery准备这些文件的本地Git预览,等待确认;停止继续查询或请求模型。最后以实际结果说明测试、构建和交付。';
|
||||
await api(route + '/messages', { content: goal });
|
||||
while (Date.now() - startedAt < maxDurationMs) {
|
||||
state = await api(route); taskId ??= state.execution.task.id; assert.equal(state.execution.task.id, taskId); assertProtected();
|
||||
if (state.execution.task.usage.requests > maxModelResponses || state.execution.task.usage.totalTokens > maxReportedTokens) throw new Error('Actual-model acceptance reached its bounded response/token scope');
|
||||
const progress = state.status + '|' + state.events.length + '|' + state.execution.checks.map(job => job.status).join(',');
|
||||
if (progress !== lastProgress) { console.log(JSON.stringify({ kind, status: state.status, elapsedSeconds: Math.round((Date.now() - startedAt) / 1000),
|
||||
responses: state.execution.task.usage.requests, reportedTokens: state.execution.task.usage.totalTokens,
|
||||
latestTools: state.events.filter(event => event.kind === 'tool').slice(-3).map(event => ({ tool: event.tool, phase: event.phase })) })); lastProgress = progress; }
|
||||
for (const proposal of state.changes.filter(change => ['pending', 'conflict'].includes(change.status))) assert.ok(allowed.includes(proposal.path), 'Out-of-scope model proposal: ' + proposal.path);
|
||||
for (const approval of state.approvals.filter(item => item.status === 'pending')) {
|
||||
const plan = state.execution.plans.find(value => value.id === approval.checkPlanId);
|
||||
const allow = !!plan && plan.taskId === taskId && plan.checks.length > 0 && plan.checks.every(check => checkNames.includes(check.name) && !check.lifecycle?.length)
|
||||
&& approval.cwd === childProject.root && approval.command === plan.checks.map(check => check.command).join('\n');
|
||||
decisions.push({ approvalId: approval.id, checkPlanId: plan?.id, command: approval.command, allow });
|
||||
await api(route + '/approvals/' + approval.id, { allow }); assert.ok(allow, 'Requested command does not match the owned engineering contract');
|
||||
}
|
||||
if (['running', 'awaiting_approval', 'stopping'].includes(state.status)) { await new Promise(resolve => setTimeout(resolve, 750)); continue; }
|
||||
if (state.status === 'failed') throw new Error(state.failure?.message ?? 'Actual-model Task failed');
|
||||
const deliveryPlan = state.execution.deliveryPlans?.findLast(plan => plan.status === 'planned' && plan.taskId === taskId);
|
||||
if (deliveryPlan) {
|
||||
assert.deepEqual(deliveryPlan.paths.slice().sort(), allowed.slice().sort()); const requestsBefore = state.execution.task.usage.requests;
|
||||
assert.ok(state.events.some(event => event.tool === 'plan_delivery' && event.phase === 'completed'), 'The model must prepare the Git review');
|
||||
await api(route + '/delivery-jobs', { planId: deliveryPlan.id });
|
||||
await wait(async () => { state = await api(route); completedDelivery = state.execution.deliveries.at(-1); return completedDelivery?.reportArtifactId && completedDelivery?.reportMarkdownArtifactId; }, 30000);
|
||||
assert.equal(completedDelivery.status, 'completed'); assert.equal(state.execution.task.usage.requests, requestsBefore); break;
|
||||
}
|
||||
const pending = state.changes.filter(change => change.status === 'pending');
|
||||
const batch = state.execution.changeSets?.findLast(set => set.taskId === taskId && set.status === 'prepared');
|
||||
if (state.status === 'paused' && !batch) {
|
||||
assert.equal(state.execution.run.stopReason, 'step_limit', 'Other stopped outcomes are recorded, not automatically replayed');
|
||||
assert.ok(continuations.length < 3); continuations.push({ runId: state.runId, reason: 'step_limit', taskId }); await api(route + '/resume', {}); continue;
|
||||
}
|
||||
if (pending.length) {
|
||||
assert.ok(state.execution.checks.some(job => job.status === 'failed' && job.results.some(value => value.tests?.failed > 0)), 'Record actual initial test failure before repairs');
|
||||
assert.ok(batch && batch.source === 'agent', 'The model must seal the reviewed batch');
|
||||
assert.deepEqual(batch.files.map(file => file.changeId).sort(), pending.map(change => change.id).sort());
|
||||
if (!applications.length) assert.deepEqual(batch.files.map(file => file.path).sort(), allowed.slice().sort());
|
||||
assert.ok(applications.length < 3); await api(route + '/change-sets/' + batch.id + '/apply', { fingerprint: batch.fingerprint });
|
||||
await wait(async () => (await api(route + '/change-sets/' + batch.id)).status === 'completed');
|
||||
for (const proposal of pending) assert.equal(sha(fs.readFileSync(path.join(childProject.root, proposal.path))), proposal.sha256);
|
||||
applications.push({ id: batch.id, fingerprint: batch.fingerprint, paths: batch.files.map(file => file.path), source: batch.source }); assertProtected();
|
||||
await api(route + '/continue-reviewed', {}); continue;
|
||||
}
|
||||
throw new Error('The model stopped before the complete check/review/Git flow: ' + state.status + '/' + state.execution.run.stopReason);
|
||||
}
|
||||
assert.ok(completedDelivery, 'Actual-model engineering acceptance timed out'); assert.equal(state.status, 'completed');
|
||||
const passing = state.execution.checks.filter(job => job.status === 'passed').at(-1); assert.ok(passing);
|
||||
assert.deepEqual(passing.results.map(value => value.name).sort(), checkNames.slice().sort());
|
||||
const tests = passing.results.find(value => value.name === (kind === 'react' ? 'test' : 'python:test')).tests;
|
||||
assert.equal(tests.total, kind === 'react' ? 9 : 6); assert.equal(tests.passed, tests.total); assert.equal(tests.failed, 0);
|
||||
assert.deepEqual(deliveryGit(childF, ['diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD']).trim().split(/\r?\n/).sort(), allowed.slice().sort()); assertProtected();
|
||||
const reportText = await readArtifact(completedDelivery.reportArtifactId), markdown = await readArtifact(completedDelivery.reportMarkdownArtifactId), report = JSON.parse(reportText);
|
||||
assert.equal(report.status, 'verified_committed'); assert.equal(report.task.id, taskId); assert.equal(report.verifiedCheckJobId, passing.id);
|
||||
fs.writeFileSync(path.join(root, 'delivery-report.json'), reportText); fs.writeFileSync(path.join(root, 'delivery-report.md'), markdown);
|
||||
const browser = kind === 'react' ? await verifyLiveReactClient(childProject.root, path.join(root, 'screenshots')) : null;
|
||||
fs.writeFileSync(path.join(root, 'session.json'), JSON.stringify(await api(route + '/export'), null, 2) + '\n');
|
||||
result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), kind, root,
|
||||
taskId, sessionId: childSession.id, projectId: childProject.id, sourceProjectId: parentProject.id, workspaceId: workspace.id, workspaceRoot: childProject.root,
|
||||
sourceCommit: workspace.source.head, commit: completedDelivery.commit, tests, checks: passing.results, usage: state.execution.task.usage,
|
||||
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), applications, decisions, continuations, browser,
|
||||
reportStatus: report.status, reportSha256: sha(reportText), sourcePreserved: true, protectedInputs: Object.fromEntries(Object.entries(protectedBytes).map(([name, bytes]) => [name, sha(bytes)])),
|
||||
scope: 'One actual configured-model engineering sample. Verifier confirms bounded checks, reviews model-written batches and confirms the model-prepared Git preview; no implementation supplied by the verifier. Not an autonomous success-rate measurement.',
|
||||
credentialsHandling: 'The original Provider executes requests in its own process; no model key is read, copied or written.' };
|
||||
succeeded = true;
|
||||
} catch (error) {
|
||||
result = { success: false, version: bootstrap.version, kind, testedAt: new Date().toISOString(), root,
|
||||
taskId, sessionId: childSession?.id, projectId: childProject?.id, workspaceId: workspace?.id, error: error.stack,
|
||||
elapsedSeconds: Math.round((Date.now() - startedAt) / 1000), usage: state?.execution?.task?.usage, state: state?.status,
|
||||
stopReason: state?.execution?.run?.stopReason, applications, decisions, continuations, checks: state?.execution?.checks }; process.exitCode = 1;
|
||||
} finally {
|
||||
if (!succeeded && route) await api(route + '/stop', {}).catch(() => {});
|
||||
let refreshError=null;
|
||||
if(route){try{state=await api(route);taskId??=state.execution?.task?.id;}catch(error){refreshError=error.message;}}
|
||||
const observation=observedTaskModels(state,taskId);
|
||||
if(refreshError){observation.status='incomplete';observation.model=null;observation.issues.push({kind:'final_state_refresh_failed',message:refreshError});}
|
||||
result.model=observation.model;result.modelObservation=observation;result.plannedModel=bootstrap.config.model;
|
||||
if(state?.execution?.task?.id===taskId)result.usage=state.execution.task.usage;
|
||||
}
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync(path.resolve('docs', 'live-' + kind + '-engineering-validation.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, kind, root, taskId, model:result.model,modelObservation:result.modelObservation.status,
|
||||
commit: result.commit, tests: result.tests, usage: result.usage, error: result.error }));
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
// Offline verification only. This script never opens a model connection or reads credentials.
|
||||
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const root = path.join(repo, 'docs/live-model-identity-j-evidence');
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
||||
const manifest = read('manifest.json');
|
||||
const listed = new Set();
|
||||
for (const file of manifest.entries) {
|
||||
assert.equal(path.posix.isAbsolute(file.path), false);
|
||||
assert.equal(file.path.includes('\\'), false);
|
||||
assert.equal(file.path.split('/').includes('..'), false);
|
||||
assert.equal(listed.has(file.path), false); listed.add(file.path);
|
||||
const bytes = fs.readFileSync(path.join(root, file.path));
|
||||
assert.equal(bytes.length, file.bytes, file.path + ' length');
|
||||
assert.equal(sha(bytes), file.sha256, file.path + ' original bytes');
|
||||
}
|
||||
const actual = [];
|
||||
function walk(relative = '') {
|
||||
for (const entry of fs.readdirSync(path.join(root, relative), { withFileTypes: true })) {
|
||||
const name = path.posix.join(relative, entry.name);
|
||||
if (entry.isDirectory()) walk(name);
|
||||
else { assert.ok(entry.isFile(), 'Only regular evidence files'); if (name !== 'manifest.json') actual.push(name); }
|
||||
}
|
||||
}
|
||||
walk(); assert.deepEqual(actual.sort(), [...listed].sort());
|
||||
assert.equal(manifest.count, listed.size);
|
||||
assert.equal(manifest.bytes, manifest.entries.reduce((n, f) => n + f.bytes, 0));
|
||||
const contract = read('contract.json'), result = read('result.json'), attempts = read('attempts.json');
|
||||
const state = read('session.json'), exported = read('session-export.json');
|
||||
const requestBytes = fs.readFileSync(path.join(root, 'request.json'));
|
||||
const request = JSON.parse(requestBytes);
|
||||
const response = fs.readFileSync(path.join(root, 'response.sse'));
|
||||
assert.equal(contract.maxRequests, 1); assert.equal(contract.maxAttempts, 1);
|
||||
assert.equal(contract.maxOutputTokens, 256); assert.equal(contract.maxTaskTokens, 16000);
|
||||
assert.equal(contract.modelIdentityPolicy, 'require-match'); assert.deepEqual(contract.aliases, []);
|
||||
assert.equal(contract.poolSwitching, false); assert.equal(contract.automaticReplay, false);
|
||||
assert.equal(attempts.length, 1);
|
||||
const attempt = attempts[0];
|
||||
assert.equal(attempt.connectionId, contract.connectionId); assert.equal(attempt.poolId, null);
|
||||
assert.equal(attempt.ordinal, 1); assert.equal(attempt.requestSha256, sha(JSON.stringify(request)));
|
||||
assert.equal(attempt.responseSha256, sha(response)); assert.equal(attempt.responseRedacted, false);
|
||||
assert.equal(request.model, contract.requestedModel); assert.equal(request.model, 'claude-opus-5-5');
|
||||
assert.equal(request.max_tokens ?? request.max_completion_tokens, 256); assert.equal(request.stream, true);
|
||||
const data = response.toString('utf8').split(/\r?\n/).filter(line => line.startsWith('data:')).map(line => line.slice(5).trim());
|
||||
assert.equal(data.at(-1), '[DONE]');
|
||||
const packets = data.filter(line => line !== '[DONE]').map(line => JSON.parse(line));
|
||||
const models = [...new Set(packets.filter(p => Object.hasOwn(p, 'model')).map(p => p.model))];
|
||||
assert.deepEqual(models, ['gpt-6.1-sol']); assert.notEqual(request.model, models[0]);
|
||||
const rawUsage = packets.findLast(p => p.usage)?.usage;
|
||||
const usage = Object.fromEntries(['prompt_tokens', 'completion_tokens', 'total_tokens'].map(key => [key, rawUsage[key]]));
|
||||
assert.deepEqual(rawUsage.prompt_tokens_details, { cache_write_tokens: 4304 });
|
||||
assert.deepEqual(usage, attempt.usage);
|
||||
assert.equal(usage.prompt_tokens + usage.completion_tokens, usage.total_tokens);
|
||||
assert.equal(usage.total_tokens, 4334); assert.ok(usage.total_tokens <= contract.maxTaskTokens);
|
||||
assert.equal(packets.flatMap(p => p.choices ?? []).flatMap(c => c.delta?.tool_calls ?? []).length, 0);
|
||||
const answer = packets.flatMap(p => p.choices ?? []).map(c => c.delta?.content ?? '').join('');
|
||||
assert.equal(answer, 'OK ' + contract.nonce);
|
||||
assert.equal(state.status, 'paused'); assert.equal(state.execution.run.stopReason, 'model_identity');
|
||||
assert.equal(state.execution.progress.totalTools, 0); assert.equal(state.execution.accountedTaskTokens, 4334);
|
||||
assert.deepEqual(state.execution.run.budget, { maxSteps: 1, maxDurationMs: 320000, maxTokens: 16000, maxTaskTokens: 16000 });
|
||||
const consumption = state.execution.run.consumption;
|
||||
assert.equal(consumption.length, 1); assert.equal(consumption[0].attemptId, attempt.id);
|
||||
assert.deepEqual(consumption[0].usage, usage); assert.deepEqual(consumption[0].modelIdentity.reportedModels, models);
|
||||
assert.equal(consumption[0].modelIdentity.policy, 'require-match'); assert.equal(consumption[0].modelIdentity.complete, true);
|
||||
assert.equal(consumption[0].modelIdentity.status, 'mismatch');
|
||||
assert.equal(consumption[0].modelIdentity.responseSource.sha256, sha(response));
|
||||
assert.ok(consumption[0].localEstimate.inputTokens + consumption[0].localEstimate.outputTokens <= contract.maxTaskTokens);
|
||||
assert.equal(attempt.status, 'failed'); assert.equal(attempt.diagnostic.kind, 'model_identity');
|
||||
assert.equal(attempt.diagnostic.retryable, false); assert.equal(attempt.diagnostic.failoverAllowed, false);
|
||||
assert.equal(result.identityPassed, false); assert.equal(result.engineeringTestAllowed, false);
|
||||
assert.equal(result.realEngineeringDeliveryTested, false); assert.equal(result.purgeBenefitTested, false);
|
||||
assert.equal(result.modelWeightsAuthenticated, false); assert.deepEqual(result.usage, usage);
|
||||
assert.equal(exported.session.id, state.id); assert.deepEqual(exported.execution, state.execution);
|
||||
for (const capture of result.captures) assert.equal(capture.sha256, sha(fs.readFileSync(path.join(root, capture.file))));
|
||||
const restoration = read('restoration.json'), audit = read('state-audit.json'), baseline = read('baseline-summary.json');
|
||||
assert.equal(restoration.passed, true); assert.equal(audit.passed, true);
|
||||
assert.equal(restoration.pid, 36552); assert.equal(restoration.addedAttempts, 1);
|
||||
assert.equal(restoration.originalExports, 70); assert.equal(restoration.originalArtifactFiles, 3566);
|
||||
assert.equal(restoration.originalCredentialRevision, restoration.restoredCredentialRevision);
|
||||
assert.equal(restoration.originalSelectionRestored, true);
|
||||
assert.equal(audit.baselineSha256, baseline.beforeSha256); assert.equal(audit.afterSha256, baseline.afterSha256);
|
||||
for (const table of audit.checks) assert.equal(table.exact + table.normalized, table.original);
|
||||
assert.equal(audit.additions.length, 60);
|
||||
assert.deepEqual(audit.diskConfigRevisions, { original: 48, actual: 52 });
|
||||
console.log(JSON.stringify({ passed: true, files: manifest.count, bytes: manifest.bytes, requests: 1,
|
||||
requestedModel: request.model, reportedModels: models, identityPassed: false, reportedTokens: usage.total_tokens,
|
||||
engineeringTested: false, purgeBenefitTested: false,
|
||||
scope: 'Offline original-artifact and receipt consistency. No model request; no authentication of supplier model weights or reconstruction of excluded private original user data.' }));
|
||||
@@ -0,0 +1,76 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
|
||||
const bootstrapResponse = await fetch(url + '/api/bootstrap');
|
||||
assert.ok(bootstrapResponse.ok, '工作台未连接');
|
||||
const bootstrap = await bootstrapResponse.json();
|
||||
assert.equal(bootstrap.config.mode, 'compatible', '请先保存兼容模型配置');
|
||||
assert.ok(bootstrap.config.model, '没有选择模型');
|
||||
const root = path.resolve('.qa', 'live-model-' + randomUUID(), '模型连接验证');
|
||||
const proof = randomUUID();
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'facts.txt'), 'PROOF_TOKEN=' + proof + '\nINPUT_NUMBER=21\n', 'utf8');
|
||||
fs.writeFileSync(path.join(root, 'README.md'), '# 模型连接验证\n\n此目录只有自动化验证的合成数据。\n', 'utf8');
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error || ('请求失败 ' + response.status));
|
||||
return body;
|
||||
};
|
||||
const project = await api('/projects', { root });
|
||||
const created = await api('/sessions', { projectId: project.id });
|
||||
let completed = false;
|
||||
const started = Date.now();
|
||||
try {
|
||||
await api('/sessions/' + created.id + '/messages', { content:
|
||||
'这是工作台的真实模型连接验证。请先调用 read_file 读取 facts.txt,取得 PROOF_TOKEN 和 INPUT_NUMBER;不能从猜测得出内容。' +
|
||||
'随后调用 propose_file,提出新文件 MODEL-RESULT.md,内容包含读取到的完整 PROOF_TOKEN,以及 INPUT_NUMBER 乘以 2 的值,格式写成 PROOF_TOKEN=... 和 DOUBLED_NUMBER=...。' +
|
||||
'只产生提案,不能声称文件已经写入。最后用一句中文说明结果。不要运行终端命令,不要读取其他项目,不要重复提出相同提案。' });
|
||||
let state, lastProgress = '';
|
||||
while (Date.now() - started < 120000) {
|
||||
state = await api('/sessions/' + created.id);
|
||||
const progress = state.status + '|' + state.events.length;
|
||||
if (progress !== lastProgress) {
|
||||
console.log(JSON.stringify({ status: state.status, tools: state.events.filter(event => event.kind === 'tool').map(event => ({ tool: event.tool, phase: event.phase })), elapsedSeconds: Math.round((Date.now() - started) / 1000) }));
|
||||
lastProgress = progress;
|
||||
}
|
||||
for (const approval of state.approvals.filter(approval => approval.status === 'pending')) {
|
||||
await api('/sessions/' + created.id + '/approvals/' + approval.id, { allow: false });
|
||||
}
|
||||
if (!['running', 'awaiting_approval', 'stopping'].includes(state.status)) break;
|
||||
if (state.events.filter(event => event.kind === 'tool').length > 6) throw new Error('验证任务超出预期工具次数,已停止');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
if (!state || ['running', 'awaiting_approval', 'stopping'].includes(state.status)) throw new Error('模型验证超时');
|
||||
if (state.status === 'failed') throw new Error(state.messages.filter(message => message.role === 'system').at(-1)?.content || '模型请求失败');
|
||||
assert.equal(state.executionMode, 'compatible');
|
||||
assert.ok(state.events.some(event => event.tool === 'read_file' && event.phase === 'completed' && event.input?.path === 'facts.txt'), '未取得真实文件读取结果');
|
||||
const proposal = state.changes.find(change => change.path === 'MODEL-RESULT.md' && change.status === 'pending');
|
||||
assert.ok(proposal, '模型没有生成预期文件提案');
|
||||
assert.ok(proposal.content.includes(proof), '提案没有包含实际读取的 token');
|
||||
assert.match(proposal.content, /DOUBLED_NUMBER\s*=\s*42/);
|
||||
assert.equal(fs.existsSync(path.join(root, 'MODEL-RESULT.md')), false, '审核前文件不应写入');
|
||||
await api('/sessions/' + created.id + '/changes/' + proposal.id + '/apply', {});
|
||||
assert.equal(fs.readFileSync(path.join(root, 'MODEL-RESULT.md'), 'utf8'), proposal.content);
|
||||
state = await api('/sessions/' + created.id);
|
||||
completed = true;
|
||||
const result = { success: true, testedAt: new Date().toISOString(), endpoint: bootstrap.config.baseUrl,
|
||||
model: bootstrap.config.model, projectRoot: root, projectId: project.id, sessionId: created.id,
|
||||
status: state.status, elapsedSeconds: Math.round((Date.now() - started) / 1000), usage: state.usage,
|
||||
tools: state.events.filter(event => event.kind === 'tool').map(event => ({ tool: event.tool, phase: event.phase })),
|
||||
evidence: ['真实配置的模型服务响应', 'read_file 读取随机合成数据', 'propose_file 使用读取结果', '审核前不写盘', '核对提案后应用成功'],
|
||||
credentialsHandling: '验证脚本使用服务内存中的连接配置,没有读取或记录 API Key' };
|
||||
fs.writeFileSync('.qa/live-model-validation.json', JSON.stringify(result, null, 2));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
fs.writeFileSync('.qa/live-model-validation.json', JSON.stringify({ success: false, testedAt: new Date().toISOString(), model: bootstrap.config.model,
|
||||
sessionId: created.id, projectRoot: root, error: error.message, elapsedSeconds: Math.round((Date.now() - started) / 1000) }, null, 2));
|
||||
throw error;
|
||||
} finally {
|
||||
if (!completed) { try { await api('/sessions/' + created.id + '/stop', {}); } catch {} }
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { completionBody, requestMetrics } from '../src/shared/contracts/model-request.mjs';
|
||||
import { nativeMessagesToChat } from '../src/shared/contracts/engine-execution.mjs';
|
||||
import { nativeSystemTexts } from '../src/core/engines/purge-strategy.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log('node scripts/verify-live-pilot-evidence.mjs — verify sealed 1.24 pilot bytes, source identities, original request/SSE usage and native input rejection; offline, no model calls.');
|
||||
process.exit(0);
|
||||
}
|
||||
const root=path.resolve('docs/live-model-purge-1.24-evidence');
|
||||
const read=file=>JSON.parse(fs.readFileSync(file,'utf8').replace(/^\uFEFF/,''));
|
||||
const sha=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const manifest=read(path.join(root,'manifest.json')),entries=new Map(manifest.entries.map(x=>[x.path,x]));
|
||||
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.formalVersion,'1.24.0');assert.equal(manifest.sampleCount,6);
|
||||
assert.equal(manifest.upstreamPluginLoaded,false);assert.equal(manifest.originalPluginBenefit,'not_tested');
|
||||
assert.equal(entries.size,manifest.count);
|
||||
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
|
||||
const from=relative=>{assert.ok(entries.has(relative),'File belongs to manifest: '+relative);return read(inside(relative));};
|
||||
let bytes=0;
|
||||
for(const entry of entries.values()){
|
||||
const file=inside(entry.path);assert.equal(fs.lstatSync(file).isFile(),true,entry.path);
|
||||
const original=fs.readFileSync(file);assert.equal(original.length,entry.bytes,entry.path);assert.equal(sha(original),entry.sha256,entry.path);bytes+=original.length;
|
||||
}
|
||||
assert.equal(bytes,manifest.bytes);
|
||||
const files=[];
|
||||
function visit(relative='') {
|
||||
for(const entry of fs.readdirSync(path.join(root,relative),{withFileTypes:true})){
|
||||
const next=path.posix.join(relative,entry.name);assert.equal(entry.isSymbolicLink(),false,next);
|
||||
if(entry.isDirectory())visit(next);else{assert.equal(entry.isFile(),true,next);files.push(next);}
|
||||
}
|
||||
}
|
||||
visit();assert.deepEqual(files.sort(),[...entries.keys(),'manifest.json'].sort(),'Exact sealed file scope');
|
||||
assert.match(manifest.sourceCommit,/^[a-f0-9]{40}$/);
|
||||
for(const input of manifest.sourceInputs){
|
||||
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$/);assert.ok(!input.path.split('/').includes('..'));
|
||||
const data=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:8*1024*1024});
|
||||
assert.equal(data.length,input.bytes,input.path);assert.equal(sha(data),input.sha256,'Historical source used by the sealed pilot: '+input.path);
|
||||
}
|
||||
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,381);
|
||||
for(const a of artifacts.values()){assert.equal(entries.get(a.file)?.sha256,a.sha256,a.id);assert.equal(entries.get(a.file)?.bytes,a.bytes,a.id);}
|
||||
const setup=from('receipts/setup-public-receipt.json'),analysis=from('receipts/analysis.json'),preservation=from('receipts/state-preservation.json');
|
||||
const complete=from('receipts/complete-owned-artifact-receipts.json'),reconciliation=from('receipts/request-and-source-reconciliation.json');
|
||||
assert.equal(complete.newUniqueFiles,342);assert.equal(complete.artifactRecords,381);assert.equal(complete.totalArtifactBytes,6426284);
|
||||
assert.equal(preservation.passed,true);assert.equal(preservation.original.exportDigestsUnchanged,33);assert.equal(preservation.original.artifactFilesUnchanged,1780);assert.equal(preservation.original.modelAttemptsUnchanged,70);
|
||||
assert.deepEqual(preservation.formal.selected,setup.originalSelection);assert.deepEqual(preservation.formal.active,[]);assert.equal(preservation.formal.pid,manifest.formalPid);
|
||||
const audit=from('receipts/process-audit.json');assert.ok(audit.processes.every(x=>x.rootExited&&x.treeConfirmed&&!x.currentlyExists));assert.deepEqual(audit.debug9229,[]);
|
||||
let requests=0,reportedTokens=0,unknown=0,sseResponses=0,approvals=0,applications=0,continuations=0;let contractReference;
|
||||
for(const sample of manifest.samples){
|
||||
const result=from(sample.result),attempts=from(sample.attempts);assert.equal(result.success,false,'Original sample must retain failed acceptance: '+sample.name);
|
||||
approvals+=(result.decisions??[]).length;applications+=(result.applications??[]).length;continuations+=(result.continuations??result.restarts??[]).length;
|
||||
assert.equal(attempts.length,result.usage.requests);
|
||||
for(const attempt of attempts){
|
||||
requests++;assert.equal(attempt.sessionId,sample.sessionId);const request=from(artifacts.get(attempt.requestArtifactId).file),metrics=requestMetrics(request);
|
||||
assert.equal(metrics.sha256,attempt.requestSha256);assert.equal(request.model,attempt.model);
|
||||
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknown++;continue;}
|
||||
const response=fs.readFileSync(inside(artifacts.get(attempt.responseArtifactId).file));assert.equal(sha(response),attempt.responseSha256);
|
||||
let usage;
|
||||
for(const line of response.toString('utf8').split(/\r?\n/)){
|
||||
if(!line.startsWith('data:'))continue;const text=line.slice(5).trim();if(!text||text==='[DONE]')continue;
|
||||
const chunk=JSON.parse(text);if(chunk.usage)usage=chunk.usage;
|
||||
}
|
||||
assert.deepEqual(usage,attempt.usage,'Stored upstream usage: '+attempt.id);reportedTokens+=usage.total_tokens;sseResponses++;
|
||||
}
|
||||
if(['react','python'].includes(sample.name)){
|
||||
const models=[...new Set(attempts.map(x=>x.model))];assert.deepEqual(models,[sample.name==='react'?'gpt-6.1-sol':'claude-opus-5-5']);
|
||||
if(sample.name==='python')assert.equal(result.model,'gpt-6.1-sol','Preserve the documented original verifier metadata error.');
|
||||
continue;
|
||||
}
|
||||
const base='samples/'+sample.name+'/',contract=from(base+'contract.json'),exp=from(base+'original-child-session.json');
|
||||
const native=from(base+'engine-'+result.engineRuns[0].id+'.json');
|
||||
const pinned={sourceTree:contract.sourceTree,sourceFiles:contract.sourceFiles,goal:contract.goal,writable:contract.writable,checks:contract.checks,budget:contract.budget,
|
||||
connectionId:contract.model.id,revision:contract.model.revision,model:contract.model.model,protocol:contract.model.protocol,streaming:contract.model.streaming,
|
||||
contextLimit:contract.model.contextLimit,maxOutputTokens:contract.model.maxOutputTokens,driverSha256:contract.driverSha256,installation:native.installationSha256};
|
||||
if(!contractReference)contractReference=pinned;else assert.deepEqual(pinned,contractReference,'Paired source/model/budget/driver identity: '+sample.name);
|
||||
assert.equal(native.native.cleanup.rootExited,true);assert.equal(native.native.cleanup.treeConfirmed,true);assert.deepEqual(native.native.cleanup.remainingProcesses,[]);
|
||||
const inputs=native.native.bridgeRequests.filter(x=>x.route==='/model'),recorded=analysis.analyses.find(x=>x.name===sample.name);
|
||||
assert.equal(inputs.length,attempts.length+1);assert.equal(result.stopReason,'engine_input_budget');assert.equal(result.applications.length,0);assert.equal(exp.execution.changeSets.length,0);
|
||||
for(let i=0;i<inputs.length;i++){
|
||||
const input=inputs[i].data,config={...exp.execution.run.model,modelRouting:exp.execution.run.model.routing,maxOutputTokens:input.maxTokens};
|
||||
const metrics=requestMetrics(completionBody(config,nativeMessagesToChat(input),input.tools.map(x=>({type:'function',function:x}))));
|
||||
assert.equal(metrics.sha256,recorded.modelInputs[i].requestSha256);assert.equal(metrics.estimatedInputTokens,recorded.modelInputs[i].estimatedInputTokens);
|
||||
assert.equal(nativeSystemTexts(input).join('\n').split(setup.purge.defaultPrompt).length-1,result.enabled?1:0);
|
||||
if(i<native.requests.length)assert.equal(metrics.sha256,native.requests[i].requestSha256);
|
||||
else{assert.equal(inputs[i].status,'failed');assert.ok(metrics.estimatedInputTokens+input.maxTokens+128>config.contextLimit);assert.equal(recorded.modelInputs[i].dispatched,false);}
|
||||
}
|
||||
assert.equal(result.upstreamPluginLoaded,false);
|
||||
}
|
||||
assert.equal(requests,20);assert.equal(sseResponses,19);assert.equal(reportedTokens,419597);assert.equal(unknown,1);
|
||||
assert.equal(approvals,4);assert.equal(applications,0);assert.equal(continuations,0);
|
||||
assert.equal(manifest.requests,requests);assert.equal(analysis.requests,requests);assert.equal(manifest.reportedTokens,reportedTokens);assert.equal(reconciliation.reportedTokens,reportedTokens);
|
||||
assert.equal(analysis.aggregate.enabled.reportedTokens,138758);assert.equal(analysis.aggregate.disabled.reportedTokens,78875);
|
||||
for(const source of reconciliation.sourceReceipts)for(const [file,identity]of Object.entries(source.files))assert.equal(entries.get('source-baselines/'+source.sample+'/'+file)?.sha256,identity.sha256);
|
||||
console.log(JSON.stringify({passed:true,manifestSha256:sha(fs.readFileSync(path.join(root,'manifest.json'))),evidenceFiles:manifest.count,evidenceBytes:manifest.bytes,
|
||||
sourceInputs:manifest.sourceInputs.length,sourceCommit:manifest.sourceCommit,samples:manifest.sampleCount,requests,sseResponses,reportedTokens,unknownUsageRequests:unknown,
|
||||
supervision:{fixedCheckApprovals:approvals,appliedBatches:applications,continuations,gitConfirmations:0,targetedImplementationHints:0},
|
||||
promptComparison:{delivered:0,enabledReportedTokens:138758,disabledReportedTokens:78875,nativeInputRejections:4},originalPluginBenefit:'not_tested',
|
||||
scope:'Offline byte/source/request/usage/input reconstruction only; no model request, no original failure rewrite, no claim that autonomous delivery passed.'}));
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {requestMetrics} from '../src/shared/contracts/model-request.mjs';
|
||||
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
|
||||
import {assertPurgeModelInput,nativeSystemTexts,ORIGINAL_PURGE_KIND} from '../src/core/engines/purge-strategy.mjs';
|
||||
|
||||
// Offline evidence only. No Runtime/Provider, model requests or plugin loading.
|
||||
const root=path.resolve('docs/live-original-purge-e-evidence'),sha=raw=>createHash('sha256').update(raw).digest('hex');
|
||||
const file=relative=>{const full=path.resolve(root,relative);assert.ok(full.startsWith(root+path.sep));return full;};
|
||||
const read=relative=>JSON.parse(fs.readFileSync(file(relative))),manifest=read('manifest.json');
|
||||
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.slice,'live-original-purge-e');
|
||||
const paths=new Set();let bytes=0;
|
||||
for(const e of manifest.entries){assert.ok(!paths.has(e.path));paths.add(e.path);const raw=fs.readFileSync(file(e.path));assert.equal(raw.length,e.bytes,e.path);assert.equal(sha(raw),e.sha256,e.path);bytes+=raw.length;}
|
||||
assert.equal(paths.size,manifest.count);assert.equal(bytes,manifest.bytes);
|
||||
assert.equal(sha(fs.readFileSync(manifest.dManifest)),manifest.dManifestSha256);
|
||||
const d=JSON.parse(fs.readFileSync(manifest.dManifest)),identity=read('receipts/runtime-identity.json');
|
||||
for(const build of d.buildInputs){const input=identity.files.find(f=>f.path===build.path);assert.equal(input?.sha256,build.sha256,build.path);}
|
||||
assert.equal(identity.dSourceCommit,d.sourceCommit);assert.equal(identity.sourceCommit,manifest.sourceCommit);
|
||||
const freeze=read('receipts/driver-freeze.json');
|
||||
for(const input of freeze.files){const sealed=input.path.startsWith('.qa/')?'drivers/'+path.basename(input.path):null;
|
||||
if(sealed)assert.equal(sha(fs.readFileSync(file(sealed))),input.sha256,input.path);}
|
||||
const records=read('receipts/owned-records.json'),owners=new Map(read('receipts/owned-sessions.json').map(s=>[s.id,s])),core={};
|
||||
for(const row of records){const value=JSON.parse(row.payload);assert.ok(owners.has(value.sessionId),row.id);(core[row.kind]??=new Map()).set(value.id,value);}
|
||||
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,core.artifact.size);
|
||||
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,kind+'/'+id);return value;},
|
||||
artifactStore:{read(a){return this.readBytes(a).toString('utf8');},readBytes(a){const input=artifacts.get(a.id),raw=fs.readFileSync(file(input.file));assert.equal(sha(raw),a.sha256);assert.equal(raw.length,a.bytes);return raw;}},
|
||||
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(t=>t.sessionId===sessionId&&t.callId===callId);assert.ok(tool);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
|
||||
const attempts=read('receipts/all-model-attempts.json'),attemptIds=new Set(attempts.map(a=>a.id));assert.equal(attemptIds.size,attempts.length);
|
||||
let reportedTokens=0,unknownRequests=0;
|
||||
for(const attempt of attempts){
|
||||
const request=core.artifact.get(attempt.requestArtifactId);assert.ok(request);assert.equal(request.sha256,attempt.requestSha256);
|
||||
const input=JSON.parse(store.artifactStore.read(request));assert.equal(requestMetrics(input).sha256,attempt.requestSha256);assert.equal(input.model,attempt.model);
|
||||
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknownRequests++;continue;}
|
||||
const response=core.artifact.get(attempt.responseArtifactId);assert.equal(response?.sha256,attempt.responseSha256);
|
||||
const raw=store.artifactStore.read(response);let usage;
|
||||
try{const value=JSON.parse(raw);usage=value.usage??value.response?.usage;}catch{
|
||||
for(const line of raw.split(/\r?\n/)){if(!line.startsWith('data:'))continue;const value=line.slice(5).trim();if(!value||value==='[DONE]')continue;
|
||||
const event=JSON.parse(value);if(event.usage??event.response?.usage)usage=event.usage??event.response.usage;}
|
||||
}
|
||||
assert.deepEqual(usage??null,attempt.usage??null,attempt.id);
|
||||
if(usage)reportedTokens+=usage.total_tokens;else unknownRequests++;
|
||||
}
|
||||
assert.equal(attempts.length,manifest.requests);assert.equal(reportedTokens,manifest.reportedTokens);assert.equal(unknownRequests,manifest.unknownRequests);
|
||||
const names=['react','python','r1-off','r1-on','r2-on','r2-off'];assert.deepEqual(manifest.samples.map(s=>s.name),names);
|
||||
let reference,onInputs=0,offInputs=0,originalProcesses=0;
|
||||
const onStrategy=[...core.engine_run.values()].find(j=>j.installation.strategy?.kind===ORIGINAL_PURGE_KIND)?.installation.strategy;assert.ok(onStrategy);
|
||||
for(const sample of manifest.samples){
|
||||
const result=read(sample.result),sampleAttempts=attempts.filter(a=>a.sessionId===sample.sessionId),session=owners.get(sample.sessionId);
|
||||
const baseline=new Map(sample.baseline.map(input=>[input.path,input]));assert.equal(baseline.size,sample.baseline.length);
|
||||
for(const input of baseline.values()){const raw=fs.readFileSync(file(input.file));assert.equal(raw.length,input.bytes);assert.equal(sha(raw),input.sha256);}
|
||||
assert.equal(result.success,sample.success);assert.equal(sampleAttempts.length,sample.requests);assert.equal(sample.requests,result.usage.requests);
|
||||
assert.equal(sampleAttempts.reduce((n,a)=>n+(a.usage?.total_tokens??0),0),sample.reportedTokens);
|
||||
assert.deepEqual([...new Set(sampleAttempts.map(a=>a.model))],sample.actualModels);assert.deepEqual(sample.actualModels,['claude-opus-5-5']);
|
||||
const budget=read('receipts/'+sample.name+'-budget.json');assert.equal(budget.contextLimit,32000);assert.equal(budget.requests,24);
|
||||
assert.equal(budget.tokens,['react','python'].includes(sample.name)?140000:105000);
|
||||
const task=[...core.task.values()].find(t=>t.id===(result.childId??result.taskId));assert.ok(task);assert.equal(task.usage.totalTokens,sample.reportedTokens);
|
||||
const consumption=[...core.run.values()].filter(r=>r.taskId===task.id).flatMap(r=>r.consumption??[]);
|
||||
assert.deepEqual(new Set(consumption.map(c=>c.attemptId)),new Set(sampleAttempts.map(a=>a.id)));
|
||||
if(sample.contract){
|
||||
const c=read(sample.contract),comparison={goal:c.goal,sourceTree:c.sourceTree,writable:c.writable,checks:c.checks,budget:c.budget,
|
||||
driverSha256:c.driverSha256,connectionId:c.model.id,revision:c.model.revision,model:c.model.model,protocol:c.model.protocol,contextLimit:c.model.contextLimit,maxOutputTokens:c.model.maxOutputTokens};
|
||||
if(reference)assert.deepEqual(comparison,reference,'Same controlled AB/BA baseline and conditions');else reference=comparison;
|
||||
assert.equal(sample.sourceTree,c.sourceTree);
|
||||
// The contract records the intentional working copies after the original
|
||||
// owner staged/unstaged edits; the sealed Git tree is their prior baseline.
|
||||
const ownerCopies={'owner-staged.txt':'Owner working copy after stage\n','owner-unstaged.txt':'Owner unstaged original\n'};
|
||||
for(const [name,sha256]of Object.entries(c.sourceFiles))assert.equal(ownerCopies[name]?sha(Buffer.from(ownerCopies[name])):baseline.get(name)?.sha256,sha256,name);
|
||||
const enabled=sample.name.endsWith('-on');assert.equal(c.enabled,enabled);assert.equal(c.upstreamPluginLoaded,enabled);
|
||||
for(const job of core.engine_run.values())if(job.sessionId===sample.sessionId){
|
||||
originalEngineRun(store,session,job);assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);
|
||||
if(enabled){assert.equal(job.installation.strategy.kind,ORIGINAL_PURGE_KIND);assert.equal(job.native.originalPurge.readiness.originalApplyCalled,true);assert.equal(job.native.originalPurge.nativeLoads.length,2);originalProcesses++;}
|
||||
else {assert.equal(job.installation.strategy??null,null);assert.equal(job.native.originalPurge??null,null);}
|
||||
for(const request of job.requests){const input=verifyEngineModelInput(store,session,job,request),texts=nativeSystemTexts(input.native);
|
||||
assert.ok(texts.every(t=>!t.includes('[Agent Workbench engineering prompt experiment]')));
|
||||
if(enabled){assertPurgeModelInput(input.native,job.installation.strategy);onInputs++;}
|
||||
else {assert.ok(texts.every(t=>!t.includes(onStrategy.originalConfig.postPrompt)));offInputs++;}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(sample.success){assert.equal(result.reportStatus??result.outcome,['react','python'].includes(sample.name)?'verified_committed':'reviewed_parent_delivered');assert.ok(result.commit);}
|
||||
}
|
||||
const restoration=read('receipts/restoration.json');assert.equal(restoration.passed,true);assert.equal(restoration.originalConfigRestored,true);
|
||||
assert.equal(read('receipts/complete-state-preservation.json').passed,true);
|
||||
assert.equal(restoration.originalConnectionParametersRestored,true);assert.deepEqual(restoration.actualSelection,restoration.originalSelection);
|
||||
for(const value of Object.values(restoration.originalRows))assert.equal(value.changed,0);
|
||||
const audit=read('receipts/process-audit.json');assert.equal(audit.passed,true);assert.equal(audit.inspectorClosed,true);assert.equal(audit.hostActive,false);
|
||||
assert.ok(audit.processes.every(p=>p.rootExited&&p.treeConfirmed&&p.currentlyExists===false));
|
||||
console.log(JSON.stringify({passed:true,files:paths.size,bytes,requests:attempts.length,reportedTokens,unknownRequests,onInputs,offInputs,originalProcesses,
|
||||
engineering:manifest.samples.slice(0,2).map(s=>({name:s.name,success:s.success})),purge:manifest.samples.slice(2).map(s=>({name:s.name,success:s.success,reportedTokens:s.reportedTokens})),
|
||||
manifestSha256:sha(fs.readFileSync(file('manifest.json'))),scope:'Offline exact source/request/upstream usage/original plugin/cleanup evidence. Six descriptive samples, no general success-rate or unverified fee claim.'}));
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {requestMetrics} from '../src/shared/contracts/model-request.mjs';
|
||||
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
|
||||
import {assertPurgeModelInput,nativeSystemTexts,ORIGINAL_PURGE_KIND} from '../src/core/engines/purge-strategy.mjs';
|
||||
|
||||
// Offline evidence only. No Runtime/Provider, model requests or plugin loading.
|
||||
const root=path.resolve('docs/live-original-purge-h-evidence'),sha=raw=>createHash('sha256').update(raw).digest('hex');
|
||||
const file=relative=>{const full=path.resolve(root,relative);assert.ok(full.startsWith(root+path.sep));return full;};
|
||||
const read=relative=>JSON.parse(fs.readFileSync(file(relative))),manifest=read('manifest.json');
|
||||
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.slice,'live-original-purge-h');
|
||||
const paths=new Set();let bytes=0;
|
||||
for(const e of manifest.entries){assert.ok(!paths.has(e.path));paths.add(e.path);const raw=fs.readFileSync(file(e.path));assert.equal(raw.length,e.bytes,e.path);assert.equal(sha(raw),e.sha256,e.path);bytes+=raw.length;}
|
||||
assert.equal(paths.size,manifest.count);assert.equal(bytes,manifest.bytes);
|
||||
assert.equal(sha(fs.readFileSync(manifest.dManifest)),manifest.dManifestSha256);
|
||||
const d=JSON.parse(fs.readFileSync(manifest.dManifest)),identity=read('receipts/runtime-identity.json');
|
||||
const release=read('receipts/release-identity.json');
|
||||
assert.equal(sha(fs.readFileSync(file('receipts/release-identity.json'))),identity.releaseIdentitySha256);
|
||||
assert.equal(identity.manifestSha256,release.manifestSha256);assert.equal(identity.runtime,release.installedRoot);
|
||||
assert.equal(identity.sourceCommit,release.sourceCommit);assert.equal(identity.sourceCommit,manifest.sourceCommit);
|
||||
const installed=read('receipts/release-manifest.json');assert.equal(sha(fs.readFileSync(file('receipts/release-manifest.json'))),release.manifestSha256);
|
||||
for(const input of identity.files){assert.deepEqual(input,installed.files.find(f=>f.path===input.path),input.path);}
|
||||
assert.equal(identity.files.filter(f=>f.path.startsWith('dist-')).length,22);
|
||||
const freeze=read('receipts/driver-freeze.json');
|
||||
for(const input of freeze.files){const sealed=input.path.startsWith('.qa/')?'drivers/'+path.basename(input.path):null;
|
||||
if(sealed)assert.equal(sha(fs.readFileSync(file(sealed))),input.sha256,input.path);}
|
||||
const records=read('receipts/owned-records.json'),owners=new Map(read('receipts/owned-sessions.json').map(s=>[s.id,s])),core={};
|
||||
for(const row of records){const value=JSON.parse(row.payload);assert.ok(owners.has(value.sessionId),row.id);(core[row.kind]??=new Map()).set(value.id,value);}
|
||||
const artifacts=new Map(manifest.artifacts.map(a=>[a.id,a]));assert.equal(artifacts.size,core.artifact.size);
|
||||
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId,kind+'/'+id);return value;},
|
||||
artifactStore:{read(a){return this.readBytes(a).toString('utf8');},readBytes(a){const input=artifacts.get(a.id),raw=fs.readFileSync(file(input.file));assert.equal(sha(raw),a.sha256);assert.equal(raw.length,a.bytes);return raw;}},
|
||||
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(t=>t.sessionId===sessionId&&t.callId===callId);assert.ok(tool);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
|
||||
const attempts=read('receipts/all-model-attempts.json'),attemptIds=new Set(attempts.map(a=>a.id));assert.equal(attemptIds.size,attempts.length);
|
||||
let reportedTokens=0,unknownRequests=0;
|
||||
for(const attempt of attempts){
|
||||
const request=core.artifact.get(attempt.requestArtifactId);assert.ok(request);assert.equal(request.sha256,attempt.requestSha256);
|
||||
const input=JSON.parse(store.artifactStore.read(request));assert.equal(requestMetrics(input).sha256,attempt.requestSha256);assert.equal(input.model,attempt.model);
|
||||
if(!attempt.responseArtifactId){assert.equal(attempt.usage??null,null);unknownRequests++;continue;}
|
||||
const response=core.artifact.get(attempt.responseArtifactId);assert.equal(response?.sha256,attempt.responseSha256);
|
||||
const raw=store.artifactStore.read(response);let usage;
|
||||
try{const value=JSON.parse(raw);usage=value.usage??value.response?.usage;}catch{
|
||||
for(const line of raw.split(/\r?\n/)){if(!line.startsWith('data:'))continue;const value=line.slice(5).trim();if(!value||value==='[DONE]')continue;
|
||||
const event=JSON.parse(value);if(event.usage??event.response?.usage)usage=event.usage??event.response.usage;}
|
||||
}
|
||||
assert.deepEqual(usage??null,attempt.usage??null,attempt.id);
|
||||
if(usage)reportedTokens+=usage.total_tokens;else unknownRequests++;
|
||||
}
|
||||
assert.equal(attempts.length,manifest.requests);assert.equal(reportedTokens,manifest.reportedTokens);assert.equal(unknownRequests,manifest.unknownRequests);
|
||||
const names=['react','python','r1-off','r1-on','r2-on','r2-off'];assert.deepEqual(manifest.samples.map(s=>s.name),names);
|
||||
let reference,onInputs=0,offInputs=0,originalProcesses=0;
|
||||
const onStrategy=[...core.engine_run.values()].find(j=>j.installation.strategy?.kind===ORIGINAL_PURGE_KIND)?.installation.strategy;assert.ok(onStrategy);
|
||||
for(const sample of manifest.samples){
|
||||
const result=read(sample.result),sampleAttempts=attempts.filter(a=>a.sessionId===sample.sessionId),session=owners.get(sample.sessionId);
|
||||
const baseline=new Map(sample.baseline.map(input=>[input.path,input]));assert.equal(baseline.size,sample.baseline.length);
|
||||
for(const input of baseline.values()){const raw=fs.readFileSync(file(input.file));assert.equal(raw.length,input.bytes);assert.equal(sha(raw),input.sha256);}
|
||||
assert.equal(result.success,sample.success);assert.equal(sampleAttempts.length,sample.requests);assert.equal(sample.requests,result.usage.requests);
|
||||
assert.equal(sampleAttempts.reduce((n,a)=>n+(a.usage?.total_tokens??0),0),sample.reportedTokens);
|
||||
assert.deepEqual([...new Set(sampleAttempts.map(a=>a.model))],sample.actualModels);assert.deepEqual(sample.actualModels,['claude-opus-5-5']);
|
||||
const budget=read('receipts/'+sample.name+'-budget.json');assert.equal(budget.contextLimit,32000);assert.equal(budget.requests,24);
|
||||
assert.equal(budget.tokens,['react','python'].includes(sample.name)?140000:105000);
|
||||
const task=[...core.task.values()].find(t=>t.id===(result.childId??result.taskId));assert.ok(task);assert.equal(task.usage.totalTokens,sample.reportedTokens);
|
||||
const consumption=[...core.run.values()].filter(r=>r.taskId===task.id).flatMap(r=>r.consumption??[]);
|
||||
assert.deepEqual(new Set(consumption.map(c=>c.attemptId)),new Set(sampleAttempts.map(a=>a.id)));
|
||||
if(sample.contract){
|
||||
const c=read(sample.contract),comparison={goal:c.goal,sourceTree:c.sourceTree,writable:c.writable,checks:c.checks,budget:c.budget,
|
||||
driverSha256:c.driverSha256,connectionId:c.model.id,revision:c.model.revision,model:c.model.model,protocol:c.model.protocol,contextLimit:c.model.contextLimit,maxOutputTokens:c.model.maxOutputTokens};
|
||||
if(reference)assert.deepEqual(comparison,reference,'Same controlled AB/BA baseline and conditions');else reference=comparison;
|
||||
assert.equal(sample.sourceTree,c.sourceTree);
|
||||
// The contract records the intentional working copies after the original
|
||||
// owner staged/unstaged edits; the sealed Git tree is their prior baseline.
|
||||
const ownerCopies={'owner-staged.txt':'Owner working copy after stage\n','owner-unstaged.txt':'Owner unstaged original\n'};
|
||||
for(const [name,sha256]of Object.entries(c.sourceFiles))assert.equal(ownerCopies[name]?sha(Buffer.from(ownerCopies[name])):baseline.get(name)?.sha256,sha256,name);
|
||||
const enabled=sample.name.endsWith('-on');assert.equal(c.enabled,enabled);assert.equal(c.upstreamPluginLoaded,enabled);
|
||||
for(const job of core.engine_run.values())if(job.sessionId===sample.sessionId){
|
||||
originalEngineRun(store,session,job);assert.equal(job.native.cleanup.rootExited,true);assert.equal(job.native.cleanup.treeConfirmed,true);
|
||||
if(enabled){assert.equal(job.installation.strategy.kind,ORIGINAL_PURGE_KIND);assert.equal(job.native.originalPurge.readiness.originalApplyCalled,true);assert.equal(job.native.originalPurge.nativeLoads.length,2);originalProcesses++;}
|
||||
else {assert.equal(job.installation.strategy??null,null);assert.equal(job.native.originalPurge??null,null);}
|
||||
for(const request of job.requests){const input=verifyEngineModelInput(store,session,job,request),texts=nativeSystemTexts(input.native);
|
||||
assert.ok(texts.every(t=>!t.includes('[Agent Workbench engineering prompt experiment]')));
|
||||
if(enabled){assertPurgeModelInput(input.native,job.installation.strategy);onInputs++;}
|
||||
else {assert.ok(texts.every(t=>!t.includes(onStrategy.originalConfig.postPrompt)));offInputs++;}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(sample.success){assert.equal(result.reportStatus??result.outcome,['react','python'].includes(sample.name)?'verified_committed':'reviewed_parent_delivered');assert.ok(result.commit);}
|
||||
}
|
||||
const restoration=read('receipts/restoration.json');assert.equal(restoration.passed,true);assert.equal(restoration.originalConfigRestored,true);
|
||||
assert.equal(read('receipts/complete-state-preservation.json').passed,true);
|
||||
assert.equal(restoration.originalConnectionParametersRestored,true);assert.deepEqual(restoration.actualSelection,restoration.originalSelection);
|
||||
for(const value of Object.values(restoration.originalRows))assert.equal(value.changed,0);
|
||||
const audit=read('receipts/process-audit.json');assert.equal(audit.passed,true);assert.equal(audit.inspectorClosed,true);assert.equal(audit.hostActive,false);
|
||||
assert.ok(audit.processes.every(p=>p.rootExited&&p.treeConfirmed&&p.currentlyExists===false));
|
||||
console.log(JSON.stringify({passed:true,files:paths.size,bytes,requests:attempts.length,reportedTokens,unknownRequests,onInputs,offInputs,originalProcesses,
|
||||
engineering:manifest.samples.slice(0,2).map(s=>({name:s.name,success:s.success})),purge:manifest.samples.slice(2).map(s=>({name:s.name,success:s.success,reportedTokens:s.reportedTokens})),
|
||||
manifestSha256:sha(fs.readFileSync(file('manifest.json'))),scope:'Offline exact source/request/upstream usage/original plugin/cleanup evidence. Six descriptive samples, no general success-rate or unverified fee claim.'}));
|
||||
@@ -0,0 +1,82 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
const url = process.env.AGENT_WORKBENCH_URL || 'http://127.0.0.1:4318';
|
||||
const sessionId = process.env.AGENT_WORKBENCH_SESSION || '569c9457-d399-46e6-b53e-4b1fdbb02109';
|
||||
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
|
||||
assert.ok(['1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.4.0', '1.5.0'].includes(bootstrap.version));
|
||||
assert.equal(bootstrap.config.mode, 'compatible');
|
||||
assert.ok(bootstrap.config.hasKey, 'The existing in-memory connection must be present.');
|
||||
const api = async (route, input) => {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }) });
|
||||
const value = await response.json(); assert.ok(response.ok, value.error); return value;
|
||||
};
|
||||
const before = await api('/sessions/' + sessionId);
|
||||
assert.equal(before.status, 'completed', 'Finish the original session before a read-only validation.');
|
||||
const project = bootstrap.projects.find(item => item.id === before.projectId);
|
||||
assert.ok(project);
|
||||
const approved = before.changes.filter(change => change.status === 'applied');
|
||||
assert.equal(approved.length, 2, 'This validation targets the original two applied changes.');
|
||||
const hash = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
const hashes = approved.map(change => ({ path: change.path, sha256: hash(path.join(project.root, change.path)) }));
|
||||
const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
const route = '/sessions/' + sessionId;
|
||||
const prompt = '刚才工作台里的 fetch failed 到底是怎么回事,跟项目的修改有关吗?src/catalog.js 和 tests/catalog.test.mjs 现在到底是什么状态?'
|
||||
+ '这次请只读诊断,不修改文件、不提出新改动、不运行终端命令。';
|
||||
const started = Date.now(); let done = false;
|
||||
fs.mkdirSync('.qa', { recursive: true });
|
||||
fs.writeFileSync('.qa/live-session-baseline.json', JSON.stringify({ sessionId, capturedAt: new Date().toISOString(),
|
||||
messageCount: before.messages.length, messagesSha256: digest(before.messages), eventCount: before.events.length,
|
||||
changesSha256: digest(before.changes), files: hashes }, null, 2));
|
||||
try {
|
||||
await api(route + '/messages', { content: prompt });
|
||||
let current, lastProgress = '';
|
||||
while (Date.now() - started < 180000) {
|
||||
current = await api(route);
|
||||
const tools = current.events.slice(before.events.length).filter(event => event.kind === 'tool');
|
||||
const progress = current.status + '|' + current.events.length;
|
||||
if (progress !== lastProgress) {
|
||||
console.log(JSON.stringify({ status: current.status, tools: tools.map(event => ({ tool: event.tool, phase: event.phase })), elapsedSeconds: Math.round((Date.now() - started) / 1000) }));
|
||||
lastProgress = progress;
|
||||
}
|
||||
assert.equal(current.approvals.filter(approval => approval.status === 'pending').length, 0, 'Read-only diagnosis must not request a terminal command.');
|
||||
assert.ok(tools.length <= 4, 'Diagnosis exceeded its bounded read-only scope.');
|
||||
if (!['running', 'awaiting_approval', 'stopping'].includes(current.status)) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
assert.ok(current && !['running', 'awaiting_approval', 'stopping'].includes(current.status), 'Actual model validation timed out.');
|
||||
assert.equal(current.status, 'completed', current.failure?.message || 'Read-only diagnosis did not complete.');
|
||||
const tools = current.events.slice(before.events.length).filter(event => event.kind === 'tool');
|
||||
assert.ok(tools.some(event => event.tool === 'inspect_session' && event.phase === 'completed'), 'The model did not inspect the actual session.');
|
||||
assert.ok(tools.every(event => event.tool === 'inspect_session'
|
||||
|| (event.tool === 'read_file' && approved.some(change => change.path === event.input?.path))),
|
||||
'Diagnosis may inspect the runtime and the named applied files; unrelated searches and mutations are outside scope.');
|
||||
assert.deepEqual(current.changes, before.changes, 'Diagnosis must preserve file proposals and their current states.');
|
||||
assert.deepEqual(current.messages.slice(0, before.messages.length), before.messages, 'Existing conversation history must be preserved.');
|
||||
for (const file of hashes) assert.equal(hash(path.join(project.root, file.path)), file.sha256);
|
||||
const answer = current.messages.slice(before.messages.length).filter(message => message.role === 'assistant').at(-1)?.content || '';
|
||||
assert.match(answer, /模型/); assert.match(answer, /请求|连接/); assert.match(answer, /已应用|applied/);
|
||||
assert.ok(!/请.{0,12}(提供|上传).{0,15}(工作台|会话|工具).{0,6}(日志|记录)/.test(answer), 'The model must use workbench records already available.');
|
||||
const health = await (await fetch(url + '/api/health')).json();
|
||||
const result = { success: true, version: bootstrap.version, testedAt: new Date().toISOString(), url, pid: health.pid,
|
||||
sessionId, projectRoot: project.root, model: bootstrap.config.model, endpoint: bootstrap.config.baseUrl,
|
||||
elapsedSeconds: Math.round((Date.now() - started) / 1000), status: current.status,
|
||||
tools: tools.map(event => ({ tool: event.tool, phase: event.phase })), fileStates: current.changes.map(change => ({ path: change.path, status: change.status })),
|
||||
answer, preservedHistory: { messageCount: before.messages.length, sha256: digest(current.messages.slice(0, before.messages.length)) }, files: hashes,
|
||||
evidence: ['Actual configured model inspected the original session', 'Historical fetch failure attributed to the model request',
|
||||
'Current applied states used; optional reads limited to the two named applied files', 'No project searches, new proposals or terminal commands', 'Original messages and file hashes preserved'],
|
||||
credentialsHandling: 'The script used the existing in-memory connection through the local service; no API Key was accessed or recorded.' };
|
||||
fs.writeFileSync('.qa/live-session-validation.json', JSON.stringify(result, null, 2)); done = true;
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
fs.writeFileSync('.qa/live-session-validation.json', JSON.stringify({ success: false, version: bootstrap.version,
|
||||
testedAt: new Date().toISOString(), sessionId, model: bootstrap.config.model, error: error.message,
|
||||
elapsedSeconds: Math.round((Date.now() - started) / 1000) }, null, 2));
|
||||
throw error;
|
||||
} finally {
|
||||
if (!done) await api(route + '/stop', {}).catch(() => {});
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
import { APP_VERSION } from '../src/server/diagnostics.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-live-task-plans.mjs — resume the prior page-only model Task with two sealed technical steps; reuse its verified desktop result and let the original Provider correct/verify mobile'); process.exit(0); }
|
||||
const prior = JSON.parse(fs.readFileSync('docs/browser-engine-live-model-validation.json', 'utf8'));
|
||||
const engineering = JSON.parse(fs.readFileSync('docs/live-react-engineering-validation.json', 'utf8'));
|
||||
const url = 'http://127.0.0.1:4318', directory = path.resolve('.qa', 'live-task-plans-' + randomUUID()); fs.mkdirSync(directory);
|
||||
const sessionId = prior.sessionId, taskId = prior.taskId, root = engineering.workspaceRoot;
|
||||
const sha = value => createHash('sha256').update(Buffer.isBuffer(value) || typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
|
||||
const canonical = value => Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object'
|
||||
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value;
|
||||
const git = args => execFileSync('git', ['--no-pager', '--no-optional-locks', '-C', root, ...args], { env: gitEnvironment(), windowsHide: true, encoding: 'utf8' }).trim();
|
||||
function sourceFiles(relative = '') {
|
||||
return fs.readdirSync(path.join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
||||
if (entry.name.startsWith('.') || ['node_modules', 'dist', 'build', 'out'].includes(entry.name)) return [];
|
||||
assert.equal(entry.isSymbolicLink(), false); const file = path.join(relative, entry.name);
|
||||
return entry.isDirectory() ? sourceFiles(file) : [{ path: file, sha256: sha(fs.readFileSync(path.join(root, file))) }];
|
||||
});
|
||||
}
|
||||
let token, current, baseline, result, beforeUsage, initialModelRequests = 0, modelStarted = false;
|
||||
const decisions = [], continuations = [], verifierProofs = [], startedAt = Date.now();
|
||||
async function api(route, input) {
|
||||
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: { Connection: 'close', 'Content-Type': 'application/json', ...(token ? { 'X-Workbench-Token': token } : {}) },
|
||||
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(20000) });
|
||||
const value = await response.json(); if (!response.ok) throw new Error(value.error ?? 'HTTP ' + response.status); return value;
|
||||
}
|
||||
const sessionRoute = '/sessions/' + sessionId;
|
||||
async function exportOriginal(job) {
|
||||
let text = '', offset = 0, page;
|
||||
do { page = await api(sessionRoute + '/browser-jobs/' + job.id + '/record?offset=' + offset + '&limit=16000'); text += page.content; offset += page.content.length; } while (page.truncated);
|
||||
assert.equal(sha(text), job.resultSha256); const original = JSON.parse(text); assert.equal(original.taskId, taskId);
|
||||
fs.writeFileSync(path.join(directory, 'original-' + job.id + '.json'), text);
|
||||
for (const image of job.screenshots) {
|
||||
const response = await fetch(url + '/api' + sessionRoute + '/browser-jobs/' + job.id + '/images/' + image.artifactId, { headers: { 'X-Workbench-Token': token }, signal: AbortSignal.timeout(15000) });
|
||||
assert.equal(response.status, 200); const bytes = Buffer.from(await response.arrayBuffer()); assert.equal(sha(bytes), image.sha256);
|
||||
fs.writeFileSync(path.join(directory, job.id + '-' + image.label + '.png'), bytes);
|
||||
}
|
||||
return original;
|
||||
}
|
||||
try {
|
||||
assert.equal(prior.success, false); assert.equal(prior.version, '1.10.1'); assert.equal(engineering.success, true);
|
||||
const before = await api('/bootstrap'); token = before.token; assert.equal(before.version, APP_VERSION); assert.equal(APP_VERSION, '1.11.0');
|
||||
assert.equal(before.config.mode, 'compatible'); assert.equal(before.config.hasKey, true);
|
||||
const project = before.projects.find(value => value.id === engineering.projectId); assert.ok(project); assert.equal(project.root, root);
|
||||
assert.ok(!before.sessions.some(value => value.projectId === project.id && ['running', 'awaiting_approval', 'stopping'].includes(value.status)));
|
||||
current = await api(sessionRoute); assert.equal(current.execution.task.id, taskId); assert.equal(current.projectId, project.id);
|
||||
assert.ok(['paused', 'failed', 'cancelled'].includes(current.status), 'Only an unfinished original Task may be resumed.');
|
||||
assert.equal(current.changes.length, 0); beforeUsage = structuredClone(current.execution.task.usage); initialModelRequests = beforeUsage.requests;
|
||||
const originalExport = await api(sessionRoute + '/export');
|
||||
const oldEngineering = await api('/sessions/' + engineering.sessionId + '/export'); delete oldEngineering.exportedAt;
|
||||
baseline = { source: sourceFiles(), head: git(['rev-parse', 'HEAD']), index: sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))),
|
||||
originalMessages: originalExport.session.providerMessages, originalJobs: originalExport.execution.browserJobs,
|
||||
engineering: sha(canonical(oldEngineering)), config: sha(canonical(before.config)) };
|
||||
fs.writeFileSync(path.join(directory, 'baseline.json'), JSON.stringify(baseline, null, 2) + '\n');
|
||||
const previousPlan = await api(sessionRoute + '/task-plan');
|
||||
const saved = await api(sessionRoute + '/task-plans', { taskId, runId: current.execution.run.id, expected_revision: previousPlan.revision,
|
||||
steps: [
|
||||
{ key: 'desktop', title: '桌面搜索、筛选与单任务切换', criterion: '1280px实际填写、筛选和点击,原断言通过,完成2项且待办工时5小时', kind: 'browser', viewport_width: 1280, actions: ['fill', 'select', 'click', 'assert_count'], depends_on: [] },
|
||||
{ key: 'mobile', title: '手机布局与控件实际操作', criterion: '390px实际点击并核对可见性及数量,原截图无横向溢出', kind: 'browser', viewport_width: 390, actions: ['click', 'assert_visible', 'assert_count'], depends_on: ['desktop'] },
|
||||
] });
|
||||
current = saved.session;
|
||||
const desktop = current.execution.browserJobs.findLast(job => job.status === 'passed' && current.execution.browserPlans.find(plan => plan.id === job.planId)?.viewport.width === 1280);
|
||||
assert.ok(desktop, 'The original desktop pass must be present; this verifier does not regenerate it.');
|
||||
const proof = await api(sessionRoute + '/task-proofs', { taskId, runId: current.execution.run.id, plan_id: saved.plan.plan_id,
|
||||
expected_revision: saved.plan.revision, step_key: 'desktop', evidence_id: desktop.id });
|
||||
assert.equal(proof.verification.status, 'completed'); verifierProofs.push({ step: 'desktop', evidenceId: desktop.id, proofId: proof.verification.proof_id, originalExecutionReplayed: false });
|
||||
await api(sessionRoute + '/resume', {}); modelStarted = true;
|
||||
const decided = new Set();
|
||||
while (Date.now() - startedAt < 210000) {
|
||||
current = await api(sessionRoute); assert.equal(current.execution.task.id, taskId); assert.equal(current.changes.length, 0);
|
||||
if (current.status === 'awaiting_approval') {
|
||||
const approval = current.approvals.find(value => value.status === 'pending' && !decided.has(value.id));
|
||||
if (approval) {
|
||||
assert.ok(approval.browserPlanId, 'Only the original page-only scope may execute.');
|
||||
const plan = current.execution.browserPlans.find(value => value.id === approval.browserPlanId);
|
||||
assert.equal(plan.taskId, taskId); assert.equal(plan.projectId, project.id); assert.equal(plan.build.directory, 'dist'); assert.equal(plan.path, '/');
|
||||
assert.ok([1280, 390].includes(plan.viewport.width)); assert.ok(plan.steps.length <= 20);
|
||||
decided.add(approval.id); decisions.push({ id: approval.id, planId: plan.id, fingerprint: plan.fingerprint, viewport: plan.viewport, steps: plan.steps.length });
|
||||
await api(sessionRoute + '/approvals/' + approval.id, { allow: true });
|
||||
}
|
||||
} else if (['completed', 'paused'].includes(current.status)) {
|
||||
const view = await api(sessionRoute + '/task-plan');
|
||||
if (current.status === 'completed' && view.all_verified) break;
|
||||
const reason = current.execution.run.stopReason;
|
||||
if (current.status !== 'paused' || continuations.length >= 2 || !['step_limit', 'task_plan_incomplete'].includes(reason)) throw new Error('Actual model task remains ' + current.status + '/' + reason);
|
||||
continuations.push({ previousRunId: current.execution.run.id, reason, at: new Date().toISOString() }); await api(sessionRoute + '/resume', {});
|
||||
} else if (['failed', 'cancelled'].includes(current.status)) throw new Error(current.failure?.message ?? current.status);
|
||||
await new Promise(resolve => setTimeout(resolve, 75));
|
||||
}
|
||||
assert.equal(current.status, 'completed', 'Actual model did not finish within the original 210-second acceptance bound.');
|
||||
const view = await api(sessionRoute + '/task-plan'); assert.equal(view.all_verified, true); assert.equal(view.verified_steps, 2);
|
||||
const records = [];
|
||||
for (const step of view.steps) {
|
||||
const job = current.execution.browserJobs.find(value => value.id === step.evidence.id); assert.ok(job); assert.equal(job.cleanup.exited, true);
|
||||
records.push({ job, original: await exportOriginal(job), viewport: current.execution.browserPlans.find(value => value.id === job.planId).viewport });
|
||||
}
|
||||
const desktopRecord = records.find(value => value.viewport.width === 1280), mobile = records.find(value => value.viewport.width === 390);
|
||||
assert.match(desktopRecord.original.pages.at(-1).text, /完成任务\s*2/); assert.match(desktopRecord.original.pages.at(-1).text, /待办工时\s*5\s*小时/);
|
||||
assert.ok(mobile.original.pages.at(-1).viewport.scrollWidth <= 390);
|
||||
assert.deepEqual(sourceFiles(), baseline.source); assert.equal(git(['rev-parse', 'HEAD']), baseline.head);
|
||||
assert.equal(sha(fs.readFileSync(path.join(git(['rev-parse', '--absolute-git-dir']), 'index'))), baseline.index);
|
||||
const afterEngineering = await api('/sessions/' + engineering.sessionId + '/export'); delete afterEngineering.exportedAt;
|
||||
assert.equal(sha(canonical(afterEngineering)), baseline.engineering); assert.equal(sha(canonical((await api('/bootstrap')).config)), baseline.config);
|
||||
const exported = await api(sessionRoute + '/export');
|
||||
assert.deepEqual(exported.session.providerMessages.slice(0, baseline.originalMessages.length), baseline.originalMessages);
|
||||
assert.deepEqual(exported.execution.browserJobs.slice(0, baseline.originalJobs.length), baseline.originalJobs);
|
||||
fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n');
|
||||
result = { success: true, version: APP_VERSION, testedAt: new Date().toISOString(), directory, model: before.config.model, projectId: project.id, root, sessionId, taskId,
|
||||
modelStatus: current.status, sourcePreserved: true, indexPreserved: true, originalEngineeringTaskPreserved: true, originalMessagesAndJobsPreserved: true,
|
||||
beforeUsage, usage: current.execution.task.usage, newModelRequests: current.execution.task.usage.requests - initialModelRequests,
|
||||
elapsedSeconds: (Date.now() - startedAt) / 1000, taskPlan: view, verifierProofs, decisions, continuations,
|
||||
jobs: records.map(value => ({ id: value.job.id, status: value.job.status, viewport: value.viewport, assertions: value.job.assertions, resultSha256: value.job.resultSha256, cleanup: value.job.cleanup })),
|
||||
scope: 'The verifier seals two technical steps and verifies the original desktop evidence once. The actual configured model resumes the same unfinished Task and must produce/verify the mobile result. This assisted page-only acceptance does not measure autonomous planning, code repair or Purge benefit.',
|
||||
credentialsHandling: 'Original in-memory Provider is retained. No model key is read, copied, printed or saved.' };
|
||||
} catch (error) {
|
||||
if (token && modelStarted) { await api(sessionRoute + '/stop', {}).catch(() => {}); current = await api(sessionRoute).catch(() => current); }
|
||||
result = { success: false, version: APP_VERSION, testedAt: new Date().toISOString(), directory, sessionId, taskId, modelStatus: current?.status,
|
||||
beforeUsage, usage: current?.execution?.task?.usage, decisions, verifierProofs, continuations, error: error.stack }; process.exitCode = 1;
|
||||
if (token) { const exported = await api(sessionRoute + '/export').catch(() => null); if (exported) fs.writeFileSync(path.join(directory, 'session-export.json'), JSON.stringify(exported, null, 2) + '\n'); }
|
||||
}
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
fs.writeFileSync('docs/task-plans-live-model-validation.json', JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, sessionId, taskId, modelStatus: result.modelStatus, newModelRequests: result.newModelRequests,
|
||||
elapsedSeconds: result.elapsedSeconds, usage: result.usage, jobs: result.jobs, error: result.error, directory }));
|
||||
@@ -0,0 +1,118 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import net from 'node:net';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { chromium, _electron, expect } from 'playwright/test';
|
||||
import { HostController } from '../src/core/desktop/host-controller.mjs';
|
||||
import { desktopExecutableIdentity } from '../src/core/desktop/app-installation.mjs';
|
||||
import { projectEnvironment } from '../src/core/process-environment.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-local-files.mjs — isolated real Edge/Electron project-picker and HTML preview acceptance; optional AGENT_WORKBENCH_QA_APP_ROOT for immutable candidate. No model requests.'); process.exit(0); }
|
||||
if (process.argv.length > 2) throw Error('Use --help.');
|
||||
const appRoot = path.resolve(process.env.AGENT_WORKBENCH_QA_APP_ROOT || '.'), root = path.resolve('.qa', 'local-files-' + randomUUID());
|
||||
const dataRoot = path.join(root, 'data'), projectRoot = path.join(root, '中文 项目'), otherRoot = path.join(root, 'other-project');
|
||||
fs.mkdirSync(path.join(projectRoot, 'pages'), { recursive: true }); fs.mkdirSync(path.join(projectRoot, 'assets')); fs.mkdirSync(otherRoot);
|
||||
fs.writeFileSync(path.join(otherRoot, 'README.md'), '# Other project');
|
||||
fs.writeFileSync(path.join(projectRoot, 'assets', 'theme.css'), 'body{font:16px system-ui;background:#f4f7fa;color:rgb(20,60,100);padding:24px}button{padding:12px}img{width:64px;height:64px}');
|
||||
fs.writeFileSync(path.join(projectRoot, 'assets', 'mark.svg'), '<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64"><rect width="64" height="64" fill="#37694d"/></svg>');
|
||||
fs.writeFileSync(path.join(projectRoot, 'assets', 'value.mjs'), 'export const label="相对模块已运行";');
|
||||
fs.writeFileSync(path.join(projectRoot, 'assets', 'data.json'), '{"message":"相对数据已加载"}');
|
||||
fs.writeFileSync(path.join(projectRoot, 'assets', 'app.mjs'), 'import {label} from "./value.mjs";document.querySelector("#module").textContent=label;fetch("./../assets/data.json").then(r=>r.json()).then(v=>document.querySelector("#data").textContent=v.message);');
|
||||
const reserve = net.createServer(); await new Promise(resolve => reserve.listen(0, '127.0.0.1', resolve)); const port = reserve.address().port; await new Promise(resolve => reserve.close(resolve));
|
||||
const controller = new HostController({ appRoot, dataRoot, port, nodePath: process.execPath, timeoutMs: 60000 });
|
||||
let browser, application, stage = 'start'; const checks = [], screenshots = [], errors = [];
|
||||
const record = name => { checks.push(name); console.log(name + ': passed'); };
|
||||
const screenshot = async (page, name) => { const file = path.join(root, name + '.png'); await page.screenshot({ path: file }); screenshots.push({ file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') }); };
|
||||
const overflow = async page => assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false);
|
||||
async function fileFlow(page, label) {
|
||||
const item = page.getByRole('button', { name: '页面.html', exact: true });
|
||||
await item.waitFor(); await item.click();
|
||||
const frame = page.frameLocator('iframe[title="HTML 预览:pages/页面.html"]');
|
||||
await expect(frame.getByRole('heading', { name: '文件预览验收' })).toBeVisible();
|
||||
await expect(frame.locator('#module')).toHaveText('相对模块已运行');
|
||||
await expect(frame.locator('#data')).toHaveText('相对数据已加载');
|
||||
await expect(frame.locator('#boundary')).toHaveText('工作台接口已隔离');
|
||||
await expect(frame.locator('body')).toHaveCSS('color', 'rgb(20, 60, 100)');
|
||||
await expect.poll(() => frame.locator('img').evaluate(image => image.complete && image.naturalWidth)).toBe(64);
|
||||
await frame.getByRole('button', { name: '计数 0' }).click();
|
||||
await expect(frame.getByRole('button', { name: '计数 1' })).toBeVisible(); record(label + ': inline JS, relative modules/CSS/image/JSON, workbench isolation and interaction');
|
||||
await screenshot(page, label + '-preview-wide');
|
||||
await page.getByRole('button', { name: '源码', exact: true }).click();
|
||||
await expect(page.locator('.code-view')).toContainText('<!doctype html>');
|
||||
await expect(page.locator('iframe')).toHaveCount(0);
|
||||
await page.getByRole('button', { name: '预览', exact: true }).click(); await expect(frame.getByRole('button', { name: '计数 0' })).toBeVisible();
|
||||
await frame.getByRole('button', { name: '计数 0' }).click(); await page.getByRole('button', { name: '刷新预览' }).click();
|
||||
await expect(frame.getByRole('button', { name: '计数 0' })).toBeVisible(); record(label + ': source switch and fresh saved-file reload');
|
||||
return frame;
|
||||
}
|
||||
try {
|
||||
const original = await controller.start();
|
||||
fs.writeFileSync(path.join(projectRoot, 'pages', '页面.html'), `<!doctype html><html lang="zh"><head><meta charset="UTF-8"><title>文件预览验收</title><link rel="stylesheet" href="../assets/theme.css"></head><body><h1>文件预览验收</h1><img src="../assets/mark.svg" alt="相对图像"><p id="module">等待模块</p><p id="data">等待数据</p><p id="boundary">核对隔离</p><button onclick="this.textContent='计数 '+(++window.count)">计数 0</button><script>window.count=0;fetch('${controller.url}/api/bootstrap').then(()=>document.querySelector('#boundary').textContent='接口可读取').catch(()=>document.querySelector('#boundary').textContent='工作台接口已隔离')</script><script type="module" src="../assets/app.mjs"></script></body></html>`);
|
||||
const initial = (await controller.api('/bootstrap')).projects.length;
|
||||
browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 960 } }); page.on('pageerror', error => errors.push(error.message));
|
||||
await page.goto(controller.url); await page.locator('.connection.connected').waitFor();
|
||||
stage = 'web directory picker';
|
||||
await page.getByRole('button', { name: '打开项目', exact: true }).click();
|
||||
await page.getByRole('button', { name: '选择文件夹', exact: true }).click();
|
||||
await page.getByLabel('浏览文件夹路径').fill(root); await page.getByRole('button', { name: '前往', exact: true }).click();
|
||||
await page.getByRole('button', { name: '中文 项目', exact: true }).click();
|
||||
await screenshot(page, 'web-directory-wide');
|
||||
await page.getByRole('button', { name: '取消选择' }).click();
|
||||
assert.equal((await controller.api('/bootstrap')).projects.length, initial);
|
||||
await page.getByLabel('项目文件夹路径').fill(path.join(root, 'missing'));
|
||||
await page.getByRole('dialog').getByRole('button', { name: '打开项目', exact: true }).click();
|
||||
await expect(page.getByRole('dialog').getByRole('alert')).toBeVisible();
|
||||
await page.getByLabel('项目文件夹路径').fill(root);
|
||||
await page.getByRole('button', { name: '选择文件夹', exact: true }).click();
|
||||
await page.getByRole('button', { name: '中文 项目', exact: true }).click();
|
||||
await page.getByRole('button', { name: '使用此文件夹' }).click();
|
||||
await expect(page.getByLabel('项目文件夹路径')).toHaveValue(projectRoot);
|
||||
await page.getByRole('dialog').getByRole('button', { name: '打开项目', exact: true }).click();
|
||||
await expect(page.getByRole('dialog')).toHaveCount(0);
|
||||
const project = (await controller.api('/bootstrap')).projects.find(item => item.root === projectRoot); assert.ok(project);
|
||||
await page.reload(); await expect(page.getByRole('combobox', { name: '当前项目', exact: true })).toHaveValue(project.id); record('web: directory selection, cancel, invalid path recovery, project persistence');
|
||||
stage = 'web HTML preview'; await fileFlow(page, 'web');
|
||||
const link = page.getByRole('link', { name: '独立页面' }), popupEvent = page.waitForEvent('popup'); await link.click(); const popup = await popupEvent;
|
||||
await expect(popup.getByRole('heading', { name: '文件预览验收' })).toBeVisible(); await popup.close(); record('web: independent page renders');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await overflow(page); await screenshot(page, 'web-preview-390');
|
||||
const other = await controller.api('/projects', { method: 'POST', input: { root: otherRoot } });
|
||||
await page.setViewportSize({ width: 1440, height: 960 }); await page.reload();
|
||||
await page.getByRole('combobox', { name: '当前项目', exact: true }).selectOption(other.id); await expect(page.locator('iframe')).toHaveCount(0); record('web: project switch clears old file preview');
|
||||
await page.getByRole('button', { name: '打开项目', exact: true }).click(); await page.getByRole('button', { name: '选择文件夹', exact: true }).click();
|
||||
await page.getByLabel('浏览文件夹路径').fill(root); await page.getByRole('button', { name: '前往', exact: true }).click();
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await overflow(page); await screenshot(page, 'web-directory-390');
|
||||
await browser.close(); browser = null;
|
||||
stage = 'actual Electron preview';
|
||||
const executable = desktopExecutableIdentity(appRoot).path, packaged = executable.toLowerCase().endsWith('agentworkbench.exe');
|
||||
application = await _electron.launch({ executablePath: executable, args: [...(packaged ? [] : [path.join(appRoot, 'dist-desktop/main.cjs')]), '--app-root', appRoot, '--data-root', dataRoot, '--node', process.execPath, '--port', String(port), '--hidden-window'], cwd: appRoot, env: projectEnvironment(), timeout: 60000 });
|
||||
const desktopPage = await application.firstWindow(); desktopPage.on('pageerror', error => errors.push(error.message));
|
||||
await (await application.browserWindow(desktopPage)).evaluate(window => { window.showInactive(); window.setContentSize(1440, 960); });
|
||||
await desktopPage.locator('.connection.connected').waitFor(); await desktopPage.getByRole('combobox', { name: '当前项目', exact: true }).selectOption(project.id);
|
||||
await fileFlow(desktopPage, 'electron'); await overflow(desktopPage);
|
||||
const rejected = await desktopPage.frameLocator('iframe').locator('body').evaluate(() => ({ bridge: typeof window.desktopWorkbench, require: typeof window.require }));
|
||||
assert.deepEqual(rejected, { bridge: 'undefined', require: 'undefined' }); record('electron: preview subframe has no native bridge or Node');
|
||||
await desktopPage.getByRole('button', { name: '源码', exact: true }).click();
|
||||
// Exercise the actual Windows openDirectory dialog contract with an explicit
|
||||
// cancelled fixture, without asking the user to select a test-only project.
|
||||
await application.evaluate(({ dialog }) => { dialog.showOpenDialog = async () => ({ canceled: true, filePaths: [] }); });
|
||||
await desktopPage.getByRole('button', { name: '打开项目', exact: true }).click(); await desktopPage.getByRole('button', { name: '选择文件夹', exact: true }).click();
|
||||
await expect(desktopPage.getByRole('dialog')).toBeVisible(); record('electron: existing native picker cancel leaves project unchanged (explicit dialog fixture)');
|
||||
await application.close(); application = null;
|
||||
assert.deepEqual(errors, []);
|
||||
const final = await controller.api('/bootstrap'); assert.equal(final.config.mode, 'demo'); assert.equal(final.config.hasKey, false); assert.equal(final.version, original.identity.version);
|
||||
const result = { success: true, version: final.version, appRoot, root, checks, screenshots, realModelRequests: 0, errors, hostPid: original.identity.pid };
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/local-files-latest.json', JSON.stringify(result, null, 2)); console.log(JSON.stringify({ success: true, checks: checks.length, root }));
|
||||
} catch (error) {
|
||||
for (const [index, page] of (browser?.contexts().flatMap(context => context.pages()) || []).entries()) {
|
||||
await screenshot(page, 'failure-' + index).catch(() => {});
|
||||
fs.writeFileSync(path.join(root, 'failure-' + index + '.html'), await page.content().catch(() => 'Page unavailable'));
|
||||
}
|
||||
if (application) {
|
||||
const page = await application.firstWindow(); await screenshot(page, 'failure-electron').catch(() => {});
|
||||
fs.writeFileSync(path.join(root, 'failure-electron.html'), await page.content().catch(() => 'Page unavailable'));
|
||||
}
|
||||
const result = { success: false, stage, root, checks, screenshots, errors, error: error.stack };
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2)); fs.writeFileSync('.qa/local-files-latest.json', JSON.stringify(result, null, 2)); console.error(JSON.stringify(result, null, 2)); process.exitCode = 1;
|
||||
} finally { await browser?.close(); await application?.close(); await controller.stop().catch(() => {}); }
|
||||
@@ -0,0 +1,70 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, compatibleFixture, completion, call } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
import { Runtime } from '../src/server/runtime.mjs';
|
||||
import { gitEnvironment } from '../src/core/git-process.mjs';
|
||||
|
||||
const directory = path.resolve('.qa', 'mcp-recovery-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t);
|
||||
const runtime = new Runtime(f.store, new Provider()), registry = runtime.mcpRegistry();
|
||||
const entry = path.join(f.root, 'owned-recovery-mcp.mjs'); fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
|
||||
const saved = registry.register(f.project.id, { name: '中断恢复工具', executable: process.execPath, entry, timeoutMs: 45000 });
|
||||
await registry.start(f.session, { id: saved.id, configurationRevision: 1, confirm: true }).promise;
|
||||
const value = registry.find(f.project.id, saved.id);
|
||||
const enabled = registry.configure(f.project.id, { id: saved.id, configurationRevision: value.configurationRevision, enabled: true, allowedTools: ['hold'] });
|
||||
const model = await compatibleFixture(t, () => completion([call('ui-recovery-crash', 'call_registered_mcp', {
|
||||
registration_id: enabled.id, configuration_revision: enabled.configurationRevision, contract_sha256: enabled.contract.sha256,
|
||||
tool: 'hold', arguments: { text: 'owned-ui-crash-effect' },
|
||||
})]));
|
||||
f.store.saveConfig({ mode: 'compatible', model: 'owned-recovery-ui', baseUrl: model.baseUrl, contextLimit: 24000 }); f.store.close();
|
||||
const crash = await promisify(execFile)(process.execPath, [path.resolve('tests/fixtures/mcp-host-crash.mjs'), f.data, f.session.id],
|
||||
{ env: gitEnvironment(), windowsHide: true, timeout: 18000 })
|
||||
.then(() => assert.fail('Expected owned host exit.'), error => { assert.equal(error.code, 85, error.stderr); return JSON.parse(error.stdout.trim().split('\n').at(-1)); });
|
||||
const host = await launchWorkbench(t, f.data), browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const pageErrors = [], requestFailures = [], assertions = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const mark = value => { assertions.push(value); console.log('PASS ' + value); };
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }), history = dialog.getByLabel('MCP运行记录', { exact: true });
|
||||
const row = history.locator('.mcp-history-row.state-unknown'); await expect(row).toHaveCount(1);
|
||||
await row.getByRole('button', { name: '查看原始输入', exact: true }).click();
|
||||
const evidence = history.getByLabel('MCP原报告'); await expect(evidence).toContainText('没有完整原报告');
|
||||
await evidence.locator('summary').click(); await expect(evidence).toContainText('owned-ui-crash-effect');
|
||||
await evidence.scrollIntoViewIfNeeded(); await page.screenshot({ path: path.join(directory, 'desktop-mcp-interrupted-input.png') });
|
||||
mark('actual host crash with no result report exposes its original verified input and unknown status');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await evidence.scrollIntoViewIfNeeded();
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
await page.screenshot({ path: path.join(directory, 'mobile-mcp-interrupted-input.png') });
|
||||
await row.getByRole('button', { name: '人工核对结果', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: '核对未知的外部结果', exact: true });
|
||||
await expect(review).toContainText('没有完整原报告'); await review.locator('summary').click(); await expect(review).toContainText('owned-ui-crash-effect');
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
assert.ok(await review.evaluate(node => node.scrollWidth <= node.clientWidth));
|
||||
await page.screenshot({ path: path.join(directory, 'mobile-mcp-review.png') });
|
||||
await review.getByRole('button', { name: '我已核对原输入和进程', exact: true }).click(); await expect(review).toBeHidden();
|
||||
const job = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === crash.jobId);
|
||||
assert.equal(job.status, 'unknown'); assert.equal(job.requiresVerification, false); assert.equal(job.reportArtifactId, undefined);
|
||||
const session = await host.api('/sessions/' + f.session.id); assert.deepEqual(session.execution.progress.requiresVerification, []);
|
||||
assert.equal(model.requests.length, 1);
|
||||
mark('390px original-input review clears the job and invocation gates while retaining unknown, no report and no automatic model replay');
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, assertions, viewportWidths: [1440, 390],
|
||||
pageErrors, requestFailures, localFixtureModelRequests: model.requests.length, realModelRequests: 0, jobId: job.id }, null, 2));
|
||||
console.log('EVIDENCE ' + directory);
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.message, assertions, pageErrors, requestFailures }, null, 2));
|
||||
console.error('EVIDENCE ' + directory); throw error;
|
||||
} finally { await browser.close(); await host.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
@@ -0,0 +1,111 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, compatibleFixture, completion, call, until } from '../tests/helpers.mjs';
|
||||
|
||||
const directory = path.resolve('.qa', 'mcp-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) }, f = fixture(t); f.store.close();
|
||||
const entry = path.join(f.root, 'registered-mcp.mjs'); fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
|
||||
let registration;
|
||||
const model = await compatibleFixture(t, () => completion([call('ui-unknown-call', 'call_registered_mcp', {
|
||||
registration_id: registration.id, configuration_revision: registration.configurationRevision, contract_sha256: registration.contract.sha256,
|
||||
tool: 'hold', arguments: { text: 'effect-can-be-unknown' },
|
||||
})]));
|
||||
const host = await launchWorkbench(t, f.data); const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const pageErrors = [], requestFailures = [], assertions = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const mark = value => { assertions.push(value); console.log('PASS ' + value); };
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); await expect(dialog).toBeVisible();
|
||||
const form = dialog.locator('.mcp-register');
|
||||
await form.getByLabel('显示名称', { exact: true }).fill('只读与工程工具');
|
||||
await form.getByLabel('可执行文件绝对路径', { exact: true }).fill(process.execPath);
|
||||
await form.getByLabel('Node 脚本入口(可选)', { exact: true }).fill(entry);
|
||||
await form.getByRole('button', { name: '登记后保持停用', exact: true }).click();
|
||||
const row = dialog.locator('.mcp-registration').filter({ hasText: '只读与工程工具' });
|
||||
await expect(row).toContainText('已停用'); await expect(row).toContainText('尚未探测');
|
||||
mark('MCP registration stays disabled and shows the hashed local executable and Node entry');
|
||||
|
||||
await row.getByRole('button', { name: '实际探测', exact: true }).click();
|
||||
const confirm = page.getByRole('dialog', { name: '启动 MCP 实际探测' });
|
||||
await expect(confirm).toContainText('不等于 OS 沙箱');
|
||||
await confirm.getByRole('button', { name: '确认启动并探测', exact: true }).click();
|
||||
await expect(confirm).toBeHidden(); await expect(row).toContainText('握手已核验'); await expect(row).toContainText('发现 3 项工具');
|
||||
await expect(row.locator('.mcp-tool-option')).toHaveCount(3);
|
||||
const echo = row.locator('.mcp-tool-option').filter({ hasText: 'echo' });
|
||||
await echo.locator('summary').click(); await expect(echo.locator('pre')).toContainText('additionalProperties');
|
||||
await echo.locator('input[type="checkbox"]').check();
|
||||
await row.getByRole('button', { name: '保存并启用所选工具', exact: true }).click();
|
||||
await expect(row).toContainText('已启用'); await expect(dialog.locator('.mcp-history-row')).toContainText('完成');
|
||||
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
|
||||
assert.equal(registration.enabled, true); assert.deepEqual(registration.allowedTools, ['echo']);
|
||||
await dialog.locator('.mcp-heading').scrollIntoViewIfNeeded();
|
||||
await page.screenshot({ path: path.join(directory, 'desktop-mcp-registry.png') });
|
||||
mark('actual stdio tools/list schema is visible; only the reviewed echo tool is enabled');
|
||||
|
||||
fs.appendFileSync(entry, '\n// Owned changed entry for disable acceptance.\n');
|
||||
await dialog.getByRole('button', { name: '刷新', exact: true }).click(); await expect(row).toContainText('来源或契约异常');
|
||||
await row.getByRole('button', { name: '立即停用', exact: true }).click(); await expect(row).toContainText('已停用');
|
||||
assert.equal((await host.api('/projects/' + f.project.id + '/mcp')).registrations[0].enabled, false);
|
||||
fs.copyFileSync(path.resolve('tests/fixtures/registered-mcp-server.mjs'), entry);
|
||||
await dialog.getByRole('button', { name: '刷新', exact: true }).click();
|
||||
mark('an enabled program whose source changes can be disabled immediately without executing the changed entry');
|
||||
|
||||
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
|
||||
await host.api('/projects/' + f.project.id + '/mcp/configure', { id: registration.id, configurationRevision: registration.configurationRevision, enabled: true, allowedTools: ['hold'] });
|
||||
registration = (await host.api('/projects/' + f.project.id + '/mcp')).registrations[0];
|
||||
await host.api('/config', { mode: 'compatible', baseUrl: model.baseUrl, model: 'owned-mcp-ui', apiKey: 'ui-mcp-fixture-key', contextLimit: 24000 });
|
||||
await host.api('/sessions/' + f.session.id + '/messages', { content: 'Use the enabled external tool and preserve its real execution record.' });
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id)).status === 'awaiting_approval');
|
||||
await page.keyboard.press('Escape');
|
||||
await page.getByRole('button', { name: '确认执行', exact: true }).click();
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).some(value => value.operation === 'call' && value.status === 'calling'));
|
||||
const job = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.operation === 'call' && value.status === 'calling');
|
||||
await until(() => fs.existsSync(path.join(job.directory, 'owned-call-dispatched.txt')));
|
||||
await page.getByRole('button', { name: '停止', exact: true }).click();
|
||||
await until(async () => (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).some(value => value.id === job.id && value.status === 'unknown'));
|
||||
const unknown = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === job.id);
|
||||
assert.equal(unknown.requiresVerification, true); assert.ok(unknown.reportArtifactId);
|
||||
mark('approved external call interrupted after dispatch remains unknown and does not replay');
|
||||
|
||||
await page.getByRole('button', { name: '查看任务记录', exact: true }).click();
|
||||
const history = page.getByLabel('MCP运行记录', { exact: true }); await expect(history).toContainText('结果待核对');
|
||||
await history.locator('.mcp-history-row.state-unknown').getByRole('button', { name: '查看原报告', exact: true }).click();
|
||||
await expect(history.getByLabel('MCP原报告')).toContainText('"status": "unknown"');
|
||||
await history.getByLabel('MCP原报告').locator('summary').click();
|
||||
await expect(history.getByLabel('MCP原报告')).toContainText('effect-can-be-unknown');
|
||||
await history.getByRole('button', { name: '人工核对结果', exact: true }).click();
|
||||
const review = page.getByRole('dialog', { name: '核对未知的外部结果' }); await expect(review).toContainText('不会把未知结果改成成功');
|
||||
await review.getByRole('button', { name: '我已查看原报告', exact: true }).click();
|
||||
await expect(review).toBeHidden();
|
||||
const acknowledged = (await host.api('/sessions/' + f.session.id + '/mcp/jobs')).find(value => value.id === job.id);
|
||||
assert.equal(acknowledged.status, 'unknown'); assert.equal(acknowledged.requiresVerification, false);
|
||||
assert.deepEqual((await host.api('/sessions/' + f.session.id)).execution.progress.requiresVerification, []);
|
||||
mark('TaskPanel exposes Task/Run source, original report, and human acknowledgement without marking unknown as success');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.getByRole('button', { name: '项目MCP工具', exact: true }).click();
|
||||
const mobileDialog = page.getByRole('dialog', { name: '项目 MCP 工具', exact: true }); await expect(mobileDialog).toBeVisible();
|
||||
await expect(mobileDialog.locator('.mcp-registration')).toHaveCount(1);
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390));
|
||||
await mobileDialog.locator('.mcp-heading').scrollIntoViewIfNeeded();
|
||||
await page.screenshot({ path: path.join(directory, 'mobile-mcp-registry.png') });
|
||||
mark('390px MCP registry, long executable paths, controls, and operation history have no horizontal overflow');
|
||||
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, assertions, viewportWidths: [1440, 390], pageErrors, requestFailures,
|
||||
localFixtureModelRequests: model.requests.length, realModelRequests: 0, unknownJobId: job.id,
|
||||
screenshots: ['desktop-mcp-registry.png', 'mobile-mcp-registry.png'] }, null, 2));
|
||||
console.log('EVIDENCE ' + directory);
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: path.join(directory, 'failure.png') }).catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.message, assertions, pageErrors, requestFailures }, null, 2));
|
||||
console.error('EVIDENCE ' + directory); throw error;
|
||||
} finally { await browser.close(); await host.close(); for (const dispose of disposers.reverse()) await dispose(); }
|
||||
@@ -0,0 +1,69 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-messages-m.mjs — offline M source/release/UI/install/state-retention and single no-tool native Messages preflight verification; no network, model or credential access.'); process.exit(0); }
|
||||
assert.equal(process.argv.length, 2);
|
||||
const root = path.resolve('docs/messages-m-evidence'), read = f => JSON.parse(fs.readFileSync(f, 'utf8')), hash = b => createHash('sha256').update(b).digest('hex');
|
||||
const manifest = read(path.join(root, 'manifest.json')); assert.equal(manifest.version, '1.24.4');
|
||||
const names = new Set(); let bytes = 0;
|
||||
for (const entry of manifest.entries) {
|
||||
assert.ok(!names.has(entry.path), entry.path); names.add(entry.path);
|
||||
const file = path.resolve(root, entry.path); assert.ok(file.startsWith(root + path.sep));
|
||||
const content = fs.readFileSync(file); assert.equal(content.length, entry.bytes, entry.path); assert.equal(hash(content), entry.sha256, entry.path); bytes += content.length;
|
||||
if (/\.(?:jpg|png)$/.test(entry.path)) {
|
||||
if (entry.path.endsWith('.jpg')) assert.equal(content.subarray(0, 3).toString('hex'), 'ffd8ff', entry.path);
|
||||
else assert.equal(content.subarray(0, 8).toString('hex'), '89504e470d0a1a0a', entry.path);
|
||||
}
|
||||
}
|
||||
const proof = name => read(path.join(root, manifest.references[name]));
|
||||
const freeze = proof('freeze'); assert.equal(freeze.sourceCommit, manifest.sourceCommit); assert.equal(freeze.files.length, 544); assert.equal(freeze.compiled.length, 22);
|
||||
const release = proof('release'); assert.equal(release.documentation.sourceCommit, manifest.sourceCommit); assert.equal(path.basename(release.portableRoot), manifest.candidate);
|
||||
for (const item of [...freeze.files, ...freeze.compiled]) assert.equal(hash(fs.readFileSync(path.join(root, 'source-bytes', item.path))), item.sha256, item.path);
|
||||
for (const name of ['sourceUI', 'portableUI', 'installedUI']) {
|
||||
const ui = proof(name); assert.equal(ui.success, true); assert.equal(ui.testedVersion, '1.24.4'); assert.equal(ui.checks.length, 7); assert.equal(ui.realModelRequests, 0); assert.deepEqual(ui.errors, []);
|
||||
}
|
||||
assert.equal(proof('regression').success, true); assert.equal(proof('regression').results.length, 4); assert.ok(proof('regression').results.every(r => r.code === 0));
|
||||
const driver = proof('probeDriver'); assert.equal(driver.passed, true); assert.equal(driver.realModelRequests, 0); assert.deepEqual(driver.results.map(r => r.mode), ['match', 'mismatch', '429']);
|
||||
assert.ok(driver.results.every(r => r.passed && r.requests === 1 && r.actualTools === 0 && r.inspectorClosed));
|
||||
const probeBinding = proof('probeSourceBinding'); assert.equal(probeBinding.passed, true); assert.equal(probeBinding.finalSourceCommit, manifest.sourceCommit); assert.equal(probeBinding.files.length, 220);
|
||||
for (const file of probeBinding.files) assert.deepEqual(freeze.files.find(f => f.path === file.path), file);
|
||||
const archives = proof('archives'); assert.equal(archives.success, true); assert.equal(archives.version, '1.24.4'); assert.equal(archives.archives.length, 3); assert.ok(archives.archives.every(a => a.crcPassed && a.manifestAndDirectoryMatch));
|
||||
for (const kind of ['portable', 'source', 'evidence']) { const archive = archives.archives.find(a => a.kind === kind); assert.equal(archive.path, release[kind].path); assert.equal(archive.sha256, release[kind].sha256); }
|
||||
const rebuild = proof('rebuild'); assert.equal(rebuild.success, true); assert.equal(rebuild.sourceCommit, manifest.sourceCommit); assert.equal(rebuild.inputs.length, 549); assert.equal(rebuild.compiled.length, 22); assert.deepEqual(rebuild.mismatches, []);
|
||||
assert.deepEqual(rebuild.sourceZip, release.source); assert.equal(proof('portable').appRoot, release.portableRoot); assert.equal(proof('portableUI').appRoot, release.portableRoot);
|
||||
for (const item of freeze.compiled) { const rebuilt = rebuild.compiled.find(r => r.path === item.path); assert.ok(rebuilt); assert.equal(rebuilt.expected.sha256, item.sha256); assert.equal(rebuilt.actual.sha256, item.sha256); }
|
||||
assert.equal(proof('portable').success, true); const install = proof('installation'); assert.equal(install.success, true); assert.equal(install.version, '1.24.4'); assert.equal(install.results.length, 12); assert.equal(install.realModelRequests, 0);
|
||||
assert.equal(install.packagedSource, release.portableRoot);
|
||||
assert.equal(install.installedUI.proofs.length, 2); assert.ok(install.installedUI.proofs.every(p => p.proof.success && p.proof.realModelRequests === 0));
|
||||
const before = proof('formalbefore'); assert.equal(before.success, true); assert.equal(before.version, '1.24.3');
|
||||
for (const name of ['formalinstall', 'formalfinal']) {
|
||||
const p = proof(name); assert.equal(p.success, true); assert.equal(p.version, '1.24.4'); assert.equal(p.pid, before.pid); assert.equal(p.hasKey, true); assert.equal(p.realModelRequests, 0);
|
||||
assert.deepEqual(p.retention, { originalRows: 'exact', originalExports: 'exact', originalArtifactBytes: 'exact', originalModelCenter: 'exact' });
|
||||
}
|
||||
assert.equal(proof('native').success, true); assert.equal(proof('browser').success, true);
|
||||
const live = proof('live'), attempts = proof('liveAttempts'), audit = proof('liveAudit');
|
||||
assert.equal(audit.passed, true); assert.equal(audit.globalConfigAndSelectionExact, true); assert.equal(audit.noCredentialExtractionOrResave, true); assert.equal(audit.originalPidRetained, true);
|
||||
assert.equal(live.requests, 1); assert.equal(live.toolDefinitions, 0); assert.equal(live.actualTools, 0); assert.equal(live.noRetry, true); assert.equal(live.noPoolSwitch, true); assert.equal(live.inspectorClosed, true);
|
||||
assert.equal(live.protocol, 'anthropic-messages'); assert.equal(live.messagesAuth, 'bearer'); assert.equal(live.requestUrl, 'https://api.apikey.fan/v1/messages'); assert.equal(live.modelWeightsAuthenticated, false);
|
||||
assert.equal(live.realEngineeringDeliveryTested, false); assert.equal(live.originalPurgeBenefitTested, false); assert.equal(live.defaultSelectionRetained, true); assert.equal(live.credentialRevisionRetained, true);
|
||||
assert.equal(attempts.length, 1); const attempt = attempts[0]; assert.equal(attempt.sessionId, live.sessionId); assert.equal(attempt.connectionId, live.connectionId); assert.equal(attempt.poolId, null); assert.equal(attempt.ordinal, 1);
|
||||
const requestCapture = live.captures.find(c => c.kind === 'request'), responseCapture = live.captures.find(c => c.kind === 'response');
|
||||
assert.ok(requestCapture && responseCapture); const requestBytes = fs.readFileSync(path.join(root, 'live', requestCapture.file)), request = JSON.parse(requestBytes);
|
||||
assert.equal(hash(requestBytes), requestCapture.sha256); assert.equal(hash(JSON.stringify(request)), attempt.requestSha256); assert.equal(request.model, live.requestedModel); assert.equal(request.max_tokens, 256); assert.equal(request.tools?.length ?? 0, 0);
|
||||
const raw = fs.readFileSync(path.join(root, 'live', responseCapture.file)); assert.equal(hash(raw), responseCapture.sha256); assert.equal(hash(raw), attempt.responseSha256); assert.equal(attempt.responseRedacted, false);
|
||||
assert.deepEqual(live.reportedModels, attempt.modelIdentity?.reportedModels ?? []); assert.equal(live.identityStatus, attempt.modelIdentity?.status ?? 'not_observed');
|
||||
const reported = [];
|
||||
if (responseCapture.file.endsWith('.sse')) {
|
||||
for (const line of raw.toString('utf8').split(/\r?\n/)) if (line.startsWith('data: ')) { const value = line.slice(6); if (value === '[DONE]') continue; const e = JSON.parse(value); if (typeof e.model === 'string') reported.push(e.model); if (typeof e.message?.model === 'string') reported.push(e.message.model); }
|
||||
} else {
|
||||
let value; try { value = JSON.parse(raw); } catch { assert.deepEqual(live.reportedModels, [], 'Non-JSON response must not fabricate a model identifier.'); }
|
||||
if (typeof value?.model === 'string') reported.push(value.model);
|
||||
}
|
||||
for (const model of live.reportedModels) assert.ok(reported.includes(typeof model === 'string' ? model : model.model), 'Reported model comes from original response bytes.');
|
||||
if (live.identityPassed) { assert.equal(live.identityStatus, 'matched'); assert.equal(attempt.status, 'completed'); assert.equal(live.usageStatus, 'reported'); assert.ok(live.usage.total_tokens <= 16000); }
|
||||
else assert.notEqual(live.sessionStatus, 'completed', 'Rejected/missing identities cannot be presented as completed.');
|
||||
assert.equal(hash(fs.readFileSync('docs/live-model-identity-j-evidence/manifest.json')), manifest.originalJManifestSha256);
|
||||
assert.equal(hash(fs.readFileSync('docs/apikey-route-l-evidence/manifest-final.json')), manifest.originalLManifestSha256);
|
||||
console.log(JSON.stringify({ success: true, files: manifest.entries.length, bytes, sourceCommit: manifest.sourceCommit, manifestSha256: hash(fs.readFileSync(path.join(root, 'manifest.json'))), realModelRequests: 1,
|
||||
identityPassed: live.identityPassed, requestedModel: live.requestedModel, reportedModels: live.reportedModels, scope: 'Offline original-byte acceptance and bounded preflight. No network/model call; no underlying-weight or engineering/Purge benefit assertion.' }));
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {execFileSync} from 'node:child_process';
|
||||
import {originalEngineRun,verifyEngineModelInput} from '../src/shared/contracts/engine-execution.mjs';
|
||||
import {observedTaskModels} from './live-model-observation.mjs';
|
||||
|
||||
// Offline only: no Provider, Runtime, credentials, model request or formal data.
|
||||
const root=path.resolve('docs/live-budget-c-evidence'),sha=value=>createHash('sha256').update(value).digest('hex');
|
||||
const inside=relative=>{const file=path.resolve(root,relative);assert.ok(file.startsWith(root+path.sep));return file;};
|
||||
const raw=fs.readFileSync(inside('manifest.json')),manifest=JSON.parse(raw),read=relative=>JSON.parse(fs.readFileSync(inside(relative),'utf8'));
|
||||
assert.equal(manifest.schemaVersion,1);assert.equal(manifest.realModelRequests,0);assert.equal(manifest.formalServiceChanged,false);
|
||||
const entries=new Map();let bytes=0;
|
||||
for(const entry of manifest.entries){assert.ok(!entries.has(entry.path));entries.set(entry.path,entry);const data=fs.readFileSync(inside(entry.path));
|
||||
assert.equal(data.length,entry.bytes,entry.path);assert.equal(sha(data),entry.sha256,entry.path);bytes+=data.length;}
|
||||
assert.equal(entries.size,manifest.count);assert.equal(bytes,manifest.bytes);
|
||||
for(const input of manifest.sourceInputs){
|
||||
assert.match(input.path,/^(?:src|scripts|tests)\/[A-Za-z0-9_./-]+$|^package(?:-lock)?\.json$/);assert.ok(!input.path.split('/').includes('..'));
|
||||
const blob=execFileSync('git',['show',manifest.sourceCommit+':'+input.path],{windowsHide:true,maxBuffer:16*1024*1024});
|
||||
const data=input.snapshot?fs.readFileSync(inside(input.snapshot)):blob;
|
||||
assert.equal(sha(blob),input.gitBlobSha256);assert.equal(sha(data),input.sha256);assert.equal(data.length,input.bytes);
|
||||
assert.equal(data.toString('utf8').replaceAll('\r\n','\n'),blob.toString('utf8').replaceAll('\r\n','\n'));
|
||||
}
|
||||
const receipt=name=>fs.readFileSync(inside(manifest.receipts.find(value=>value.name===name).file),'utf8');
|
||||
assert.match(receipt('final-related'),/tests 149\b/);assert.match(receipt('final-related'),/pass 149\b/);assert.match(receipt('final-related'),/fail 0\b/);
|
||||
assert.match(receipt('final-build'),/Server bundle built\./);
|
||||
const ui=read(manifest.ui.result),session=read(manifest.ui.session);
|
||||
assert.equal(ui.passed,true);assert.equal(ui.cases.length,2);assert.equal(ui.ownedHttpRequests,1);assert.deepEqual(ui.errors,[]);
|
||||
assert.equal(session.status,'paused');assert.equal(session.execution.run.stopReason,'token_limit');
|
||||
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.reportedTokens,55017);
|
||||
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.exceededTokens,20017);
|
||||
assert.equal(session.execution.run.budgetPause.tokenAccounting.run.unknownRequests,0);
|
||||
assert.equal(session.execution.run.budgetPause.modelFailure.httpStatus,503);
|
||||
assert.equal(observedTaskModels(session).model,'owned-budget-ui');
|
||||
let originalRequests=0,artifactRecords=0;
|
||||
for(const sample of manifest.samples){
|
||||
const result=read(sample.result),records=read(sample.records),sessions=new Map(read(sample.sessions).map(value=>[value.id,value])),core={};
|
||||
for(const row of records){const value=JSON.parse(row.payload);(core[row.kind]??=new Map()).set(value.id,value);}
|
||||
artifactRecords+=core.artifact.size;const artifacts=new Map(sample.artifacts.map(value=>[value.id,value]));
|
||||
const store={core,owned(kind,id,sessionId){const value=core[kind]?.get(id);assert.equal(value?.sessionId,sessionId);return value;},
|
||||
artifactStore:{read(artifact){const data=fs.readFileSync(inside(artifacts.get(artifact.id).file));assert.equal(sha(data),artifact.sha256);assert.equal(data.length,artifact.bytes);return data.toString('utf8');}},
|
||||
savedToolResult(sessionId,callId){const tool=[...core.tool.values()].find(value=>value.sessionId===sessionId&&value.callId===callId);return JSON.parse(this.artifactStore.read(core.artifact.get(tool.resultArtifactId)));}};
|
||||
assert.equal(result.success,sample.passed);assert.equal(result.realModelRequests,0);
|
||||
if(!sample.passed){assert.ok(result.error);continue;}
|
||||
assert.equal(result.metrics.contextLimit,16000);assert.equal(result.parentCheck.results[0].tests.passed,9);
|
||||
assert.ok(result.checks.some(job=>job.status==='failed'&&job.results[0].tests.failed===9));assert.equal(result.writable.length,3);
|
||||
for(const job of core.engine_run.values()){
|
||||
const owner=sessions.get(job.sessionId);originalEngineRun(store,owner,job);
|
||||
assert.equal(job.status,'completed');assert.equal(job.native.cleanup.treeConfirmed,true);assert.equal(job.native.cleanup.rootExited,true);
|
||||
for(const request of job.requests){
|
||||
verifyEngineModelInput(store,owner,job,request);assert.equal(request.reservationVersion,1);originalRequests++;
|
||||
assert.ok(request.tokenReservation.routes.every(route=>route.inputRatio>=1));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.deepEqual(manifest.samples.filter(value=>value.passed).map(value=>value.name).sort(),['chat','responses']);
|
||||
for(const name of ['a-offline','b-offline','pilot-offline'])assert.equal(JSON.parse(receipt(name)).passed,true);
|
||||
console.log(JSON.stringify({passed:true,sourceCommit:manifest.sourceCommit,manifestSha256:sha(raw),files:entries.size,bytes,
|
||||
sourceInputs:manifest.sourceInputs.length,artifactRecords,originalRequests,realModelRequests:0,
|
||||
scope:'Offline raw bytes/source, known usage reservation, actual Task model identity and owned native engineering evidence; original failures retained, real model and original Purge not measured.'}));
|
||||
@@ -0,0 +1,56 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import {randomUUID} from 'node:crypto';
|
||||
import {chromium,expect} from 'playwright/test';
|
||||
import {fixture,compatibleFixture,launchWorkbench,until} from '../tests/helpers.mjs';
|
||||
import {observedTaskModels} from './live-model-observation.mjs';
|
||||
|
||||
const directory=path.resolve('.qa','model-budget-ui-'+randomUUID());fs.mkdirSync(directory,{recursive:true});
|
||||
const disposers=[],t={after:fn=>disposers.push(fn)},f=fixture(t),screenshots=[],cases=[],errors=[];
|
||||
let host,browser,report,state;
|
||||
try{
|
||||
const model=await compatibleFixture(t,()=>({status:503,body:{error:{message:'Owned UI budget failure'},
|
||||
usage:{prompt_tokens:55000,completion_tokens:17,total_tokens:55017}}}));
|
||||
f.store.close();host=await launchWorkbench(t,f.data);assert.notEqual(new URL(host.url).port,'4318');
|
||||
await host.api('/config',{...host.bootstrap.config,contextLimit:16000,maxOutputTokens:512,maxRunTokens:35000,maxTaskTokens:1000000});
|
||||
const connection=await host.api('/model-center/connections',{name:'自有预算验收模型',model:'owned-budget-ui',baseUrl:model.baseUrl,requiresKey:false});
|
||||
await host.api('/model-center/select',{type:'connection',id:connection.id});
|
||||
const route='/sessions/'+f.session.id;
|
||||
await host.api(route+'/messages',{content:'读取README,不修改文件,不运行检查。'});
|
||||
await until(async()=>{state=await host.api(route);return !['running','awaiting_approval','stopping'].includes(state.status);},15000);
|
||||
assert.equal(state.status,'paused',JSON.stringify({status:state.status,failure:state.failure?.message,stopReason:state.execution.run?.stopReason,usage:state.execution.task?.usage,budget:state.execution.run?.budget}));
|
||||
assert.equal(state.execution.run.stopReason,'token_limit');assert.equal(model.requests.length,1);
|
||||
assert.equal(observedTaskModels(state).model,'owned-budget-ui');
|
||||
browser=await chromium.launch({channel:'msedge',headless:true});
|
||||
for(const width of [1440,390]){
|
||||
const page=await browser.newPage({viewport:{width,height:1000}});page.on('pageerror',error=>errors.push(error.message));
|
||||
await page.goto(host.url);await page.waitForLoadState('networkidle');
|
||||
const project=page.getByLabel('当前项目',{exact:true});
|
||||
if(!await project.isVisible())await page.getByRole('button',{name:'展开项目栏',exact:true}).click();
|
||||
await project.selectOption(f.project.id);
|
||||
const panel=page.getByLabel('任务预算诊断',{exact:true});await expect(panel).toBeVisible();
|
||||
for(const text of ['本次运行上游报告用量','55,017','35,000','20,017','用量未知请求 / 本地预留','0 / 0','单个在途响应可能超过预估'])await expect(panel).toContainText(text);
|
||||
await panel.getByRole('button',{name:'核对预算并续接原任务',exact:true}).click();
|
||||
const dialog=page.getByRole('dialog',{name:'续接原任务',exact:true});
|
||||
await dialog.getByLabel('新Run累计tokens',{exact:true}).fill('100000');
|
||||
await dialog.getByRole('button',{name:'准备续接预览',exact:true}).click();await expect(dialog.getByRole('status')).toContainText('将创建原Task的新Run');
|
||||
assert.equal(model.requests.length,1,'A budget preview cannot dispatch another model request.');
|
||||
const box=await dialog.boundingBox();assert.ok(box.x>=0&&box.x+box.width<=width+1);
|
||||
await dialog.getByRole('button',{name:'关闭窗口',exact:true}).click();await page.reload();await expect(panel).toContainText('20,017');
|
||||
const current=await host.api(route);assert.equal(current.execution.task.id,state.execution.task.id);assert.equal(current.execution.task.usage.requests,1);
|
||||
assert.equal(current.execution.run.consumption.length,1);assert.equal(model.requests.length,1);
|
||||
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+1),'No viewport overflow');
|
||||
await panel.scrollIntoViewIfNeeded();const screenshot=path.join(directory,'reported-overrun-'+width+'.png');
|
||||
await page.screenshot({path:screenshot,fullPage:true});screenshots.push(screenshot);
|
||||
fs.writeFileSync(path.join(directory,'reported-overrun-'+width+'.html'),await page.content());
|
||||
cases.push(width+': actual reported overrun persists, zero unknown use stays distinct, and preview does not dispatch or replace the original Task');await page.close();
|
||||
}
|
||||
assert.deepEqual(errors,[]);fs.writeFileSync(path.join(directory,'session.json'),JSON.stringify(await host.api(route),null,2));
|
||||
report={passed:true,cases,screenshots,errors,realModelRequests:0,ownedHttpRequests:model.requests.length,
|
||||
modelObservation:observedTaskModels(state),scope:'Actual compiled host and owned HTTP reported overrun, persistence and preview UI; real-model autonomy/Purge benefit remain separate.'};
|
||||
}catch(error){if(state)fs.writeFileSync(path.join(directory,'failed-state.json'),JSON.stringify(state,null,2));report={passed:false,cases,screenshots,errors,error:error.stack,realModelRequests:0};process.exitCode=1;}
|
||||
finally{
|
||||
await browser?.close();await host?.close();for(const dispose of disposers.reverse())await dispose();
|
||||
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify({...report,directory}));
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench } from '../tests/helpers.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-model-center-ui.mjs — isolated model connections, pool, import preview, local inspection and 390px layout'); process.exit(0); }
|
||||
const evidenceDir = path.resolve('.qa', 'model-center-ui-' + randomUUID()); fs.mkdirSync(evidenceDir, { recursive: true });
|
||||
const disposers = [], t = { after: dispose => disposers.push(dispose) };
|
||||
const f = fixture(t), inspectionArtifacts = [];
|
||||
for (let index = 0; index < 65; index++) { const value = f.store.artifactStore.write(f.session, null, { ownedInspection: index }); f.store.core.artifact.set(value.id, value); inspectionArtifacts.push(value); }
|
||||
f.store.save(f.session); const oldestArtifact = inspectionArtifacts[0]; fs.appendFileSync(path.join(f.store.artifactStore.root, oldestArtifact.relative), 'owned-original-corruption'); f.store.close();
|
||||
const importFile = path.join(evidenceDir, 'owned-import.json');
|
||||
fs.writeFileSync(importFile, JSON.stringify({ schema: 1, connections: [{ name: 'Imported Fixture', brand: '自定义', model: 'fixture-imported',
|
||||
baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'ui-owned-import-key' }] }));
|
||||
const host = await launchWorkbench(t, f.data);
|
||||
assert.notEqual(new URL(host.url).port, '4318'); assert.equal(host.bootstrap.config.hasKey, false);
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
const pageErrors = [], requestFailures = [], marks = [];
|
||||
page.on('pageerror', error => pageErrors.push(error.message));
|
||||
page.on('requestfailed', request => requestFailures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const mark = value => { marks.push(value); console.log('PASS ' + value); };
|
||||
try {
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
await page.getByRole('button', { name: '打开模型中心', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '模型中心', exact: true }); await expect(dialog).toBeVisible();
|
||||
await dialog.getByRole('button', { name: '添加连接', exact: true }).click();
|
||||
const editor = dialog.getByRole('form', { name: '编辑模型连接', exact: true });
|
||||
await editor.getByLabel('连接名称', { exact: true }).fill('Fixture Primary');
|
||||
await editor.getByLabel('接口根地址', { exact: true }).fill('http://127.0.0.1:9/v1');
|
||||
await editor.getByLabel('请求模型名称', { exact: true }).fill('fixture-primary');
|
||||
await editor.getByLabel('临时 API Key', { exact: true }).fill('ui-owned-primary-key');
|
||||
await editor.getByRole('button', { name: '保存模型连接', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-connection').filter({ hasText: 'Fixture Primary' })).toContainText('已设置,重启需重新填写');
|
||||
assert.equal(JSON.stringify(await host.api('/model-center')).includes('ui-owned-primary-key'), false);
|
||||
assert.equal((await host.api('/model-center')).connections.find(value => value.name === 'Fixture Primary').hasKey, true);
|
||||
mark('connection form stores only the volatile credential and renders configured status without exposing it');
|
||||
|
||||
await dialog.getByRole('button', { name: '添加连接', exact: true }).click();
|
||||
const second = dialog.getByRole('form', { name: '编辑模型连接', exact: true });
|
||||
await second.getByLabel('连接名称', { exact: true }).fill('Fixture Reserve');
|
||||
await second.getByLabel('接口根地址', { exact: true }).fill('http://127.0.0.1:9/v1');
|
||||
await second.getByLabel('请求模型名称', { exact: true }).fill('fixture-reserve');
|
||||
await second.getByRole('button', { name: '保存模型连接', exact: true }).click();
|
||||
await dialog.getByRole('button', { name: '模型池', exact: true }).click();
|
||||
await dialog.getByRole('button', { name: '新建模型池', exact: true }).click();
|
||||
const poolForm = dialog.getByRole('form', { name: '编辑模型池', exact: true });
|
||||
await poolForm.getByLabel('模型池名称', { exact: true }).fill('Fixture failover pool');
|
||||
await poolForm.locator('.mc-member-choice').filter({ hasText: 'Fixture Primary' }).locator('input[type=checkbox]').check();
|
||||
await poolForm.locator('.mc-member-choice').filter({ hasText: 'Fixture Reserve' }).locator('input[type=checkbox]').check();
|
||||
await poolForm.getByLabel('Fixture Primary权重', { exact: true }).fill('2');
|
||||
await poolForm.getByRole('button', { name: '保存模型池', exact: true }).click();
|
||||
const poolCard = dialog.locator('.mc-pool').filter({ hasText: 'Fixture failover pool' }); await expect(poolCard).toContainText('2 个成员');
|
||||
await poolCard.getByRole('button', { name: '使用此模型池', exact: true }).click();
|
||||
const center = await host.api('/model-center'); assert.equal(center.selected.type, 'pool');
|
||||
assert.equal(center.pools.find(value => value.id === center.selected.id).members.length, 2);
|
||||
mark('pool editor saves weighted members and binds selection for new Runs');
|
||||
|
||||
await dialog.getByRole('button', { name: '配置导入', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-tabs button[aria-current]')).toContainText('配置导入');
|
||||
await dialog.getByLabel('配置格式').selectOption('workbench-json');
|
||||
await dialog.getByLabel('本地文件路径', { exact: true }).fill(importFile);
|
||||
await dialog.getByRole('button', { name: '预览可导入连接', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-editor').filter({ hasText: '本次预览' })).toContainText('Imported Fixture');
|
||||
assert.equal((await page.locator('body').innerText()).includes('ui-owned-import-key'), false);
|
||||
await dialog.getByRole('button', { name: '导入这些连接', exact: true }).click();
|
||||
assert.ok((await host.api('/model-center')).connections.some(value => value.name === 'Imported Fixture'));
|
||||
const sourceCard = dialog.locator('.mc-import-source').filter({ hasText: 'workbench-json' });
|
||||
await sourceCard.getByRole('button', { name: '启用自动同步', exact: true }).click();
|
||||
let view = await host.api('/model-center'); assert.equal(view.imports[0].enabled, true);
|
||||
fs.writeFileSync(importFile, JSON.stringify({ schema: 1, connections: [{ name: 'Imported Fixture', brand: '自定义', model: 'fixture-imported-v2',
|
||||
baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'ui-owned-import-key' }] }));
|
||||
await dialog.getByRole('button', { name: '立即同步已启用来源', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-notice')).toContainText('更新 1 项');
|
||||
view = await host.api('/model-center'); assert.equal(view.connections.find(value => value.name === 'Imported Fixture').model, 'fixture-imported-v2');
|
||||
await sourceCard.getByRole('button', { name: '关闭自动同步', exact: true }).click();
|
||||
assert.equal((await host.api('/model-center')).imports[0].enabled, false);
|
||||
assert.equal(JSON.stringify(view).includes('ui-owned-import-key'), false);
|
||||
mark('schema-bound import preview, opt-in automatic sync, revision update and sync shutdown keep credentials hidden');
|
||||
|
||||
await dialog.getByRole('button', { name: '运行巡检', exact: true }).click();
|
||||
await dialog.getByRole('button', { name: '开始本地巡检', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-checks')).toContainText('SQLite quick_check通过');
|
||||
const operations = await host.api('/operations'); assert.equal(operations.modelRequests, 0);
|
||||
assert.equal(operations.artifacts.total, 65); assert.equal(operations.artifacts.checked, 50); assert.equal(operations.checks.find(value => value.id === 'artifacts').status, 'passed');
|
||||
await dialog.getByRole('button', { name: '检查更早工件', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-operations-pages')).toContainText(oldestArtifact.id); await expect(dialog.getByRole('button', { name: '检查更早工件', exact: true })).toBeDisabled();
|
||||
const olderInspection = await host.api('/operations?artifactOffset=0'); assert.equal(olderInspection.checks.find(value => value.id === 'artifacts').status, 'failed'); assert.equal(olderInspection.artifacts.results[0].id, oldestArtifact.id);
|
||||
mark('real historical pagination detects the owned corrupted older artifact, retains its identity and leaves original bytes unchanged');
|
||||
await page.screenshot({ path: path.join(evidenceDir, 'desktop-operations.png'), fullPage: false });
|
||||
mark('local inspection shows real database and artifact checks and emits zero model requests');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390), '390px viewport must not overflow horizontally');
|
||||
await page.screenshot({ path: path.join(evidenceDir, 'mobile-operations.png'), fullPage: false });
|
||||
await dialog.getByRole('button', { name: '模型连接', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-connection').filter({ hasText: 'Fixture Primary' })).toBeVisible();
|
||||
assert.ok(await page.locator('body').evaluate(node => node.scrollWidth <= 390), '390px connections must not overflow horizontally');
|
||||
await page.screenshot({ path: path.join(evidenceDir, 'mobile-connections.png'), fullPage: false });
|
||||
mark('390px model-center modal and connection cards remain operable without horizontal page overflow');
|
||||
|
||||
assert.deepEqual(pageErrors, []); assert.deepEqual(requestFailures, []);
|
||||
const report = { url: host.url, viewportWidths: [1440, 390], assertions: marks, pageErrors, requestFailures,
|
||||
modelRequests: operations.modelRequests, screenshots: ['desktop-operations.png', 'mobile-operations.png', 'mobile-connections.png'] };
|
||||
fs.writeFileSync(path.join(evidenceDir, 'result.json'), JSON.stringify(report, null, 2));
|
||||
console.log('EVIDENCE ' + evidenceDir);
|
||||
} finally {
|
||||
await browser.close();
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
assert.ok(args.length === 0 || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
||||
const appRoot = args[1] ?? process.cwd(), directory = path.resolve('.qa', 'model-fidelity-ui-' + randomUUID());
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: action => disposers.push(action) }, f = fixture(t), requests = [], screenshots = [], checks = [], errors = [];
|
||||
let mode = 'quota', browser, page;
|
||||
const model = http.createServer(async (request, response) => {
|
||||
const parts = []; for await (const part of request) parts.push(part); requests.push({ route: request.url, body: JSON.parse(Buffer.concat(parts)) });
|
||||
if (mode === 'quota') {
|
||||
response.writeHead(429, { 'Content-Type': 'application/json', 'X-Request-ID': 'owned-upstream-quota-ui', 'Retry-After': '180' });
|
||||
response.end(JSON.stringify({ error: { code: 'insufficient_quota', type: 'billing_error', param: 'account', message: 'Owned quota error; owned-ui-private-key' } }));
|
||||
} else {
|
||||
response.writeHead(200, { 'Content-Type': 'text/event-stream', 'X-Request-ID': 'owned-upstream-stream-ui' });
|
||||
const event = value => 'data: ' + JSON.stringify(value) + '\n\n';
|
||||
response.end(event({ type: 'response.created', sequence_number: 1, response: { id: 'resp-owned-ui' } }) +
|
||||
event({ type: 'response.output_item.added', sequence_number: 2, item: { id: 'fc-owned-ui', call_id: 'owned-ui-tool', type: 'function_call', name: 'propose_file', arguments: '' } }) +
|
||||
event({ type: 'response.function_call_arguments.delta', sequence_number: 3, item_id: 'fc-owned-ui', delta: '{"path":' }) +
|
||||
event({ type: 'response.failed', sequence_number: 4, response: { id: 'resp-owned-ui', status: 'failed', error: { code: 'server_error', type: 'api_error', message: 'Owned stream failed' }, usage: { input_tokens: 20, output_tokens: 4 } } }));
|
||||
}
|
||||
});
|
||||
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
||||
const provider = new Provider(), hub = provider.modelHub(f.store);
|
||||
let member = hub.upsert({ name: 'Owned UI 模型', model: 'owned-ui-model', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', apiKey: 'owned-ui-private-key', requiresKey: true });
|
||||
hub.select({ type: 'connection', id: member.id }); f.store.close();
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath: fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath });
|
||||
// The test credential is intentionally volatile; set it in this new test host.
|
||||
member = await host.api('/model-center/connections', { ...member, apiKey: 'owned-ui-private-key' });
|
||||
const mark = text => { checks.push(text); console.log('PASS ' + text); };
|
||||
async function capture(name) { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file }); screenshots.push({ path: file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') }); }
|
||||
async function openCenter() { await page.getByRole('button', { name: '打开模型中心', exact: true }).click(); return page.getByRole('dialog', { name: '模型中心', exact: true }); }
|
||||
async function run(prompt) {
|
||||
await page.getByLabel('任务内容').fill(prompt); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
let id; await until(async () => { id = (await host.api('/sessions')).find(value => value.title === prompt)?.id; return id && (await host.api('/sessions/' + id)).status === 'failed'; }, 15000);
|
||||
return host.api('/sessions/' + id);
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
page.on('pageerror', error => errors.push(error.message)); await page.goto(host.url); await page.getByLabel('当前项目').selectOption(f.project.id);
|
||||
let dialog = await openCenter(); const card = dialog.locator('.mc-connection').filter({ has: page.getByRole('heading', { name: 'Owned UI 模型', exact: true }) });
|
||||
await card.getByRole('button', { name: '原生参数', exact: true }).click();
|
||||
await page.getByLabel('原生参数 JSON', { exact: true }).fill('{"reasoning_effort":"high","temperature":0.4,"parallel_tool_calls":false}');
|
||||
await page.getByLabel('输出上限字段', { exact: true }).selectOption('max_completion_tokens');
|
||||
await page.getByRole('button', { name: '保存原生参数', exact: true }).click(); await expect(dialog.getByRole('status')).toContainText('原生参数已保存');
|
||||
assert.equal(requests.length, 0); await page.reload(); dialog = await openCenter();
|
||||
await dialog.locator('.mc-connection').filter({ has: page.getByRole('heading', { name: 'Owned UI 模型', exact: true }) }).getByRole('button', { name: '原生参数', exact: true }).click();
|
||||
await expect(page.getByLabel('原生参数 JSON', { exact: true })).toContainText('reasoning_effort');
|
||||
await capture('native-desktop'); await page.setViewportSize({ width: 390, height: 844 });
|
||||
await page.getByRole('button', { name: '保存原生参数', exact: true }).scrollIntoViewIfNeeded();
|
||||
const bounds = await page.getByRole('button', { name: '保存原生参数', exact: true }).boundingBox(); assert.ok(bounds && bounds.x >= 0 && bounds.x + bounds.width <= 390);
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); await capture('native-390'); mark('native parameters persist after reload and the narrow form remains usable without calling a model');
|
||||
await page.getByLabel('原生参数 JSON', { exact: true }).fill('{"tools":[]}'); await page.getByRole('button', { name: '保存原生参数', exact: true }).click();
|
||||
await expect(page.getByRole('form', { name: '原生请求参数' }).getByRole('alert')).toContainText('尚未接入原生参数:tools');
|
||||
assert.equal(requests.length, 0); await page.keyboard.press('Escape'); await page.setViewportSize({ width: 1440, height: 1000 }); mark('unsupported field is rejected visibly before any dispatch');
|
||||
const first = await run('读取当前项目,HTTP错误原文验收'); assert.equal(requests.length, 1); assert.equal(first.changes.length, 0);
|
||||
assert.equal(requests[0].body.reasoning_effort, 'high'); assert.equal(requests[0].body.temperature, 0.4); assert.equal(requests[0].body.parallel_tool_calls, false);
|
||||
assert.ok(requests[0].body.max_completion_tokens > 0); assert.equal(requests[0].body.max_tokens, undefined);
|
||||
const failureCard = page.locator('.failure-card').last(); await failureCard.getByText('查看上游原文与请求信息', { exact: true }).click();
|
||||
await expect(failureCard).toContainText('owned-upstream-quota-ui'); await expect(failureCard).toContainText('insufficient_quota'); await expect(failureCard).toContainText('180 秒');
|
||||
assert.ok(!(await failureCard.innerText()).includes('owned-ui-private-key')); await capture('http-error-desktop');
|
||||
await failureCard.getByLabel('上游错误原文').scrollIntoViewIfNeeded(); await capture('http-raw-desktop');
|
||||
const downloadEvent = page.waitForEvent('download'); await failureCard.getByRole('button', { name: '下载已保存的原文', exact: true }).click();
|
||||
const download = await downloadEvent, downloaded = path.join(directory, 'owned-http-error.txt'); await download.saveAs(downloaded);
|
||||
assert.equal(createHash('sha256').update(fs.readFileSync(downloaded)).digest('hex'), first.failure.upstream.artifactSha256);
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await failureCard.scrollIntoViewIfNeeded(); await capture('http-error-390');
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); mark('actual HTTP error shows raw fields and original wait, keeps secrets hidden and downloads SHA-verified original at both widths');
|
||||
dialog = await openCenter(); await dialog.getByRole('button', { name: '请求账本', exact: true }).click();
|
||||
const attempt = dialog.locator('.mc-attempts article').first(); await attempt.getByText('查看上游原文与请求信息', { exact: true }).click();
|
||||
await expect(attempt).toContainText('owned-upstream-quota-ui'); await expect(attempt.getByRole('button', { name: '下载实际请求', exact: true })).toBeVisible();
|
||||
await capture('ledger-390'); await page.keyboard.press('Escape'); mark('physical request ledger preserves the same error origin and offers the sealed actual request');
|
||||
mode = 'stream'; let center = await host.api('/model-center'), current = center.connections.find(value => value.id === member.id);
|
||||
member = await host.api('/model-center/connections', { ...current, protocol: 'responses', streaming: true, capabilityDeclarations: { ...current.capabilityDeclarations, streaming: 'supported' },
|
||||
nativeParameters: { reasoning: { effort: 'medium' }, text: { verbosity: 'high' } }, outputTokenField: 'max_output_tokens' });
|
||||
const secondSession = await host.api('/sessions', { projectId: f.project.id });
|
||||
await page.evaluate(value => { localStorage.setItem('awb-project', value.projectId); localStorage.setItem('awb-session', value.id); }, secondSession);
|
||||
await page.reload(); await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
const second = await run('读取当前项目,Responses流失败验收');
|
||||
assert.equal(requests.length, 2); assert.equal(requests[1].route, '/v1/responses'); assert.deepEqual(requests[1].body.reasoning, { effort: 'medium' });
|
||||
assert.equal(second.execution.run.usage.totalTokens, 24); assert.equal(second.changes.length, 0); assert.equal(second.execution.run.usage.requests, 1);
|
||||
const streamCard = page.locator('.failure-card').last(); await streamCard.getByText('查看上游原文与请求信息', { exact: true }).click();
|
||||
await expect(streamCard).toContainText('response.failed'); await expect(streamCard).toContainText('resp-owned-ui'); await expect(streamCard).toContainText('owned-upstream-stream-ui');
|
||||
await expect(streamCard).toContainText('流式响应已开始'); await capture('sse-error-desktop');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await streamCard.scrollIntoViewIfNeeded(); await capture('sse-error-390');
|
||||
await streamCard.getByLabel('上游错误原文').scrollIntoViewIfNeeded(); await capture('sse-raw-390');
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); assert.deepEqual(errors, []);
|
||||
mark('Responses failed event retains original identity and reported use at both widths; partial tool is never dispatched or retried');
|
||||
mode = 'quota';
|
||||
await until(async () => (await host.api('/model-center')).connections.find(value => value.id === member.id).status === 'configured');
|
||||
dialog = await openCenter();
|
||||
await dialog.locator('.mc-connection').filter({ has: page.getByRole('heading', { name: 'Owned UI 模型', exact: true }) }).getByRole('button', { name: '验证响应(调用模型)', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-notice')).toContainText('额度或消费限制');
|
||||
await dialog.locator('.mc-notice').getByText('查看上游原文与请求信息', { exact: true }).click();
|
||||
await expect(dialog.locator('.mc-notice')).toContainText('owned-upstream-quota-ui');
|
||||
await expect(dialog.locator('.mc-notice')).toContainText('仅保留以上诊断预览');
|
||||
assert.equal(requests.length, 3); assert.deepEqual(errors, []); await capture('probe-error-390');
|
||||
mark('explicit synthetic probe also shows the API diagnostic and labels its preview-only retention without inventing a Task artifact');
|
||||
result = { success: true, appRoot, checks, screenshots, errors, modelRequests: requests.length, realModelRequests: 0 };
|
||||
} catch (error) { if (page) await capture('failure').catch(() => {}); result = { success: false, appRoot, error: error.stack, checks, screenshots, errors, realModelRequests: 0 }; throw error; }
|
||||
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2)); console.log(JSON.stringify({ success: result.success, directory, checks: checks.length, screenshots: screenshots.length }));
|
||||
await browser?.close(); await new Promise(resolve => { model.closeAllConnections(); model.close(resolve); }); for (const action of disposers.reverse()) await action(); }
|
||||
@@ -0,0 +1,78 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createHash,randomUUID} from 'node:crypto';
|
||||
import {chromium,expect} from 'playwright/test';
|
||||
import {fixture,launchWorkbench,until} from '../tests/helpers.mjs';
|
||||
import {Provider} from '../src/server/provider.mjs';
|
||||
|
||||
if(process.argv.includes('--help')){console.log('node scripts/verify-model-identity-ui.mjs [--app-root ABSOLUTE_ROOT] — compiled or packaged host, owned identity fixture and desktop/390px UI. No configured-model requests.');process.exit(0);}
|
||||
const args=process.argv.slice(2);assert.ok(!args.length||args.length===2&&args[0]==='--app-root'&&path.isAbsolute(args[1]),'Use --app-root with an absolute directory.');
|
||||
const appRoot=args.length?fs.realpathSync.native(args[1]):process.cwd();
|
||||
const targetVersion=JSON.parse(fs.readFileSync(path.join(appRoot,fs.existsSync(path.join(appRoot,'MANIFEST.json'))?'MANIFEST.json':'package.json'))).version;
|
||||
const directory=path.resolve('.qa','model-identity-ui-'+randomUUID()),disposers=[],checks=[],screenshots=[],errors=[],requests=[];
|
||||
fs.mkdirSync(directory,{recursive:true});const t={after:fn=>disposers.push(fn)},f=fixture(t);
|
||||
const upstream=http.createServer(async(req,res)=>{
|
||||
const chunks=[];for await(const chunk of req)chunks.push(chunk);requests.push(JSON.parse(Buffer.concat(chunks)));
|
||||
res.writeHead(200,{'Content-Type':'application/json'});
|
||||
res.end(JSON.stringify({model:'owned-upstream-snapshot',choices:[{message:{role:'assistant',content:'',tool_calls:[
|
||||
{id:'owned-ui-call',type:'function',function:{name:'propose_file',arguments:JSON.stringify({path:'unsafe.txt',content:'must not execute',reason:'fixture'})}}]}}],
|
||||
usage:{prompt_tokens:13,completion_tokens:7,total_tokens:20}}));
|
||||
});
|
||||
await new Promise(resolve=>upstream.listen(0,'127.0.0.1',resolve));
|
||||
const provider=new Provider(),hub=provider.modelHub(f.store),connection=hub.upsert({name:'身份核对 · 自有接口夹具',model:'owned-requested-model',
|
||||
baseUrl:'http://127.0.0.1:'+upstream.address().port+'/v1',requiresKey:false});
|
||||
hub.select({type:'connection',id:connection.id});f.store.close();
|
||||
const nodePath=fs.existsSync(path.join(appRoot,'runtime/node.exe'))?path.join(appRoot,'runtime/node.exe'):process.execPath;
|
||||
const host=await launchWorkbench(t,f.data,{}, {appRoot,nodePath});assert.notEqual(new URL(host.url).port,'4318');
|
||||
let browser,page,result;
|
||||
const capture=async name=>{const file=path.join(directory,name+'.png');await page.screenshot({path:file});screenshots.push({path:file,sha256:createHash('sha256').update(fs.readFileSync(file)).digest('hex')});};
|
||||
const mark=name=>{checks.push(name);console.log('PASS '+name);};
|
||||
const openCenter=async()=>{await page.getByRole('button',{name:'打开模型中心',exact:true}).click();return page.getByRole('dialog',{name:'模型中心',exact:true});};
|
||||
try{
|
||||
browser=await chromium.launch({channel:'msedge',headless:true});page=await browser.newPage({viewport:{width:1440,height:1000}});
|
||||
page.on('pageerror',e=>errors.push(e.message));await page.goto(host.url);await page.waitForLoadState('networkidle');
|
||||
assert.equal((await host.api('/health')).version,targetVersion);await page.getByLabel('当前项目').selectOption(f.project.id);
|
||||
let center=await openCenter();const card=()=>center.locator('.mc-connection').filter({has:page.getByRole('heading',{name:connection.name,exact:true})});
|
||||
await card().getByRole('button',{name:'编辑',exact:true}).click();const form=()=>page.getByRole('form',{name:'编辑模型连接',exact:true});
|
||||
await form().getByLabel('型号核对',{exact:true}).selectOption('require-match');
|
||||
await form().getByLabel('允许的返回型号',{exact:false}).fill('snapshot-a, snapshot-b');
|
||||
await form().getByLabel('型号映射依据',{exact:true}).fill('自有接口映射约定 revision 1');
|
||||
await capture('settings-desktop');await page.setViewportSize({width:390,height:844});
|
||||
await form().getByRole('button',{name:'保存模型连接',exact:true}).scrollIntoViewIfNeeded();
|
||||
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth));await capture('settings-390');
|
||||
await form().getByRole('button',{name:'保存模型连接',exact:true}).click();await expect(form()).toHaveCount(0);
|
||||
await page.reload();center=await openCenter();await card().getByRole('button',{name:'编辑',exact:true}).click();
|
||||
await expect(form().getByLabel('型号核对',{exact:true})).toHaveValue('require-match');
|
||||
await expect(form().getByLabel('允许的返回型号',{exact:false})).toHaveValue('snapshot-a, snapshot-b');
|
||||
await expect(form().getByLabel('型号映射依据',{exact:true})).toHaveValue('自有接口映射约定 revision 1');
|
||||
mark('strict policy and multiple declared aliases save and reload at desktop and 390px without a model call');assert.equal(requests.length,0);
|
||||
await form().getByLabel('允许的返回型号',{exact:false}).fill('');await form().getByLabel('型号映射依据',{exact:true}).fill('');
|
||||
await form().getByRole('button',{name:'保存模型连接',exact:true}).click();await expect(form()).toHaveCount(0);await page.keyboard.press('Escape');
|
||||
await page.setViewportSize({width:1440,height:1000});const prompt='核对当前项目,身份不一致应暂停';
|
||||
await page.getByLabel('任务内容').fill(prompt);await page.getByRole('button',{name:'发送任务',exact:true}).click();let session;
|
||||
await until(async()=>{const id=(await host.api('/sessions')).find(v=>v.title===prompt)?.id;session=id&&await host.api('/sessions/'+id);return session?.status==='paused';},15000);
|
||||
assert.equal(requests.length,1);assert.equal(session.changes.length,0);assert.equal(session.execution.run.stopReason,'model_identity');
|
||||
await expect(page.locator('.tool-name strong').filter({hasText:'模型返回型号需要核对'})).toHaveCount(1);
|
||||
await expect(page.locator('.model-identity-details').first()).toContainText('请求与返回型号不一致');await page.reload();
|
||||
const failure=page.locator('.failure-card').last();await failure.locator('.model-identity-details summary').click();
|
||||
await expect(failure).toContainText('owned-requested-model');await expect(failure).toContainText('owned-upstream-snapshot');
|
||||
await expect(failure).toContainText('不一致或无法核对时暂停');await capture('paused-desktop');
|
||||
await page.setViewportSize({width:390,height:844});await failure.locator('.model-identity-details').scrollIntoViewIfNeeded();await capture('paused-390');
|
||||
assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth));
|
||||
const downloadPromise=page.waitForEvent('download');await failure.getByRole('button',{name:'下载原响应',exact:true}).click();
|
||||
const download=await downloadPromise,saved=path.join(directory,'original-response.json');await download.saveAs(saved);
|
||||
const original=fs.readFileSync(saved),identity=session.execution.run.consumption[0].modelIdentity;
|
||||
assert.equal(createHash('sha256').update(original).digest('hex'),identity.responseSource.sha256);assert.equal(JSON.parse(original).model,'owned-upstream-snapshot');
|
||||
mark('strict mismatch pauses before any proposal, retains reported usage, persists on refresh and downloads SHA-verified original');
|
||||
center=await openCenter();await center.getByRole('button',{name:'请求账本',exact:true}).click();
|
||||
const attempt=center.locator('.mc-attempts article').first();await attempt.locator('.model-identity-details').first().locator('summary').click();
|
||||
await expect(attempt).toContainText('owned-upstream-snapshot');await expect(attempt).toContainText(identity.responseSource.sha256);
|
||||
await capture('ledger-390');assert.deepEqual(errors,[]);mark('physical request ledger retains the same identity and source without page errors');
|
||||
result={success:true,appRoot,targetVersion,testedVersion:targetVersion,checks,screenshots,errors,modelRequests:requests.length,realModelRequests:0};
|
||||
}catch(error){if(page)await capture('failure').catch(()=>{});result={success:false,error:error.stack,checks,screenshots,errors,realModelRequests:0};throw error;}
|
||||
finally{
|
||||
fs.writeFileSync(path.join(directory,'result.json'),JSON.stringify(result,null,2));console.log(JSON.stringify({success:result.success,directory,checks:checks.length}));
|
||||
await browser?.close();upstream.closeAllConnections();await new Promise(resolve=>upstream.close(resolve));for(const action of disposers.reverse())await action();
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID, X509Certificate } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-model-protocols-ui.mjs [--app-root <absolute-package>] — owned HTTP Responses/SSE, connection persistence, scoped detection, pool rejection, original stream download and desktop/mobile stop. Explicit package uses contained Node/server. No real model calls or OS trust changes.'); process.exit(0); }
|
||||
const args = process.argv.slice(2); assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
||||
const appRoot = fs.realpathSync.native(args.length ? args[1] : process.cwd()), nodePath = args.length ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
|
||||
const directory = path.resolve('.qa', 'model-protocols-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: action => disposers.push(action) }, f = fixture(t); f.store.close();
|
||||
const requests = [], sockets = new Set(), held = new Set(); let mode = 'success', closedStreams = 0;
|
||||
const item = text => ({ type: 'message', id: 'msg_owned', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text }] });
|
||||
const responseBody = output => ({ id: 'resp_owned', status: 'completed', output, usage: { input_tokens: 60, output_tokens: 20, total_tokens: 80 } });
|
||||
const modelServer = http.createServer(async (request, response) => {
|
||||
try {
|
||||
const parts = []; for await (const part of request) parts.push(part);
|
||||
const body = JSON.parse(Buffer.concat(parts)); requests.push({ route: request.url, body });
|
||||
assert.equal(request.url, '/v1/responses');
|
||||
const prompt = body.input.findLast(value => value.role === 'user')?.content ?? '';
|
||||
const probe = prompt === 'Reply OK.' || prompt.startsWith('Call workbench_capability_echo');
|
||||
const nonce = prompt.match(/nonce ([a-f0-9-]+)/)?.[1];
|
||||
const tool = { type: 'function_call', id: 'fc_owned', call_id: 'owned_call', name: nonce ? 'workbench_capability_echo' : 'list_files', arguments: JSON.stringify(nonce ? { nonce } : { path: '', depth: 1 }), status: 'completed' };
|
||||
const output = probe ? nonce ? [tool] : [item('OK')] : body.input.some(value => value.type === 'function_call_output') ? [item('项目读取完成。')] : [item('正在核对当前项目。'), tool];
|
||||
if (!body.stream) { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify(responseBody(output))); return; }
|
||||
response.writeHead(200, { 'Content-Type': 'text/event-stream' }); let sequence = 0;
|
||||
const send = (type, data) => response.write('data: ' + JSON.stringify({ type, sequence_number: sequence++, ...data }) + '\r\n\r\n');
|
||||
send('response.created', { response: { id: 'resp_owned' } });
|
||||
if (!probe && mode !== 'success') {
|
||||
send('response.output_text.delta', { delta: '正在核对当前项目,尚未执行工具。' });
|
||||
send('response.output_item.added', { item: { ...tool, arguments: '', status: 'in_progress' }, output_index: 1 });
|
||||
send('response.function_call_arguments.delta', { item_id: tool.id, delta: '{"path":' });
|
||||
if (mode === 'truncate') response.end();
|
||||
else { held.add(response); response.on('close', () => { held.delete(response); closedStreams++; }); }
|
||||
return;
|
||||
}
|
||||
for (const value of output) {
|
||||
if (value.type === 'message') send('response.output_text.delta', { delta: value.content[0].text });
|
||||
else { send('response.output_item.added', { item: { ...value, arguments: '', status: 'in_progress' }, output_index: 1 }); send('response.function_call_arguments.delta', { item_id: value.id, delta: value.arguments }); send('response.function_call_arguments.done', { item_id: value.id, arguments: value.arguments }); }
|
||||
}
|
||||
send('response.completed', { response: responseBody(output) }); response.end();
|
||||
} catch { response.destroy(); }
|
||||
});
|
||||
modelServer.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
|
||||
await new Promise(resolve => modelServer.listen(0, '127.0.0.1', resolve));
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath }), browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
assert.equal(fs.realpathSync.native(host.bootstrap.runtimeRoot), appRoot); assert.notEqual(new URL(host.url).port, '4318');
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }), errors = [], failures = [], passed = [], screenshots = [];
|
||||
page.on('pageerror', error => errors.push(error.message)); page.on('requestfailed', request => failures.push(request.url() + ': ' + request.failure()?.errorText));
|
||||
const mark = text => { passed.push(text); console.log('PASS ' + text); };
|
||||
async function shot(target, name) { const file = path.join(directory, name + '.png'); await target.screenshot({ path: file, fullPage: false }); screenshots.push(file); assert.ok(await target.locator('body').evaluate(node => node.scrollWidth <= innerWidth), name + ' page overflow'); }
|
||||
const getSession = () => host.api('/sessions/' + f.session.id);
|
||||
let mobile, relay, cookie = '', gateway;
|
||||
try {
|
||||
await page.addInitScript(({ projectId, sessionId }) => { localStorage.setItem('awb-project', projectId); localStorage.setItem('awb-session', sessionId); }, { projectId: f.project.id, sessionId: f.session.id });
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle');
|
||||
await page.getByRole('button', { name: '打开模型中心', exact: true }).click();
|
||||
const dialog = page.getByRole('dialog', { name: '模型中心', exact: true });
|
||||
await dialog.getByRole('button', { name: '添加连接', exact: true }).click();
|
||||
const form = dialog.getByRole('form', { name: '编辑模型连接', exact: true });
|
||||
await form.getByLabel('连接名称', { exact: true }).fill('验收 Responses');
|
||||
await form.getByLabel('接口协议', { exact: true }).selectOption('responses');
|
||||
await expect(form.getByLabel('使用流式输出', { exact: true })).toBeDisabled();
|
||||
await form.getByLabel('流式能力', { exact: true }).selectOption('supported'); await form.getByLabel('使用流式输出', { exact: true }).check();
|
||||
await form.getByLabel('接口根地址', { exact: true }).fill('http://127.0.0.1:' + modelServer.address().port + '/v1');
|
||||
await form.getByLabel('请求模型名称', { exact: true }).fill('owned-responses'); await form.getByLabel('服务需要 API Key', { exact: true }).uncheck();
|
||||
await shot(page, 'desktop-editor'); await page.setViewportSize({ width: 390, height: 844 }); await form.scrollIntoViewIfNeeded(); await shot(page, 'narrow-editor');
|
||||
await form.getByRole('button', { name: '保存模型连接', exact: true }).click(); await expect(form).toHaveCount(0);
|
||||
await page.reload(); await page.getByRole('button', { name: '打开模型中心', exact: true }).click();
|
||||
const card = dialog.locator('.mc-connection').filter({ hasText: '验收 Responses' }); await expect(card).toContainText('Responses · 流式开启');
|
||||
let center = await host.api('/model-center'), connection = center.connections.find(value => value.name === '验收 Responses');
|
||||
assert.equal(connection.protocol, 'responses'); assert.equal(connection.streaming, true);
|
||||
mark('actual form saves Responses and declared stream, refresh preserves both and narrow editor remains operable');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
for (const [scope, label] of [['text', '文本'], ['tools', '工具'], ['streaming', '流式']]) {
|
||||
await card.getByLabel('验收 Responses检测范围', { exact: true }).selectOption(scope);
|
||||
await card.getByRole('button', { name: '验证响应(调用模型)', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-notice')).toContainText(label + '检测通过');
|
||||
await expect(card.locator('.mc-capabilities')).toContainText(label + ':已验证');
|
||||
if (scope === 'text') await expect(card.locator('.mc-capabilities')).toContainText('工具:声明支持');
|
||||
}
|
||||
await shot(page, 'desktop-capabilities'); await page.setViewportSize({ width: 390, height: 844 }); await shot(page, 'narrow-capabilities'); await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
mark('separate text, nonce tool and actual SSE probes bind only their requested capability and own attempts');
|
||||
const text = await host.api('/model-center/connections', { name: '只支持文本', model: 'text-only', baseUrl: connection.baseUrl, requiresKey: false, capabilityDeclarations: { tools: 'unsupported' } });
|
||||
const textId = text.connection?.id ?? (await host.api('/model-center')).connections.find(value => value.name === '只支持文本').id;
|
||||
await page.reload(); await page.getByRole('button', { name: '打开模型中心', exact: true }).click();
|
||||
await dialog.getByRole('button', { name: '模型池', exact: true }).click(); await dialog.getByRole('button', { name: '新建模型池', exact: true }).click();
|
||||
const poolForm = dialog.getByRole('form', { name: '编辑模型池', exact: true }); await poolForm.getByLabel('模型池名称', { exact: true }).fill('按能力选择的验收池');
|
||||
for (const name of ['验收 Responses', '只支持文本']) await poolForm.locator('.mc-member-choice').filter({ hasText: name }).locator('input[type=checkbox]').check();
|
||||
await poolForm.getByRole('button', { name: '保存模型池', exact: true }).click(); await dialog.locator('.mc-pool').filter({ hasText: '按能力选择的验收池' }).getByRole('button', { name: '使用此模型池', exact: true }).click();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('.model-button')).toContainText('按能力选择的验收池');
|
||||
await page.reload(); await expect(page.locator('.model-button')).toContainText('按能力选择的验收池');
|
||||
assert.equal((await host.api('/bootstrap')).config.model, '', 'The unselected legacy model remains untouched.');
|
||||
await page.getByLabel('任务内容', { exact: true }).fill('受控完整流程:读取项目后结束。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await until(async () => (await getSession()).status === 'completed', 15000);
|
||||
const completed = await getSession(); assert.equal(completed.events.filter(value => value.tool === 'list_files' && value.phase === 'completed').length, 1);
|
||||
center = await host.api('/model-center'); const attempt = center.attempts.find(value => value.runId === completed.runId);
|
||||
assert.equal(attempt.protocol, 'responses'); assert.ok(attempt.rejections.some(value => value.connectionId === textId && value.capability === 'tools')); assert.ok(requests.every(value => value.body.model !== 'text-only'));
|
||||
await page.getByRole('button', { name: '打开模型中心', exact: true }).click(); await dialog.getByRole('button', { name: '请求账本', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-route-rejections').first()).toContainText('只支持文本');
|
||||
const downloadEvent = page.waitForEvent('download'); await dialog.getByRole('button', { name: '下载已保存的响应', exact: true }).first().click();
|
||||
const download = await downloadEvent, saved = path.join(directory, 'original-stream.sse'); await download.saveAs(saved);
|
||||
assert.equal(createHash('sha256').update(fs.readFileSync(saved)).digest('hex'), attempt.responseSha256); await shot(page, 'desktop-ledger');
|
||||
mark('actual pooled task excludes text-only before dispatch, executes original tool once and downloads SHA-matched original SSE');
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
const initial = await host.api('/companion'); assert.ok(initial.addresses.length);
|
||||
const reservation = http.createServer(); await new Promise(resolve => reservation.listen(0, initial.addresses[0].address, resolve)); const port = reservation.address().port; await new Promise(resolve => reservation.close(resolve));
|
||||
await host.api('/companion/start', { address: initial.addresses[0].address, port }); gateway = await host.api('/companion');
|
||||
const cert = await fetch(host.url + '/api/companion/certificate', { headers: { 'X-Workbench-Token': host.bootstrap.token } }); assert.ok(cert.ok);
|
||||
const ca = new X509Certificate(Buffer.from(await cert.arrayBuffer())).toString(), target = new URL(gateway.origin);
|
||||
const ticket = await host.api('/companion/pair', { projectIds: [f.project.id], permissions: { observe: true, approve: false, stop: true }, expiresInHours: 1 });
|
||||
relay = http.createServer((request, response) => {
|
||||
const forwarded = https.request(new URL(request.url, target), { method: request.method, ca, servername: '', headers: { Host: target.host, Origin: target.origin, Connection: 'close', ...(cookie ? { Cookie: cookie } : {}), ...(request.headers['content-type'] ? { 'Content-Type': request.headers['content-type'] } : {}) } }, incoming => {
|
||||
if (incoming.headers['set-cookie']?.length) cookie = incoming.headers['set-cookie'][0].split(';')[0]; const headers = { ...incoming.headers }; delete headers['set-cookie']; response.writeHead(incoming.statusCode, headers); incoming.pipe(response);
|
||||
}); forwarded.on('error', () => response.destroy()); request.pipe(forwarded);
|
||||
});
|
||||
await new Promise(resolve => relay.listen(0, '127.0.0.1', resolve));
|
||||
mobile = await browser.newPage({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true }); mobile.on('pageerror', error => errors.push(error.message));
|
||||
await mobile.goto('http://127.0.0.1:' + relay.address().port + '/#pair=' + new URLSearchParams(new URL(ticket.url).hash.slice(1)).get('pair'));
|
||||
await mobile.getByLabel('设备名称', { exact: true }).fill('协议验收观察窗口'); await mobile.getByRole('button', { name: '连接这台电脑', exact: true }).click(); await expect(mobile.getByRole('heading', { name: '项目任务', exact: true })).toBeVisible();
|
||||
mode = 'hold'; const count = requests.length;
|
||||
await page.getByLabel('任务内容', { exact: true }).fill('受控停止:保留片段。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await expect(page.getByLabel('模型输出片段', { exact: true }).last()).toContainText('正在核对当前项目');
|
||||
const active = await getSession(); assert.equal(active.events.filter(value => value.kind === 'tool' && value.runId === active.runId).length, 0);
|
||||
await shot(page, 'desktop-streaming'); await page.setViewportSize({ width: 390, height: 844 }); await shot(page, 'narrow-streaming');
|
||||
await mobile.getByRole('button', { name: '刷新任务', exact: true }).click(); await mobile.locator('.mobile-task-row').first().click();
|
||||
await expect(mobile.locator('.mobile-stream')).toContainText('正在核对当前项目'); await mobile.locator('.mobile-stream').scrollIntoViewIfNeeded(); await shot(mobile, 'mobile-streaming');
|
||||
await mobile.getByRole('button', { name: '停止原任务', exact: true }).click(); await mobile.getByRole('dialog').getByRole('button', { name: '确认', exact: true }).click();
|
||||
await until(async () => (await getSession()).status === 'paused' && closedStreams === 1, 15000);
|
||||
await expect(mobile.locator('.mobile-stream')).toContainText('输出未完成');
|
||||
assert.equal(requests.length, count + 1); await mobile.locator('.mobile-stream').scrollIntoViewIfNeeded(); await shot(mobile, 'mobile-stopped');
|
||||
await expect(page.getByLabel('模型输出片段', { exact: true }).last()).toContainText('输出未完成'); await shot(page, 'narrow-stopped');
|
||||
mark('desktop and actual paired gateway show same bounded stream; mobile stops original HTTP with zero fragment tool dispatch or replay');
|
||||
await page.setViewportSize({ width: 1440, height: 1000 }); mode = 'truncate';
|
||||
await page.getByLabel('任务内容', { exact: true }).fill('受控断流:半截参数不得执行。'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await until(async () => (await getSession()).status === 'failed', 15000);
|
||||
const broken = await getSession(), latest = (await host.api('/model-center')).attempts.find(value => value.runId === broken.runId);
|
||||
assert.equal(latest.status, 'unknown'); assert.equal(latest.stream.terminal, false); assert.equal(requests.length, count + 2);
|
||||
await page.getByRole('button', { name: '打开模型中心', exact: true }).click(); await dialog.getByRole('button', { name: '请求账本', exact: true }).click();
|
||||
await expect(dialog.locator('.mc-attempts article').first()).toContainText('结果待核对'); await expect(dialog.locator('.mc-attempts article').first()).toContainText('响应未完整'); await shot(page, 'desktop-incomplete');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await shot(page, 'narrow-incomplete');
|
||||
mark('truncated output keeps explicit unknown ledger, partial evidence and no automatic failover or incomplete tool execution');
|
||||
assert.deepEqual(errors, []); assert.deepEqual(failures, []);
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, version: host.bootstrap.version, appRoot, nodePath, containedRuntime: args.length > 0, nodeSha256: createHash('sha256').update(fs.readFileSync(nodePath)).digest('hex'), passed, screenshots, pageErrors: errors, requestFailures: failures, fixtureModelRequests: requests.length, realModelRequests: 0, osTrustChanged: false, nativeIPhone: 'not exercised; paired Edge viewport uses CA-verifying owned relay', taskId: completed.taskId, runIds: [completed.runId, active.runId, broken.runId] }, null, 2));
|
||||
} catch (error) {
|
||||
await shot(page, 'failure-desktop').catch(() => {}); if (mobile) await shot(mobile, 'failure-mobile').catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.stack, passed, screenshots, pageErrors: errors, requestFailures: failures }, null, 2)); throw error;
|
||||
} finally {
|
||||
console.log('EVIDENCE ' + directory); await browser.close();
|
||||
if (relay) await new Promise(resolve => { relay.closeAllConnections(); relay.close(resolve); });
|
||||
if (gateway) await host.api('/companion/close', {});
|
||||
for (const response of held) response.destroy(); for (const socket of sockets) socket.destroy(); await new Promise(resolve => modelServer.close(resolve));
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
if(process.argv.includes('--help')){console.log('node scripts/verify-model-selection-k.mjs — offline original-byte evidence, routing/UI, archive/rebuild/install/formal-retention and J identity preservation; no network or model calls.');process.exit(0);}
|
||||
assert.equal(process.argv.length,2);
|
||||
const root=path.resolve('docs/model-selection-k-evidence'),read=file=>JSON.parse(fs.readFileSync(file,'utf8')),hash=bytes=>createHash('sha256').update(bytes).digest('hex');
|
||||
const manifest=read(path.join(root,'manifest.json'));assert.equal(manifest.version,'1.24.3');
|
||||
const names=new Set();let bytes=0;
|
||||
for(const entry of manifest.entries){assert.ok(!names.has(entry.path),entry.path);names.add(entry.path);const file=path.resolve(root,entry.path);assert.ok(file.startsWith(root+path.sep));const contents=fs.readFileSync(file);assert.equal(contents.length,entry.bytes,entry.path);assert.equal(hash(contents),entry.sha256,entry.path);bytes+=contents.length;}
|
||||
const proof=name=>read(path.join(root,manifest.references[name]));
|
||||
for(const name of ['sourceUI','portableUI','installedUI']){const ui=proof(name);assert.equal(ui.success,true);assert.equal(ui.testedVersion,'1.24.3');assert.equal(ui.checks.length,7);assert.equal(ui.realModelRequests,0);assert.deepEqual(ui.errors,[]);assert.equal(ui.requests.length,2);assert.ok(ui.requests.every(item=>item.path==='/selected/v1/chat/completions'&&item.model==='owned-claude'));}
|
||||
assert.equal(proof('regression').success,true);assert.ok(proof('regression').results.every(item=>item.code===0));
|
||||
assert.equal(proof('archives').success,true);assert.equal(proof('archives').archives.length,3);assert.ok(proof('archives').archives.every(item=>item.crcPassed&&item.manifestAndDirectoryMatch));
|
||||
const rebuild=proof('rebuild');assert.equal(rebuild.success,true);assert.equal(rebuild.sourceCommit,manifest.sourceCommit);assert.equal(rebuild.compiled.length,22);assert.deepEqual(rebuild.mismatches,[]);
|
||||
assert.equal(proof('portable').success,true);assert.equal(proof('installation').success,true);
|
||||
const before=proof('formalbefore');assert.equal(before.success,true);assert.equal(before.version,'1.24.2');
|
||||
for(const name of ['formalinstall','formalfinal']){const p=proof(name);assert.equal(p.success,true);assert.equal(p.version,'1.24.3');assert.equal(p.pid,before.pid);assert.equal(p.hasKey,true);assert.equal(p.realModelRequests,0);assert.deepEqual(p.retention,{originalRows:'exact',originalExports:'exact',originalArtifactBytes:'exact',originalModelCenter:'exact'});}
|
||||
assert.equal(proof('native').success,true);
|
||||
assert.equal(hash(fs.readFileSync('docs/live-model-identity-j-evidence/manifest.json')),manifest.originalJManifestSha256);
|
||||
console.log(JSON.stringify({success:true,files:manifest.entries.length,bytes,sourceCommit:manifest.sourceCommit,manifestSha256:hash(fs.readFileSync(path.join(root,'manifest.json'))),realModelRequests:0,scope:'Offline verification of preserved original bytes and acceptance receipts. No inference of underlying model weights or Purge benefit.'}));
|
||||
@@ -0,0 +1,149 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-model-selection-ui.mjs [--app-root ABS] — isolated compiled UI, two owned HTTP routes, add/switch/budget/reload/demo/locking at desktop and 390px; no configured-model calls'); process.exit(0); }
|
||||
const args = process.argv.slice(2);
|
||||
assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
||||
const appRoot = args[1] ?? process.cwd();
|
||||
const directory = path.resolve('.qa/model-selection-k-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t), requests = [];
|
||||
let release, block = false;
|
||||
const upstream = http.createServer(async (req, res) => {
|
||||
const chunks = []; for await (const chunk of req) chunks.push(chunk);
|
||||
const body = JSON.parse(Buffer.concat(chunks)); requests.push({ url: req.url, body });
|
||||
if (block) await new Promise(resolve => { release = resolve; });
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ model: body.model, choices: [{ message: { role: 'assistant', content: 'Owned route receipt.' } }],
|
||||
usage: { prompt_tokens: 13, completion_tokens: 7, total_tokens: 20 } }));
|
||||
});
|
||||
await new Promise(resolve => upstream.listen(0, '127.0.0.1', resolve));
|
||||
const endpoint = 'http://127.0.0.1:' + upstream.address().port;
|
||||
f.store.saveConfig({ mode: 'compatible', baseUrl: endpoint + '/legacy/v1', model: 'owned-legacy-gpt', maxSteps: 1 });
|
||||
const provider = new Provider(), hub = provider.modelHub(f.store);
|
||||
const selected = hub.upsert({ name: 'Claude · 自有接口', model: 'owned-claude', baseUrl: endpoint + '/selected/v1', requiresKey: false, modelIdentityPolicy: 'require-match' });
|
||||
const noKey = hub.upsert({ name: '待填凭据', model: 'no-credential', baseUrl: endpoint + '/no-key/v1' });
|
||||
const pool = hub.pool({ name: '工程模型池', members: [{ connectionId: selected.id, weight: 1 }], strategy: 'round-robin', sticky: true, maxAttempts: 1, queueTimeoutMs: 1000 });
|
||||
hub.select({ type: 'pool', id: pool.id }); f.store.close();
|
||||
const nodePath = fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath });
|
||||
assert.notEqual(new URL(host.url).port, '4318');
|
||||
let browser, page, result; const checks = [], screenshots = [], errors = [];
|
||||
const mark = name => { checks.push(name); console.log('PASS ' + name); };
|
||||
const capture = async name => { const file = path.join(directory, name + '.png'); await page.screenshot({ path: file }); screenshots.push({ path: file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') }); };
|
||||
const dialog = () => page.getByRole('dialog', { name: '模型中心', exact: true });
|
||||
const select = () => dialog().getByLabel('新任务默认模型', { exact: true });
|
||||
const close = () => dialog().getByRole('button', { name: '关闭窗口', exact: true }).click();
|
||||
try {
|
||||
browser = await chromium.launch({ channel: 'msedge', headless: true }); page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); await page.getByLabel('当前项目').selectOption(f.project.id);
|
||||
await page.getByRole('button', { name: '选择与管理模型', exact: true }).click();
|
||||
await expect(dialog()).toBeVisible(); await expect(select()).toHaveValue('pool:' + pool.id);
|
||||
// This Playwright build reports disabled <option> as enabled. Inspect the
|
||||
// native option property instead; the original DOM/error is kept in QA.
|
||||
await expect(select().locator('option[value="connection:' + noKey.id + '"]')).toHaveJSProperty('disabled', true);
|
||||
await capture('unified-entry-desktop');
|
||||
await select().selectOption('connection:' + selected.id);
|
||||
await expect(dialog().locator('.mc-current')).toContainText(selected.name);
|
||||
assert.equal((await host.api('/model-center/selection')).modelSelection.id, selected.id);
|
||||
await close(); await expect(page.getByRole('button', { name: '选择与管理模型' })).toContainText(selected.name);
|
||||
await expect(page.getByRole('button', { name: '切换当前模型' })).toContainText(selected.name);
|
||||
mark('sidebar and header share the authoritative multi-model selector; selection persists without a request');
|
||||
|
||||
await page.getByRole('button', { name: '切换当前模型', exact: true }).click();
|
||||
await dialog().getByRole('button', { name: '添加模型', exact: true }).click();
|
||||
const form = dialog().getByRole('form', { name: '编辑模型连接', exact: true });
|
||||
const longName = 'DeepSeek · 本机工程备用连接与长名称适配';
|
||||
await form.getByLabel('连接名称', { exact: true }).fill(longName);
|
||||
await form.getByLabel('接口根地址', { exact: true }).fill(endpoint + '/new/v1');
|
||||
await form.getByLabel('请求模型名称', { exact: true }).fill('owned-new-model');
|
||||
await form.getByLabel('服务需要 API Key', { exact: true }).uncheck();
|
||||
await form.getByRole('button', { name: '保存模型连接', exact: true }).click(); await expect(form).toHaveCount(0);
|
||||
assert.equal((await host.api('/model-center/selection')).modelSelection.id, selected.id);
|
||||
const added = (await host.api('/model-center')).connections.find(c => c.name === longName); assert.ok(added);
|
||||
await select().selectOption('connection:' + added.id); await expect(select()).toHaveValue('connection:' + added.id);
|
||||
await close(); await page.reload(); await page.getByRole('button', { name: '选择与管理模型' }).click();
|
||||
await expect(select()).toHaveValue('connection:' + added.id); assert.equal(requests.length, 0);
|
||||
mark('add from primary entry keeps the previous default, supports a new selection and survives reload without paid probes');
|
||||
|
||||
await select().selectOption('pool:' + pool.id); await expect(select()).toHaveValue('pool:' + pool.id);
|
||||
await dialog().getByRole('button', { name: '运行预算', exact: true }).click();
|
||||
const budgets = dialog().getByRole('form', { name: '运行预算设置', exact: true });
|
||||
await budgets.getByLabel('上下文预算', { exact: true }).fill('32000');
|
||||
await budgets.getByLabel('单次运行 tokens', { exact: true }).fill('90000');
|
||||
await budgets.getByRole('button', { name: '保存运行预算', exact: true }).click();
|
||||
await expect(dialog().getByRole('status')).toContainText('当前模型选择保持');
|
||||
assert.equal((await host.api('/model-center/selection')).modelSelection.id, pool.id);
|
||||
await dialog().getByRole('button', { name: '模型连接', exact: true }).click();
|
||||
await dialog().locator('.mc-connection').filter({ has: page.getByRole('heading', { name: '原模型连接', exact: true }) }).getByRole('button', { name: '编辑', exact: true }).click();
|
||||
const legacy = page.getByRole('dialog', { name: '原连接与演示设置', exact: true });
|
||||
await expect(legacy.getByRole('button', { name: '保存并切换原连接', exact: true })).toBeVisible();
|
||||
await legacy.getByRole('button', { name: '取消', exact: true }).click();
|
||||
assert.equal((await host.api('/model-center/selection')).modelSelection.id, pool.id);
|
||||
mark('budget-only saves preserve the pool and original connection editing is explicitly secondary and cancellable');
|
||||
|
||||
await page.getByLabel('任务内容').fill('仅回复一次,用于自有接口路由验收,不执行工具。');
|
||||
await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
await until(async () => { const current = await host.api('/sessions/' + f.session.id); return current.status !== 'running' && current.execution.run?.consumption?.length === 1; });
|
||||
const firstRun = (await host.api('/sessions/' + f.session.id)).execution.run; const frozen = structuredClone(firstRun);
|
||||
assert.equal(requests.length, 1); assert.equal(requests[0].url, '/selected/v1/chat/completions'); assert.equal(requests[0].body.model, selected.model);
|
||||
await page.getByRole('button', { name: '切换当前模型' }).click();
|
||||
await select().selectOption('legacy:legacy-default'); await expect(select()).toHaveValue('legacy:legacy-default');
|
||||
await expect(dialog().locator('.mc-run-model')).toContainText(selected.model);
|
||||
assert.deepEqual((await host.api('/sessions/' + f.session.id)).execution.run, frozen);
|
||||
await capture('default-versus-run-desktop'); await close();
|
||||
mark('actual pool request reaches only the selected endpoint; changing defaults preserves the sealed old Run and shows both identities');
|
||||
|
||||
await host.api('/model-center/select', { type: 'connection', id: selected.id });
|
||||
await page.getByRole('button', { name: '打开模型中心', exact: true }).click();
|
||||
await expect(select()).toHaveValue('connection:' + selected.id); await close();
|
||||
await expect(page.getByRole('button', { name: '切换当前模型' })).toContainText(selected.name);
|
||||
await page.getByRole('button', { name: '切换当前模型' }).click(); await select().selectOption('demo'); await expect(select()).toHaveValue('demo');
|
||||
assert.equal((await host.api('/model-center')).selected.id, selected.id);
|
||||
await select().selectOption('legacy:legacy-default'); await expect(select()).toHaveValue('legacy:legacy-default');
|
||||
assert.equal((await host.api('/model-center/selection')).config.mode, 'compatible');
|
||||
await close();
|
||||
mark('opening refreshes another tab selection and choosing a real connection exits demo without losing profiles');
|
||||
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await page.getByRole('button', { name: '切换当前模型' }).click();
|
||||
await expect(select()).toHaveValue('legacy:legacy-default');
|
||||
await select().selectOption('connection:' + added.id); await expect(select()).toHaveValue('connection:' + added.id);
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth));
|
||||
await capture('selection-390'); await close();
|
||||
await page.getByRole('button', { name: '展开项目栏' }).click();
|
||||
await expect(page.getByRole('button', { name: '选择与管理模型' })).toContainText(longName);
|
||||
await page.getByRole('button', { name: '选择与管理模型' }).click(); await expect(select()).toHaveValue('connection:' + added.id);
|
||||
await capture('sidebar-entry-390'); await close();
|
||||
mark('390px header and sidebar both add and switch with long model labels and no page overflow');
|
||||
|
||||
await page.setViewportSize({ width: 1440, height: 1000 }); await host.api('/model-center/select', { type: 'connection', id: selected.id });
|
||||
block = true; const second = await host.api('/sessions', { projectId: f.project.id });
|
||||
await host.api('/sessions/' + second.id + '/messages', { content: 'Owned held request.' });
|
||||
await until(() => requests.length === 2 && release);
|
||||
await page.reload(); await page.getByRole('button', { name: '切换当前模型' }).click();
|
||||
// The host also rejects edits if the active task belongs to a different visible session.
|
||||
const attempted = await fetch(host.url + '/api/model-center/runtime-budgets', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Workbench-Token': host.bootstrap.token }, body: JSON.stringify({ maxSteps: 4 }) });
|
||||
assert.equal(attempted.status, 409);
|
||||
await expect(select()).toBeDisabled(); await expect(dialog().getByRole('button', { name: '添加模型', exact: true })).toBeDisabled();
|
||||
await capture('active-request-lock'); release(); block = false;
|
||||
await until(async () => (await host.api('/sessions/' + second.id)).status !== 'running');
|
||||
assert.deepEqual(errors, []);
|
||||
mark('active request locks route and add controls and server rejects budget overrides across sessions');
|
||||
result = { success: true, appRoot, testedVersion: (await host.api('/health')).version, directory, checks, screenshots, errors,
|
||||
requests: requests.map(r => ({ path: r.url, model: r.body.model })), realModelRequests: 0 };
|
||||
} catch (error) {
|
||||
if (page) fs.writeFileSync(path.join(directory, 'failure.html'), await page.content());
|
||||
fs.writeFileSync(path.join(directory, 'failure-center.json'), JSON.stringify(await host.api('/model-center'), null, 2));
|
||||
if (page) await capture('failure').catch(() => {}); result = { success: false, directory, error: error.stack, checks, screenshots, errors, realModelRequests: 0 }; throw error;
|
||||
} finally {
|
||||
release?.(); fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(result, null, 2) + '\n');
|
||||
console.log(JSON.stringify({ success: result.success, directory, checks: checks.length }));
|
||||
await browser?.close(); upstream.closeAllConnections(); await new Promise(resolve => upstream.close(resolve));
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import http from 'node:http';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
assert.ok(args.length === 0 || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
||||
const appRoot = args[1] ?? process.cwd();
|
||||
const directory = path.resolve('.qa', 'model-wait-ui-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
||||
const disposers = [], t = { after: action => disposers.push(action) }, f = fixture(t), requests = [], sockets = new Set();
|
||||
const model = http.createServer(async (request, response) => {
|
||||
const pieces = []; for await (const piece of request) pieces.push(piece); requests.push(JSON.parse(Buffer.concat(pieces)));
|
||||
response.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
||||
response.write('data: ' + JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'owned-incomplete', type: 'function', function: { name: 'propose_file', arguments: '{"path":"incomplete.html",' } }] }, finish_reason: null }] }) + '\n\n');
|
||||
});
|
||||
model.on('connection', socket => { sockets.add(socket); socket.once('close', () => sockets.delete(socket)); });
|
||||
await new Promise(resolve => model.listen(0, '127.0.0.1', resolve));
|
||||
const provider = new Provider(), hub = provider.modelHub(f.store);
|
||||
const member = hub.upsert({ name: '本地受控流', model: 'owned-wait-model', baseUrl: 'http://127.0.0.1:' + model.address().port + '/v1', requiresKey: false, streaming: true, capabilityDeclarations: { streaming: 'supported' } });
|
||||
const pool = hub.pool({ name: '保持当前模型池', members: [{ connectionId: member.id }] }); hub.select({ type: 'pool', id: pool.id }); f.store.close();
|
||||
const host = await launchWorkbench(t, f.data, {}, { appRoot, nodePath: fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath });
|
||||
const checks = [], screenshots = [], pageErrors = [];
|
||||
let browser, page;
|
||||
const mark = text => { checks.push(text); console.log('PASS ' + text); };
|
||||
const capture = async name => {
|
||||
const file = path.join(directory, name + '.png'); await page.screenshot({ path: file });
|
||||
fs.writeFileSync(path.join(directory, name + '.html'), await page.content());
|
||||
screenshots.push({ path: file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') });
|
||||
};
|
||||
const openWaits = async () => {
|
||||
await page.getByRole('button', { name: '打开模型中心', exact: true }).click();
|
||||
await page.getByRole('button', { name: '等待时限', exact: true }).click();
|
||||
await expect(page.getByRole('form', { name: '模型等待时限' })).toBeVisible();
|
||||
};
|
||||
try {
|
||||
browser = await chromium.launch({ channel: 'msedge', headless: true });
|
||||
page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); page.on('pageerror', error => pageErrors.push(error.message));
|
||||
await page.goto(host.url); await page.getByLabel('当前项目').selectOption(f.project.id); await openWaits();
|
||||
await expect(page.getByLabel('单次模型响应上限(秒)', { exact: true })).toHaveValue('300');
|
||||
await expect(page.getByLabel('流式无数据上限(秒)', { exact: true })).toHaveValue('120');
|
||||
await page.getByLabel('单次模型响应上限(秒)', { exact: true }).fill('480');
|
||||
await page.getByLabel('流式无数据上限(秒)', { exact: true }).fill('180');
|
||||
await page.getByRole('button', { name: '保存等待时限', exact: true }).click();
|
||||
await expect(page.getByRole('status')).toContainText('等待时限已保存');
|
||||
let center = await host.api('/model-center'); assert.equal(center.selected.id, pool.id);
|
||||
assert.deepEqual(center.waitSettings, { modelResponseTimeoutMs: 480000, modelStreamIdleTimeoutMs: 180000 });
|
||||
assert.equal(center.connections.find(value => value.id === member.id).revision, member.revision); assert.equal(requests.length, 0);
|
||||
await capture('settings-desktop'); mark('desktop saves seconds, preserves original pool/member and sends no model request');
|
||||
await assert.rejects(host.api('/model-center/wait-settings', { modelStreamIdleTimeoutMs: 0 }), error => error.status === 400);
|
||||
assert.deepEqual((await host.api('/model-center')).waitSettings, center.waitSettings); mark('invalid timeout rejected without saving');
|
||||
await page.reload(); await openWaits();
|
||||
await expect(page.getByLabel('单次模型响应上限(秒)', { exact: true })).toHaveValue('480');
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
const narrowSave = page.getByRole('button', { name: '保存等待时限', exact: true });
|
||||
await narrowSave.scrollIntoViewIfNeeded(); await expect(narrowSave).toBeVisible();
|
||||
const saveBounds = await narrowSave.boundingBox();
|
||||
assert.ok(saveBounds && saveBounds.x >= 0 && saveBounds.y >= 0 && saveBounds.x + saveBounds.width <= 390 && saveBounds.y + saveBounds.height <= 844, 'Full narrow save button is reachable within viewport');
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); await capture('settings-390');
|
||||
mark('saved values survive reload; narrow screen remains usable');
|
||||
await page.keyboard.press('Escape'); await page.setViewportSize({ width: 1440, height: 1000 });
|
||||
await host.api('/model-center/wait-settings', { modelResponseTimeoutMs: 15000, modelStreamIdleTimeoutMs: 5000 });
|
||||
await page.reload();
|
||||
await page.getByLabel('任务内容').fill('读取当前项目,受控超时验收'); await page.getByRole('button', { name: '发送任务', exact: true }).click();
|
||||
let sessionId; await until(async () => { sessionId = (await host.api('/sessions')).find(value => value.title === '读取当前项目,受控超时验收')?.id; return !!sessionId && requests.length === 1; });
|
||||
await assert.rejects(host.api('/model-center/wait-settings', { modelResponseTimeoutMs: 60000 }), error => error.status === 409);
|
||||
await openWaits(); await expect(page.getByRole('button', { name: '保存等待时限', exact: true })).toBeDisabled();
|
||||
await page.keyboard.press('Escape'); mark('active request locks wait changes in both API and UI');
|
||||
await until(async () => (await host.api('/sessions/' + sessionId)).status === 'failed', 20000);
|
||||
const session = await host.api('/sessions/' + sessionId); assert.equal(session.failure.timeout.kind, 'idle');
|
||||
assert.equal(session.failure.timeout.limitMs, 5000); assert.equal(session.changes.length, 0); assert.equal(requests.length, 1);
|
||||
assert.equal(session.execution.run.usage.unknownRequests, 1); assert.equal(session.execution.run.model.modelResponseTimeoutMs, 15000);
|
||||
await expect(page.getByText(/流式响应连续 5 秒未收到新数据/).first()).toBeVisible();
|
||||
await capture('idle-error-desktop'); await page.setViewportSize({ width: 390, height: 844 }); await capture('idle-error-390');
|
||||
assert.equal(pageErrors.length, 0); mark('actual stalled stream shows exact idle reason, saves unknown use, never writes or retries');
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: true, checks, screenshots, pageErrors, requests: requests.length, realModelRequests: 0, appRoot }, null, 2));
|
||||
console.log(JSON.stringify({ success: true, directory, checks: checks.length, screenshots: screenshots.length }));
|
||||
} catch (error) {
|
||||
if (page) await capture('failure').catch(() => {});
|
||||
fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify({ success: false, error: error.stack, checks, screenshots, pageErrors, requests: requests.length, appRoot }, null, 2)); throw error;
|
||||
} finally {
|
||||
await browser?.close(); for (const socket of sockets) socket.destroy(); await new Promise(resolve => model.close(resolve));
|
||||
for (const action of disposers.reverse()) await action();
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import net from 'node:net';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { chromium, expect } from 'playwright/test';
|
||||
import { fixture, launchWorkbench, until } from '../tests/helpers.mjs';
|
||||
import { Provider } from '../src/server/provider.mjs';
|
||||
|
||||
if (process.argv.includes('--help')) { console.log('node scripts/verify-network-diagnostics-ui.mjs [--app-root ABSOLUTE_ROOT] — owned TCP/TLS-only diagnostics, stop/reload/revision/proxy scopes, desktop and 390px. Zero configured-model calls.'); process.exit(0); }
|
||||
const args = process.argv.slice(2); assert.ok(!args.length || args.length === 2 && args[0] === '--app-root' && path.isAbsolute(args[1]));
|
||||
const appRoot = args.length ? fs.realpathSync.native(args[1]) : process.cwd();
|
||||
const targetVersion = JSON.parse(fs.readFileSync(path.join(appRoot, fs.existsSync(path.join(appRoot, 'MANIFEST.json')) ? 'MANIFEST.json' : 'package.json'))).version;
|
||||
const root = path.resolve('.qa', 'network-q-ui-' + randomUUID()); fs.mkdirSync(root);
|
||||
const disposers = [], f = fixture({ after: action => disposers.push(action) }), checks = [], screenshots = [], errors = [], traffic = [];
|
||||
const sockets = new Set(); let plainBytes = 0, tlsOpen = 0;
|
||||
const listen = async server => { server.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); return server.address().port; };
|
||||
const plain = net.createServer(socket => socket.on('data', chunk => { plainBytes += chunk.length; }));
|
||||
const stalled = net.createServer(socket => { tlsOpen++; socket.on('data', chunk => { traffic.push({ bytes: chunk.length, tls: chunk[0] === 22 }); }); socket.on('close', () => tlsOpen--); });
|
||||
const plainPort = await listen(plain), tlsPort = await listen(stalled), refused = net.createServer(), refusedPort = await listen(refused);
|
||||
await new Promise(resolve => refused.close(resolve));
|
||||
const provider = new Provider(), hub = provider.modelHub(f.store);
|
||||
const add = (name, scheme, port) => hub.upsert({ name, model: 'owned-no-model', baseUrl: scheme + '://127.0.0.1:' + port + '/v1', requiresKey: true });
|
||||
const good = add('本机 TCP · 不需要模型凭据', 'http', plainPort), bad = add('本机拒绝连接 · 失败样例', 'https', refusedPort), waiting = add('本机 TLS 等待 · 可停止样例', 'https', tlsPort);
|
||||
const initialConfig = structuredClone(f.store.config), initialCenter = hub.view(); f.store.close();
|
||||
const nodePath = fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
|
||||
const host = await launchWorkbench({ after: action => disposers.push(action) }, f.data,
|
||||
{ HTTPS_PROXY: 'http://owned-proxy-user:owned-proxy-secret@127.0.0.1:9', NO_PROXY: '127.0.0.1' }, { appRoot, nodePath });
|
||||
assert.notEqual(new URL(host.url).port, '4318');
|
||||
const browser = await chromium.launch({ channel: 'msedge', headless: true }), page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
let result;
|
||||
const mark = value => { checks.push(value); console.log('PASS ' + value); };
|
||||
const capture = async name => { const file = path.join(root, name + '.png'); await page.screenshot({ path: file, fullPage: true }); screenshots.push(file); };
|
||||
const open = async () => { await page.getByRole('button', { name: '打开模型中心', exact: true }).click(); const dialog = page.getByRole('dialog', { name: '模型中心', exact: true }); await expect(dialog).toBeVisible(); return dialog; };
|
||||
const tab = dialog => dialog.getByRole('navigation', { name: '模型中心分类' }).getByRole('button', { name: '连接诊断', exact: true });
|
||||
const panel = () => page.getByRole('region', { name: '模型连接诊断', exact: true });
|
||||
const start = async () => { await panel().getByRole('button', { name: '开始直连检查', exact: true }).click(); };
|
||||
try {
|
||||
await page.goto(host.url); await page.waitForLoadState('networkidle'); let dialog = await open();
|
||||
const card = dialog.locator('.mc-connection').filter({ hasText: good.name }); await card.getByRole('button', { name: '连接诊断', exact: true }).click();
|
||||
await expect(panel().getByLabel('诊断连接', { exact: true })).toHaveValue(good.id); await expect(panel()).toContainText('这条连接还没有检查记录');
|
||||
await expect(panel().getByRole('button', { name: '开始直连检查', exact: true })).toBeEnabled(); await capture('empty-desktop');
|
||||
mark('connection-card entry binds the exact saved target, works without a Key and shows an honest empty state');
|
||||
await start(); await expect(panel().getByRole('status')).toContainText('直连握手通过', { timeout: 10000 });
|
||||
await expect(panel()).toContainText('HTTP 端点无需 TLS'); await expect(panel()).toContainText('DNS 未返回此地址族');
|
||||
await expect(panel()).toContainText('实际 API 路线尚未核验'); await panel().getByText('查看原诊断范围与运输参数', { exact: true }).click();
|
||||
await expect(panel()).toContainText('未请求;能力与连接健康结论保持原记录'); await capture('passed-desktop');
|
||||
mark('real loopback TCP success separates missing IPv6, HTTPS proxy scope and untested model API capability');
|
||||
let view = await host.api('/model-center/network-diagnostics'); const goodId = view.jobs[0].id;
|
||||
await page.reload(); dialog = await open(); await tab(dialog).click();
|
||||
await panel().getByLabel('诊断连接', { exact: true }).selectOption(good.id);
|
||||
await expect(panel().getByRole('status')).toContainText('直连握手通过');
|
||||
assert.equal((await host.api('/model-center/network-diagnostics')).jobs[0].id, goodId); mark('refresh restores the original saved result without repeating network work');
|
||||
await panel().getByLabel('诊断连接', { exact: true }).selectOption(bad.id); await start();
|
||||
await expect(panel().getByRole('status')).toContainText('未通过', { timeout: 10000 }); await expect(panel()).toContainText('ECONNREFUSED'); await capture('refused-desktop');
|
||||
mark('connection failure stays attached to the selected endpoint and shows its real TCP error code');
|
||||
await panel().getByLabel('诊断连接', { exact: true }).selectOption(waiting.id); await start();
|
||||
await expect(panel()).toContainText('TLS 握手中'); await expect(panel().getByRole('button', { name: '开始直连检查', exact: true })).toBeDisabled();
|
||||
assert.equal((await host.api('/host/status')).activity.counts.networkDiagnostics, 1);
|
||||
await panel().getByRole('button', { name: '停止诊断', exact: true }).click(); await expect(panel().getByRole('status')).toContainText('已停止'); await until(() => tlsOpen === 0);
|
||||
mark('running TLS check disables duplicate start and explicit stop closes owned sockets before reporting stopped');
|
||||
await start(); await expect(panel()).toContainText('TLS 握手中'); view = await host.api('/model-center/network-diagnostics'); const runningId = view.jobs[0].id;
|
||||
await dialog.locator('.modal-heading .icon-button').click(); dialog = await open(); await tab(dialog).click();
|
||||
await expect(panel()).toContainText('正在检查'); await panel().getByRole('button', { name: '查看正在运行的诊断', exact: true }).click();
|
||||
await panel().getByRole('button', { name: '停止诊断', exact: true }).click(); await expect(panel().getByRole('status')).toContainText('已停止');
|
||||
assert.equal((await host.api('/model-center/network-diagnostics')).jobs[0].id, runningId); mark('closing and reopening the model center recovers the same active check and retains explicit stop');
|
||||
await host.api('/model-center/toggle', { id: good.id, enabled: false }); await panel().getByLabel('诊断连接', { exact: true }).selectOption(good.id);
|
||||
await panel().getByRole('button', { name: '刷新连接诊断', exact: true }).click(); await expect(panel()).toContainText('连接配置已变化');
|
||||
await start(); await expect(panel().getByRole('alert')).toContainText('连接已变化');
|
||||
await dialog.getByRole('button', { name: '刷新模型中心', exact: true }).click();
|
||||
await start();
|
||||
await expect(panel().locator('.mn-result-heading .mn-revision')).toHaveText('修订 2');
|
||||
await expect(panel().getByRole('status')).toContainText('直连握手通过');
|
||||
await until(async () => (await host.api('/model-center/network-diagnostics')).active === 0);
|
||||
mark('old revisions remain visible, stale starts reject before any socket, and refreshing permits a new bounded check');
|
||||
await page.setViewportSize({ width: 390, height: 844 }); await capture('passed-390');
|
||||
await panel().locator('.mn-result-heading').scrollIntoViewIfNeeded(); await capture('passed-result-390');
|
||||
await panel().locator('.mn-paths').scrollIntoViewIfNeeded(); await capture('passed-paths-390');
|
||||
await panel().getByRole('complementary', { name: '连接诊断历史' }).scrollIntoViewIfNeeded(); await capture('history-390');
|
||||
assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth));
|
||||
assert.ok(await panel().evaluate(element => element.scrollWidth <= element.clientWidth + 1));
|
||||
const startBox = await panel().getByRole('button', { name: '开始直连检查', exact: true }).boundingBox(); assert.ok(startBox.height >= 44);
|
||||
await panel().getByLabel('诊断连接', { exact: true }).selectOption(bad.id); await expect(panel()).toContainText('ECONNREFUSED');
|
||||
await panel().locator('.mn-result-heading').scrollIntoViewIfNeeded(); await capture('refused-390');
|
||||
mark('390px success and failure states fit without page or panel overflow and retain touch-sized actions');
|
||||
view = await host.api('/model-center/network-diagnostics'); const center = await host.api('/model-center'), boot = await host.api('/bootstrap');
|
||||
assert.deepEqual(center.selected, initialCenter.selected); assert.deepEqual(center.pools, initialCenter.pools); assert.deepEqual(boot.config, { ...initialConfig, hasKey: false });
|
||||
assert.deepEqual(center.attempts, []); assert.ok(center.connections.every(value => Object.values(value.capabilities).every(state => state !== 'verified')));
|
||||
assert.equal(plainBytes, 0); assert.ok(traffic.length > 0 && traffic.every(value => value.tls)); assert.equal(JSON.stringify(view).includes('owned-proxy-secret'), false);
|
||||
assert.equal((await page.locator('body').innerText()).includes('owned-proxy-secret'), false); assert.deepEqual(errors, []);
|
||||
mark('original routing, budgets, model ledger and capability results remain unchanged; no HTTP or credential text is sent');
|
||||
result = { success: true, appRoot, targetVersion, checks, screenshots, errors, httpRequests: 0, realModelRequests: 0,
|
||||
plainBytes, tlsFrames: traffic.length, jobs: view.jobs, originalSelectionPreserved: true, originalBudgetsPreserved: true };
|
||||
} catch (error) { await capture('failure').catch(() => {}); result = { success: false, error: error.stack, checks, screenshots, errors, realModelRequests: 0 }; throw error; }
|
||||
finally {
|
||||
fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result, null, 2)); console.log(JSON.stringify({ success: result.success, root, checks: checks.length }));
|
||||
await browser.close(); for (const socket of sockets) socket.destroy();
|
||||
for (const server of [plain, stalled]) await new Promise(resolve => server.close(resolve));
|
||||
for (const dispose of disposers.reverse()) await dispose();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user