135 lines
15 KiB
JavaScript
135 lines
15 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { performance } from 'node:perf_hooks';
|
|
import { fixture, compatibleFixture, call, completion, until, assertToolPairs } from '../tests/helpers.mjs';
|
|
import { specification } from '../tests/fixtures/engineering-specification.mjs';
|
|
import { deliveryGit as git, writeDeliveryFile as write } from '../tests/fixtures/delivery-project.mjs';
|
|
import { ownerBytes } from '../tests/fixtures/workers-project.mjs';
|
|
import { Runtime } from '../src/server/runtime.mjs';
|
|
import { Provider } from '../src/server/provider.mjs';
|
|
import { ProjectIndex } from '../src/core/context/project-index.mjs';
|
|
import { verifyPlanEvidence } from '../src/core/context/request-plan.mjs';
|
|
|
|
if (process.argv.includes('--help')) { console.log('node scripts/verify-project-cache-projects.mjs - identical source cold/disk/memory/invalidated index semantics, then fixed HTTP coding Task with original failing/passing Node8 React9 Python10 checks, Edge and Git. No real inference or token saving claim. Set isolated QA Python.'); process.exit(0); }
|
|
assert.equal(process.argv.length, 2);
|
|
const version = JSON.parse(fs.readFileSync('package.json', 'utf8')).version, directory = path.resolve('.qa', 'project-cache-projects-' + randomUUID()); fs.mkdirSync(directory, { recursive: true });
|
|
const python = process.env.AGENT_WORKBENCH_TEST_PYTHON; assert.ok(python && path.isAbsolute(python) && fs.existsSync(python));
|
|
const hash = value => createHash('sha256').update(value).digest('hex'), cases = [], results = [];
|
|
const mark = value => { cases.push(value); console.log('PASS ' + value); };
|
|
async function sample(kind) {
|
|
const disposers = [], t = { after: fn => disposers.push(fn) }, f = fixture(t), saved = path.join(directory, kind); fs.mkdirSync(saved);
|
|
let runtime, result;
|
|
const reviews = [], comparisons = [];
|
|
try {
|
|
const baseline = kind === 'react' ? 'benchmarks/react-delivery-v1/baseline' : 'benchmarks/workers-delivery-v1/' + kind;
|
|
fs.cpSync(path.resolve(baseline), f.projectRoot, { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
|
write(f, '.gitignore', 'node_modules/\ndist/\n.awb-*\n__pycache__/\n.venv/\n.pytest_cache/\n');
|
|
write(f, 'owner-staged.txt', 'Owner baseline\n'); write(f, 'owner-unstaged.txt', 'Owner baseline\n');
|
|
git(f, ['init', '--initial-branch=main']); git(f, ['config', 'core.autocrlf', 'false']); git(f, ['config', 'user.name', 'Workbench QA']); git(f, ['config', 'user.email', 'qa@workbench.invalid']);
|
|
git(f, ['add', '--', '.']); git(f, ['commit', '--no-gpg-sign', '-m', 'Seal project-cache engineering baseline']);
|
|
write(f, 'owner-staged.txt', 'Owner staged change\n'); git(f, ['add', '--', 'owner-staged.txt']);
|
|
write(f, 'owner-staged.txt', 'Owner working copy after staging\n'); write(f, 'owner-unstaged.txt', 'Owner unstaged change\n'); write(f, 'owner-untracked.txt', 'Owner untracked file\n');
|
|
const original = ownerBytes(f), spec = specification(kind, f.projectRoot), writable = [...new Set([...Object.keys(spec.initial ?? {}), ...Object.keys(spec.fixes)])].sort();
|
|
const protectedFiles = Object.fromEntries(git(f, ['ls-files', '-z']).split('\0').filter(file => file && !writable.includes(file) && !file.startsWith('owner-')).map(file => [file, fs.readFileSync(path.join(f.projectRoot, file))]));
|
|
const verify = () => { for (const [file, bytes] of Object.entries(protectedFiles)) assert.deepEqual(fs.readFileSync(path.join(f.projectRoot, file)), bytes, 'Protected ' + file); };
|
|
if (kind === 'react') {
|
|
const npm = path.join(path.dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js');
|
|
const child = spawnSync(process.execPath, [npm, 'ci', '--offline', '--no-audit', '--no-fund'], { cwd: f.projectRoot, windowsHide: true, timeout: 120000, maxBuffer: 8 * 1024 * 1024 });
|
|
fs.writeFileSync(path.join(saved, 'npm-ci-log.txt'), child.stdout ?? ''); fs.writeFileSync(path.join(saved, 'npm-ci-stderr.txt'), child.stderr ?? '');
|
|
if (child.error) throw child.error; assert.equal(child.status, 0); verify();
|
|
}
|
|
const cacheDirectory = path.join(f.data, 'cache', 'project-index'), create = () => new ProjectIndex({ cacheDirectory });
|
|
let indexer = create();
|
|
const measure = async name => {
|
|
const started = performance.now(), index = await indexer.refresh(f.project), durationMs = performance.now() - started;
|
|
assert.equal(index.cache.sourceVerified, true); assert.equal(index.coverage.snapshotComplete, true);
|
|
comparisons.push({ case: name, durationMs, cache: index.cache, sourceFingerprint: index.sourceFingerprint, indexFingerprint: index.indexFingerprint });
|
|
return index;
|
|
};
|
|
const cold = await measure('cold'); indexer = create(); const disk = await measure('disk-after-new-indexer'), warm = await measure('memory');
|
|
assert.equal(disk.cache.origin, 'disk'); assert.equal(disk.cache.parsedFiles, 0); assert.equal(warm.cache.parsedFiles, 0);
|
|
const target = Object.keys(spec.fixes).find(file => fs.existsSync(path.join(f.projectRoot, file))) ?? cold.files.find(file => /\.py$/.test(file.path))?.path; assert.ok(target);
|
|
const filename = path.join(f.projectRoot, target), bytes = fs.readFileSync(filename), stat = fs.statSync(filename);
|
|
fs.appendFileSync(filename, kind === 'python' ? '\n# owned cache invalidation\n' : '\n// owned cache invalidation\n');
|
|
const changed = await measure('changed-source'); assert.notEqual(changed.sourceFingerprint, cold.sourceFingerprint); assert.equal(changed.cache.parsedFiles, 1);
|
|
fs.writeFileSync(filename, bytes); fs.utimesSync(filename, stat.atime, stat.mtime);
|
|
const invalidated = await measure('restored-original-invalidates-cache'); assert.equal(invalidated.cache.parsedFiles, 1);
|
|
for (const index of [disk, warm, invalidated]) {
|
|
assert.equal(index.sourceFingerprint, cold.sourceFingerprint); assert.equal(index.indexFingerprint, cold.indexFingerprint);
|
|
assert.deepEqual(index.symbols, cold.symbols); assert.deepEqual(index.imports, cold.imports);
|
|
assert.deepEqual(indexer.query(index, { query: '', kind: 'files' }).results, indexer.query(cold, { query: '', kind: 'files' }).results);
|
|
}
|
|
verify(); mark(kind + ': cold, disk, memory and restored-source invalidation produce identical original index/query semantics with fresh SHA checks');
|
|
let phase = 'new', observed = false;
|
|
const model = await compatibleFixture(t, (_route, input) => {
|
|
assertToolPairs(input.messages);
|
|
const job = [...f.store.core.check_job.values()].filter(value => value.taskId === f.session.taskId).at(-1);
|
|
const plan = [...f.store.core.check_plan.values()].filter(value => value.taskId === f.session.taskId).at(-1);
|
|
const propose = files => Object.entries(files).map(([file, content]) => call(randomUUID(), 'propose_file', { path: file, content, reason: 'Fixed engineering acceptance; retain tests and declarations.' }));
|
|
let calls = [];
|
|
if (!observed) { observed = true; calls = [call(randomUUID(), 'index_project', {}), call(randomUUID(), 'query_code', { query: '', kind: 'symbols', limit: 30 })]; }
|
|
else if (f.session.changes.some(value => value.status === 'pending')) { /* Files require a separate explicit review. */ }
|
|
else if (phase === 'new' && spec.initial) { calls = propose(spec.initial); phase = 'initial'; }
|
|
else if (phase !== 'fixed' && job) { assert.equal(job.status, 'failed', job.summary); assert.ok(job.results.some(value => value.tests?.failed > 0)); calls = propose(spec.fixes); phase = 'fixed'; }
|
|
else if (phase === 'fixed' && job?.status === 'passed') assert.equal(job.results[0].tests.passed, spec.tests);
|
|
else if (!plan || job?.planId === plan.id) calls = [call(randomUUID(), 'plan_checks', { names: spec.checks })];
|
|
else calls = [call(randomUUID(), 'run_checks', { plan_id: plan.id })];
|
|
return completion(calls, calls.length ? '' : '实际检查完成,仍须独立审查页面和Git。');
|
|
});
|
|
f.store.saveConfig({ mode: 'compatible', model: 'owned-cache-engineering', baseUrl: model.baseUrl, contextLimit: 128000, maxSteps: 32, maxTaskTokens: 400000 });
|
|
runtime = new Runtime(f.store, new Provider()); runtime.checkRunner().pythonExecutable = python;
|
|
runtime.start(f.session.id, '定位实际工程声明,修复工程功能,保留原始测试和已有修改,完成实际检查、构建及审查交付。');
|
|
const taskId = f.session.taskId;
|
|
for (let turn = 0; turn < 5; turn++) {
|
|
await until(() => {
|
|
for (const approval of f.session.approvals.filter(value => value.status === 'pending')) {
|
|
assert.equal(f.store.core.tool.get(approval.invocationId).tool, 'run_checks'); reviews.push({ kind: 'check-approval', id: approval.id }); runtime.decide(f.session.id, approval.id, true);
|
|
}
|
|
return !runtime.isActive(f.session.id);
|
|
}, 180000);
|
|
const ids = f.session.changes.filter(value => value.status === 'pending').map(value => value.id); if (!ids.length) break;
|
|
const set = await runtime.changeSetManager().prepare(f.session, ids); reviews.push({ kind: 'file-review', id: set.id, fingerprint: set.fingerprint });
|
|
await (await runtime.changeSetManager().start(f.session, set.id, set.fingerprint)).promise; assert.equal(set.status, 'completed', set.summary); verify(); runtime.continueReviewed(f.session.id);
|
|
}
|
|
assert.equal(f.session.status, 'completed', JSON.stringify(f.session.messages));
|
|
const checks = [...f.store.core.check_job.values()].filter(value => value.taskId === taskId), check = checks.at(-1);
|
|
assert.ok(checks.some(value => value.status === 'failed' && value.results.some(row => row.tests?.failed > 0))); assert.equal(check.status, 'passed', check.summary); assert.equal(check.results[0].tests.passed, spec.tests); verify();
|
|
const plans = [...f.store.core.context_plan.values()].filter(value => value.taskId === taskId); assert.equal(plans.length, model.requests.length);
|
|
for (const plan of plans) verifyPlanEvidence(f.store, f.session, plan); assertToolPairs(f.session.providerMessages);
|
|
const indexed = [...f.store.core.artifact.values()].filter(value => value.kind === 'project-index' && value.taskId === taskId);
|
|
assert.equal(indexed.length, 1); assert.equal(JSON.parse(f.store.artifactStore.read(indexed[0])).cache.origin, 'disk');
|
|
mark(kind + ': cached index participates in sealed original requests; retained failing checks then ' + spec.tests + ' functional tests and build/compile pass');
|
|
const browserJobs = [];
|
|
if (kind === 'react') for (const viewport of [{ width: 1280, height: 900 }, { width: 390, height: 844 }]) {
|
|
const steps = [{ action: 'assert_count', selector: '.task-list article', count: 3 }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'pending' }, { action: 'assert_count', selector: '.task-list article', count: 2 },
|
|
{ action: 'fill', selector: 'input[aria-label="任务搜索"]', value: ' 编写 ' }, { action: 'assert_count', selector: '.task-list article', count: 1 }, { action: 'fill', selector: 'input[aria-label="任务搜索"]', value: '' }, { action: 'select', selector: 'select[aria-label="任务状态"]', value: 'all' },
|
|
{ action: 'click', selector: '.task-list article[data-task="api"] button' }, { action: 'assert_text', selector: '.task-list article[data-task="api"] .tag', text: '已完成' }, { action: 'assert_text', selector: '.task-list article[data-task="tests"] .tag', text: '待完成' },
|
|
{ action: 'assert_text', selector: '.summary > div:nth-child(2) strong', text: '2' }, { action: 'assert_text', selector: '.summary > div:nth-child(3) strong', text: viewport.width > 600 ? '5小时' : '5\n小时' }];
|
|
const plan = await runtime.browserManager().plan(f.session, { viewport, steps }), job = await runtime.browserManager().start(f.session, plan.id).promise;
|
|
assert.equal(job.status, 'passed', job.summary); assert.equal(job.cleanup.exited, true); browserJobs.push(job);
|
|
for (const image of job.screenshots) fs.writeFileSync(path.join(saved, 'page-' + viewport.width + '-' + image.artifactId + '.png'), f.store.artifactStore.readBytes(f.store.core.artifact.get(image.artifactId)));
|
|
}
|
|
const deliveryPlan = await runtime.gitDelivery().plan(f.session, { message: 'Reviewed cache ' + kind + ' engineering delivery' }), delivery = await (await runtime.gitDelivery().start(f.session, deliveryPlan.id)).promise;
|
|
assert.equal(delivery.status, 'completed', delivery.summary); assert.deepEqual(git(f, ['show', '--format=', '--name-only', delivery.commit]).trim().split(/\r?\n/).sort(), writable);
|
|
assert.deepEqual(ownerBytes(f).files, original.files); assert.equal(git(f, ['show', ':owner-staged.txt']), 'Owner staged change\n'); verify();
|
|
mark(kind + ': reviewed local Git, original staged/unstaged/untracked bytes and actual page checks remain intact');
|
|
result = { success: true, version, kind, taskId, comparisons, check, checks, reviews, browserJobs, commit: delivery.commit,
|
|
requests: model.requests.map(value => ({ sha256: hash(JSON.stringify(value.input)), route: value.url })),
|
|
protectedFiles: Object.fromEntries(Object.entries(protectedFiles).map(([file, bytes]) => [file, hash(bytes)])), realModelRequests: 0 };
|
|
return result;
|
|
} catch (error) { result = { success: false, version, kind, comparisons, error: error.stack, reviews, messages: f.session.messages, checks: [...f.store.core.check_job.values()] }; throw error; }
|
|
finally {
|
|
await runtime?.stopAll(); f.store.storage.database.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
fs.cpSync(f.root, path.join(saved, 'original-fixture'), { recursive: true, filter: file => !file.split(path.sep).includes('node_modules') });
|
|
fs.writeFileSync(path.join(saved, 'result.json'), JSON.stringify(result, null, 2) + '\n'); for (const dispose of disposers.reverse()) await dispose();
|
|
}
|
|
}
|
|
let report;
|
|
try { for (const kind of ['node', 'react', 'python']) results.push(await sample(kind)); report = { success: true, version, directory, cases, results, realModelRequests: 0,
|
|
scope: 'Deterministic local parsing and fixed HTTP engineering execution. Source/index/query semantic equality, original request artifact validation, checks, real Edge, Git and modification protection. No inference-quality or token-saving claim.' }; }
|
|
catch (error) { report = { success: false, version, directory, cases, results, error: error.stack, realModelRequests: 0 }; process.exitCode = 1; }
|
|
finally { fs.writeFileSync(path.join(directory, 'result.json'), JSON.stringify(report, null, 2) + '\n'); fs.writeFileSync('.qa/project-cache-engineering-location.json', JSON.stringify({ directory, result: path.join(directory, 'result.json') })); console.log('EVIDENCE ' + directory); console.log(JSON.stringify({ success: report.success, cases: cases.length, error: report.error ?? null })); }
|