Files
Agent-Workbench/scripts/verify-tianshu-research.mjs
2026-11-04 10:03:32 +08:00

119 lines
6.6 KiB
JavaScript

import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
// Research validation only: no fetch, credentials, model calls or third-party execution.
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const evidence = path.join(root, 'docs/tianshu-research-evidence');
const pin = '2d0b22e0b3554686e1e4131f8bdc568ef768e060';
const args = process.argv.slice(2);
let sourceRoot = null, useIndex = false;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--index') useIndex = true;
else if (args[i] === '--source-root' && args[i + 1]) sourceRoot = path.resolve(args[++i]);
else throw new Error('Usage: node scripts/verify-tianshu-research.mjs [--source-root <checkout>] [--index]');
}
const sha = data => createHash('sha256').update(data).digest('hex');
const git = (cwd, ...gitArgs) => execFileSync('git', ['-C', cwd, ...gitArgs], { maxBuffer: 16 * 1024 * 1024, windowsHide: true });
function contained(base, relative) {
assert.equal(typeof relative, 'string');
assert.ok(relative.length && !relative.includes('\\') && !path.isAbsolute(relative));
const target = path.resolve(base, relative), rel = path.relative(base, target);
assert.ok(rel && !rel.startsWith('..') && !path.isAbsolute(rel), 'Path escapes research root');
assert.ok(fs.lstatSync(target).isFile(), 'Expected an ordinary file: ' + relative);
return target;
}
function checkBytes(bytes, row) {
assert.equal(bytes.length, row.bytes, 'Byte count: ' + row.path);
assert.equal(sha(bytes), row.sha256, 'SHA-256: ' + row.path);
}
const manifest = JSON.parse(fs.readFileSync(path.join(evidence, 'manifest.json'), 'utf8'));
assert.equal(manifest.sourceCommit, pin);
assert.equal(manifest.modelRequests, 0);
assert.equal(manifest.productCodeChanged, false);
assert.equal(manifest.files.length, new Set(manifest.files.map(row => row.path)).size);
let bytes = 0;
for (const row of manifest.files) {
checkBytes(fs.readFileSync(contained(root, row.path)), row);
if (useIndex) checkBytes(git(root, 'show', ':' + row.path), row);
bytes += row.bytes;
}
assert.equal(bytes, manifest.bytes);
const index = JSON.parse(fs.readFileSync(path.join(evidence, 'source-index.json'), 'utf8'));
assert.equal(index.commit, pin);
assert.equal(index.packageVersion, '3.37.4');
assert.equal(index.publicTree.desktop, false);
assert.equal(index.publicTree['src/pro'], false);
assert.equal(index.sourceFiles.length, 63);
assert.equal(index.sourceFiles.length, new Set(index.sourceFiles.map(row => row.id)).size);
assert.equal(index.productFiles.length, 14);
const report = fs.readFileSync(path.join(root, 'docs/TIANSHU-HARNESS-SOURCE-REVIEW.md'), 'utf8');
assert.ok(!report.includes('<!-- SOURCE_REFERENCES -->'), 'Unexpanded source links');
for (const row of index.sourceFiles) assert.ok(report.includes(`[${row.id}]: ${row.url}`), 'Missing pinned link ' + row.id);
for (const [, id] of report.matchAll(/\[(S\d+)\]/g)) assert.ok(index.sourceFiles.some(row => row.id === id), 'Unknown citation ' + id);
const probe = JSON.parse(fs.readFileSync(path.join(evidence, 'pure-probe-original.json'), 'utf8'));
assert.equal(probe.sourceCommit, pin);
assert.equal(probe.modelRequests, 0);
assert.equal(probe.fullApplicationStarted, false);
assert.equal(probe.thirdPartyDependenciesInstalled, false);
assert.equal(probe.productChanged, false);
const expectedModules = new Set(['src/agent/convergence-detector.ts', 'src/agent/convergence-message.ts',
'src/agent/score-history.ts', 'src/agent/runtime-advice-facts.ts', 'src/tools/write-tool-helpers.ts']);
assert.equal(probe.sources.length, expectedModules.size);
for (const module of probe.sources) {
assert.ok(expectedModules.delete(module.path), 'Unexpected or duplicate executable module');
const indexed = index.sourceFiles.find(row => row.path === module.path);
assert.ok(indexed);
assert.equal(indexed.sha256, module.sha256);
assert.equal(indexed.bytes, module.bytes);
}
const expected = [
['upstream-W4-high-error-fixture', 0.5164603132291642, 1, false],
['upstream-W4-repeated-read-baseline', 0.082125, 2, false],
['same-repeated-read-plus-high-error-ratio', 0.082125, 2, false],
['repeated-failing-read-actual-high-error-window', 0, 2, false],
['verification-waiting-protection', 0.082125, 1, false],
['no-tool-hard-stop-preserved', 0.048556689040178395, 3, true],
];
assert.equal(probe.cases.length, expected.length);
for (let i = 0; i < expected.length; i++) {
const [name, score, level, abort] = expected[i], row = probe.cases[i];
assert.equal(row.name, name);
assert.ok(Math.abs(row.result.score - score) < 1e-12);
assert.equal(row.result.level, level);
assert.equal(row.result.shouldAbort, abort);
}
// Research describes the frozen product baseline, not future working-tree code.
// Read Git blobs so later implementation never invalidates or rewrites old evidence.
assert.match(manifest.productBaselineCommit, /^[a-f0-9]{40}$/);
for (const row of index.productFiles) {
const blob = git(root, 'show', manifest.productBaselineCommit + ':' + row.path);
// The original index captured one CRLF Markdown checkout; Git stores LF.
// Restore only that line-ending representation, then require the original SHA.
const original = sha(blob) === row.sha256 ? blob : Buffer.from(blob.toString('utf8').replace(/\r?\n/g, '\r\n'));
checkBytes(original, row);
}
if (sourceRoot) {
assert.equal(git(sourceRoot, 'rev-parse', 'HEAD').toString('utf8').trim(), pin);
assert.equal(git(sourceRoot, 'status', '--porcelain').toString('utf8'), '');
for (const row of index.sourceFiles) {
const data = fs.readFileSync(contained(sourceRoot, row.path));
checkBytes(data, row);
const lines = data.toString('utf8').split(/\r\n|\n|\r/);
if (lines.at(-1) === '') lines.pop();
assert.equal(lines.length, row.lines, 'Source line count ' + row.id);
assert.ok(row.anchorLine >= 1 && row.anchorLine <= lines.length);
assert.ok(lines[row.anchorLine - 1].includes(row.anchorNeedle));
assert.equal(git(sourceRoot, 'rev-parse', 'HEAD:' + row.path).toString('utf8').trim(), row.gitBlob);
}
for (const [directory, present] of Object.entries(index.publicTree)) {
assert.equal(git(sourceRoot, 'ls-tree', '-d', 'HEAD', directory).length > 0, present);
}
}
console.log(JSON.stringify({ ok: true, sourceCommit: pin, files: manifest.files.length, bytes,
indexedSourceFiles: index.sourceFiles.length, productInputs: index.productFiles.length, pureCases: probe.cases.length,
sourceCheckoutChecked: !!sourceRoot, gitIndexChecked: useIndex, thirdPartyCodeExecutedByVerifier: false, modelRequests: 0 }));