test(live): seal AD autonomous Node success and preserved provider failures

This commit is contained in:
Agent Workbench Dev
2026-11-06 12:57:20 +08:00
parent 3b1510f83e
commit f5b0ffee1c
9 changed files with 302 additions and 2 deletions
+1 -1
View File
@@ -94,7 +94,7 @@ try {
if (original.path === 'config.json' && actual.sha256 !== original.sha256) {
const actualConfig = JSON.parse(fs.readFileSync(path.join(host.identity.dataRoot,original.path),'utf8'));
assert.deepEqual(actualConfig,metaAfter.config);
assert.equal(sha(JSON.stringify({...actualConfig,configRevision:metaBefore.config.configRevision},null,2)+'\n'),original.sha256,'Original config bytes reconstructed exactly after the declared revision increment.');
assert.equal(sha(JSON.stringify({...actualConfig,configRevision:metaBefore.config.configRevision},null,2)),original.sha256,'Original config bytes reconstructed exactly with atomicJson serialization after the declared revision increment.');
} else assert.equal(actual.sha256,original.sha256);
}
const newArtifactCount = after.persistent.artifacts.length-before.persistent.artifacts.length;
+37
View File
@@ -0,0 +1,37 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, read, save, sha } from './common.mjs';
const node = read('node/result.json'), state = read('node/session.json'), compatibility = read('compatibility/result.json');
const audit = read('audits/node-577d9308-d5b6-4bd2-85d3-0ad8cb5255bf/result.json'); assert.equal(audit.success,true);
const prefixes = compatibility.results.flatMap(row=>row.preflights.map(value=>'compatibility/'+row.connectionId+'/'+value.scope)).concat(['node']);
const all = [], proofs = [];
for (const prefix of prefixes) {
const rows = read(prefix+'/attempts.json'); all.push(...rows);
for (const row of rows) for (const kind of ['request','response']) {
const names = fs.readdirSync(path.join(root,prefix,'captures')).filter(file=>file.startsWith(row.id+'-'+kind)); assert.equal(names.length,1);
const name=prefix+'/captures/'+names[0],bytes=fs.readFileSync(path.join(root,name)); assert.equal(sha(bytes),row[kind+'Sha256']);
if (kind==='request') {
const body=JSON.parse(bytes); assert.ok(['max_tokens','max_output_tokens','max_completion_tokens'].every(key=>!Object.hasOwn(body,key)));
if (prefix==='node') {
const plan=state.execution.contextPlans.find(value=>value.id===row.requestId),variant=plan.prefixAudit.variants.find(value=>value.connectionId===row.connectionId);
assert.equal(sha(JSON.stringify(body.tools)),variant.toolsSha256); assert.equal(Buffer.byteLength(JSON.stringify(body.tools)),variant.toolBytes);
proofs.push({attemptId:row.id,planId:plan.id,toolCount:body.tools.length,toolSha256:variant.toolsSha256,commonPrefixBytes:variant.commonPrefixBytes,firstChangedPath:variant.firstChange.path});
}
}
}
}
const nodeAttempts=read('node/attempts.json'),reads=state.events.filter(row=>row.tool==='read_file'&&row.phase==='completed').map(row=>row.input);
const summary={schema:1,driverCommit:read('current.json').driverCommit,version:'1.25.8',realModelRequests:all.length,reportedRequests:all.filter(row=>row.usageStatus==='reported').length,
reportedTokens:all.reduce((sum,row)=>sum+(row.usage?.total_tokens??0),0),unknownRequests:all.filter(row=>row.usageStatus==='unknown').length,
unknownLocalEstimate:compatibility.results.flatMap(row=>row.preflights).filter(row=>row.usageStatus==='unknown').reduce((sum,row)=>sum+read('compatibility/'+compatibility.results.find(value=>value.preflights.includes(row)).connectionId+'/'+row.scope+'/invocation.json').result.usage.estimatedUnknownTokens,0),
node:{success:node.success,commit:node.commit,requests:node.requests,usage:node.usage,independentChecks:node.final,allIdentitiesMatched:node.identitiesPassed,ownerBytesAndIndexPreserved:node.allOriginalOwnerBytesAndIndexPreserved,
reads,duplicateReadInputCount:reads.length-new Set(reads.map(row=>JSON.stringify(row))).size,reportedCachedTokens:nodeAttempts.reduce((sum,row)=>sum+(row.usage?.prompt_tokens_details?.cached_tokens??0),0),inputTokens:node.usage.inputTokens,
terminalGit:true,internalDeliveryReportStatus:'in_progress',internalReportIsNotIndependentGitProof:true,supervisorApprovals:0,supervisorContinuations:0,elapsedMs:Date.parse(state.execution.run.finishedAt)-Date.parse(state.execution.run.startedAt)},
prefixProofs:proofs,originalStateAudit:audit,reactPythonExecuted:false,purgeExecuted:false,cacheCausalBenefit:'unknown',historicalSamplesReplayed:false,keysReadOrResaved:false};
save('summary.json',summary);
const files=[];function walk(dir){for(const entry of fs.readdirSync(dir,{withFileTypes:true})){const file=path.join(dir,entry.name);if(entry.isDirectory())walk(file);else if(entry.isFile()){const bytes=fs.readFileSync(file);files.push({path:path.relative(root,file).replaceAll('\\','/'),bytes:bytes.length,sha256:sha(bytes)});}else throw Error('Nonregular evidence entry.');}}
walk(root);files.sort((a,b)=>a.path.localeCompare(b.path));save('FINAL-MANIFEST.json',{schema:1,files,bytes:files.reduce((sum,row)=>sum+row.bytes,0),scope:'All original AD source/requests/responses/snapshots/audits, including failed audits. Previous historical evidence is not repackaged.'});
for(const row of files){const bytes=fs.readFileSync(path.join(root,row.path));assert.equal(bytes.length,row.bytes);assert.equal(sha(bytes),row.sha256);}
const manifest=fs.readFileSync(path.join(root,'FINAL-MANIFEST.json'));fs.writeFileSync('docs/live-validation-ad-evidence.json',JSON.stringify({summary,evidence:{root,files:files.length,bytes:files.reduce((sum,row)=>sum+row.bytes,0),manifestSha256:sha(manifest)}},null,2)+'\n',{flag:'wx'});
console.log(JSON.stringify({requests:summary.realModelRequests,reportedTokens:summary.reportedTokens,unknown:summary.unknownRequests,nodePassed:node.success,files:files.length,bytes:files.reduce((sum,row)=>sum+row.bytes,0)}));