docs(qa): retain Claude identity typo and DeepSeek reasoning cap failure

This commit is contained in:
Agent Workbench Dev
2026-11-04 23:34:08 +08:00
parent 409d30f6e2
commit 324ec46623
8 changed files with 204 additions and 1 deletions
+51
View File
@@ -0,0 +1,51 @@
// Offline analysis of the original receipts; never reruns a failed request.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, save, sha, experiment } from './common.mjs';
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
const preflight = read('preflight/result.json');
const requests = [];
const add = (stage, attempts, captures) => {
for (const attempt of attempts) {
const response = captures.find(value => value.attemptId === attempt.id && value.kind === 'response');
let done = false, finishReasons = [], reasoningDeltas = 0;
if (response) {
const raw = fs.readFileSync(path.join(root, response.path));
assert.equal(raw.length, response.bytes); assert.equal(sha(raw), response.sha256);
const text = raw.toString('utf8'); done = text.includes('[DONE]') || text.includes('event: message_stop');
for (const line of text.split(/\r?\n/).filter(value => value.startsWith('data: ') && value !== 'data: [DONE]')) {
const chunk = JSON.parse(line.slice(6));
for (const choice of chunk.choices ?? []) {
if (choice.finish_reason) finishReasons.push(choice.finish_reason);
if (choice.delta?.reasoning_content) reasoningDeltas++;
}
if (chunk.delta?.stop_reason) finishReasons.push(chunk.delta.stop_reason);
}
}
requests.push({ stage, id: attempt.id, status: attempt.status, requested: attempt.model,
reported: attempt.modelIdentity?.reportedModels, identity: attempt.modelIdentity?.status,
identityComplete: attempt.modelIdentity?.complete, usageStatus: attempt.usageStatus, usage: attempt.usage,
requestUrl: attempt.requestUrl, protocol: attempt.protocol, done, finishReasons, reasoningDeltas });
}
};
add('preflight', read('preflight/attempts.json'), preflight.captures);
let node;
if (fs.existsSync(path.join(root, 'node/result.json'))) {
node = read('node/result.json');
assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905');
add('node', read('node/attempts.json'), node.captures);
}
const result = { route: experiment.name, preflightPassed: preflight.passed,
engineeringStarted: !!node, engineeringSuccess: node?.success ?? false, requests,
requestCount: requests.length, reportedTokens: requests.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0),
everyReportedIdentityMatched: requests.every(value => value.identity === 'matched' && value.identityComplete),
everyAttemptCompleted: requests.every(value => value.status === 'completed'),
note: 'Model identity, transport/stream completion and engineering success are separate observations.',
toolCallsOnPreflight: preflight.actualTools, failure: node?.failure, noAutomaticReplay: true, verifierModelRequests: 0 };
save('analysis.json', result);
console.log(JSON.stringify({ route: result.route, requestCount: result.requestCount,
reportedTokens: result.reportedTokens, engineeringSuccess: result.engineeringSuccess,
everyReportedIdentityMatched: result.everyReportedIdentityMatched,
everyAttemptCompleted: result.everyAttemptCompleted, lastFinish: requests.at(-1)?.finishReasons,
lastReasoningTokens: requests.at(-1)?.usage?.completion_tokens_details?.reasoning_tokens }));
+42
View File
@@ -0,0 +1,42 @@
// Verify already-saved preflight snapshots; no host/model request or config write.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, save, sha, experiment } from './common.mjs';
const read = file => JSON.parse(fs.readFileSync(path.join(root, file)));
const before = read('preflight/before.json'), after = read('preflight/after.json');
const result = read('preflight/result.json'), owned = read('preflight/owned.json');
const attempts = read('preflight/attempts.json'), session = read('preflight/session.json');
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey', 'configSha256']) assert.equal(after[field], before[field], field);
assert.deepEqual(before.active, []); assert.deepEqual(after.active, []);
assert.equal(after.projects, before.projects + 1); assert.equal(after.exports.length, before.exports.length + 1);
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
for (const table of ['sessions', 'records', 'events', 'outbox']) {
assert.equal(after.database.tables[table].schemaSha256, before.database.tables[table].schemaSha256);
const rows = new Map(after.database.tables[table].rows.map(value => [value.id, value]));
for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, table + '/' + row.id);
}
const metadata = new Map(after.database.tables.metadata.rows.map(value => [value.id, value]));
for (const row of before.database.tables.metadata.rows.filter(value => value.id !== 'modelAttempts')) {
assert.deepEqual(metadata.get(row.id), row, 'No preflight config or diagnostic write: ' + row.id);
}
for (const row of before.persistent.artifacts) assert.deepEqual(after.persistent.artifacts.find(value => value.path === row.path), row);
assert.deepEqual(after.modelCenter.selected, before.modelCenter.selected);
assert.deepEqual(after.modelCenter.pools, before.modelCenter.pools);
assert.equal(attempts.length, 1); assert.equal(result.requests, 1);
assert.equal(attempts[0].sessionId, owned.session.id); assert.equal(attempts[0].poolId, null); assert.equal(attempts[0].ordinal, 1);
assert.equal(attempts[0].model, experiment.model); assert.equal(attempts[0].protocol, experiment.protocol);
assert.equal(result.actualTools, 0); assert.equal(session.execution.run.toolCalls, 0);
assert.equal(read('preflight/inspector-final.json').closed, true);
for (const capture of result.captures) {
const bytes = fs.readFileSync(path.join(root, capture.path));
assert.equal(bytes.length, capture.bytes); assert.equal(sha(bytes), capture.sha256);
}
const verification = { passed: true, route: experiment.name, engineeringAllowed: result.passed,
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length,
originalRowsExact: ['sessions', 'records', 'events', 'outbox'], publicMetadataExactExceptOneNewAttempt: true,
configSelectionPoolAndCredentialsUnchanged: true, originalPidPreserved: after.pid, inspectorClosed: true,
preflightRequests: 1, preflightReportedTokens: result.usage.total_tokens, toolsExecuted: 0,
verificationModelRequests: 0, stopReason: result.stopReason };
save('preflight/verification.json', verification);
console.log(JSON.stringify(verification));
+28
View File
@@ -0,0 +1,28 @@
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { root, sha, experiment } from './common.mjs';
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
const manifest = read('MANIFEST.json');
for (const row of manifest.files) {
const file = path.resolve(root, row.path); assert.ok(file.startsWith(root + path.sep));
const bytes = fs.readFileSync(file); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
}
assert.equal(read('preflight/verification.json').passed, true);
const analysis = read('analysis.json');
for (const stage of ['preflight', 'node']) {
const file = path.join(root, stage, 'result.json'); if (!fs.existsSync(file)) continue;
const result = JSON.parse(fs.readFileSync(file));
for (const row of result.captures) {
const bytes = fs.readFileSync(path.join(root, row.path));
assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
}
if (stage === 'node') {
assert.equal(read('node/restoration.json').passed, true);
assert.equal(result.usage.requests <= 8, true); assert.equal(result.usage.totalTokens <= 90000, true);
assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905');
}
}
console.log(JSON.stringify({ passed: true, route: experiment.name, files: manifest.files.length, bytes: manifest.totalBytes,
requests: analysis.requestCount, reportedTokens: analysis.reportedTokens, engineeringSuccess: analysis.engineeringSuccess,
modelRequestsExecutedByVerifier: 0 }));