docs(qa): retain Claude identity typo and DeepSeek reasoning cap failure
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
// Offline analysis of the original receipts; never reruns a failed request.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { root, save, sha, experiment } from './common.mjs';
|
||||
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
||||
const preflight = read('preflight/result.json');
|
||||
const requests = [];
|
||||
const add = (stage, attempts, captures) => {
|
||||
for (const attempt of attempts) {
|
||||
const response = captures.find(value => value.attemptId === attempt.id && value.kind === 'response');
|
||||
let done = false, finishReasons = [], reasoningDeltas = 0;
|
||||
if (response) {
|
||||
const raw = fs.readFileSync(path.join(root, response.path));
|
||||
assert.equal(raw.length, response.bytes); assert.equal(sha(raw), response.sha256);
|
||||
const text = raw.toString('utf8'); done = text.includes('[DONE]') || text.includes('event: message_stop');
|
||||
for (const line of text.split(/\r?\n/).filter(value => value.startsWith('data: ') && value !== 'data: [DONE]')) {
|
||||
const chunk = JSON.parse(line.slice(6));
|
||||
for (const choice of chunk.choices ?? []) {
|
||||
if (choice.finish_reason) finishReasons.push(choice.finish_reason);
|
||||
if (choice.delta?.reasoning_content) reasoningDeltas++;
|
||||
}
|
||||
if (chunk.delta?.stop_reason) finishReasons.push(chunk.delta.stop_reason);
|
||||
}
|
||||
}
|
||||
requests.push({ stage, id: attempt.id, status: attempt.status, requested: attempt.model,
|
||||
reported: attempt.modelIdentity?.reportedModels, identity: attempt.modelIdentity?.status,
|
||||
identityComplete: attempt.modelIdentity?.complete, usageStatus: attempt.usageStatus, usage: attempt.usage,
|
||||
requestUrl: attempt.requestUrl, protocol: attempt.protocol, done, finishReasons, reasoningDeltas });
|
||||
}
|
||||
};
|
||||
add('preflight', read('preflight/attempts.json'), preflight.captures);
|
||||
let node;
|
||||
if (fs.existsSync(path.join(root, 'node/result.json'))) {
|
||||
node = read('node/result.json');
|
||||
assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905');
|
||||
add('node', read('node/attempts.json'), node.captures);
|
||||
}
|
||||
const result = { route: experiment.name, preflightPassed: preflight.passed,
|
||||
engineeringStarted: !!node, engineeringSuccess: node?.success ?? false, requests,
|
||||
requestCount: requests.length, reportedTokens: requests.reduce((sum, value) => sum + (value.usage?.total_tokens ?? 0), 0),
|
||||
everyReportedIdentityMatched: requests.every(value => value.identity === 'matched' && value.identityComplete),
|
||||
everyAttemptCompleted: requests.every(value => value.status === 'completed'),
|
||||
note: 'Model identity, transport/stream completion and engineering success are separate observations.',
|
||||
toolCallsOnPreflight: preflight.actualTools, failure: node?.failure, noAutomaticReplay: true, verifierModelRequests: 0 };
|
||||
save('analysis.json', result);
|
||||
console.log(JSON.stringify({ route: result.route, requestCount: result.requestCount,
|
||||
reportedTokens: result.reportedTokens, engineeringSuccess: result.engineeringSuccess,
|
||||
everyReportedIdentityMatched: result.everyReportedIdentityMatched,
|
||||
everyAttemptCompleted: result.everyAttemptCompleted, lastFinish: requests.at(-1)?.finishReasons,
|
||||
lastReasoningTokens: requests.at(-1)?.usage?.completion_tokens_details?.reasoning_tokens }));
|
||||
@@ -0,0 +1,42 @@
|
||||
// Verify already-saved preflight snapshots; no host/model request or config write.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { root, save, sha, experiment } from './common.mjs';
|
||||
const read = file => JSON.parse(fs.readFileSync(path.join(root, file)));
|
||||
const before = read('preflight/before.json'), after = read('preflight/after.json');
|
||||
const result = read('preflight/result.json'), owned = read('preflight/owned.json');
|
||||
const attempts = read('preflight/attempts.json'), session = read('preflight/session.json');
|
||||
for (const field of ['pid', 'version', 'runtimeRoot', 'hasKey', 'configSha256']) assert.equal(after[field], before[field], field);
|
||||
assert.deepEqual(before.active, []); assert.deepEqual(after.active, []);
|
||||
assert.equal(after.projects, before.projects + 1); assert.equal(after.exports.length, before.exports.length + 1);
|
||||
for (const item of before.exports) assert.deepEqual(after.exports.find(value => value.id === item.id), item);
|
||||
for (const table of ['sessions', 'records', 'events', 'outbox']) {
|
||||
assert.equal(after.database.tables[table].schemaSha256, before.database.tables[table].schemaSha256);
|
||||
const rows = new Map(after.database.tables[table].rows.map(value => [value.id, value]));
|
||||
for (const row of before.database.tables[table].rows) assert.deepEqual(rows.get(row.id), row, table + '/' + row.id);
|
||||
}
|
||||
const metadata = new Map(after.database.tables.metadata.rows.map(value => [value.id, value]));
|
||||
for (const row of before.database.tables.metadata.rows.filter(value => value.id !== 'modelAttempts')) {
|
||||
assert.deepEqual(metadata.get(row.id), row, 'No preflight config or diagnostic write: ' + row.id);
|
||||
}
|
||||
for (const row of before.persistent.artifacts) assert.deepEqual(after.persistent.artifacts.find(value => value.path === row.path), row);
|
||||
assert.deepEqual(after.modelCenter.selected, before.modelCenter.selected);
|
||||
assert.deepEqual(after.modelCenter.pools, before.modelCenter.pools);
|
||||
assert.equal(attempts.length, 1); assert.equal(result.requests, 1);
|
||||
assert.equal(attempts[0].sessionId, owned.session.id); assert.equal(attempts[0].poolId, null); assert.equal(attempts[0].ordinal, 1);
|
||||
assert.equal(attempts[0].model, experiment.model); assert.equal(attempts[0].protocol, experiment.protocol);
|
||||
assert.equal(result.actualTools, 0); assert.equal(session.execution.run.toolCalls, 0);
|
||||
assert.equal(read('preflight/inspector-final.json').closed, true);
|
||||
for (const capture of result.captures) {
|
||||
const bytes = fs.readFileSync(path.join(root, capture.path));
|
||||
assert.equal(bytes.length, capture.bytes); assert.equal(sha(bytes), capture.sha256);
|
||||
}
|
||||
const verification = { passed: true, route: experiment.name, engineeringAllowed: result.passed,
|
||||
originalExportsExact: before.exports.length, originalArtifactFilesExact: before.persistent.artifacts.length,
|
||||
originalRowsExact: ['sessions', 'records', 'events', 'outbox'], publicMetadataExactExceptOneNewAttempt: true,
|
||||
configSelectionPoolAndCredentialsUnchanged: true, originalPidPreserved: after.pid, inspectorClosed: true,
|
||||
preflightRequests: 1, preflightReportedTokens: result.usage.total_tokens, toolsExecuted: 0,
|
||||
verificationModelRequests: 0, stopReason: result.stopReason };
|
||||
save('preflight/verification.json', verification);
|
||||
console.log(JSON.stringify(verification));
|
||||
@@ -0,0 +1,28 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import assert from 'node:assert/strict';
|
||||
import { root, sha, experiment } from './common.mjs';
|
||||
const read = name => JSON.parse(fs.readFileSync(path.join(root, name)));
|
||||
const manifest = read('MANIFEST.json');
|
||||
for (const row of manifest.files) {
|
||||
const file = path.resolve(root, row.path); assert.ok(file.startsWith(root + path.sep));
|
||||
const bytes = fs.readFileSync(file); assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
|
||||
}
|
||||
assert.equal(read('preflight/verification.json').passed, true);
|
||||
const analysis = read('analysis.json');
|
||||
for (const stage of ['preflight', 'node']) {
|
||||
const file = path.join(root, stage, 'result.json'); if (!fs.existsSync(file)) continue;
|
||||
const result = JSON.parse(fs.readFileSync(file));
|
||||
for (const row of result.captures) {
|
||||
const bytes = fs.readFileSync(path.join(root, row.path));
|
||||
assert.equal(bytes.length, row.bytes); assert.equal(sha(bytes), row.sha256);
|
||||
}
|
||||
if (stage === 'node') {
|
||||
assert.equal(read('node/restoration.json').passed, true);
|
||||
assert.equal(result.usage.requests <= 8, true); assert.equal(result.usage.totalTokens <= 90000, true);
|
||||
assert.equal(sha(fs.readFileSync(path.join(root, 'node/goal.txt'))), 'b5421d0bb502126f49a5153729027f13ef82db86ab19a9e5b0c23a49070bb905');
|
||||
}
|
||||
}
|
||||
console.log(JSON.stringify({ passed: true, route: experiment.name, files: manifest.files.length, bytes: manifest.totalBytes,
|
||||
requests: analysis.requestCount, reportedTokens: analysis.reportedTokens, engineeringSuccess: analysis.engineeringSuccess,
|
||||
modelRequestsExecutedByVerifier: 0 }));
|
||||
Reference in New Issue
Block a user