Files
Agent-Workbench/docs/live-validation-aa-evidence/compatibility-summary.json
T
2026-11-05 03:21:49 +08:00

565 lines
22 KiB
JSON

{
"passed": true,
"purpose": "AA compatibility and preservation audit, not a claim all providers passed",
"observations": [
{
"key": "deepseek",
"catalog": {
"success": true,
"connectionId": "1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd",
"startedAt": "2026-11-04T19:15:16.575Z",
"finishedAt": "2026-11-04T19:15:16.776Z",
"models": 2,
"complete": true,
"selectedModel": "deepseek-flash",
"selectedModelListed": true,
"declaredCapabilities": 2,
"declaredReasoning": 0,
"normalizedPreviewOnly": true,
"rawHttpResponseArtifactAvailable": false,
"realModelRequests": 0,
"configurationSaved": false
},
"network": "passed",
"modes": {
"off": {
"passed": false,
"engineeringTestAllowed": false,
"testedAt": "2026-11-04T19:15:31.713Z",
"version": "1.25.3",
"nativeParameters": {
"thinking": {
"type": "disabled"
}
},
"reasoningMode": "off",
"connectionId": "1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd",
"sessionId": "d4e7d411-d013-4ab9-b5ee-ac5b1e0f98c5",
"projectId": "bd328b25-14df-4dd7-8c61-828f818a6850",
"taskId": "dcc030db-4f59-4658-9a74-fde8244f9133",
"requestedModel": "deepseek-flash",
"reportedModels": [
"deepseek-flash"
],
"identity": {
"schemaVersion": 1,
"requestedModel": "deepseek-flash",
"reportedModels": [
"deepseek-flash"
],
"status": "matched",
"complete": true,
"invalidIdentifiers": false,
"policy": "require-match",
"scope": "Identifiers in the outgoing request and upstream response; neither equal identifiers nor a declared mapping authenticate model weights.",
"responseSource": {
"artifactId": "35b61023-dd8b-4317-9377-8c54a3d74a4b",
"sha256": "4515967f82a028142ea53b1d3f5b18f2b238cc13efbec33b5992a8f07a8a90f2",
"sessionId": "d4e7d411-d013-4ab9-b5ee-ac5b1e0f98c5",
"scope": "sse-stream"
}
},
"requestUrl": "https://api.deepseek.com/v1/chat/completions",
"requests": 1,
"usage": {
"prompt_tokens": 81,
"completion_tokens": 17,
"total_tokens": 98,
"prompt_tokens_details": {
"cached_tokens": 0
},
"prompt_cache_hit_tokens": 0,
"prompt_cache_miss_tokens": 81
},
"usageStatus": "reported",
"actualTools": 0,
"stopReason": "identity_probe_complete",
"captures": [
{
"attemptId": "b8d21833-f054-44cf-a6dd-378a38d06bf3",
"kind": "request",
"artifactId": "2afbe377-048e-4f0f-b963-10bac7efacc4",
"path": "preflight-off/captures/b8d21833-f054-44cf-a6dd-378a38d06bf3-request.json",
"sha256": "00eb6cfa26dcfad8ea3bae9d9b56685f83df00851ef601e40b48113fb50931b2",
"bytes": 521
},
{
"attemptId": "b8d21833-f054-44cf-a6dd-378a38d06bf3",
"kind": "response",
"artifactId": "35b61023-dd8b-4317-9377-8c54a3d74a4b",
"path": "preflight-off/captures/b8d21833-f054-44cf-a6dd-378a38d06bf3-response.sse",
"sha256": "4515967f82a028142ea53b1d3f5b18f2b238cc13efbec33b5992a8f07a8a90f2",
"bytes": 5642
}
],
"noRetryOrPoolSwitch": true,
"modelWeightsAuthenticated": false,
"engineeringDeliveryTested": false,
"purgeBenefitTested": false,
"defaultSelectionChanged": false,
"connectionConfigChanged": false,
"credentialValuesReadOrSaved": false
},
"on": {
"passed": true,
"engineeringTestAllowed": true,
"testedAt": "2026-11-04T19:15:57.964Z",
"version": "1.25.3",
"nativeParameters": {
"thinking": {
"type": "enabled"
},
"reasoning_effort": "low"
},
"reasoningMode": "on",
"connectionId": "1a7a55a9-29f2-4b9f-8ea2-5a55afd7eacd",
"sessionId": "ebb5e199-fce0-4477-aedc-8942ec47cea3",
"projectId": "64c4285f-2afd-4869-ade6-f9a19cc697d6",
"taskId": "b7a73849-eb9a-4b3c-8294-a4df3e104e45",
"requestedModel": "deepseek-flash",
"reportedModels": [
"deepseek-flash"
],
"identity": {
"schemaVersion": 1,
"requestedModel": "deepseek-flash",
"reportedModels": [
"deepseek-flash"
],
"status": "matched",
"complete": true,
"invalidIdentifiers": false,
"policy": "require-match",
"scope": "Identifiers in the outgoing request and upstream response; neither equal identifiers nor a declared mapping authenticate model weights.",
"responseSource": {
"artifactId": "f353781f-7963-41fe-a5c4-ad73731c925e",
"sha256": "f57f176a1fe879613e52869b3f68c06d90a864fbd82bce70d0cf4f0bed3636bb",
"sessionId": "ebb5e199-fce0-4477-aedc-8942ec47cea3",
"scope": "sse-stream"
}
},
"requestUrl": "https://api.deepseek.com/v1/chat/completions",
"requests": 1,
"usage": {
"prompt_tokens": 106,
"completion_tokens": 66,
"total_tokens": 172,
"prompt_tokens_details": {
"cached_tokens": 0
},
"completion_tokens_details": {
"reasoning_tokens": 40
},
"prompt_cache_hit_tokens": 0,
"prompt_cache_miss_tokens": 106
},
"usageStatus": "reported",
"actualTools": 0,
"stopReason": "identity_probe_complete",
"captures": [
{
"attemptId": "c0a35bc8-2449-4723-89e6-7a07aeffa8bb",
"kind": "request",
"artifactId": "89c4269e-cc5f-429d-a1be-fe518a5127a7",
"path": "preflight-on/captures/c0a35bc8-2449-4723-89e6-7a07aeffa8bb-request.json",
"sha256": "116da4e17fbc1eaf5aa3af94333edc75689104a49fa39985c50f462f2e621fac",
"bytes": 545
},
{
"attemptId": "c0a35bc8-2449-4723-89e6-7a07aeffa8bb",
"kind": "response",
"artifactId": "f353781f-7963-41fe-a5c4-ad73731c925e",
"path": "preflight-on/captures/c0a35bc8-2449-4723-89e6-7a07aeffa8bb-response.sse",
"sha256": "f57f176a1fe879613e52869b3f68c06d90a864fbd82bce70d0cf4f0bed3636bb",
"bytes": 21154
}
],
"noRetryOrPoolSwitch": true,
"modelWeightsAuthenticated": false,
"engineeringDeliveryTested": false,
"purgeBenefitTested": false,
"defaultSelectionChanged": false,
"connectionConfigChanged": false,
"credentialValuesReadOrSaved": false
}
}
},
{
"key": "claude",
"catalog": {
"success": true,
"connectionId": "9ae348a5-d0a6-42ed-8664-0da5f9effe34",
"startedAt": "2026-11-04T19:16:10.096Z",
"finishedAt": "2026-11-04T19:16:10.790Z",
"models": 9,
"complete": true,
"selectedModel": "claude-opus-5-5",
"selectedModelListed": true,
"declaredCapabilities": 0,
"declaredReasoning": 0,
"normalizedPreviewOnly": true,
"rawHttpResponseArtifactAvailable": false,
"realModelRequests": 0,
"configurationSaved": false
},
"network": "passed",
"modes": {
"off": {
"passed": false,
"engineeringTestAllowed": false,
"testedAt": "2026-11-04T19:16:27.535Z",
"version": "1.25.3",
"nativeParameters": {
"thinking": {
"type": "disabled"
}
},
"reasoningMode": "off",
"connectionId": "9ae348a5-d0a6-42ed-8664-0da5f9effe34",
"sessionId": "7287fa1b-af05-453b-a25d-2c10b38af3d8",
"projectId": "952d29a0-112f-471f-b817-4fba4abf1ff4",
"taskId": "57210154-1c8e-46c7-b4a1-509461b58dd2",
"requestedModel": "claude-opus-5-5",
"reportedModels": [
"claude-opus-5-5"
],
"identity": {
"schemaVersion": 1,
"requestedModel": "claude-opus-5-5",
"reportedModels": [
"claude-opus-5-5"
],
"status": "incomplete",
"complete": false,
"invalidIdentifiers": false,
"policy": "require-match",
"scope": "Identifiers in the outgoing request and upstream response; neither equal identifiers nor a declared mapping authenticate model weights.",
"responseSource": {
"artifactId": "3f6b0f65-228a-429f-9f38-13ebb1d688e3",
"sha256": "e2b8825c31698cfd5a2fbe5d7d0a443dce9c0962299caa1fde8f888eddb4e49e",
"sessionId": "7287fa1b-af05-453b-a25d-2c10b38af3d8",
"scope": "sse-stream"
}
},
"requestUrl": "https://api.apikey.fan/v1/messages",
"requests": 1,
"usage": {
"prompt_tokens": 176,
"completion_tokens": 0,
"total_tokens": 176,
"prompt_tokens_details": {
"cached_tokens": 114,
"cache_creation_tokens": 0
},
"raw": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 114,
"input_tokens": 62,
"output_tokens": 0
},
"protocol": "anthropic-messages"
},
"usageStatus": "reported",
"actualTools": 0,
"stopReason": "protocol",
"captures": [
{
"attemptId": "bfd01d34-5dd0-4b77-a945-da74ade4c7ac",
"kind": "request",
"artifactId": "0b2bd89f-4d5a-4935-8628-846656a9d94c",
"path": "preflight-off/captures/bfd01d34-5dd0-4b77-a945-da74ade4c7ac-request.json",
"sha256": "7183b944384954db3682a9ec3b0f6c36a45d15952d0bee4a9f7c5cded61b63d2",
"bytes": 481
},
{
"attemptId": "bfd01d34-5dd0-4b77-a945-da74ade4c7ac",
"kind": "response",
"artifactId": "3f6b0f65-228a-429f-9f38-13ebb1d688e3",
"path": "preflight-off/captures/bfd01d34-5dd0-4b77-a945-da74ade4c7ac-response.sse",
"sha256": "e2b8825c31698cfd5a2fbe5d7d0a443dce9c0962299caa1fde8f888eddb4e49e",
"bytes": 668
}
],
"noRetryOrPoolSwitch": true,
"modelWeightsAuthenticated": false,
"engineeringDeliveryTested": false,
"purgeBenefitTested": false,
"defaultSelectionChanged": false,
"connectionConfigChanged": false,
"credentialValuesReadOrSaved": false
},
"on": {
"passed": true,
"engineeringTestAllowed": true,
"testedAt": "2026-11-04T19:16:48.334Z",
"version": "1.25.3",
"nativeParameters": {
"thinking": {
"type": "enabled",
"budget_tokens": 1024
},
"output_config": {
"effort": "low"
}
},
"reasoningMode": "on",
"connectionId": "9ae348a5-d0a6-42ed-8664-0da5f9effe34",
"sessionId": "325371f1-5e94-4fd6-8758-05574ce6c479",
"projectId": "b2b48f0c-6e45-40e6-be9b-bf4e7911cf1a",
"taskId": "f31538cf-aa0e-4ff3-9813-96e851fd25c5",
"requestedModel": "claude-opus-5-5",
"reportedModels": [
"claude-opus-5-5"
],
"identity": {
"schemaVersion": 1,
"requestedModel": "claude-opus-5-5",
"reportedModels": [
"claude-opus-5-5"
],
"status": "matched",
"complete": true,
"invalidIdentifiers": false,
"policy": "require-match",
"scope": "Identifiers in the outgoing request and upstream response; neither equal identifiers nor a declared mapping authenticate model weights.",
"responseSource": {
"artifactId": "791dd1d9-7714-4537-9c00-454b0e520c44",
"sha256": "f47c2bbe01bdfa5f6d08f1b435825b088449282b54f9f1f5e9ad6d0d3737e839",
"sessionId": "325371f1-5e94-4fd6-8758-05574ce6c479",
"scope": "sse-stream"
}
},
"requestUrl": "https://api.apikey.fan/v1/messages",
"requests": 1,
"usage": {
"prompt_tokens": 176,
"completion_tokens": 33,
"total_tokens": 209,
"prompt_tokens_details": {
"cached_tokens": 114,
"cache_creation_tokens": 0
},
"raw": {
"cache_creation": {
"ephemeral_1h_input_tokens": 0,
"ephemeral_5m_input_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 114,
"input_tokens": 62,
"output_tokens": 33
},
"protocol": "anthropic-messages"
},
"usageStatus": "reported",
"actualTools": 0,
"stopReason": "identity_probe_complete",
"captures": [
{
"attemptId": "6f0c76ad-997a-4a54-8e32-d606d41338da",
"kind": "request",
"artifactId": "4fd31f00-3130-49eb-8e22-348c5a362c54",
"path": "preflight-on/captures/6f0c76ad-997a-4a54-8e32-d606d41338da-request.json",
"sha256": "889ae4f8f4cff6a6f445ee77d67b51c6def6380041136f1f8e4cc4af03090691",
"bytes": 534
},
{
"attemptId": "6f0c76ad-997a-4a54-8e32-d606d41338da",
"kind": "response",
"artifactId": "791dd1d9-7714-4537-9c00-454b0e520c44",
"path": "preflight-on/captures/6f0c76ad-997a-4a54-8e32-d606d41338da-response.sse",
"sha256": "f47c2bbe01bdfa5f6d08f1b435825b088449282b54f9f1f5e9ad6d0d3737e839",
"bytes": 1358
}
],
"noRetryOrPoolSwitch": true,
"modelWeightsAuthenticated": false,
"engineeringDeliveryTested": false,
"purgeBenefitTested": false,
"defaultSelectionChanged": false,
"connectionConfigChanged": false,
"credentialValuesReadOrSaved": false
}
}
},
{
"key": "gpt",
"catalog": {
"success": true,
"connectionId": "93d0cf6d-4f32-4918-9da8-6f125aa8a6e5",
"startedAt": "2026-11-04T19:17:13.722Z",
"finishedAt": "2026-11-04T19:17:14.431Z",
"models": 5,
"complete": true,
"selectedModel": "gpt-6.1-sol",
"selectedModelListed": true,
"declaredCapabilities": 0,
"declaredReasoning": 0,
"normalizedPreviewOnly": true,
"rawHttpResponseArtifactAvailable": false,
"realModelRequests": 0,
"configurationSaved": false
},
"network": "passed",
"modes": {
"off": {
"passed": false,
"engineeringTestAllowed": false,
"testedAt": "2026-11-04T19:17:24.253Z",
"version": "1.25.3",
"nativeParameters": {
"reasoning_effort": "none"
},
"reasoningMode": "off",
"connectionId": "93d0cf6d-4f32-4918-9da8-6f125aa8a6e5",
"sessionId": "83aa2f1b-a028-441d-8eb9-a4c85eeec492",
"projectId": "3a8df553-cf55-4a3b-8007-d9c04f9cad1a",
"taskId": "ea042356-cb46-4e62-9911-db7af18a48b7",
"requestedModel": "gpt-6.1-sol",
"reportedModels": [],
"identity": {
"schemaVersion": 1,
"requestedModel": "gpt-6.1-sol",
"reportedModels": [],
"status": "incomplete",
"complete": false,
"invalidIdentifiers": false,
"policy": "require-match",
"scope": "Identifiers in the outgoing request and upstream response; neither equal identifiers nor a declared mapping authenticate model weights."
},
"requestUrl": "https://api.apikey.fan/v1/chat/completions",
"requests": 1,
"usage": null,
"usageStatus": "unknown",
"actualTools": 0,
"stopReason": "service",
"captures": [
{
"attemptId": "4329b0e0-874d-41bd-841b-cedec13a3a83",
"kind": "request",
"artifactId": "7f07baca-f2ea-4728-8338-45551a0a561f",
"path": "preflight-off/captures/4329b0e0-874d-41bd-841b-cedec13a3a83-request.json",
"sha256": "0eaa2bc198dca4cfdbab7f7fbee4cc35368bcd7ffe3df5942266405ff51f035a",
"bytes": 513
},
{
"attemptId": "4329b0e0-874d-41bd-841b-cedec13a3a83",
"kind": "response",
"artifactId": "28dcff7b-7ccc-4d3a-baef-62b4fa27ab0c",
"path": "preflight-off/captures/4329b0e0-874d-41bd-841b-cedec13a3a83-response.sse",
"sha256": "48d982e523a29c3f22019b6a313f5e1506f62659d7f088bf1371f268a60727cb",
"bytes": 146
}
],
"noRetryOrPoolSwitch": true,
"modelWeightsAuthenticated": false,
"engineeringDeliveryTested": false,
"purgeBenefitTested": false,
"defaultSelectionChanged": false,
"connectionConfigChanged": false,
"credentialValuesReadOrSaved": false
},
"on": {
"passed": true,
"engineeringTestAllowed": true,
"testedAt": "2026-11-04T19:17:45.249Z",
"version": "1.25.3",
"nativeParameters": {
"reasoning_effort": "low"
},
"reasoningMode": "on",
"connectionId": "93d0cf6d-4f32-4918-9da8-6f125aa8a6e5",
"sessionId": "0fe3dccc-d7eb-4174-b252-08572fd1f4c1",
"projectId": "092f1b26-6421-4198-96d4-80ff791a7d44",
"taskId": "45757292-e121-4ba3-af5a-4b665a8e818c",
"requestedModel": "gpt-6.1-sol",
"reportedModels": [
"gpt-6.1-sol"
],
"identity": {
"schemaVersion": 1,
"requestedModel": "gpt-6.1-sol",
"reportedModels": [
"gpt-6.1-sol"
],
"status": "matched",
"complete": true,
"invalidIdentifiers": false,
"policy": "require-match",
"scope": "Identifiers in the outgoing request and upstream response; neither equal identifiers nor a declared mapping authenticate model weights.",
"responseSource": {
"artifactId": "49194a39-e99e-4c15-a550-a4abc15b44d1",
"sha256": "7f7022ef9da9332c4e23de838a9334a887b8613909361918607e816825864ece",
"sessionId": "0fe3dccc-d7eb-4174-b252-08572fd1f4c1",
"scope": "sse-stream"
}
},
"requestUrl": "https://api.apikey.fan/v1/chat/completions",
"requests": 1,
"usage": {
"prompt_tokens": 88,
"completion_tokens": 28,
"total_tokens": 116
},
"usageStatus": "reported",
"actualTools": 0,
"stopReason": "identity_probe_complete",
"captures": [
{
"attemptId": "7a4a4406-0b91-4b21-bae1-9bfc4c914c84",
"kind": "request",
"artifactId": "15445504-f933-4ddd-a261-8a93794596e2",
"path": "preflight-on/captures/7a4a4406-0b91-4b21-bae1-9bfc4c914c84-request.json",
"sha256": "c2ea335d89b8ebe8773b4442a266436a2451dcfc11a5fc00e3e9a34c70959542",
"bytes": 512
},
{
"attemptId": "7a4a4406-0b91-4b21-bae1-9bfc4c914c84",
"kind": "response",
"artifactId": "49194a39-e99e-4c15-a550-a4abc15b44d1",
"path": "preflight-on/captures/7a4a4406-0b91-4b21-bae1-9bfc4c914c84-response.sse",
"sha256": "7f7022ef9da9332c4e23de838a9334a887b8613909361918607e816825864ece",
"bytes": 5557
}
],
"noRetryOrPoolSwitch": true,
"modelWeightsAuthenticated": false,
"engineeringDeliveryTested": false,
"purgeBenefitTested": false,
"defaultSelectionChanged": false,
"connectionConfigChanged": false,
"credentialValuesReadOrSaved": false
}
}
}
],
"usage": {
"requests": 6,
"tokens": 771,
"unknown": 1
},
"engineeringRoute": null,
"nodeExecuted": false,
"reactPythonExecuted": false,
"purgeExecuted": false,
"blockedReason": "No off-mode passed; AA engineering gate not satisfied. No replay or change of mode within AA.",
"originalExportsExact": 91,
"originalArtifactFilesExact": 3926,
"originalRowsExceptSixOwnedProjectPositionPrependsAndEventSequenceExact": true,
"eventSequenceFrom": 14320,
"eventSequenceTo": 14439,
"originalProjectPayloadsExact": true,
"originalAttemptsExact": 252,
"originalDiagnosticsExact": true,
"originalConfigAndRegistryExact": true,
"credentialsReadOrResaved": false,
"pid": 36552,
"instanceId": "4c910dd5-82cb-4c8a-92f6-328338ea0662",
"windows": 0,
"inspectorClosed": true
}