Files
Agent-Workbench/tests/ci-pipeline.test.mjs
2026-11-07 22:59:31 +08:00

93 lines
9.1 KiB
JavaScript

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import net from 'node:net';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import YAML from 'yaml';
import { createCiRun, ciEnvironment, loggedCommand, tapTotals, requireCompleteTests, unpackVerifiedSource, fileDigest, binaryMirror, loopbackPortStatus } from '../scripts/ci-utils.mjs';
import { packageSource } from '../scripts/package-source.mjs';
import { verifyCiWorkflow } from '../scripts/verify-ci-workflow.mjs';
import { verifyReplayFixtures, replayFixtureJson } from '../scripts/verify-replay-fixtures.mjs';
const workspace=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..');
function owned(t){const root=fs.mkdtempSync(path.join(os.tmpdir(),'awb-ci-test-'));t.after(()=>{assert.equal(path.dirname(path.resolve(root)),path.resolve(os.tmpdir()));assert.ok(path.basename(root).startsWith('awb-ci-test-'));fs.rmSync(root,{recursive:true,force:true,maxRetries:5,retryDelay:100});});return root;}
test('fresh output runs are separate and excluded credential getters are never read or forwarded',t=>{
const root=owned(t),first=createCiRun(root,'contract'),second=createCiRun(root,'contract');assert.notEqual(first.root,second.root);
const input={PATH:'owned-original-path',SystemRoot:'owned-system',AGENT_WORKBENCH_TEST_PYTHON:'owned-not-approved-interpreter',ELECTRON_SKIP_BINARY_DOWNLOAD:'1'};
Object.defineProperty(input,'AGENT_WORKBENCH_API_KEY',{enumerable:true,get(){throw new Error('Credential value must never be read');}});
Object.defineProperty(input,'OPENAI_API_KEY',{enumerable:true,get(){throw new Error('Credential value must never be read');}});
const env=ciEnvironment(first,input);assert.ok(env.PATH.startsWith(path.dirname(process.execPath)));assert.equal(env.SystemRoot,'owned-system');
for(const name of ['AGENT_WORKBENCH_API_KEY','OPENAI_API_KEY','AGENT_WORKBENCH_TEST_PYTHON','ELECTRON_SKIP_BINARY_DOWNLOAD']) assert.equal(Object.hasOwn(env,name),false);
assert.equal(fs.readFileSync(env.npm_config_userconfig,'utf8'),''); assert.notEqual(env.npm_config_userconfig,env.npm_config_globalconfig);
});
test('a download mirror is an explicit public HTTPS directory and cannot carry credentials in URL or query',()=>{
assert.equal(binaryMirror(undefined),null);assert.equal(binaryMirror('https://npmmirror.com/mirrors/electron/'),'https://npmmirror.com/mirrors/electron/');
for(const value of ['http://example.invalid/electron/','https://owned:credential@example.invalid/','https://example.invalid/?token=owned','https://example.invalid/electron/#owned','https://example.invalid/file.zip'])assert.throws(()=>binaryMirror(value));
});
test('CI cleanup observes the recorded TCP listener instead of treating a missing health response as a closed port',async t=>{
const server=net.createServer(socket=>socket.destroy());t.after(()=>{if(server.listening)return new Promise(resolve=>server.close(resolve));});
await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const port=server.address().port;
assert.deepEqual(await loopbackPortStatus(port),{host:'127.0.0.1',port,status:'accepting'});
await new Promise(resolve=>server.close(resolve));assert.deepEqual(await loopbackPortStatus(port),{host:'127.0.0.1',port,status:'refused'});
});
test('actual nonzero command, original stdout/stderr and spawn failure survive without replay or log overwrite',async t=>{
const run=createCiRun(owned(t),'contract'),env=ciEnvironment(run);
const result=await loggedCommand({id:'original-failure',executable:process.execPath,args:['-e',"process.stdout.write('original\\n');process.stderr.write('failure\\n');process.exitCode=7"],cwd:run.source,env,evidence:run.evidence,check:false});
assert.equal(result.exitCode,7);assert.equal(fs.readFileSync(result.stdout.file,'utf8'),'original\n');assert.equal(fs.readFileSync(result.stderr.file,'utf8'),'failure\n');
const hash=fileDigest(result.stdout.file);await assert.rejects(loggedCommand({id:'original-failure',executable:process.execPath,args:['-e',"console.log('replayed')"],cwd:run.source,env,evidence:run.evidence}),/EEXIST/);assert.equal(fileDigest(result.stdout.file),hash);
const missing=await loggedCommand({id:'missing-environment',executable:path.join(run.root,'missing-python.exe'),cwd:run.source,env,evidence:run.evidence,check:false});
assert.equal(missing.spawnError.code,'ENOENT');assert.notEqual(missing.exitCode,0);assert.ok(fs.existsSync(path.join(run.evidence,'missing-environment.json')));
});
test('zero process exit with a real skipped Node test is still an incomplete CI result',async t=>{
const run=createCiRun(owned(t),'contract'),env=ciEnvironment(run),file=path.join(run.source,'skipped.test.mjs');
fs.writeFileSync(file,"import test from 'node:test';test('executed',()=>{});test.skip('missing dependency is not success',()=>{});\n");
const result=await loggedCommand({id:'skipped-tests',executable:process.execPath,args:['--test','--test-reporter=tap',file],cwd:run.source,env,evidence:run.evidence});
assert.equal(result.exitCode,0);const totals=tapTotals(fs.readFileSync(result.stdout.file,'utf8'));assert.equal(totals.skipped,1);assert.throws(()=>requireCompleteTests(totals),/skipped/);
assert.throws(()=>tapTotals('incomplete output'),/Missing full Node TAP totals/);
});
test('actual YAML, immutable action refs and original failure upload are checked instead of relying on ignored Gitea syntax',t=>{
assert.equal(verifyCiWorkflow(workspace).passed,true);
const root=owned(t);fs.mkdirSync(path.join(root,'.gitea/workflows'),{recursive:true});fs.mkdirSync(path.join(root,'scripts'));
fs.copyFileSync(path.join(workspace,'scripts/ci-environment.json'),path.join(root,'scripts/ci-environment.json'));
const document=YAML.parse(fs.readFileSync(path.join(workspace,'.gitea/workflows/ci.yml'),'utf8'));
document.jobs['windows-source'].steps.at(-1).if='success()';fs.writeFileSync(path.join(root,'.gitea/workflows/ci.yml'),YAML.stringify(document));
assert.throws(()=>verifyCiWorkflow(root));
document.jobs['windows-source'].steps.at(-1).if='always()';document.jobs['windows-source']['timeout-minutes']=30;fs.writeFileSync(path.join(root,'.gitea/workflows/ci.yml'),YAML.stringify(document));
assert.throws(()=>verifyCiWorkflow(root));
delete document.jobs['windows-source']['timeout-minutes'];document.jobs['windows-source'].steps.at(-1).with['include-hidden-files']=false;fs.writeFileSync(path.join(root,'.gitea/workflows/ci.yml'),YAML.stringify(document));
assert.throws(()=>verifyCiWorkflow(root));
});
test('a real committed source ZIP is verified before extraction and tampering cannot replace existing input',t=>{
const root=owned(t),source=path.join(root,'source');fs.mkdirSync(source);
const git=(...args)=>execFileSync('git',args,{cwd:source,windowsHide:true,encoding:'utf8'});
git('init');fs.mkdirSync(path.join(source,'src'));fs.writeFileSync(path.join(source,'src/main.mjs'),'export const original = true;\n');
fs.writeFileSync(path.join(source,'package.json'),JSON.stringify({name:'owned',version:'1.0.0'}));fs.writeFileSync(path.join(source,'package-lock.json'),'{}\n');
git('add','--all');git('-c','user.name=CI QA','-c','user.email=ci-qa@local.invalid','commit','-m','Owned source');
const archive=packageSource({root:source,outputRoot:path.join(root,'archives')}),destination=path.join(root,'extracted');fs.mkdirSync(destination);
assert.equal(unpackVerifiedSource(archive,destination).passed,true);const original=fileDigest(path.join(destination,'src/main.mjs'));
assert.throws(()=>unpackVerifiedSource(archive,destination),/EEXIST/);assert.equal(fileDigest(path.join(destination,'src/main.mjs')),original);
fs.appendFileSync(archive.path,'tampered');const neverRun=path.join(root,'never-executed');fs.mkdirSync(neverRun);assert.throws(()=>unpackVerifiedSource(archive,neverRun),/Source ZIP SHA changed/);assert.deepEqual(fs.readdirSync(neverRun),[]);
});
test('declared replay inputs verify without historical archives and reject altered bytes, missing inputs or escaping paths',t=>{
assert.equal(verifyReplayFixtures(workspace).passed,true);
const root=owned(t),directory=path.join(root,'tests/fixtures/ci-replay');fs.mkdirSync(directory,{recursive:true});
const name='owned.json',file=path.join(directory,name),original=Buffer.from(JSON.stringify({schemaVersion:1,value:'original'})+'\n');fs.writeFileSync(file,original);
const index={schemaVersion:1,sourceCommit:'a'.repeat(40),files:[{file:name,bytes:original.length,sha256:fileDigest(file),originalSources:[{sha256:'b'.repeat(64),gitBlob:'c'.repeat(40)}]}]};
const save=()=>fs.writeFileSync(path.join(directory,'INDEX.json'),JSON.stringify(index));save();
assert.equal(replayFixtureJson(name,root).value,'original');assert.throws(()=>replayFixtureJson('undeclared.json',root),/Undeclared/);
fs.writeFileSync(file,original.toString().replace('original','modified'));assert.throws(()=>verifyReplayFixtures(root),/SHA changed/);
fs.writeFileSync(file,original);fs.unlinkSync(file);assert.throws(()=>verifyReplayFixtures(root),/ENOENT/);
index.files[0].file='../outside.json';save();assert.throws(()=>verifyReplayFixtures(root),/Invalid replay fixture path/);
});