Files
Agent-Workbench/scripts/verify-credential-persistence.mjs
T

233 lines
18 KiB
JavaScript

// New synthetic credentials only. Never discovers the production host or a user Key.
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
import { DatabaseSync } from 'node:sqlite';
import { chromium, _electron } from 'playwright';
import { expect } from 'playwright/test';
import { fixture, compatibleFixture, until } from '../tests/helpers.mjs';
const args = process.argv.slice(2);
if (args.includes('--help')) {
console.log('node scripts/verify-credential-persistence.mjs --output NEW_DIRECTORY [--app-root DIRECTORY] [--desktop] — synthetic Windows DPAPI, three actual UI entry paths, process restart and local authentication.');
process.exit(0);
}
const options = {};
while (args.length) {
const flag = args.shift();
if (flag === '--desktop') { options.desktop = true; continue; }
if (!['--output', '--app-root'].includes(flag) || !args.length || options[flag]) throw new Error('Use --help.');
options[flag] = args.shift();
}
assert.equal(process.platform, 'win32');
assert.ok(options['--output']);
const workspace = process.cwd(), appRoot = path.resolve(options['--app-root'] ?? workspace), output = path.resolve(options['--output']);
assert.ok(output.startsWith(path.join(workspace, '.qa') + path.sep));
fs.mkdirSync(output, { recursive: false });
const nodePath = fs.existsSync(path.join(appRoot, 'runtime/node.exe')) ? path.join(appRoot, 'runtime/node.exe') : process.execPath;
const disposers = [], t = { after: dispose => disposers.push(dispose) };
const f = fixture(t, { modernDefaults: true }); f.store.close();
const keys = Object.fromEntries(['manual', 'temporary', 'current', 'catalog', 'import', 'legacy'].map(name => [name, 'owned-restart-' + name + '-' + randomUUID()]));
const marks = [], screenshots = [], starts = [], exits = [];
const sha = value => createHash('sha256').update(value).digest('hex');
const mark = step => { marks.push({ step, passed: true }); console.log(JSON.stringify({ step, passed: true })); };
const write = (name, value) => fs.writeFileSync(path.join(output, name), JSON.stringify(value, null, 2));
let browser, electron, report;
const mock = await compatibleFixture(t, (url, input, requests) => {
const auth = requests.at(-1).headers.authorization;
if (!Object.values(keys).some(key => auth === 'Bearer ' + key)) return { status: 401, body: { error: { message: 'Synthetic receiver did not accept the restored credential.' } } };
if (url === '/v1/models') return { data: ['owned-catalog-one', 'owned-catalog-two'].map(id => ({ id, capabilities: { text: true, tools: true } })) };
assert.equal(url, '/v1/chat/completions'); assert.ok(input);
return { model: input.model, choices: [{ message: { role: 'assistant', content: 'Owned restart authentication verified.' }, finish_reason: 'stop' }], usage: { prompt_tokens: 12, completion_tokens: 8, total_tokens: 20 } };
});
assert.equal(new URL(mock.baseUrl).hostname, '127.0.0.1');
async function start(label) {
const env = { ...process.env, AGENT_WORKBENCH_APP_DIR: appRoot, AGENT_WORKBENCH_DATA_DIR: f.data, AGENT_WORKBENCH_PORT: '0' };
delete env.AGENT_WORKBENCH_API_KEY;
const child = spawn(nodePath, [path.join(appRoot, 'dist-server/server.mjs')], { cwd: appRoot, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
let stdout = '', stderr = '', url = '';
child.stdout.on('data', bytes => {
stdout += bytes.toString('utf8'); fs.writeFileSync(path.join(output, label + '-stdout.log'), stdout);
for (const line of stdout.split('\n')) { try { const value = JSON.parse(line); if (value.ready) url = value.url; } catch {} }
});
child.stderr.on('data', bytes => { stderr += bytes.toString('utf8'); fs.writeFileSync(path.join(output, label + '-stderr.log'), stderr); });
disposers.push(async () => { if (child.exitCode === null && child.signalCode === null) child.kill(); });
await until(() => url || child.exitCode !== null, 60000);
assert.ok(url, 'Owned host did not start; inspect its retained logs.'); assert.notEqual(new URL(url).port, '4318');
const bootstrap = await (await fetch(url + '/api/bootstrap')).json();
assert.equal(bootstrap.version, '1.25.14');
const api = async (route, input) => {
const response = await fetch(url + '/api' + route, { method: input === undefined ? 'GET' : 'POST', headers: {
'Content-Type': 'application/json', 'X-Workbench-Token': bootstrap.token, Connection: 'close' },
...(input === undefined ? {} : { body: JSON.stringify(input) }), signal: AbortSignal.timeout(30000) });
const value = await response.json(); if (!response.ok) throw Object.assign(new Error(value.error), { status: response.status }); return value;
};
const status = await api('/host/status');
assert.equal(status.identity.pid, child.pid); assert.equal(path.resolve(status.identity.dataRoot), path.resolve(f.data));
starts.push({ label, pid: child.pid, url, identity: status.identity });
const close = async () => {
if (child.exitCode !== null || child.signalCode !== null) return;
await api('/shutdown', {}); await until(() => child.exitCode !== null || child.signalCode !== null, 10000);
exits.push({ label, pid: child.pid, exitCode: child.exitCode, signalCode: child.signalCode });
assert.equal(child.exitCode, 0); assert.equal(child.signalCode, null);
};
disposers.push(close);
return { url, api, bootstrap, child, close };
}
async function openModels(page) {
await page.getByRole('button', { name: '选择与管理模型', exact: true }).click();
const dialog = page.getByRole('dialog', { name: '模型中心', exact: true }); await expect(dialog).toBeVisible(); return dialog;
}
async function picture(page, name) {
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
assert.equal(await page.evaluate(() => document.body.scrollWidth <= window.innerWidth), true);
const filename = name + '.png'; await page.screenshot({ path: path.join(output, filename) });
screenshots.push({ filename, sha256: sha(fs.readFileSync(path.join(output, filename))), viewport: page.viewportSize() });
}
async function manual(dialog, name, key, temporary = false) {
await dialog.getByRole('button', { name: '手动填写(兼容)', exact: true }).click();
const editor = dialog.getByRole('form', { name: '编辑模型连接', exact: true });
await expect(editor.getByRole('checkbox', { name: '加密保存到本机,重启后自动恢复', exact: true })).toBeChecked();
await editor.getByLabel('连接名称', { exact: true }).fill(name);
await editor.getByLabel('接口根地址', { exact: true }).fill(mock.baseUrl);
await editor.getByLabel('请求模型名称', { exact: true }).fill(name);
await editor.getByLabel('API Key', { exact: true }).fill(key);
if (temporary) await editor.getByRole('checkbox', { name: '加密保存到本机,重启后自动恢复', exact: true }).uncheck();
await editor.getByRole('button', { name: '保存模型连接', exact: true }).click();
const card = dialog.locator('.mc-connection').filter({ hasText: name });
await expect(card).toContainText(temporary ? '仅本次使用' : '重启自动恢复'); return card;
}
function vaultSnapshot() {
const database = new DatabaseSync(path.join(f.data, 'workbench.sqlite'), { readOnly: true });
try {
const row = database.prepare("SELECT value FROM metadata WHERE key = 'modelCredentials'").get();
const state = JSON.parse(row.value);
return { sha256: sha(row.value), entries: Object.keys(state.entries), plaintextAbsent: Object.values(keys).every(key => !row.value.includes(key)) };
} finally { database.close(); }
}
try {
const first = await start('before-restart');
browser = await chromium.launch({ channel: 'msedge', headless: true });
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
const errors = []; page.on('pageerror', error => errors.push(error.message));
await page.goto(first.url); await page.waitForLoadState('networkidle');
const dialog = await openModels(page);
const card = await manual(dialog, 'owned-manual', keys.manual); mark('manual form defaults to encrypted restart recovery');
await card.getByRole('button', { name: '编辑', exact: true }).click();
const editor = dialog.getByRole('form', { name: '编辑模型连接', exact: true });
await expect(editor.getByLabel('API Key', { exact: true })).toHaveValue('');
await expect(editor.getByRole('button', { name: '保存模型连接', exact: true })).toBeEnabled();
await editor.getByLabel('连接名称', { exact: true }).fill('owned-manual-edited');
await editor.getByRole('button', { name: '保存模型连接', exact: true }).click();
await expect(dialog.locator('.mc-connection').filter({ hasText: 'owned-manual-edited' })).toContainText('重启自动恢复');
mark('editing without a new key preserves the saved credential and permits submission');
await manual(dialog, 'owned-temporary', keys.temporary, true); mark('explicit temporary choice is reflected accurately');
const currentCard = await manual(dialog, 'owned-current', keys.current, true);
const currentBefore = (await first.api('/model-center')).connections.find(value => value.name === 'owned-current');
await currentCard.getByRole('button', { name: '保存当前 Key', exact: true }).click();
await expect(currentCard).toContainText('重启自动恢复');
const currentAfter = (await first.api('/model-center')).connections.find(value => value.name === 'owned-current');
assert.equal(currentAfter.credentialRevision, currentBefore.credentialRevision); assert.equal(currentAfter.revision, currentBefore.revision);
mark('one-click current-key save needs no key input and keeps its credential and connection revisions');
await dialog.getByRole('button', { name: '从上游添加模型', exact: true }).click();
const catalog = dialog.getByRole('region', { name: '上游模型目录', exact: true });
await catalog.getByLabel('上游名称', { exact: true }).fill('owned-catalog');
await catalog.getByLabel('目录接口根地址', { exact: true }).fill(mock.baseUrl);
await catalog.getByLabel('目录 API Key', { exact: true }).fill(keys.catalog);
await expect(catalog.getByRole('checkbox', { name: '保存 Key 到本机,重启后自动恢复', exact: true })).toBeChecked();
await catalog.getByRole('button', { name: '获取上游模型', exact: true }).click();
for (const id of ['owned-catalog-one', 'owned-catalog-two']) await catalog.getByLabel('选择模型 ' + id, { exact: true }).check();
await catalog.getByLabel('模型使用方式', { exact: true }).selectOption('auto');
await catalog.getByRole('button', { name: '保存所选 2 个模型', exact: true }).click();
await expect(dialog.locator('.mc-connection').filter({ hasText: 'owned-catalog-two' })).toContainText('重启自动恢复');
await dialog.locator('.mc-connection').filter({ hasText: 'owned-catalog-two' }).scrollIntoViewIfNeeded();
await picture(page, 'web-desktop-saved'); mark('actual upstream selection creates two models sharing one saved key');
const importFile = path.join(output, 'synthetic-import.private.json');
fs.writeFileSync(importFile, JSON.stringify({ schema: 1, connections: [{ name: 'owned-import', model: 'owned-import', baseUrl: mock.baseUrl, apiKey: keys.import }] }));
await dialog.getByRole('button', { name: '配置导入', exact: true }).click();
await dialog.getByLabel('配置格式').selectOption('workbench-json');
await dialog.getByLabel('本地文件路径', { exact: true }).fill(importFile);
await dialog.getByRole('button', { name: '预览可导入连接', exact: true }).click();
await expect(dialog.getByRole('checkbox', { name: '本机加密保存导入的 Key,重启后自动恢复', exact: true })).toBeChecked();
await dialog.getByRole('button', { name: '导入这些连接', exact: true }).click();
await expect(dialog.getByRole('status')).toContainText('重启后自动恢复'); mark('confirmed import defaults to encrypted persistence');
await first.api('/config', { ...first.bootstrap.config, mode: 'compatible', model: 'owned-legacy', baseUrl: mock.baseUrl, apiKey: keys.legacy });
const center = await first.api('/model-center');
assert.equal(center.attemptCount, 0); assert.equal(center.connections.filter(value => value.hasKey).length, 7);
const saved = vaultSnapshot(); assert.equal(saved.entries.length, 5); assert.equal(saved.plaintextAbsent, true);
write('before-restart.json', { saved, connections: center.connections.map(({ id, name, hasKey, savedCredential, credentialSource }) => ({ id, name, hasKey, savedCredential, credentialSource })) });
assert.equal(Object.values(keys).some(key => JSON.stringify(center).includes(key)), false);
await page.goto('about:blank'); await first.close();
const second = await start('after-restart'); assert.notEqual(second.child.pid, first.child.pid);
const restored = await second.api('/model-center');
assert.equal(restored.attemptCount, 0); assert.equal(restored.connections.filter(value => value.hasKey).length, 6);
assert.equal(restored.connections.find(value => value.name === 'owned-temporary').hasKey, false);
assert.equal(restored.connections.filter(value => value.name !== 'owned-temporary').every(value => value.hasKey), true);
assert.equal(vaultSnapshot().sha256, saved.sha256);
mark('new process restores saved manual, catalog, import and legacy credentials, while temporary key remains absent');
const manualConnection = restored.connections.find(value => value.name === 'owned-manual-edited');
assert.equal((await second.api('/model-center/probe', { id: manualConnection.id, scope: 'text' })).ok, true);
assert.equal(mock.requests.filter(value => value.url === '/v1/chat/completions').length, 1);
assert.equal(mock.requests.at(-1).headers.authorization, 'Bearer ' + keys.manual);
mark('restored key authenticates one synthetic local model request without re-entry');
await page.goto(second.url); await page.waitForLoadState('networkidle');
const freshDialog = await openModels(page);
await expect(freshDialog.locator('.mc-connection').filter({ hasText: 'owned-manual-edited' })).toContainText('重启自动恢复');
await page.setViewportSize({ width: 390, height: 844 });
await freshDialog.locator('.mc-connection').filter({ hasText: 'owned-manual-edited' }).scrollIntoViewIfNeeded();
await picture(page, 'web-mobile-restored');
await freshDialog.getByRole('button', { name: '手动填写(兼容)', exact: true }).click();
const invalid = freshDialog.getByRole('form', { name: '编辑模型连接', exact: true });
await invalid.getByLabel('连接名称', { exact: true }).fill('owned-invalid');
await invalid.getByLabel('请求模型名称', { exact: true }).fill('owned-invalid');
await invalid.getByLabel('接口根地址', { exact: true }).fill('ftp://127.0.0.1/invalid');
await invalid.getByLabel('API Key', { exact: true }).fill(keys.manual);
await invalid.getByRole('button', { name: '保存模型连接', exact: true }).click();
await expect(freshDialog.getByRole('status')).toContainText('模型地址只支持');
await expect(invalid.getByLabel('API Key', { exact: true })).toHaveValue(keys.manual);
await invalid.getByLabel('API Key', { exact: true }).scrollIntoViewIfNeeded();
await picture(page, 'web-mobile-save-error'); mark('mobile save failure preserves masked key input and gives an actionable error');
assert.deepEqual(errors, []);
assert.equal(Object.values(keys).some(key => JSON.stringify(restored).includes(key)), false);
if (options.desktop) {
const executable = path.join(appRoot, 'desktop/AgentWorkbench.exe'); assert.ok(fs.existsSync(executable));
const cleanEnv = { ...process.env }; delete cleanEnv.AGENT_WORKBENCH_API_KEY;
electron = await _electron.launch({ executablePath: executable, args: [path.join(appRoot, 'dist-desktop/main.cjs'), '--hidden-window', '--app-root', appRoot, '--data-root', f.data, '--node', nodePath, '--port', new URL(second.url).port], cwd: appRoot, env: cleanEnv, timeout: 60000 });
const desktop = await electron.firstWindow(); await desktop.waitForLoadState('networkidle');
const nativeDialog = await openModels(desktop);
await expect(nativeDialog.locator('.mc-connection').filter({ hasText: 'owned-manual-edited' })).toContainText('重启自动恢复');
const visible = await electron.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().some(window => window.isVisible())); assert.equal(visible, false);
await desktop.setViewportSize({ width: 1440, height: 1000 });
await nativeDialog.locator('.mc-connection').filter({ hasText: 'owned-manual-edited' }).scrollIntoViewIfNeeded();
await picture(desktop, 'electron-desktop-restored');
await desktop.setViewportSize({ width: 390, height: 844 });
await nativeDialog.locator('.mc-connection').filter({ hasText: 'owned-manual-edited' }).scrollIntoViewIfNeeded();
await picture(desktop, 'electron-mobile-restored');
await electron.close(); electron = null;
assert.equal((await second.api('/host/status')).identity.pid, second.child.pid); mark('actual packaged desktop shows restored credentials at both widths and closing it preserves the host');
}
const final = await second.api('/model-center');
assert.equal(final.attemptCount, 1);
report = { status: 'passed', version: '1.25.14', appRoot, starts, exits, assertions: marks, screenshots,
savedCiphertextUnchangedAcrossRestart: true, plaintextAbsent: saved.plaintextAbsent, restoredSavedConnections: 6,
syntheticModelRequests: 1, realProviderRequests: 0, localCatalogRequests: mock.requests.filter(value => value.url === '/v1/models').length,
temporaryCredentialRestored: false, pageErrors: errors };
} catch (error) {
write('failure.json', { status: 'failed', message: error.message, stack: error.stack, starts, exits, assertions: marks, screenshots });
throw error;
} finally {
await electron?.close(); await browser?.close();
for (const dispose of disposers.reverse()) await dispose();
}
if (report) write('result.json', { ...report, exits, ownedHostsExited: exits.length === starts.length });