Files
Agent-Workbench/docs/network-q-evidence/source-bytes/src/desktop/browsing-view.mjs
T
2026-11-03 23:53:31 +08:00

198 lines
16 KiB
JavaScript

import { WebContentsView, session } from 'electron';
import { lookup } from 'node:dns/promises';
import { isIP } from 'node:net';
import { assertDesktopFrame } from '../shared/contracts/desktop-bridge.mjs';
import { browsingUrl } from '../shared/contracts/browsing.mjs';
// This function runs only in an isolated page world. No renderer-supplied JS,
// selectors, CDP methods or Node APIs cross this boundary.
function pageOperation(operation, input) {
const visible = element => { const style = getComputedStyle(element), rect = element.getBoundingClientRect(); return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden'; };
let state = globalThis.__awbPageObservation;
if (!state || state.document !== document) {
state = { document, epoch: crypto.randomUUID(), revision: 0, elements: new Map() };
state.observer = new MutationObserver(() => state.revision++);
state.observer.observe(document, { subtree: true, attributes: true, childList: true, characterData: true });
globalThis.__awbPageObservation = state;
}
if (state.observer.takeRecords().length) state.revision++;
const revision = () => state.epoch + ':' + state.revision;
if (operation === 'revision') return { revision: revision(), url: location.href };
if (operation === 'act') {
if (revision() !== input.revision) return { stale: true, reason: '页面在原观察后变化,请重新观察。' };
if (input.action === 'scroll') { scrollBy(0, input.amount); return { performed: true }; }
if (!['click','fill'].includes(input.action)) return { performed: false };
const element = state.elements.get(input.ref);
if (!element || !element.isConnected || !visible(element) || element.disabled) return { stale: true, reason: '原可见控件已失效。' };
if (input.action === 'click') { element.click(); return { performed: true }; }
if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) || element.readOnly
|| element instanceof HTMLInputElement && !['text','search','email','tel','url','number'].includes(element.type)) return { stale: true, reason: '只支持普通文本字段,密码、文件和受限控件不通过代理输入。' };
const prototype = element instanceof HTMLInputElement ? HTMLInputElement.prototype : HTMLTextAreaElement.prototype;
Object.getOwnPropertyDescriptor(prototype, 'value').set.call(element, input.value);
element.dispatchEvent(new Event('input', { bubbles: true })); element.dispatchEvent(new Event('change', { bubbles: true })); return { performed: true };
}
state.elements.clear();
const candidates = document.querySelectorAll('a[href],button,input,textarea,select,[role="button"],[role="link"]'), controls = []; let visited = 0;
for (const element of candidates) {
if (++visited > 6000 || controls.length === 120) break;
if (!visible(element) || element.type === 'hidden') continue;
const ref = 'e' + (controls.length + 1); state.elements.set(ref, element);
const label = element.labels?.[0]?.innerText || element.getAttribute('aria-label') || element.innerText || element.getAttribute('placeholder') || element.getAttribute('name') || element.tagName;
controls.push({ ref, tag: element.tagName.toLowerCase(), type: element.type || '', label: label.slice(0, 180), disabled: !!element.disabled,
...(element instanceof HTMLAnchorElement ? { href: element.href.slice(0, 2000) } : {}),
...(element instanceof HTMLInputElement && ['password','file'].includes(element.type) ? { restricted: true } : {}) });
}
const text = document.body?.innerText ?? '';
return { revision: revision(), url: location.href, title: document.title.slice(0, 300), text: text.slice(0, 16000), controls,
scope: { textCharacters: text.length, controlsSeen: candidates.length, complete: text.length <= 16000 && candidates.length <= 6000 && controls.length < 120 }, sampledAt: new Date().toISOString() };
}
const publicAddress = address => {
if (isIP(address) === 4) { const [a,b] = address.split('.').map(Number); return !(a === 0 || a === 10 || a === 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a >= 224 || a === 100 && b >= 64 && b <= 127); }
return isIP(address) === 6 && !/^(::|fe[89ab]|f[cd]|::ffff:)/i.test(address);
};
export class BrowsingView {
constructor({ window, controller, windowOwner }) {
Object.assign(this, { window, controller, windowOwner }); this.views = new Map(); this.partitions = new Map(); this.current = null; this.disposed = false;
for (const event of ['minimize','resize','blur']) window.on(event, () => this.hide());
window.webContents.on('did-start-loading', () => this.hide());
void this.loop();
}
async loop() {
while (!this.disposed && !this.window.isDestroyed()) {
try {
const states = [...this.views.values()].map(value => ({ id: value.tab.id, url: value.view.webContents.getURL(), title: value.view.webContents.getTitle() }));
const next = await this.controller.api('/browsing-desktop/poll', { method: 'POST', input: { owner: this.windowOwner, states } });
for (const [id] of this.views) if (!next.retainedTabs.includes(id) && !next.commands.some(command => command.tab.id === id && ['close','clear'].includes(command.kind))) this.destroy(id);
for (const command of next.commands) {
if (this.disposed) break;
let reply;
try {
await this.controller.api('/browsing-desktop/validate', { method: 'POST', input: { owner: this.windowOwner, id: command.id, nonce: command.nonce } });
reply = { result: await this.execute(command) };
} catch (error) { reply = { error: error.message, uncertain: command.kind === 'act' && error.noEffect !== true }; }
await this.controller.api('/browsing-desktop/complete', { method: 'POST', input: { owner: this.windowOwner, id: command.id, nonce: command.nonce, ...reply }, timeoutMs: 15000 });
}
} catch { this.hide(); }
await new Promise(resolve => setTimeout(resolve, 350));
}
}
policy(tab, value) { return browsingUrl(value, { ...tab.scope, workbenchOrigin: this.controller.url }); }
async permitted(tab, value) {
if (value === 'about:blank') return false;
// Vite HMR uses a WebSocket on the same explicitly owned origin.
try { const resource = new URL(value); if (resource.protocol === 'ws:' || resource.protocol === 'wss:') { resource.protocol = resource.protocol === 'ws:' ? 'http:' : 'https:'; value = resource.href; } } catch { return false; }
let url; try { url = this.policy(tab, value); } catch { return false; }
if (url.origin === tab.previewOrigin || tab.scope.allowHttpLoopback && url.hostname === '127.0.0.1') return true;
try { const addresses = await lookup(url.hostname, { all: true }); return addresses.length > 0 && addresses.every(item => publicAddress(item.address)); } catch { return false; }
}
async create(tab) {
if (this.views.has(tab.id)) throw new Error('原标签页已打开,不能重放打开。');
let partition = this.partitions.get(tab.profileId);
if (!partition) {
const partitionName = (tab.persistent ? 'persist:' : '') + 'awb-browsing-' + tab.profileId;
partition = session.fromPartition(partitionName, { cache: true }); await partition.setProxy({ mode: 'direct' });
partition.setPermissionRequestHandler((_contents, _permission, callback) => callback(false)); partition.setPermissionCheckHandler(() => false);
partition.on('will-download', (event, item) => { event.preventDefault(); item.cancel(); });
partition.webRequest.onBeforeRequest((details, callback) => {
// All users of one profile have exactly the same authorized origins.
void this.permitted(tab, details.url).then(allowed => callback({ cancel: !allowed }), () => callback({ cancel: true }));
});
this.partitions.set(tab.profileId, partition);
}
const view = new WebContentsView({ webPreferences: { session: partition, sandbox: true, contextIsolation: true, nodeIntegration: false,
webSecurity: true, allowRunningInsecureContent: false, navigateOnDragDrop: false, spellcheck: false, backgroundThrottling: false } });
const value = { tab, view, history: [], historyIndex: -1, historyTarget: null }; this.views.set(tab.id, value); view.setBounds({ x: 0, y: 0, width: 800, height: 600 }); view.setVisible(false);
this.window.contentView.addChildView(view);
const contents = view.webContents; contents.setWindowOpenHandler(() => ({ action: 'deny' }));
for (const name of ['did-navigate','did-navigate-in-page']) contents.on(name, () => this.recordNavigation(value));
contents.on('will-attach-webview', event => event.preventDefault());
for (const name of ['will-navigate','will-frame-navigate','will-redirect']) contents.on(name, (event, url) => {
try { this.policy(tab, typeof url === 'string' ? url : event.url); } catch { event.preventDefault(); }
});
contents.on('render-process-gone', () => { this.hide(); this.destroy(tab.id); });
return value;
}
async script(contents, operation, input = {}) {
return contents.executeJavaScriptInIsolatedWorld(999, [{ code: '(' + pageOperation.toString() + ')(' + JSON.stringify(operation) + ',' + JSON.stringify(input) + ')' }]);
}
async load(contents, url) {
let timer; try { await Promise.race([contents.loadURL(url), new Promise((_,reject) => { timer = setTimeout(() => { contents.stop(); reject(new Error('所属页面加载超过15秒,请核对原网址和请求。')); },15000); })]); }
finally { clearTimeout(timer); }
}
async observe(value) {
const contents = value.view.webContents;
this.recordNavigation(value);
const observation = await this.script(contents, 'observe'); this.policy(value.tab, observation.url);
const png = (await contents.capturePage()).toPNG(); if (png.length > 6 * 1024 * 1024) throw new Error('页面截图超过6MiB上限,请缩小浏览区。');
const after = await this.script(contents, 'revision');
if (after.revision !== observation.revision || after.url !== observation.url) throw Object.assign(new Error('页面在观察/截图期间变化,请重新观察。'), { noEffect: true });
observation.navigation = { mode: 'owned-url-history', canGoBack: value.historyIndex > 0, canGoForward: value.historyIndex < value.history.length - 1 };
return { observation, png: png.toString('base64') };
}
recordNavigation(value) {
const url = value.view.webContents.getURL(); if (!url || url === 'about:blank') return;
this.policy(value.tab,url);
if (value.historyTarget !== null) { value.historyIndex = value.historyTarget; value.history[value.historyTarget] = url; return; }
if (value.history[value.historyIndex] === url) return;
value.history.splice(value.historyIndex + 1); value.history.push(url); if (value.history.length > 100) value.history.shift(); value.historyIndex = value.history.length - 1;
}
async execute(command) {
const { tab, kind, input } = command;
if (kind === 'clear') {
for (const [id, value] of this.views) if (value.tab.profileId === tab.profileId) this.destroy(id);
let partition = this.partitions.get(tab.profileId);
if (!partition) partition = session.fromPartition((tab.persistent ? 'persist:' : '') + 'awb-browsing-' + tab.profileId);
await partition.clearStorageData(); await partition.clearCache(); this.partitions.delete(tab.profileId); return { cleared: true };
}
if (kind === 'close') { this.destroy(tab.id); if (!tab.persistent) { const partition = this.partitions.get(tab.profileId); if (partition) { await partition.clearStorageData(); await partition.clearCache(); this.partitions.delete(tab.profileId); } } return { closed: true }; }
let value = this.views.get(tab.id);
if (kind === 'open') { this.policy(tab, input.url); value = await this.create(tab); await this.load(value.view.webContents,input.url); return this.observe(value); }
if (!value || value.view.webContents.isDestroyed()) throw Object.assign(new Error('原桌面标签页已结束,请明确打开新页面。'), { noEffect: true });
if (value.tab.grantRevision !== tab.grantRevision) throw Object.assign(new Error('原授权已变化。'), { noEffect: true });
if (kind === 'observe') return this.observe(value);
const contents = value.view.webContents, current = await this.script(contents, 'revision');
if (current.revision !== input.revision || current.url !== tab.url) throw Object.assign(new Error('页面在原观察后变化,请重新观察。'), { noEffect: true });
if (['click','fill','scroll'].includes(input.action)) {
const result = await this.script(contents, 'act', input); if (result.stale) throw Object.assign(new Error(result.reason), { noEffect: true });
} else if (input.action === 'navigate') { this.policy(tab, input.url); await this.load(contents,input.url); }
else if (input.action === 'reload') { await this.load(contents,current.url); }
else if (['back','forward'].includes(input.action)) {
const target = value.historyIndex + (input.action === 'back' ? -1 : 1);
if (target < 0 || target >= value.history.length) throw Object.assign(new Error('没有对应的页面历史。'), { noEffect: true });
const url = value.history[target]; this.policy(tab,url); value.historyTarget = target;
try { await this.load(contents,url); } finally { value.historyTarget = null; }
} else if (input.action === 'wait') await new Promise(resolve => setTimeout(resolve, input.milliseconds));
else if (input.action === 'download') {
this.policy(tab, input.url);
const response = await contents.session.fetch(input.url, { redirect: 'error', signal: AbortSignal.timeout(10000) });
if (!response.ok) throw new Error('下载返回HTTP ' + response.status);
let length = 0; const chunks = []; for await (const chunk of response.body) { length += chunk.length; if (length > 6 * 1024 * 1024) throw new Error('下载超过6MiB上限。'); chunks.push(Buffer.from(chunk)); }
return { download: { url: input.url, bytes: Buffer.concat(chunks).toString('base64'), mediaType: response.headers.get('content-type') } };
}
// Let navigation and framework input handlers finish a bounded turn. This
// is observation after one action, never an automatic second action.
await new Promise(resolve => setTimeout(resolve, 150));
return this.observe(value);
}
layout(event, input) {
assertDesktopFrame(event, this.window, this.controller.url);
const reject = message => { this.hide(); throw new Error(message); };
const fields = ['projectId','sessionId','taskId','tabId','bounds','visible'];
if (!input || Array.isArray(input) || Object.keys(input).some(key => !fields.includes(key)) || typeof input.visible !== 'boolean') reject('浏览区域字段无效。');
if (!input.visible) { this.hide(); return { visible: false }; }
const value = this.views.get(input.tabId), zoom = this.window.webContents.getZoomFactor(), available = this.window.getContentBounds();
const bounds = input.bounds && Object.fromEntries(Object.entries(input.bounds).map(([key,value]) => [key, Math.round(value * zoom)]));
if (!value || value.tab.projectId !== input.projectId || value.tab.sessionId !== input.sessionId || value.tab.taskId !== input.taskId) reject('原浏览区域归属已变化。');
if (!bounds || Object.keys(bounds).sort().join() !== 'height,width,x,y' || Object.values(bounds).some(number => !Number.isSafeInteger(number))
|| bounds.x < 0 || bounds.y < 0 || bounds.width < 80 || bounds.height < 80 || bounds.x + bounds.width > available.width + 1 || bounds.y + bounds.height > available.height + 1) reject('原生浏览区域超过当前窗口。');
if (this.window.isMinimized() || !this.window.isVisible()) { this.hide(); return { visible: false }; }
if (this.current !== input.tabId) { this.hide(); this.window.contentView.addChildView(value.view); this.current = input.tabId; }
value.view.setBounds(bounds); value.view.setVisible(true); return { visible: true };
}
hide() { if (this.current) { const value = this.views.get(this.current); if (value) { value.view.setVisible(false); this.window.contentView.removeChildView(value.view); } this.current = null; } }
destroy(id) { if (this.current === id) this.hide(); const value = this.views.get(id); this.views.delete(id); if (value) { this.window.contentView.removeChildView(value.view); if (!value.view.webContents.isDestroyed()) value.view.webContents.close(); } }
dispose() { this.disposed = true; this.hide(); for (const id of [...this.views.keys()]) this.destroy(id); }
}