feat: add project index and exact patch review

This commit is contained in:
Agent Workbench Dev
2026-10-29 17:59:35 +08:00
parent 5ec85b0319
commit 14ce622028
45 changed files with 1290 additions and 82 deletions
+228
View File
@@ -0,0 +1,228 @@
import path from 'node:path';
import { createHash } from 'node:crypto';
import { parse } from '@babel/parser';
import { inspectProject } from '../project-inspector.mjs';
import { readText } from '../../server/files.mjs';
const hash = value => createHash('sha256').update(value).digest('hex');
const conflict = message => Object.assign(new Error(message), { status: 409 });
const js = /\.(?:[mc]?js|jsx|[mc]?ts|tsx)$/i;
const functionTypes = new Set(['FunctionExpression', 'ArrowFunctionExpression']);
const names = { FunctionDeclaration: 'function', FunctionExpression: 'function', TSDeclareFunction: 'function', ClassDeclaration: 'class', ClassExpression: 'class', TSTypeAliasDeclaration: 'type', TSInterfaceDeclaration: 'interface', TSEnumDeclaration: 'enum' };
const parserVersion = 'babel-parser-7.29.11';
const nodeName = node => node?.type === 'Identifier' ? node.name : node?.type === 'PrivateName' ? '#' + node.id.name : node?.type === 'StringLiteral' ? node.value : null;
function javascript(source, filename, maximumNodes, maximumSymbols) {
const plugins = [...(/\.(?:[mc]?ts|tsx)$/i.test(filename) ? ['typescript'] : []), ...(/\.(?:[mc]?js|jsx|tsx)$/i.test(filename) ? ['jsx'] : []), 'decorators-legacy'];
const ast = parse(source, { sourceType: 'unambiguous', plugins, createImportExpressions: true, allowReturnOutsideFunction: true, allowAwaitOutsideFunction: true });
const exported = new Set(), exportNames = new Set(), topLevel = new Set();
for (const entry of ast.program.body) {
const declaration = entry.declaration ?? entry;
const declarations = declaration.type === 'VariableDeclaration' ? declaration.declarations : [declaration];
for (const node of declarations) {
topLevel.add(node);
if (entry.type.startsWith('Export')) exported.add(node);
}
if (entry.type === 'ExportNamedDeclaration') for (const specifier of entry.specifiers) if (specifier.local?.name) exportNames.add(specifier.local.name);
if (entry.type === 'ExportDefaultDeclaration' && declaration.type === 'Identifier') exportNames.add(declaration.name);
}
const symbols = [], imports = [], stack = [{ node: ast.program, parent: null, scope: [] }];
let visited = 0, truncated = false, opaqueImports = 0;
while (stack.length) {
const { node, parent, scope } = stack.pop();
if (++visited > maximumNodes) { truncated = true; break; }
let name = nodeName(node.id), kind = names[node.type] ?? null;
if (functionTypes.has(node.type) && parent?.type === 'ExportDefaultDeclaration') kind = 'function';
if (!name && kind && parent?.type === 'ExportDefaultDeclaration') name = 'default';
if (node.type === 'VariableDeclarator' && node.id.type === 'Identifier') {
name = node.id.name;
kind = functionTypes.has(node.init?.type) ? 'function' : topLevel.has(node) ? 'binding' : null;
}
if (['ClassMethod', 'ClassPrivateMethod', 'ObjectMethod', 'TSMethodSignature'].includes(node.type)
|| ['ClassProperty', 'ClassPrivateProperty'].includes(node.type) && functionTypes.has(node.value?.type)) {
name = !node.computed ? nodeName(node.key) : null; kind = name ? 'method' : null;
}
if (name && kind) {
const cut = node.body?.start ?? node.init?.body?.start ?? node.value?.body?.start ?? node.id?.end ?? node.key?.end ?? node.end;
const signature = source.slice(node.start, Math.min(cut, node.start + 240)).replace(/\s+/g, ' ').trim();
if (symbols.length < maximumSymbols) symbols.push({ name, qualifiedName: [...scope, name].join('.'), kind, line: node.loc.start.line,
column: node.loc.start.column + 1, endLine: node.loc.end.line, signature, exported: exported.has(node) || topLevel.has(node) && exportNames.has(name), evidenceMethod: 'javascript-typescript-ast' });
else truncated = true;
}
let sourceNode, importKind;
if (node.type === 'ImportDeclaration') { sourceNode = node.source; importKind = node.importKind === 'type' ? 'type-import' : 'import'; }
else if (['ExportNamedDeclaration', 'ExportAllDeclaration'].includes(node.type) && node.source) { sourceNode = node.source; importKind = 're-export'; }
else if (node.type === 'ImportExpression') { sourceNode = node.source; importKind = 'literal-dynamic-import'; }
else if (node.type === 'CallExpression' && node.callee?.type === 'Identifier' && node.callee.name === 'require') { sourceNode = node.arguments[0]; importKind = 'literal-require-call'; }
else if (node.type === 'TSImportEqualsDeclaration' && node.moduleReference?.type === 'TSExternalModuleReference') { sourceNode = node.moduleReference.expression; importKind = 'import-equals'; }
if (sourceNode?.type === 'StringLiteral' && imports.length < 400) imports.push({ source: sourceNode.value.slice(0, 500), line: node.loc.start.line,
kind: importKind, bindings: node.type === 'TSImportEqualsDeclaration' ? [nodeName(node.id)].filter(Boolean) : (node.specifiers ?? []).map(value => nodeName(value.local)).filter(Boolean).slice(0, 30), evidenceMethod: 'javascript-typescript-ast' });
else if (sourceNode) { truncated = true; if (sourceNode.type !== 'StringLiteral') opaqueImports++; }
let childScope = scope;
if (name && (names[node.type] && ['function', 'class'].includes(kind) || node.type === 'VariableDeclarator' && kind === 'function' || kind === 'method')) childScope = [...scope, name];
if (node.type === 'ObjectExpression' && parent?.type === 'VariableDeclarator' && parent.id.type === 'Identifier') childScope = [...scope, parent.id.name];
const children = Object.values(node).flatMap(value => Array.isArray(value) ? value : value && typeof value === 'object' ? [value] : []).filter(value => typeof value?.type === 'string');
for (let i = children.length - 1; i >= 0; i--) stack.push({ node: children[i], parent: node, scope: childScope });
}
return { symbols, imports, method: 'javascript-typescript-ast', truncated, opaqueImports, visitedNodes: visited };
}
// This lexer only removes comments and string bodies so declaration candidates
// do not come from docstrings. It is deliberately not a Python compiler.
function python(source, maximumSymbols) {
let triple = null, quote = null, escaped = false, truncated = false, bracketDepth = 0, lineContinuation = false;
const symbols = [], imports = [], scopes = [];
const lines = source.split('\n');
for (let index = 0; index < lines.length; index++) {
const line = lines[index], continued = bracketDepth > 0 || lineContinuation; let cleaned = '';
for (let i = 0; i < line.length;) {
const char = line[i], three = line.slice(i, i + 3);
if (triple) {
if (three === triple && !escaped) { triple = null; cleaned += ' '; i += 3; }
else { cleaned += ' '; escaped = char === '\\' && !escaped; i++; }
} else if (quote) {
cleaned += ' '; if (char === quote && !escaped) quote = null;
escaped = char === '\\' && !escaped; i++;
} else if (char === '#') break;
else if (three === '"""' || three === "'''") { triple = three; cleaned += ' '; escaped = false; i += 3; }
else if (char === '"' || char === "'") { quote = char; cleaned += ' '; escaped = false; i++; }
else { cleaned += char; i++; }
}
escaped = false;
if (!continued && cleaned.trim()) {
const indent = cleaned.match(/^\s*/)[0].replaceAll('\t', ' ').length;
while (scopes.length && scopes.at(-1).indent >= indent) scopes.pop();
}
const declaration = !continued && cleaned.match(/^(\s*)(?:async\s+)?(def|class)\s+([\p{ID_Start}_][\p{ID_Continue}]*)\s*(?:\(|:)/u);
if (declaration) {
const indent = declaration[1].replaceAll('\t', ' ').length;
const name = declaration[3];
if (symbols.length < maximumSymbols) symbols.push({ name, qualifiedName: [...scopes.map(value => value.name), name].join('.'), kind: declaration[2] === 'class' ? 'class' : 'function',
line: index + 1, column: declaration[1].length + 1, endLine: index + 1, signature: cleaned.trim().slice(0, 180), exported: false, evidenceMethod: 'python-declaration-candidate' });
else truncated = true;
scopes.push({ indent, name });
}
const imported = !continued && cleaned.match(/^\s*(?:from\s+([.\p{ID_Continue}]+)\s+import\s+(.+)|import\s+(.+))/u);
if (imported) {
const sources = imported[1] ? [imported[1]] : imported[3].split(',').map(value => value.trim().split(/\s+/)[0]);
for (const source of sources) if (source) {
if (imports.length < 400) imports.push({ source: source.slice(0, 500), line: index + 1, kind: 'python-import-candidate', bindings: [], evidenceMethod: 'python-declaration-candidate' });
else truncated = true;
}
}
for (const char of cleaned) {
if ('([{'.includes(char)) bracketDepth++;
else if (')]}'.includes(char)) bracketDepth = Math.max(0, bracketDepth - 1);
}
lineContinuation = cleaned.trimEnd().endsWith('\\');
}
return { symbols, imports, method: 'python-declaration-candidate', truncated, visitedNodes: 0 };
}
function resolveImport(from, dependency, files) {
let candidate = null;
if (dependency.kind === 'python-import-candidate') {
if (dependency.source.startsWith('.')) {
const dots = dependency.source.match(/^\.+/)[0].length;
candidate = path.posix.join(path.posix.dirname(from), ...Array(dots - 1).fill('..'), dependency.source.slice(dots).replaceAll('.', '/'));
} else candidate = dependency.source.replaceAll('.', '/');
} else if (dependency.source.startsWith('.')) candidate = path.posix.join(path.posix.dirname(from), dependency.source);
if (!candidate || candidate === '..' || candidate.startsWith('../')) return { ...dependency, from, status: 'external-or-alias', target: null, targetSha256: null };
const extensions = dependency.kind === 'python-import-candidate' ? ['.py', '/__init__.py'] : ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts', '/index.ts', '/index.tsx', '/index.js', '/index.mjs'];
const candidates = extensions.map(extension => candidate + extension);
if (/\.[mc]?js$/i.test(candidate)) candidates.push(candidate.replace(/\.[mc]?js$/i, '.ts'), candidate.replace(/\.[mc]?js$/i, '.tsx'));
if (/\.mjs$/i.test(candidate)) candidates.splice(1, 0, candidate.replace(/\.mjs$/i, '.mts'));
if (/\.cjs$/i.test(candidate)) candidates.splice(1, 0, candidate.replace(/\.cjs$/i, '.cts'));
const target = candidates.map(value => files.get(process.platform === 'win32' ? value.toLowerCase() : value)).find(Boolean);
return { ...dependency, from, status: target ? 'local-file' : 'unresolved-local', target: target?.path ?? null, targetSha256: target?.sha256 ?? null,
resolutionMethod: 'bounded-local-file-candidates', runtimeResolution: 'not_verified' };
}
export class ProjectIndex {
constructor({ maximumProjects = 8, maximumFileBytes = 256000, maximumSymbols = 6000, maximumSymbolsPerFile = 300, maximumAstNodes = 80000 } = {}) {
this.maximumProjects = maximumProjects; this.maximumFileBytes = maximumFileBytes; this.maximumSymbols = maximumSymbols;
this.maximumSymbolsPerFile = maximumSymbolsPerFile; this.maximumAstNodes = maximumAstNodes;
this.projects = new Map(); this.pending = new Map();
}
refresh(project) {
const key = project.id + '\0' + project.root;
if (!this.pending.has(key)) {
const promise = this.build(project, key).finally(() => this.pending.delete(key));
this.pending.set(key, promise);
}
return this.pending.get(key);
}
async build(project, key) {
const inspection = await inspectProject(project, { includeFiles: true });
const previous = this.projects.get(key) ?? new Map(), next = new Map(), entries = [];
let updatedFiles = 0, reusedFiles = 0;
for (const file of inspection.files) {
if (file.missing) continue;
let entry = previous.get(file.path);
if (entry?.sha256 === file.sha256) reusedFiles++;
else {
updatedFiles++;
entry = { ...file, symbols: [], imports: [], status: 'metadata-only', method: 'file-sha256', truncated: false };
if ((js.test(file.path) || /\.py$/i.test(file.path)) && file.bytes <= this.maximumFileBytes) {
try {
const text = readText(project, file.path);
if (text.sha256 !== file.sha256) throw conflict('建立索引期间文件已变化,请刷新工程索引:' + file.path);
Object.assign(entry, js.test(file.path) ? javascript(text.content, file.path, this.maximumAstNodes, this.maximumSymbolsPerFile) : python(text.content, this.maximumSymbolsPerFile));
entry.status = entry.truncated ? 'partial' : 'indexed';
} catch (error) {
if (error.status === 409) throw error;
entry.status = 'unindexed'; entry.error = String(error.message).slice(0, 180); entry.errorLine = error.loc?.line ?? null;
}
} else if (js.test(file.path) || /\.py$/i.test(file.path)) { entry.status = 'outside-index-budget'; entry.truncated = true; }
}
next.set(file.path, entry); entries.push(entry);
}
const available = new Map(entries.map(file => [process.platform === 'win32' ? file.path.toLowerCase() : file.path, file]));
const allSymbols = entries.flatMap(file => file.symbols.map(value => ({ ...value, path: file.path, sha256: file.sha256, language: file.language })));
const symbols = allSymbols.slice(0, this.maximumSymbols).sort((a, b) => a.path.localeCompare(b.path) || a.line - b.line || a.name.localeCompare(b.name));
const allImports = entries.flatMap(file => file.imports.map(value => resolveImport(file.path, value, available)));
const imports = allImports.slice(0, 6000);
const removedFiles = [...previous.keys()].filter(file => !next.has(file)).length;
this.projects.delete(key); this.projects.set(key, next);
while (this.projects.size > this.maximumProjects) this.projects.delete(this.projects.keys().next().value);
const warnings = entries.filter(file => ['unindexed', 'partial', 'outside-index-budget'].includes(file.status)).map(file => ({ path: file.path, sha256: file.sha256, status: file.status,
message: file.error ?? (file.opaqueImports ? file.opaqueImports + '处动态导入无法确定静态文件目标;其余声明结果保留。' : '该文件达到索引读取或分析上限;按范围提供已有结果。'), line: file.errorLine ?? 1 }));
const files = entries.map(({ symbols: ignoredSymbols, imports: ignoredImports, error, ...file }) => file);
const coverage = { snapshotComplete: inspection.scope.complete, fileCount: entries.length, symbolCount: symbols.length, importCount: imports.length,
astFiles: entries.filter(file => file.method === 'javascript-typescript-ast').length, pythonCandidateFiles: entries.filter(file => file.method === 'python-declaration-candidate').length,
unindexedFiles: warnings.length, truncated: !inspection.scope.complete || warnings.length > 0 || allSymbols.length > symbols.length || allImports.length > imports.length,
maximumFileBytes: this.maximumFileBytes, maximumSymbols: this.maximumSymbols };
return { schemaVersion: 1, projectId: project.id, projectName: project.name, parser: parserVersion, generatedAt: new Date().toISOString(),
sourceFingerprint: inspection.sourceFingerprint, indexFingerprint: hash(JSON.stringify({ projectId: project.id, parser: parserVersion, source: inspection.sourceFingerprint, symbols, imports, coverage })),
sourceScope: inspection.scope, coverage, refresh: { updatedFiles, reusedFiles, removedFiles }, files, symbols, imports, warnings,
notes: ['JS/TS/React符号来自静态AST;局部文件候选不证明实际运行时模块解析。', 'Python结果为声明候选,需要读取源文件核对;未执行项目代码。', '只覆盖已读取范围,不解析动态绑定、路径别名或整个语言的引用关系。'] };
}
summary(index) {
return { schemaVersion: index.schemaVersion, projectId: index.projectId, projectName: index.projectName, parser: index.parser, generatedAt: index.generatedAt,
sourceFingerprint: index.sourceFingerprint, indexFingerprint: index.indexFingerprint, coverage: index.coverage, refresh: index.refresh,
sourceScope: index.sourceScope, warnings: index.warnings.slice(0, 20), notes: index.notes };
}
query(index, { query = '', kind = 'all', limit = 30 } = {}) {
if (typeof query !== 'string' || query.length > 200 || !['all', 'symbols', 'imports', 'files'].includes(kind)
|| !Number.isInteger(limit) || limit < 1 || limit > 60) throw new Error('索引查询需要200字以内文本、有效类型与1到60条结果上限。');
const terms = query.trim().toLocaleLowerCase().split(/\s+/).filter(Boolean);
const score = (name, filename) => {
const lower = name.toLocaleLowerCase(), all = (name + ' ' + filename).toLocaleLowerCase();
if (!terms.every(term => all.includes(term))) return 0;
return !terms.length ? 1 : lower === query.trim().toLocaleLowerCase() ? 100 : lower.startsWith(terms[0]) ? 80 : lower.includes(terms[0]) ? 60 : 30;
};
const results = [];
if (kind === 'all' || kind === 'symbols') for (const symbol of index.symbols) {
const rank = score(symbol.qualifiedName, symbol.path); if (rank) results.push({ ...symbol, resultKind: 'symbol', score: rank });
}
if (kind === 'all' || kind === 'imports') for (const dependency of index.imports) {
const rank = score([dependency.source, ...dependency.bindings].join(' '), dependency.from); if (rank) results.push({ ...dependency, name: dependency.source,
path: dependency.from, sha256: index.files.find(file => file.path === dependency.from)?.sha256, resultKind: 'import', score: rank });
}
if (kind === 'files' || kind === 'all' && terms.length) for (const file of index.files) {
const rank = score(path.posix.basename(file.path), file.path); if (rank) results.push({ ...file, name: path.posix.basename(file.path), line: 1, resultKind: 'file', score: rank, evidenceMethod: 'file-sha256' });
}
results.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path) || a.line - b.line);
return { query, kind, results: results.slice(0, limit), totalMatches: results.length, truncated: results.length > limit || index.coverage.truncated, index: this.summary(index) };
}
}