import path from 'node:path'; import { createHash } from 'node:crypto'; import { parse } from '@babel/parser'; import { inspectProject, verifyIndexSnapshot } from '../project-inspector.mjs'; import { readText } from '../../server/files.mjs'; import { performance } from 'node:perf_hooks'; import { IndexCache } from './index-cache.mjs'; import { projectWorkspaceIdentity } from '../../shared/contracts/workspaces.mjs'; const hash = value => createHash('sha256').update(value).digest('hex'); const conflict = message => Object.assign(new Error(message), { status: 409 }); const js = /\.(?:[mc]?js|jsx|[mc]?ts|tsx)$/i; const functionTypes = new Set(['FunctionExpression', 'ArrowFunctionExpression']); const names = { FunctionDeclaration: 'function', FunctionExpression: 'function', TSDeclareFunction: 'function', ClassDeclaration: 'class', ClassExpression: 'class', TSTypeAliasDeclaration: 'type', TSInterfaceDeclaration: 'interface', TSEnumDeclaration: 'enum' }; const parserVersion = 'babel-parser-7.29.11'; const nodeName = node => node?.type === 'Identifier' ? node.name : node?.type === 'PrivateName' ? '#' + node.id.name : node?.type === 'StringLiteral' ? node.value : null; function javascript(source, filename, maximumNodes, maximumSymbols, open = false) { const plugins = [...(/\.(?:[mc]?ts|tsx)$/i.test(filename) ? ['typescript'] : []), ...(/\.(?:[mc]?js|jsx|tsx)$/i.test(filename) ? ['jsx'] : []), 'decorators-legacy']; const ast = parse(source, { sourceType: 'unambiguous', plugins, createImportExpressions: true, allowReturnOutsideFunction: true, allowAwaitOutsideFunction: true }); const exported = new Set(), exportNames = new Set(), topLevel = new Set(); for (const entry of ast.program.body) { const declaration = entry.declaration ?? entry; const declarations = declaration.type === 'VariableDeclaration' ? declaration.declarations : [declaration]; for (const node of declarations) { topLevel.add(node); if (entry.type.startsWith('Export')) exported.add(node); } if (entry.type === 'ExportNamedDeclaration') for (const specifier of entry.specifiers) if (specifier.local?.name) exportNames.add(specifier.local.name); if (entry.type === 'ExportDefaultDeclaration' && declaration.type === 'Identifier') exportNames.add(declaration.name); } const symbols = [], imports = [], stack = [{ node: ast.program, parent: null, scope: [] }]; let visited = 0, truncated = false, opaqueImports = 0; while (stack.length) { const { node, parent, scope } = stack.pop(); if (++visited > maximumNodes) { truncated = true; break; } let name = nodeName(node.id), kind = names[node.type] ?? null; if (functionTypes.has(node.type) && parent?.type === 'ExportDefaultDeclaration') kind = 'function'; if (!name && kind && parent?.type === 'ExportDefaultDeclaration') name = 'default'; if (node.type === 'VariableDeclarator' && node.id.type === 'Identifier') { name = node.id.name; kind = functionTypes.has(node.init?.type) ? 'function' : topLevel.has(node) ? 'binding' : null; } if (['ClassMethod', 'ClassPrivateMethod', 'ObjectMethod', 'TSMethodSignature'].includes(node.type) || ['ClassProperty', 'ClassPrivateProperty'].includes(node.type) && functionTypes.has(node.value?.type)) { name = !node.computed ? nodeName(node.key) : null; kind = name ? 'method' : null; } if (name && kind) { const cut = node.body?.start ?? node.init?.body?.start ?? node.value?.body?.start ?? node.id?.end ?? node.key?.end ?? node.end; const signature = source.slice(node.start, Math.min(cut, node.start + 240)).replace(/\s+/g, ' ').trim(); if (symbols.length < maximumSymbols) symbols.push({ name, qualifiedName: [...scope, name].join('.'), kind, line: node.loc.start.line, column: node.loc.start.column + 1, endLine: node.loc.end.line, signature, exported: exported.has(node) || topLevel.has(node) && exportNames.has(name), evidenceMethod: 'javascript-typescript-ast' }); else truncated = true; } let sourceNode, importKind; if (node.type === 'ImportDeclaration') { sourceNode = node.source; importKind = node.importKind === 'type' ? 'type-import' : 'import'; } else if (['ExportNamedDeclaration', 'ExportAllDeclaration'].includes(node.type) && node.source) { sourceNode = node.source; importKind = 're-export'; } else if (node.type === 'ImportExpression') { sourceNode = node.source; importKind = 'literal-dynamic-import'; } else if (node.type === 'CallExpression' && node.callee?.type === 'Identifier' && node.callee.name === 'require') { sourceNode = node.arguments[0]; importKind = 'literal-require-call'; } else if (node.type === 'TSImportEqualsDeclaration' && node.moduleReference?.type === 'TSExternalModuleReference') { sourceNode = node.moduleReference.expression; importKind = 'import-equals'; } if (sourceNode?.type === 'StringLiteral' && imports.length < (open ? Infinity : 400)) imports.push({ source: sourceNode.value.slice(0, open ? Infinity : 500), line: node.loc.start.line, kind: importKind, bindings: node.type === 'TSImportEqualsDeclaration' ? [nodeName(node.id)].filter(Boolean) : (node.specifiers ?? []).map(value => nodeName(value.local)).filter(Boolean).slice(0, open ? Infinity : 30), evidenceMethod: 'javascript-typescript-ast' }); else if (sourceNode) { truncated = true; if (sourceNode.type !== 'StringLiteral') opaqueImports++; } let childScope = scope; if (name && (names[node.type] && ['function', 'class'].includes(kind) || node.type === 'VariableDeclarator' && kind === 'function' || kind === 'method')) childScope = [...scope, name]; if (node.type === 'ObjectExpression' && parent?.type === 'VariableDeclarator' && parent.id.type === 'Identifier') childScope = [...scope, parent.id.name]; const children = Object.values(node).flatMap(value => Array.isArray(value) ? value : value && typeof value === 'object' ? [value] : []).filter(value => typeof value?.type === 'string'); for (let i = children.length - 1; i >= 0; i--) stack.push({ node: children[i], parent: node, scope: childScope }); } return { symbols, imports, method: 'javascript-typescript-ast', truncated, opaqueImports, visitedNodes: visited }; } // This lexer only removes comments and string bodies so declaration candidates // do not come from docstrings. It is deliberately not a Python compiler. function python(source, maximumSymbols, open = false) { let triple = null, quote = null, escaped = false, truncated = false, bracketDepth = 0, lineContinuation = false; const symbols = [], imports = [], scopes = []; const lines = source.split('\n'); for (let index = 0; index < lines.length; index++) { const line = lines[index], continued = bracketDepth > 0 || lineContinuation; let cleaned = ''; for (let i = 0; i < line.length;) { const char = line[i], three = line.slice(i, i + 3); if (triple) { if (three === triple && !escaped) { triple = null; cleaned += ' '; i += 3; } else { cleaned += ' '; escaped = char === '\\' && !escaped; i++; } } else if (quote) { cleaned += ' '; if (char === quote && !escaped) quote = null; escaped = char === '\\' && !escaped; i++; } else if (char === '#') break; else if (three === '"""' || three === "'''") { triple = three; cleaned += ' '; escaped = false; i += 3; } else if (char === '"' || char === "'") { quote = char; cleaned += ' '; escaped = false; i++; } else { cleaned += char; i++; } } escaped = false; if (!continued && cleaned.trim()) { const indent = cleaned.match(/^\s*/)[0].replaceAll('\t', ' ').length; while (scopes.length && scopes.at(-1).indent >= indent) scopes.pop(); } const declaration = !continued && cleaned.match(/^(\s*)(?:async\s+)?(def|class)\s+([\p{ID_Start}_][\p{ID_Continue}]*)\s*(?:\(|:)/u); if (declaration) { const indent = declaration[1].replaceAll('\t', ' ').length; const name = declaration[3]; if (symbols.length < maximumSymbols) symbols.push({ name, qualifiedName: [...scopes.map(value => value.name), name].join('.'), kind: declaration[2] === 'class' ? 'class' : 'function', line: index + 1, column: declaration[1].length + 1, endLine: index + 1, signature: cleaned.trim().slice(0, 180), exported: false, evidenceMethod: 'python-declaration-candidate' }); else truncated = true; scopes.push({ indent, name }); } const imported = !continued && cleaned.match(/^\s*(?:from\s+([.\p{ID_Continue}]+)\s+import\s+(.+)|import\s+(.+))/u); if (imported) { const sources = imported[1] ? [imported[1]] : imported[3].split(',').map(value => value.trim().split(/\s+/)[0]); for (const source of sources) if (source) { if (imports.length < (open ? Infinity : 400)) imports.push({ source: source.slice(0, open ? Infinity : 500), line: index + 1, kind: 'python-import-candidate', bindings: [], evidenceMethod: 'python-declaration-candidate' }); else truncated = true; } } for (const char of cleaned) { if ('([{'.includes(char)) bracketDepth++; else if (')]}'.includes(char)) bracketDepth = Math.max(0, bracketDepth - 1); } lineContinuation = cleaned.trimEnd().endsWith('\\'); } return { symbols, imports, method: 'python-declaration-candidate', truncated, visitedNodes: 0 }; } function resolveImport(from, dependency, files) { let candidate = null; if (dependency.kind === 'python-import-candidate') { if (dependency.source.startsWith('.')) { const dots = dependency.source.match(/^\.+/)[0].length; candidate = path.posix.join(path.posix.dirname(from), ...Array(dots - 1).fill('..'), dependency.source.slice(dots).replaceAll('.', '/')); } else candidate = dependency.source.replaceAll('.', '/'); } else if (dependency.source.startsWith('.')) candidate = path.posix.join(path.posix.dirname(from), dependency.source); if (!candidate || candidate === '..' || candidate.startsWith('../')) return { ...dependency, from, status: 'external-or-alias', target: null, targetSha256: null }; const extensions = dependency.kind === 'python-import-candidate' ? ['.py', '/__init__.py'] : ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts', '/index.ts', '/index.tsx', '/index.js', '/index.mjs']; const candidates = extensions.map(extension => candidate + extension); if (/\.[mc]?js$/i.test(candidate)) candidates.push(candidate.replace(/\.[mc]?js$/i, '.ts'), candidate.replace(/\.[mc]?js$/i, '.tsx')); if (/\.mjs$/i.test(candidate)) candidates.splice(1, 0, candidate.replace(/\.mjs$/i, '.mts')); if (/\.cjs$/i.test(candidate)) candidates.splice(1, 0, candidate.replace(/\.cjs$/i, '.cts')); const target = candidates.map(value => files.get(process.platform === 'win32' ? value.toLowerCase() : value)).find(Boolean); return { ...dependency, from, status: target ? 'local-file' : 'unresolved-local', target: target?.path ?? null, targetSha256: target?.sha256 ?? null, resolutionMethod: 'bounded-local-file-candidates', runtimeResolution: 'not_verified' }; } export class ProjectIndex { constructor({ maximumProjects = 8, maximumFileBytes = 256000, maximumSymbols = 6000, maximumSymbolsPerFile = 300, maximumAstNodes = 80000, cacheDirectory, maximumCacheBytes = 32 * 1024 * 1024, maximumEntryBytes = 8 * 1024 * 1024, executionMode = 'restricted', parserRevision = 'index-2-python-candidates-1' } = {}) { this.open = executionMode === 'unrestricted'; if (this.open) maximumProjects = maximumFileBytes = maximumSymbols = maximumSymbolsPerFile = maximumAstNodes = maximumCacheBytes = maximumEntryBytes = Infinity; this.maximumProjects = maximumProjects; this.maximumFileBytes = maximumFileBytes; this.maximumSymbols = maximumSymbols; this.maximumSymbolsPerFile = maximumSymbolsPerFile; this.maximumAstNodes = maximumAstNodes; this.projects = new Map(); this.pending = new Map(); this.identities = new Map(); this.latest = new Map(); this.memoryBytes = new Map(); this.generation = 0; this.cache = new IndexCache({ directory: cacheDirectory, maximumProjects, maximumBytes: maximumCacheBytes, maximumEntryBytes }); this.contract = { parser: parserVersion, revision: parserRevision, maximumFileBytes, maximumSymbols, maximumSymbolsPerFile, maximumAstNodes }; } status(project) { return { ...this.cache.status(), memoryProjects: this.projects.size, memoryBytes: [...this.memoryBytes.values()].reduce((sum, bytes) => sum + bytes, 0), pending: this.pending.size, latest: project ? this.latest.get(project.id + '\0' + project.root) ?? null : null }; } configure(enabled) { this.cache.configure(enabled); this.generation++; this.projects.clear(); this.identities.clear(); this.memoryBytes.clear(); this.latest.clear(); return this.status(); } clear(project) { const removedProjects = this.cache.clear(project), key = project.id + '\0' + project.root; this.generation++; this.projects.delete(key); this.identities.delete(key); this.memoryBytes.delete(key); this.latest.delete(key); return { ...this.status(project), clearedAt: new Date().toISOString(), removedProjects }; } refresh(project) { const key = project.id + '\0' + project.root; if (!this.pending.has(key)) { const promise = this.build(project, key).finally(() => this.pending.delete(key)); this.pending.set(key, promise); } return this.pending.get(key); } async build(project, key) { const started = performance.now(), generation = this.generation; const inspection = await inspectProject(project, { includeFiles: true, strictSnapshot: true }); const identity = { project: projectWorkspaceIdentity(project), contract: this.contract, git: { available: inspection.git.available, head: inspection.git.head, branch: inspection.git.branch } }; const memory = this.projects.get(key), sameIdentity = JSON.stringify(this.identities.get(key)) === JSON.stringify(identity); const loaded = !this.cache.enabled ? { entries: null, reason: 'disabled' } : !inspection.scope.complete ? { entries: null, reason: 'incomplete-scan' } : memory && sameIdentity ? { entries: memory, reason: 'memory' } : this.cache.load(project, identity); const previous = loaded.entries ?? new Map(), next = new Map(), entries = []; let updatedFiles = 0, reusedFiles = 0, parsedFiles = 0, parseMs = 0, invalidatedFiles = 0; for (const file of inspection.files) { if (file.missing) continue; let entry = previous.get(file.path); if (entry?.sha256 === file.sha256 && entry.access === file.access && entry.bytes === file.bytes && this.validEntry(entry)) reusedFiles++; else { if (entry) invalidatedFiles++; updatedFiles++; entry = { ...file, symbols: [], imports: [], status: 'metadata-only', method: 'file-sha256', truncated: false }; if ((js.test(file.path) || /\.py$/i.test(file.path)) && file.bytes <= this.maximumFileBytes) { try { const text = readText(project, file.path); if (text.sha256 !== file.sha256) throw conflict('建立索引期间文件已变化,请刷新工程索引:' + file.path); const parseStarted = performance.now(); parsedFiles++; try { Object.assign(entry, js.test(file.path) ? javascript(text.content, file.path, this.maximumAstNodes, this.maximumSymbolsPerFile, this.open) : python(text.content, this.maximumSymbolsPerFile, this.open)); } finally { parseMs += performance.now() - parseStarted; } entry.status = entry.truncated ? 'partial' : 'indexed'; } catch (error) { if (error.status === 409) throw error; entry.status = 'unindexed'; entry.error = String(error.message).slice(0, 180); entry.errorLine = error.loc?.line ?? null; } } else if (js.test(file.path) || /\.py$/i.test(file.path)) { entry.status = 'outside-index-budget'; entry.truncated = true; } } next.set(file.path, entry); entries.push(entry); } // Recheck the bounded snapshot after parsing/loading, including names, Git // identity and access scope. A cache hit cannot close a concurrent change. let verified; try { verified = await verifyIndexSnapshot(project, inspection); } catch (error) { this.projects.delete(key); this.identities.delete(key); this.memoryBytes.delete(key); throw error; } const available = new Map(entries.map(file => [process.platform === 'win32' ? file.path.toLowerCase() : file.path, file])); const allSymbols = entries.flatMap(file => file.symbols.map(value => ({ ...value, path: file.path, sha256: file.sha256, language: file.language }))); const symbols = allSymbols.slice(0, this.maximumSymbols).sort((a, b) => a.path.localeCompare(b.path) || a.line - b.line || a.name.localeCompare(b.name)); const allImports = entries.flatMap(file => file.imports.map(value => resolveImport(file.path, value, available))); const imports = allImports.slice(0, this.open ? Infinity : 6000); const removedFiles = [...previous.keys()].filter(file => !next.has(file)).length; const entryBytes = Buffer.byteLength(JSON.stringify([...next.values()])), withinCapacity = entryBytes <= Math.min(this.cache.maximumEntryBytes, this.cache.maximumBytes); const canStore = this.cache.enabled && inspection.scope.complete && generation === this.generation && withinCapacity; let stored = { stored: false, reason: generation !== this.generation ? 'cleared-during-refresh' : !withinCapacity ? 'capacity' : loaded.reason }; if (canStore) { this.projects.delete(key); this.projects.set(key, next); this.identities.set(key, identity); this.memoryBytes.set(key, entryBytes); while (this.projects.size > this.maximumProjects || [...this.memoryBytes.values()].reduce((sum, bytes) => sum + bytes, 0) > this.cache.maximumBytes) { const oldest = this.projects.keys().next().value; this.projects.delete(oldest); this.identities.delete(oldest); this.memoryBytes.delete(oldest); this.latest.delete(oldest); } stored = !updatedFiles && !removedFiles && ['memory', 'disk'].includes(loaded.reason) ? this.cache.touch(project) : this.cache.save(project, identity, next); } else { this.projects.delete(key); this.identities.delete(key); this.memoryBytes.delete(key); } const warnings = entries.filter(file => ['unindexed', 'partial', 'outside-index-budget'].includes(file.status)).map(file => ({ path: file.path, sha256: file.sha256, status: file.status, message: file.error ?? (file.opaqueImports ? file.opaqueImports + '处动态导入无法确定静态文件目标;其余声明结果保留。' : '该文件达到索引读取或分析上限;按范围提供已有结果。'), line: file.errorLine ?? 1 })); const files = entries.map(({ symbols: ignoredSymbols, imports: ignoredImports, error, ...file }) => file); const coverage = { snapshotComplete: inspection.scope.complete, fileCount: entries.length, symbolCount: symbols.length, importCount: imports.length, astFiles: entries.filter(file => file.method === 'javascript-typescript-ast').length, pythonCandidateFiles: entries.filter(file => file.method === 'python-declaration-candidate').length, unindexedFiles: warnings.length, truncated: !inspection.scope.complete || warnings.length > 0 || allSymbols.length > symbols.length || allImports.length > imports.length, maximumFileBytes: this.maximumFileBytes, maximumSymbols: this.maximumSymbols }; const cache = { ...this.cache.status(), checkedAt: verified.inspectedAt, sourceVerified: inspection.scope.complete && verified.scope.complete, origin: loaded.reason, invalidation: !inspection.scope.complete ? 'incomplete-scan' : memory && !sameIdentity ? 'identity-changed' : invalidatedFiles ? 'source-or-access-changed' : previous.size && (removedFiles || updatedFiles) ? 'files-added-or-removed' : ['corrupt', 'identity-changed', 'unavailable'].includes(loaded.reason) ? loaded.reason : null, reusedFiles, rebuiltFiles: updatedFiles, parsedFiles, invalidatedFiles, removedFiles, verifiedFiles: verified.files.filter(file => file.sha256).length, verificationPasses: 2, sourceBytesRead: inspection.scope.bytes + verified.scope.bytes, parseMs, durationMs: performance.now() - started, stored: stored.stored, storageReason: stored.reason, evictedProjects: stored.evictedProjects ?? 0 }; this.latest.delete(key); if (generation === this.generation) this.latest.set(key, cache); while (this.latest.size > this.maximumProjects) this.latest.delete(this.latest.keys().next().value); return { schemaVersion: 1, projectId: project.id, projectName: project.name, parser: parserVersion, generatedAt: new Date().toISOString(), cache, sourceFingerprint: inspection.sourceFingerprint, indexFingerprint: hash(JSON.stringify({ projectId: project.id, parser: parserVersion, source: inspection.sourceFingerprint, symbols, imports, coverage })), sourceScope: inspection.scope, coverage, refresh: { updatedFiles, reusedFiles, removedFiles }, files, symbols, imports, warnings, notes: ['JS/TS/React符号来自静态AST;局部文件候选不证明实际运行时模块解析。', 'Python结果为声明候选,需要读取源文件核对;未执行项目代码。', '只覆盖已读取范围,不解析动态绑定、路径别名或整个语言的引用关系。'] }; } summary(index) { return { schemaVersion: index.schemaVersion, projectId: index.projectId, projectName: index.projectName, parser: index.parser, generatedAt: index.generatedAt, sourceFingerprint: index.sourceFingerprint, indexFingerprint: index.indexFingerprint, coverage: index.coverage, refresh: index.refresh, sourceScope: index.sourceScope, warnings: index.warnings.slice(0, 20), notes: index.notes, ...(index.cache ? { cache: index.cache } : {}) }; } validEntry(entry) { const bounded = value => typeof value === 'string' && value.length <= (this.open ? Infinity : 10000); return Array.isArray(entry.symbols) && entry.symbols.length <= this.maximumSymbolsPerFile && entry.symbols.every(value => value && bounded(value.name) && bounded(value.qualifiedName) && bounded(value.kind) && bounded(value.signature) && Number.isInteger(value.column) && value.column > 0 && Number.isInteger(value.line) && value.line > 0 && Number.isInteger(value.endLine) && value.endLine >= value.line && typeof value.exported === 'boolean' && bounded(value.evidenceMethod)) && Array.isArray(entry.imports) && entry.imports.length <= (this.open ? Infinity : 400) && entry.imports.every(value => value && bounded(value.source) && bounded(value.kind) && bounded(value.evidenceMethod) && Number.isInteger(value.line) && value.line > 0 && Array.isArray(value.bindings) && value.bindings.length <= (this.open ? Infinity : 30) && value.bindings.every(bounded)) && ['metadata-only', 'indexed', 'partial', 'outside-index-budget'].includes(entry.status) && ['file-sha256', 'javascript-typescript-ast', 'python-declaration-candidate'].includes(entry.method) && typeof entry.truncated === 'boolean'; } query(index, { query = '', kind = 'all', limit = this.open ? Infinity : 30 } = {}) { if (typeof query !== 'string' || query.length > (this.open ? Infinity : 200) || !['all', 'symbols', 'imports', 'files'].includes(kind) || !(this.open && limit === Infinity) && (!Number.isSafeInteger(limit) || limit < 1 || !this.open && limit > 60)) throw new Error('索引查询需要200字以内文本、有效类型与1到60条结果上限。'); const terms = query.trim().toLocaleLowerCase().split(/\s+/).filter(Boolean); const score = (name, filename) => { const lower = name.toLocaleLowerCase(), all = (name + ' ' + filename).toLocaleLowerCase(); if (!terms.every(term => all.includes(term))) return 0; return !terms.length ? 1 : lower === query.trim().toLocaleLowerCase() ? 100 : lower.startsWith(terms[0]) ? 80 : lower.includes(terms[0]) ? 60 : 30; }; const results = []; if (kind === 'all' || kind === 'symbols') for (const symbol of index.symbols) { const rank = score(symbol.qualifiedName, symbol.path); if (rank) results.push({ ...symbol, resultKind: 'symbol', score: rank }); } if (kind === 'all' || kind === 'imports') for (const dependency of index.imports) { const rank = score([dependency.source, ...dependency.bindings].join(' '), dependency.from); if (rank) results.push({ ...dependency, name: dependency.source, path: dependency.from, sha256: index.files.find(file => file.path === dependency.from)?.sha256, resultKind: 'import', score: rank }); } if (kind === 'files' || kind === 'all' && terms.length) for (const file of index.files) { const rank = score(path.posix.basename(file.path), file.path); if (rank) results.push({ ...file, name: path.posix.basename(file.path), line: 1, resultKind: 'file', score: rank, evidenceMethod: 'file-sha256' }); } results.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path) || a.line - b.line); return { query, kind, results: results.slice(0, limit), totalMatches: results.length, truncated: results.length > limit || index.coverage.truncated, index: this.summary(index) }; } }