From 44b2eaf2733d558e5ab71991a261ddf411ffa2af Mon Sep 17 00:00:00 2001 From: lanyizi Date: Sun, 2 Aug 2026 19:11:54 +0200 Subject: [PATCH] 0.1.1 --- README.md | 9 +- docs/analysis-issues.md | 77 +++++++ docs/plan.md | 28 ++- docs/requirements.md | 3 +- package-lock.json | 4 +- package.json | 2 +- src/extension.ts | 12 +- src/indexer/caches.ts | 168 ++++++++++++-- src/indexer/fileScanner.ts | 23 +- src/indexer/indexer.ts | 442 +++++++++++++++++++------------------ src/indexer/records.ts | 177 +++++++++++++++ src/indexer/shallowScan.ts | 16 +- src/indexer/types.ts | 37 +++- src/workspace.ts | 77 ++++++- test/caches.test.mjs | 73 ++++++ test/indexer.test.mjs | 72 +++++- test/records.test.mjs | 65 ++++++ test/shallowScan.test.mjs | 28 ++- 18 files changed, 1035 insertions(+), 278 deletions(-) create mode 100644 src/indexer/records.ts create mode 100644 test/caches.test.mjs create mode 100644 test/records.test.mjs diff --git a/README.md b/README.md index 2d9204e..1ad3b07 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ `Model@Name`、`Hierarchy`、`Mesh` 等引用可以解析、悬停与跳转。超大模型 （几十 MB 的顶点/三角形数据）采用浅扫描——只提取顶层资产记录、不建 DOM 树， 结果在 workspace 级缓存并跨重建复用，保存文件触发的重建不会重读未变化的模型文件。 +- **大项目性能**：索引记录（资产 / Define / Include / 行号）与 include 解析结果 + 跨重建缓存，保存触发的重建零 stat、零重读（Corona 实测约 2 秒）；DOM 树只按需 + 保留并设元素预算，避免内存膨胀。 ## 使用 @@ -78,8 +81,10 @@ src/ fileScanner.ts 目录扫描与 Include source 候选 refs.ts 引用目标解析（按引用类型过滤） shallowScan.ts .w3x 等大体积美术资产顶层浅扫描（纯 TS，不建 DOM） - caches.ts 跨重建持久缓存（DocumentCache / ShallowScanCache） - indexer.ts 工作区索引器（后台、缓存、增量重建） + records.ts 每文件紧凑索引记录（资产/Define/Include/xi + 行号） + caches.ts 跨重建持久缓存（DocumentCache / IndexRecordsCache / + IncludeResolveCache） + indexer.ts 工作区索引器（后台、缓存、记录驱动重建） features/ completion / hover / navigation / diagnostics / semanticTokens syntaxes/ TextMate 注入语法 tools/ XSD → 模型、AssetType 枚举提取 diff --git a/docs/analysis-issues.md b/docs/analysis-issues.md index 2666076..cce1246 100644 --- a/docs/analysis-issues.md +++ b/docs/analysis-issues.md @@ -685,3 +685,80 @@ AttachTest `Harbinger Gunship\GameObject.xml` 中 ` 关联为 xml，完整解析仍会发生在打开的文档上（VS Code 自身行为）。 - UTF-16 XML 的 w3x 会被嗅探判定为二进制（文件头含 NUL）；实测生态中不存在， 如遇可再扩展解码。 + +--- + +## 十四、问题分析（第九轮，2026-08-02）：Corona 重建性能与内存 + +### 目标与基线 + +第八轮后 Corona 的实测基线： + +| 场景 | 耗时 | +|---|---| +| 首次全量建索引 | ~180-290s（机械盘波动） | +| 信任二次构建（保存触发） | 8-21s | +| 强制 reindex | 26-38s | +| 首建后保留堆 | ~2.5GB（原因不明） | + +### 调查 1：二次构建为何仍有 8-21s + +插桩 `fs.statSync` 后发现：**每次构建（含信任重建）都会执行约 11 万次同步 statSync**， +全部来自 `resolveSource` 的 include 目标存在性检查（BAB 搜索路径逐 base `statSync`）。 +机械盘上这就是 10-20s 的来源——即使文件内容全部命中缓存，include 解析仍在逐路径 stat。 + +修复：新增 `IncludeResolveCache`（workspace 级、跨重建复用）： +- 键 = 当前文件目录 + source 字符串；命中直接返回，不 stat； +- 内容编辑不影响"文件是否存在"，因此保存触发的重建**不清除**解析缓存； +- 文件创建/删除（watcher `onDidCreate` / `onDidDelete`）与 `ra3modxml.reindex` + （强制校验）清除缓存； +- `reference` 的 manifest 查找（`builtmods/*.manifest` 存在性）同样缓存； +- 配置变更（搜索路径 / builtmods 目录）时清除。 + +### 调查 2：首建后保留堆 2.5GB 是什么 + +逐级清空测量（构建 → 清 DOM 缓存 → 清 records → 清 walker → 清索引 Map）： + +- `DocumentCache`（元素预算 1M、64 条）：实际只保留 64 条 / 1788 个元素 ≈ 1MB； +- `IndexRecordsCache`（8976 个文件的紧凑记录）：约 11MB； +- 目录 walker：约 4MB； +- 全部索引 Map（manifests + assets + assetsById + defines + files + streams + + candidates + diagnostics）：约 75MB； +- 全部清空并强制 GC 后，堆回到基线（0MB 保留）。 + +结论：2.5GB 是**构建期可回收垃圾**（60MB XML 的 DOM 瞬态 + 2.6GB w3x 扫描文本/行映射 +的分配压力），不是常驻泄漏；VSCode 正常 GC 压力下会回收。扩展常驻内存约为 +基线 + ~100MB。顺带修复了一个潜在常驻风险：w3x 的 `LineMap`（Corona 全量约 700MB） +不再随 `ShallowScanCache` 保留，浅扫描结果以"带行号的紧凑记录"形式缓存。 + +### 其他改动 + +1. **`IndexRecordsCache` 取代 DOM 依赖的重建**：新增 `src/indexer/records.ts`， + 每个文件解析时提取紧凑索引记录（顶层资产 / Define / Include / xi:include + + 1-based 行号），跨重建缓存；信任重建完全不接触 DOM。 +2. **`DocumentCache` 双重淘汰**：条数（LRU）+ 元素预算（超预算先淘汰最大树）， + 把 DOM 常驻内存封顶；DOM 只服务于按需特性（跳转精确范围等）。 +3. **候选目录扫描并行化**（`collectSourceCandidates` 各目录 `Promise.all`）。 +4. **阶段计时**：`stats.candidatesMs` / `stats.walkMs` / `stats.resolveCalls` / + `stats.resolveCacheHits` 进入索引报告，便于后续定位耗时。 +5. 版本 0.1.0 → **0.1.1**。 + +### 实测（Corona，D: 机械盘） + +| 场景 | 优化前 | 优化后 | +|---|---|---| +| 首次全量建索引 | ~180-290s | ~250s（含 2.6GB w3x 读取+浅扫描、~7.6 万次冷 statSync） | +| 信任二次构建 | 8-21s | **2.0s**（statSync 0 次，resolveHits 15,333） | +| 强制 reindex | 26-38s | ~5s（保留解析缓存时）；显式命令会清解析缓存，约 15-25s | +| 首建后保留堆 | ~2.5GB（疑为泄漏） | 确认是可回收垃圾；常驻 ~100MB | + +单元测试 **63 → 73 全绿**：新增 `records.test.mjs`（DOM→记录提取、浅扫描→记录）、 +`caches.test.mjs`（元素预算淘汰、LRU、records/resolve 缓存），indexer 测试补充 +resolve 缓存命中与"信任重建 0 次重解析"断言。 + +### 遗留 + +- 首次全量建索引仍受限于机械盘读 2.6GB + 冷 statSync（~4 分钟）；后续可考虑 + 并行浅扫描（worker）或把 w3x 顶层记录持久化到磁盘缓存。 +- `ra3modxml.reindex` 出于正确性会清空解析缓存（可能 ~15-25s）；如接受 watcher + 可靠性可改为保留。 diff --git a/docs/plan.md b/docs/plan.md index 52485e2..adb8c58 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -86,7 +86,9 @@ src/ fileScanner.ts 目录遍历缓存 + Include source 候选收集 refs.ts 引用目标解析（按 refType / isRef / inheritFrom 过滤，纯 TS） shallowScan.ts 大体积美术资产（.w3x 等）顶层浅扫描（纯 TS，不建 DOM） - caches.ts 跨重建持久缓存（DocumentCache / ShallowScanCache） + records.ts 每文件紧凑索引记录（资产/Define/Include/xi + 行号） + caches.ts 跨重建持久缓存（DocumentCache / IndexRecordsCache / + IncludeResolveCache） indexer.ts 工作区索引器（资产/Define/流/manifest/w3x 合并） types.ts 共享类型 features/ @@ -149,6 +151,19 @@ test/ w3x 重扫数为 0（4,829 次缓存命中）。 - 读取整个文件不可避免（顶层边界需要全量扫描），但建 DOM 不是；优化的是 "不分配子节点对象"与"跨重建不重读"，两者叠加后方案可行。 +15. **重建零 stat + 记录驱动索引**（第九轮，2026-08-02，v0.1.1）： + - 插桩发现每次重建（含信任重建）都有约 11 万次同步 `statSync` + （`resolveSource` 的 include 存在性检查），机械盘上占 10-20s； + `IncludeResolveCache` 按（目录 + source）缓存解析结果，内容编辑不清、 + 创建/删除文件与强制 reindex 才清。信任重建 statSync 降到 0。 + - `IndexRecordsCache` 缓存每文件紧凑索引记录（顶层资产 / Define / Include / + xi:include + 1-based 行号），信任重建完全不接触 DOM；`DocumentCache` + 双重淘汰（条数 LRU + 元素预算，超预算先淘汰最大树）把 DOM 常驻内存封顶。 + - w3x 缓存不再保留 LineMap（Corona 全量约 700MB），浅扫描直接产出带行号的记录。 + - 实测：Corona 信任二次构建 21s → **2.0s**（statSync 0）；首建后 2.5GB + 堆保留确认为构建期可回收垃圾，常驻 ~100MB；强制 reindex ~5-25s。 + - 候选目录扫描并行化；`stats` 新增 `candidatesMs` / `walkMs` / `resolveCalls` / + `resolveCacheHits` 供索引报告定位耗时。 ## 三、实施步骤 @@ -157,13 +172,16 @@ test/ 3. [x] 生成模型：`schema-model.json`（XSD，295 顶层元素 / 1851 类型）与 `asset-types.json`（79 个 AssetType 哈希）。 4. [x] 纯 TS 核心：include 解析、manifest 解析、XML 解析封装、索引器、引用解析。 5. [x] 功能层：补全、hover、导航、诊断、大纲、高亮 grammar。 -6. [x] 单测（fixture Mod，63 个用例全绿：含 xs:list 枚举、未闭合引号恢复、list 多值 +6. [x] 单测（fixture Mod，73 个用例全绿：含 xs:list 枚举、未闭合引号恢复、list 多值 分段、带 vscode stub 的补全集成、语义 token 兜底）+ 编译 + `vsce package` 打包 - （ra3-mod-xml-0.1.0.vsix，约 495KB）。 + （ra3-mod-xml-0.1.1.vsix，约 499KB）。 7. [x] 在 AttachTest / GenEvoTest / Corona 上做冒烟验证，并按真实项目反馈修复问题（详见 `docs/analysis-issues.md` 八轮分析）。 8. [x] w3x 美术资产索引（第八轮）：新增 `shallowScan.ts` / `caches.ts`，w3x 与内容嗅探 XML 走浅扫描，Include source 补全候选加入 w3x，BOM 剥离，缓存跨重建持久化； AttachTest 报错场景（`AUGunship_SKN`）修复，Corona 首次/二次构建实测验证。 +9. [x] Corona 性能与内存优化（第九轮，v0.1.1）：`records.ts` 记录驱动索引、 + `IncludeResolveCache` 零 stat 重建、DOM 元素预算淘汰、w3x LineMap 移除、 + 候选并行扫描、阶段计时；Corona 信任重建 2.0s；确认首建 2.5GB 为可回收垃圾。 ## 四、验证结果（实测） @@ -171,11 +189,11 @@ test/ |---|---|---|---|---| | AttachTest | 88 文件 | ~1.2s | 35,607（manifest 35,322，w3x 浅扫 62） | 2 个流（static + mapmetadata）；二次构建 354ms / 62 缓存命中 | | GenEvoTest | 66 文件 | 首次 ~2.6s / 二次 ~0.4s | 35,502（manifest 35,322，w3x 浅扫 38） | 2 个流、73 个 Define；二次构建 0 重扫 / 38 缓存命中 | -| Corona | 8,976 文件 | 首次 ~241s / 二次 ~38s | 64,868（manifest 35,322，w3x 浅扫 4,829） | 3 个流、183 个 Define、0 诊断；二次构建 0 重扫 / 4,829 缓存命中 | +| Corona | 8,976 文件 | 首次 ~250s / 信任二次 ~2s / 强制 ~5-25s | 64,868（manifest 35,322，w3x 浅扫 4,829） | 3 个流、183 个 Define、0 诊断；二次构建 statSync 0、resolveHits 15,333 | 单元测试覆盖：XML 解析（自闭合/容错/偏移/未闭合引号行尾恢复）、补全上下文（未闭合引号仍为 attribute-value、list 多值分段）、补全集成（vscode stub 下 `LocomotorTemplate@Surfaces` 未闭合引号枚举补全、空格后第二段过滤与替换范围）、语义 token（标签/属性/值范围、合法文档返回空、malformed 返回兜底 token）、include 解析（BAB 顺序、SDK 根优先于 SageXml）、manifest 二进制解析（合成 v5 样本、类型/ID 推导）、索引器（资产/Define/流/缺失 include/嵌套 xi:include）、XSD 模型（上下文类型、`childTypeOf`、大小写规范化、属性级 refType、外来命名空间判定、`xs:list` 枚举继承与 `isList` 标记）、引用过滤（`Weapon="X"` 只跳 `WeaponTemplate`、模块 `id` 定义点、Poid 局部引用、`xi:include` 不校验、`Side="Allies"` 命中 manifest 的 `PlayerTemplate`）。 -> 注：D: 盘移动硬盘已恢复连接；Corona 已按上述新数据回归（第八轮）。 +> 注：D: 盘移动硬盘已恢复连接；Corona 已在第八 / 九轮按上述新数据回归。 ## 五、假设与开放问题 diff --git a/docs/requirements.md b/docs/requirements.md index ca62a5a..cc5ca5b 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -76,7 +76,8 @@ XML 之间的组织靠 `` 标签，共有三种语义： 补全、悬停、导航与诊断——`Model@Name`、`Hierarchy`、`Mesh` 等引用依赖这些定义； - 大模型文件（实测 Corona 最大 22.8 MB，顶点/三角形数据占大头）采用**顶层浅扫描** （不建 DOM 树），结果在 workspace 级缓存并跨重建复用，避免每次保存都重读整个 - 美术资产目录（Corona 约 2.6 GB）。 + 美术资产目录（Corona 约 2.6 GB）；索引记录与 include 解析结果同样跨重建缓存， + 保存触发的重建零 stat、零重读（Corona 实测约 2 秒）。 ### P1：非近期目标（本期不做，但预留扩展点） diff --git a/package-lock.json b/package-lock.json index 9a4c71f..4ef5a32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ra3-mod-xml", - "version": "0.1.0", + "version": "0.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ra3-mod-xml", - "version": "0.1.0", + "version": "0.1.1", "license": "MIT", "dependencies": { "fast-xml-parser": "^4.5.0" diff --git a/package.json b/package.json index b2bccc2..7e8d3d8 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "ra3-mod-xml", "displayName": "RA3 Mod XML", "description": "Red Alert 3 Mod XML tooling: syntax highlighting, completions, reference navigation and diagnostics for SAGE/BinaryAssetBuilder XML.", - "version": "0.1.0", + "version": "0.1.1", "publisher": "ra3-mod-xml", "license": "MIT", "engines": { diff --git a/src/extension.ts b/src/extension.ts index 07e5618..b73c088 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -109,18 +109,24 @@ export function activate(context: vscode.ExtensionContext): void { context.subscriptions.push( vscode.workspace.onDidSaveTextDocument((doc) => { if (doc.languageId !== "xml") return; + ws.invalidate(doc.uri.fsPath); ws.scheduleRebuild(); void diagnostics.update(doc); }), ); context.subscriptions.push( vscode.workspace.onDidChangeConfiguration((e) => { - if (e.affectsConfiguration("ra3modxml")) ws.scheduleRebuild(); + if (e.affectsConfiguration("ra3modxml")) { + // Search paths / builtmods locations may have changed: cached include + // resolutions and manifest lookups are no longer valid. + ws.invalidateExistence(); + ws.scheduleRebuild(); + } }), ); context.subscriptions.push( - vscode.commands.registerCommand("ra3modxml.reindex", () => ws.rebuild()), + vscode.commands.registerCommand("ra3modxml.reindex", () => ws.rebuild(true)), ); context.subscriptions.push( vscode.commands.registerCommand("ra3modxml.openIndexReport", () => { @@ -135,7 +141,7 @@ export function activate(context: vscode.ExtensionContext): void { void vscode.window.showInformationMessage( `RA3 Mod XML index\n` + `Project: ${s.projectDir}\n` + - `Files: ${s.indexedFiles} (${s.parsedFiles} parsed, ${s.shallowScannedFiles} shallow-scanned, ${s.shallowCacheHits} cached)\n` + + `Files: ${s.indexedFiles} (${s.parsedFiles} parsed, ${s.shallowScannedFiles} shallow-scanned, ${s.shallowCacheHits + s.recordsCacheHits} cache hits)\n` + `Assets: ${s.assetCount} (${s.manifestAssetCount} from ${s.manifestFiles} manifests)\n` + `Defines: ${s.defineCount} · Streams: ${s.streams} · Candidates: ${s.sourceCandidates}\n` + `Indexed in ${(s.elapsedMs / 1000).toFixed(1)}s`, diff --git a/src/indexer/caches.ts b/src/indexer/caches.ts index 010e92f..d798fc7 100644 --- a/src/indexer/caches.ts +++ b/src/indexer/caches.ts @@ -10,7 +10,9 @@ */ import { resolve } from "node:path"; -import type { ParsedFile } from "./types"; +import type { IndexedFile, ParsedFile } from "./types"; +import type { IndexRecords } from "./records"; +import type { ResolveResult } from "./includeResolver"; /** Case-insensitive absolute path key. */ export function normKey(path: string): string { @@ -18,14 +20,21 @@ export function normKey(path: string): string { } /** - * LRU cache for fully parsed XML documents. Parse trees of huge mods can be - * memory-heavy, so only a bounded number of recent documents is retained; - * evicted entries are re-read from disk on demand. + * LRU cache for fully parsed XML documents. + * + * Parse trees of huge mods can be memory-heavy (~17x the source text), so + * retention is bounded twice: by entry count (least-recently-used eviction) + * and by a total element budget (largest trees are evicted first). Evicted + * entries are re-read from disk on demand. */ export class DocumentCache { private map = new Map(); + private totalElements = 0; - constructor(private capacity = 64) {} + constructor( + private capacity = 64, + private elementBudget = 2_000_000, + ) {} get(path: string): ParsedFile | undefined { const key = normKey(path); @@ -36,10 +45,111 @@ export class DocumentCache { return hit; } + /** Number of cached documents (for diagnostics). */ + get size(): number { + return this.map.size; + } + + /** Total elements held by cached parse trees (for diagnostics). */ + get elements(): number { + return this.totalElements; + } + set(parsed: ParsedFile): void { const key = normKey(parsed.file.path); + const prev = this.map.get(key); + if (prev) this.totalElements -= elementCount(prev); this.map.delete(key); this.map.set(key, parsed); + this.totalElements += elementCount(parsed); + this.evict(); + } + + invalidate(path: string): void { + const key = normKey(path); + const prev = this.map.get(key); + if (prev) this.totalElements -= elementCount(prev); + this.map.delete(key); + } + + clear(): void { + this.map.clear(); + this.totalElements = 0; + } + + private evict(): void { + while (this.map.size > this.capacity || this.totalElements > this.elementBudget) { + if (this.map.size === 0) break; + if (this.map.size > this.capacity) { + // Over capacity: drop the least recently used entry. + const oldest = this.map.keys().next().value; + if (oldest === undefined) break; + this.remove(oldest); + } else { + // Over the element budget: drop the largest tree, which frees the + // most memory per eviction. + let largestKey: string | undefined; + let largest = -1; + for (const [key, value] of this.map) { + const n = elementCount(value); + if (n > largest) { + largest = n; + largestKey = key; + } + } + if (largestKey === undefined || largest <= 0) break; + this.remove(largestKey); + } + } + } + + private remove(key: string): void { + const prev = this.map.get(key); + if (prev) this.totalElements -= elementCount(prev); + this.map.delete(key); + } +} + +function elementCount(parsed: ParsedFile): number { + return parsed.parse?.elements.length ?? 0; +} + +export interface IndexRecordsCacheEntry { + stat: IndexedFile["stat"]; + records: IndexRecords; + /** "shallow" for art-asset scans (.w3x), "full" for parsed XML. */ + kind: "shallow" | "full"; +} + +/** + * Cache for per-file index records (top-level assets, defines, includes, + * xi:include targets). Records are tiny compared to DOM trees or line maps of + * multi-megabyte model files, so the capacity comfortably covers a whole mod + * (Corona: ~9k files) and rebuilds never re-read unchanged files. + */ +export class IndexRecordsCache { + private map = new Map(); + + constructor(private capacity = 16384) {} + + get(path: string): IndexRecordsCacheEntry | undefined { + const key = normKey(path); + const hit = this.map.get(key); + if (!hit) return undefined; + this.map.delete(key); + this.map.set(key, hit); + return hit; + } + + /** Number of cached record sets (for diagnostics). */ + get size(): number { + return this.map.size; + } + + set(path: string, entry: IndexRecordsCacheEntry): void { + const key = normKey(path); + this.map.delete(key); + this.map.set(key, entry); if (this.map.size > this.capacity) { const oldest = this.map.keys().next().value; if (oldest !== undefined) this.map.delete(oldest); @@ -56,17 +166,21 @@ export class DocumentCache { } /** - * Cache for shallow-scanned art-asset documents (.w3x and content-sniffed - * XML files). The retained records are tiny (top-level assets, includes, - * defines) compared to a full DOM, so the capacity can be much larger. + * Cache for Include/@source (and xi:include/@href) resolution results. + * + * Resolving a source performs synchronous statSync existence checks against + * every search base; a Corona build issues ~110k of them (tens of seconds on + * a mechanical drive). Content edits never change *existence*, so this cache + * survives content rebuilds and is only cleared when files are created or + * deleted (or on a forced reindex). */ -export class ShallowScanCache { - private map = new Map(); +export class IncludeResolveCache { + private map = new Map(); + private manifestMap = new Map(); - constructor(private capacity = 8192) {} + constructor(private capacity = 262144) {} - get(path: string): ParsedFile | undefined { - const key = normKey(path); + get(key: string): ResolveResult | undefined { const hit = this.map.get(key); if (!hit) return undefined; this.map.delete(key); @@ -74,21 +188,39 @@ export class ShallowScanCache { return hit; } - set(parsed: ParsedFile): void { - const key = normKey(parsed.file.path); + set(key: string, result: ResolveResult): void { this.map.delete(key); - this.map.set(key, parsed); + this.map.set(key, result); if (this.map.size > this.capacity) { const oldest = this.map.keys().next().value; if (oldest !== undefined) this.map.delete(oldest); } } - invalidate(path: string): void { - this.map.delete(normKey(path)); + getManifest(key: string): string | null | undefined { + const hit = this.manifestMap.get(key); + if (hit === undefined) return undefined; + this.manifestMap.delete(key); + this.manifestMap.set(key, hit); + return hit; + } + + setManifest(key: string, path: string | null): void { + this.manifestMap.delete(key); + this.manifestMap.set(key, path); + if (this.manifestMap.size > this.capacity) { + const oldest = this.manifestMap.keys().next().value; + if (oldest !== undefined) this.manifestMap.delete(oldest); + } } clear(): void { this.map.clear(); + this.manifestMap.clear(); + } + + /** Number of cached resolutions (for diagnostics). */ + get size(): number { + return this.map.size; } } diff --git a/src/indexer/fileScanner.ts b/src/indexer/fileScanner.ts index 7fc430d..e74a3c2 100644 --- a/src/indexer/fileScanner.ts +++ b/src/indexer/fileScanner.ts @@ -87,8 +87,15 @@ export async function collectSourceCandidates( out.push({ source, path: resolve(path), prefix, baseDir: resolve(baseDir) }); }; - for (const dir of dataDirs) { - const files = await walker.listFiles(dir); + // List directories in parallel; the walker caches each directory by its + // mtime, so rebuilds after the first are still cheap. + const [dataLists, artLists, audioLists] = await Promise.all([ + Promise.all(dataDirs.map(async (dir) => ({ dir, files: await walker.listFiles(dir) }))), + Promise.all(artDirs.map(async (dir) => ({ dir, files: await walker.listFiles(dir) }))), + Promise.all(audioDirs.map(async (dir) => ({ dir, files: await walker.listFiles(dir) }))), + ]); + + for (const { dir, files } of dataLists) { for (const f of files) { if (!isDataCandidate(f)) continue; add(f, dir, "DATA"); @@ -101,15 +108,11 @@ export async function collectSourceCandidates( } } - for (const dir of artDirs) { - for (const f of await walker.listFiles(dir)) { - add(f, dir, "ART"); - } + for (const { dir, files } of artLists) { + for (const f of files) add(f, dir, "ART"); } - for (const dir of audioDirs) { - for (const f of await walker.listFiles(dir)) { - add(f, dir, "AUDIO"); - } + for (const { dir, files } of audioLists) { + for (const f of files) add(f, dir, "AUDIO"); } return out; diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 81b3012..c6d681f 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -12,6 +12,7 @@ */ import { open, readFile, readdir, stat } from "node:fs/promises"; +import type { Stats } from "node:fs"; import { basename, dirname, extname, join, resolve } from "node:path"; import { LineMap, @@ -24,6 +25,7 @@ import { buildSearchPaths, manifestPathForReference, resolveSource, + type ResolveResult, type SearchPaths, } from "./includeResolver"; import { @@ -34,8 +36,14 @@ import { } from "./manifestParser"; import { canonicalTypeName } from "../model/schemaModel"; import { collectSourceCandidates } from "./fileScanner"; -import { DocumentCache, ShallowScanCache, normKey } from "./caches"; +import { DocumentCache, IncludeResolveCache, IndexRecordsCache, normKey } from "./caches"; +import type { IndexRecordsCacheEntry } from "./caches"; import { scanXmlShallow } from "./shallowScan"; +import { + extractIndexRecords, + recordsFromShallow, + type IndexRecordXi, +} from "./records"; import type { AssetDef, DefineDef, @@ -69,8 +77,16 @@ type XmlMode = "full" | "shallow" | "binary"; export class ModIndexer { private searchPaths: SearchPaths; private docs: DocumentCache; - private shallowDocs: ShallowScanCache; - private scanCounters = { shallowScannedFiles: 0, shallowCacheHits: 0 }; + private recordsCache: IndexRecordsCache; + private resolveCache: IncludeResolveCache; + private scanCounters = { + shallowScannedFiles: 0, + shallowCacheHits: 0, + recordsCacheHits: 0, + resolveCacheHits: 0, + resolveCalls: 0, + }; + private phase = { candidatesMs: 0, walkMs: 0 }; private assets = new Map>(); private assetsById = new Map(); private defines = new Map(); @@ -89,7 +105,8 @@ export class ModIndexer { }); // Caches may be owned by the workspace so they survive rebuilds. this.docs = opts.documentCache ?? new DocumentCache(); - this.shallowDocs = opts.shallowCache ?? new ShallowScanCache(); + this.recordsCache = opts.recordsCache ?? new IndexRecordsCache(); + this.resolveCache = opts.resolveCache ?? new IncludeResolveCache(); } /** @@ -105,22 +122,30 @@ export class ModIndexer { */ async readDocument(path: string): Promise { const key = normKey(path); - const mode = await this.detectXmlMode(path); - if (mode === "shallow") return this.scanShallow(path); - if (mode === "binary") { - try { - const st = await stat(path); - const file: IndexedFile = { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }; - this.files.set(key, file); - return { file, parse: null, shallow: null, lineMap: null }; - } catch { - const file: IndexedFile = { path: resolve(path), stat: null }; - this.files.set(key, file); - return { file, parse: null, shallow: null, lineMap: null }; + const trust = + this.opts.trustUnchanged === true && !this.opts.changedFiles?.has(key); + + // Trusted fast path: a file that the watcher has not reported as changed + // is reused without any stat / content sniff / read at all. This is what + // makes save-triggered rebuilds cheap on huge corpora (Corona: ~9k files, + // ~2.6 GB of art assets on a mechanical drive). + if (trust) { + const rec = this.recordsCache.get(key); + if (rec) return this.recordsParsed(path, rec); + const cached = this.docs.get(key); + if (cached) { + this.files.set(key, cached.file); + return cached; } } + + // Untrusted / cache miss: verify the stat against caches, then read. try { const st = await stat(path); + const rec = this.recordsCache.get(key); + if (rec?.stat && rec.stat.mtimeMs === st.mtimeMs && rec.stat.size === st.size) { + return this.recordsParsed(path, rec); + } const hit = this.docs.get(key); if ( hit?.file.stat && @@ -130,29 +155,41 @@ export class ModIndexer { this.files.set(key, hit.file); return hit; } + const mode = await this.detectXmlMode(path); + if (mode === "shallow") return this.scanShallow(path, st); + if (mode === "binary") { + const file: IndexedFile = { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }; + const parsed: ParsedFile = { file, parse: null, records: null, lineMap: null }; + this.docs.set(parsed); + this.files.set(key, file); + return parsed; + } if (st.size > MAX_PARSE_BYTES) { const file: IndexedFile = { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }; - const parsed: ParsedFile = { file, parse: null, shallow: null, lineMap: null }; + const parsed: ParsedFile = { file, parse: null, records: null, lineMap: null }; this.docs.set(parsed); this.files.set(key, file); return parsed; } const text = stripBom(await readFile(path, "utf8")); + const lineMap = new LineMap(text); const parse = parseXml(text); + const records = extractIndexRecords(parse, lineMap); const parsed: ParsedFile = { file: { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }, parse, - shallow: null, - lineMap: new LineMap(text), + records, + lineMap, }; this.docs.set(parsed); + this.recordsCache.set(key, { stat: parsed.file.stat, records, kind: "full" }); this.files.set(key, parsed.file); return parsed; } catch { const parsed: ParsedFile = { file: { path: resolve(path), stat: null }, parse: null, - shallow: null, + records: null, lineMap: null, }; this.docs.set(parsed); @@ -163,32 +200,23 @@ export class ModIndexer { /** * Shallow-scans a large art-asset XML document (no DOM built) and caches - * the top-level records. Cache hits are counted separately so tests and - * the index report can verify that rebuilds skip unchanged files. + * its compact index records. The transient LineMap used to compute record + * lines is discarded, so the cache never retains megabytes of line offsets + * for model files (Corona w3x alone would otherwise keep ~700 MB). */ - private async scanShallow(path: string): Promise { + private async scanShallow(path: string, st: Stats): Promise { const key = normKey(path); try { - const st = await stat(path); - const hit = this.shallowDocs.get(key); - if ( - hit?.file.stat && - hit.file.stat.mtimeMs === st.mtimeMs && - hit.file.stat.size === st.size - ) { - this.scanCounters.shallowCacheHits++; - this.files.set(key, hit.file); - return hit; - } const text = stripBom(await readFile(path, "utf8")); - const shallow = scanXmlShallow(text); + const lineMap = new LineMap(text); + const records = recordsFromShallow(scanXmlShallow(text), lineMap); const parsed: ParsedFile = { file: { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }, parse: null, - shallow, - lineMap: new LineMap(text), + records, + lineMap: null, }; - this.shallowDocs.set(parsed); + this.recordsCache.set(key, { stat: parsed.file.stat, records, kind: "shallow" }); this.files.set(key, parsed.file); this.scanCounters.shallowScannedFiles++; return parsed; @@ -197,6 +225,59 @@ export class ModIndexer { } } + /** Builds a lean ParsedFile from cached index records (no DOM / line map). */ + private recordsParsed(path: string, entry: IndexRecordsCacheEntry): ParsedFile { + if (entry.kind === "shallow") this.scanCounters.shallowCacheHits++; + else this.scanCounters.recordsCacheHits++; + const file: IndexedFile = { path: resolve(path), stat: entry.stat }; + this.files.set(normKey(path), file); + return { file, parse: null, records: entry.records, lineMap: null }; + } + + /** + * Reads a document and guarantees a DOM parse tree. Used only for + * root-level xpointer selection (rare), where the target's + * container children are needed. + */ + private async readDom(path: string): Promise { + const key = normKey(path); + const cached = this.docs.get(key); + if (cached?.parse?.root) { + this.files.set(key, cached.file); + return cached; + } + try { + const st = await stat(path); + const hit = this.docs.get(key); + if ( + hit?.parse?.root && + hit.file.stat && + hit.file.stat.mtimeMs === st.mtimeMs && + hit.file.stat.size === st.size + ) { + this.files.set(key, hit.file); + return hit; + } + if (st.size > MAX_PARSE_BYTES) return null; + const text = stripBom(await readFile(path, "utf8")); + const lineMap = new LineMap(text); + const parse = parseXml(text); + const records = extractIndexRecords(parse, lineMap); + const parsed: ParsedFile = { + file: { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }, + parse, + records, + lineMap, + }; + this.docs.set(parsed); + this.recordsCache.set(key, { stat: parsed.file.stat, records, kind: "full" }); + this.files.set(key, parsed.file); + return parsed; + } catch { + return null; + } + } + /** Decides how a resolved include target should be consumed. */ private async detectXmlMode(path: string): Promise { const ext = extname(path).toLowerCase(); @@ -205,6 +286,38 @@ export class ModIndexer { return (await looksLikeXml(path)) ? "shallow" : "binary"; } + /** + * Resolves an include source through the cross-rebuild cache. Existence + * does not change on content edits, so trusted rebuilds pay zero statSync + * for include resolution (Corona does ~110k checks per build otherwise). + */ + private resolveCached(source: string, currentDir: string | null): ResolveResult { + const key = `${normKey(currentDir ?? "")}\u0000${source}`; + const hit = this.resolveCache.get(key); + if (hit) { + this.scanCounters.resolveCacheHits++; + return hit; + } + this.scanCounters.resolveCalls++; + const result = resolveSource(source, currentDir, this.searchPaths); + this.resolveCache.set(key, result); + return result; + } + + /** Cached manifest lookup for `reference` includes. */ + private manifestPathCached(source: string): string | null { + const key = source.toLowerCase(); + const hit = this.resolveCache.getManifest(key); + if (hit !== undefined) { + this.scanCounters.resolveCacheHits++; + return hit; + } + this.scanCounters.resolveCalls++; + const path = manifestPathForReference(source, this.opts.builtmodsDirs); + this.resolveCache.setManifest(key, path); + return path; + } + /** Returns the cached parse if present (does not read from disk). */ cachedDocument(path: string): ParsedFile | undefined { return this.docs.get(path); @@ -218,6 +331,7 @@ export class ModIndexer { : null; // ── Streams ── + const walkStart = Date.now(); const staticEntry = projectData ? join(projectData, "Mod.xml") : null; if (staticEntry) { const stream: StreamInfo = { name: "static", entry: staticEntry, files: new Set() }; @@ -246,8 +360,10 @@ export class ModIndexer { await this.walk(entry, "all", stream, 0); } } + this.phase.walkMs = Date.now() - walkStart; // ── Source completion candidates ── + const candidatesStart = Date.now(); const dataDirs = [ projectData ?? join(this.opts.projectDir, "Data"), join(this.opts.sdkDir, "SageXml"), @@ -292,6 +408,7 @@ export class ModIndexer { ...sdkRootCandidates, ...this.sourceCandidates, ]); + this.phase.candidatesMs = Date.now() - candidatesStart; const manifestAssetCount = [...this.manifests.values()].reduce( (sum, m) => sum + m.assets.length, @@ -318,6 +435,11 @@ export class ModIndexer { ).length, shallowScannedFiles: this.scanCounters.shallowScannedFiles, shallowCacheHits: this.scanCounters.shallowCacheHits, + recordsCacheHits: this.scanCounters.recordsCacheHits, + resolveCacheHits: this.scanCounters.resolveCacheHits, + resolveCalls: this.scanCounters.resolveCalls, + candidatesMs: this.phase.candidatesMs, + walkMs: this.phase.walkMs, assetCount: [...this.assets.values()].reduce((sum, byId) => sum + byId.size, 0), defineCount: this.defines.size, manifestFiles: this.manifests.size, @@ -359,167 +481,64 @@ export class ModIndexer { stream.files.add(key); - // readDocument handles every mode: full XML parse, shallow scan for - // art-asset XML (.w3x / content-sniffed), or binary registration only. + // readDocument returns compact index records for every indexable XML + // document (full parse or shallow scan), or a bare file registration + // for binary / unparseable targets. const parsed = await this.readDocument(path); if (!parsed) return; - if (parsed.shallow) { - await this.walkShallow(parsed, stream, depth, mode === "instance"); + if (parsed.records) { + await this.applyRecords(parsed, stream, depth, mode === "instance"); return; } - if (!parsed.parse?.root) return; - const root = parsed.parse.root; - - for (const child of root.children) { - const local = localName(child.name); - if (local === "Tags" || local === "Includes" || local === "Defines") continue; - if (local === "include") { - await this.handleXiInclude(child, parsed, stream, depth); - continue; - } - const idAttr = child.attrs.find((a) => a.name === "id"); - if (idAttr) { - this.addAsset({ - type: local, - id: idAttr.value, - file: parsed.file.path, - line: lineOf(parsed, idAttr.valueStart), - origin: this.originOf(parsed.file.path), - stream: stream.name, - viaInstance: mode === "instance", - }); - } - } - - for (const child of root.children) { - if (localName(child.name) !== "Defines") continue; - for (const define of child.children) { - if (localName(define.name) !== "Define") continue; - const name = define.attrs.find((a) => a.name === "name")?.value; - const value = define.attrs.find((a) => a.name === "value")?.value; - if (!name) continue; - const entry: DefineDef = { - name, - value: value ?? "", - file: parsed.file.path, - line: lineOf(parsed, define.start), - origin: this.originOf(parsed.file.path), - }; - const arr = this.defines.get(name.toLowerCase()); - if (arr) arr.push(entry); - else this.defines.set(name.toLowerCase(), [entry]); - } - } - - const includesElem = root.children.find((c) => localName(c.name) === "Includes"); - if (includesElem) { - for (const inc of includesElem.children) { - if (localName(inc.name) !== "Include") continue; - const type = inc.attrs.find((a) => a.name === "type")?.value; - const source = inc.attrs.find((a) => a.name === "source")?.value; - if (!source) continue; - const resolved = resolveSource(source, dirname(parsed.file.path), this.searchPaths); - if (!resolved.path) { - this.diagnostics.push({ - file: parsed.file.path, - line: lineOf(parsed, inc.start), - message: `Include target not found: ${source}`, - severity: "warning", - code: "include-not-found", - }); - continue; - } - if (type === "all" || type === "instance") { - await this.walk(resolved.path, type === "all" ? "all" : "instance", stream, depth + 1); - } else if (type === "reference") { - const manifestPath = manifestPathForReference(source, this.opts.builtmodsDirs); - if (manifestPath) { - const loaded = await this.loadManifest(manifestPath, stream.name); - if (!loaded) { - // The manifest could not be parsed (missing/invalid): fall back - // to the placeholder target so its content is still available. - await this.walk(resolved.path, "instance", stream, depth + 1); - } - } else { - // reference to a real XML file: treat its assets as available - await this.walk(resolved.path, "instance", stream, depth + 1); - } - } - } - } - - // Nested anywhere in the tree (not just under the root): - // the target content is inlined into the parent element. We make the - // target file available and surface missing targets instead of ignoring - // them silently. - for (const el of parsed.parse.elements) { - if (localName(el.name) !== "include") continue; - if (el.parent === root) continue; // already handled in the loop above - const href = el.attrs.find((a) => a.name === "href")?.value; - if (!href) continue; - const resolved = resolveSource(href, dirname(parsed.file.path), this.searchPaths); - if (!resolved.path) { - this.diagnostics.push({ - file: parsed.file.path, - line: lineOf(parsed, el.start), - message: `xi:include target not found: ${href}`, - severity: "warning", - code: "include-not-found", - }); - continue; - } - stream.files.add(normKey(resolved.path)); - if ((await this.detectXmlMode(resolved.path)) !== "binary") { - await this.walk(resolved.path, "all", stream, depth + 1); - } - } } /** - * Consumes a shallow-scanned art-asset document: top-level assets, - * top-level , and nested targets. + * Applies a document's compact index records: top-level assets, defines, + * , nested and root-level targets. + * Works identically for fully parsed XML and shallow-scanned art assets. */ - private async walkShallow( + private async applyRecords( parsed: ParsedFile, stream: StreamInfo, depth: number, viaInstance: boolean, ): Promise { - const scan = parsed.shallow; - if (!scan) return; + const records = parsed.records; + if (!records) return; const file = parsed.file.path; + const origin = this.originOf(file); - for (const asset of scan.assets) { + for (const asset of records.assets) { this.addAsset({ - type: asset.name, + type: asset.type, id: asset.id, file, - line: lineOf(parsed, asset.idValueStart), - origin: this.originOf(file), + line: asset.line, + origin, stream: stream.name, viaInstance, }); } - for (const define of scan.defines) { + for (const define of records.defines) { const entry: DefineDef = { name: define.name, value: define.value, file, - line: lineOf(parsed, define.start), - origin: this.originOf(file), + line: define.line, + origin, }; const arr = this.defines.get(define.name.toLowerCase()); if (arr) arr.push(entry); else this.defines.set(define.name.toLowerCase(), [entry]); } - for (const inc of scan.includes) { - const resolved = resolveSource(inc.source, dirname(file), this.searchPaths); + for (const inc of records.includes) { + const resolved = this.resolveCached(inc.source, dirname(file)); if (!resolved.path) { this.diagnostics.push({ file, - line: lineOf(parsed, inc.start), + line: inc.line, message: `Include target not found: ${inc.source}`, severity: "warning", code: "include-not-found", @@ -534,7 +553,7 @@ export class ModIndexer { depth + 1, ); } else if (inc.type === "reference") { - const manifestPath = manifestPathForReference(inc.source, this.opts.builtmodsDirs); + const manifestPath = this.manifestPathCached(inc.source); if (manifestPath) { const loaded = await this.loadManifest(manifestPath, stream.name); if (!loaded) await this.walk(resolved.path, "instance", stream, depth + 1); @@ -544,12 +563,12 @@ export class ModIndexer { } } - for (const xi of scan.xiIncludes) { - const resolved = resolveSource(xi.href, dirname(file), this.searchPaths); + for (const xi of records.nestedXiIncludes) { + const resolved = this.resolveCached(xi.href, dirname(file)); if (!resolved.path) { this.diagnostics.push({ file, - line: lineOf(parsed, xi.start), + line: xi.line, message: `xi:include target not found: ${xi.href}`, severity: "warning", code: "include-not-found", @@ -557,68 +576,65 @@ export class ModIndexer { continue; } stream.files.add(normKey(resolved.path)); - if ((await this.detectXmlMode(resolved.path)) !== "binary") { - await this.walk(resolved.path, "all", stream, depth + 1); - } + await this.walk(resolved.path, "all", stream, depth + 1); + } + + for (const xi of records.rootXiIncludes) { + await this.handleRootXiInclude(xi, file, stream, depth); } } - private async handleXiInclude( - xi: XmlElement, - parent: ParsedFile, + /** + * Root-level with an xpointer selects a named container's + * children in the target document, so the target's DOM is needed. These + * targets are rare, so they are parsed on demand (the tree is also cached). + */ + private async handleRootXiInclude( + xi: IndexRecordXi, + parentFile: string, stream: StreamInfo, depth: number, ): Promise { - const href = xi.attrs.find((a) => a.name === "href")?.value; - if (!href) return; - const resolved = resolveSource(href, dirname(parent.file.path), this.searchPaths); + const resolved = this.resolveCached(xi.href, dirname(parentFile)); if (!resolved.path) { this.diagnostics.push({ - file: parent.file.path, - line: lineOf(parent, xi.start), - message: `xi:include target not found: ${href}`, + file: parentFile, + line: xi.line, + message: `xi:include target not found: ${xi.href}`, severity: "warning", code: "include-not-found", }); return; } - if ((await this.detectXmlMode(resolved.path)) === "binary") return; - const target = await this.readDocument(resolved.path); - if (!target) return; - if (target.shallow) { - // Shallow-scanned targets have no tree to select xpointer children - // from; index their top-level content as a whole. - stream.files.add(normKey(target.file.path)); - await this.walk(target.file.path, "all", stream, depth + 1); - return; - } - if (!target.parse?.root) return; - const xpointer = xi.attrs.find((a) => a.name === "xpointer")?.value ?? ""; - let candidates: XmlElement[]; - if (xpointer) { - const container = findXPointerContainer(target.parse, xpointer); - candidates = container ? container.children : []; - } else { - candidates = target.parse.root.children; - } - for (const el of candidates) { - const local = localName(el.name); - if (local === "Tags" || local === "Includes" || local === "Defines") continue; - const idAttr = el.attrs.find((a) => a.name === "id"); - if (idAttr) { - this.addAsset({ - type: local, - id: idAttr.value, - file: target.file.path, - line: lineOf(target, idAttr.valueStart), - origin: this.originOf(target.file.path), - stream: stream.name, - }); + const target = await this.readDom(resolved.path); + if (target?.parse?.root) { + const xpointer = xi.xpointer ?? ""; + let candidates: XmlElement[]; + if (xpointer) { + const container = findXPointerContainer(target.parse, xpointer); + candidates = container ? container.children : []; + } else { + candidates = target.parse.root.children; + } + for (const el of candidates) { + const local = localName(el.name); + if (local === "Tags" || local === "Includes" || local === "Defines") continue; + const idAttr = el.attrs.find((a) => a.name === "id"); + if (idAttr) { + this.addAsset({ + type: local, + id: idAttr.value, + file: target.file.path, + line: lineOf(target, idAttr.valueStart), + origin: this.originOf(target.file.path), + stream: stream.name, + }); + } } } - stream.files.add(normKey(target.file.path)); - await this.walk(target.file.path, "all", stream, depth + 1); + stream.files.add(normKey(resolved.path)); + await this.walk(resolved.path, "all", stream, depth + 1); } // ── Manifest loading ────────────────────────────────────────────── diff --git a/src/indexer/records.ts b/src/indexer/records.ts new file mode 100644 index 0000000..822565d --- /dev/null +++ b/src/indexer/records.ts @@ -0,0 +1,177 @@ +/** + * Compact per-file index records. + * + * A rebuild only needs each file's top-level assets, defines, includes and + * xi:include targets — not the DOM. Extracting these records at parse time + * and caching them across rebuilds lets Corona-scale rebuilds skip both the + * DOM and the file I/O for unchanged files, while keeping the DOM cache small + * for on-demand features (hover / navigation / outline). + * + * Pure TypeScript: no vscode dependency. + */ + +import type { LineMap, XmlDocument } from "../language/xmlParser"; +import type { ShallowDocument } from "./shallowScan"; + +export interface IndexRecordAsset { + /** Top-level element name, e.g. "W3DContainer". */ + type: string; + id: string; + /** 1-based line of the id attribute value. */ + line: number; +} + +export interface IndexRecordDefine { + name: string; + value: string; + /** 1-based line of the element. */ + line: number; +} + +export interface IndexRecordInclude { + type: "all" | "instance" | "reference" | null; + source: string; + /** 1-based line of the element. */ + line: number; +} + +export interface IndexRecordXi { + href: string; + xpointer: string | null; + /** 1-based line of the element. */ + line: number; +} + +export interface IndexRecords { + assets: IndexRecordAsset[]; + defines: IndexRecordDefine[]; + includes: IndexRecordInclude[]; + /** elements that are direct children of the root. */ + rootXiIncludes: IndexRecordXi[]; + /** elements nested anywhere else in the document. */ + nestedXiIncludes: IndexRecordXi[]; +} + +const INCLUDE_TYPES = new Set(["all", "instance", "reference"]); + +function lineOf(lineMap: LineMap, offset: number): number { + return lineMap.positionAt(offset).line + 1; +} + +function localName(tag: string): string { + const idx = tag.lastIndexOf(":"); + return idx >= 0 ? tag.slice(idx + 1) : tag; +} + +/** + * Extracts the index records of a fully parsed document. Mirrors the walk + * semantics of the indexer exactly: top-level assets (excluding + * Tags/Includes/Defines), $DEFINE constants, the top-level block + * and root/nested elements. + */ +export function extractIndexRecords(parse: XmlDocument, lineMap: LineMap): IndexRecords { + const assets: IndexRecordAsset[] = []; + const defines: IndexRecordDefine[] = []; + const includes: IndexRecordInclude[] = []; + const rootXiIncludes: IndexRecordXi[] = []; + const nestedXiIncludes: IndexRecordXi[] = []; + const root = parse.root; + if (!root) return { assets, defines, includes, rootXiIncludes, nestedXiIncludes }; + + for (const child of root.children) { + const local = localName(child.name); + if (local === "Tags" || local === "Includes" || local === "Defines") continue; + if (local === "include") { + const href = child.attrs.find((a) => a.name === "href")?.value; + if (href) { + rootXiIncludes.push({ + href, + xpointer: child.attrs.find((a) => a.name === "xpointer")?.value ?? null, + line: lineOf(lineMap, child.start), + }); + } + continue; + } + const idAttr = child.attrs.find((a) => a.name === "id"); + if (idAttr) { + assets.push({ type: local, id: idAttr.value, line: lineOf(lineMap, idAttr.valueStart) }); + } + } + + for (const child of root.children) { + if (localName(child.name) !== "Defines") continue; + for (const define of child.children) { + if (localName(define.name) !== "Define") continue; + const name = define.attrs.find((a) => a.name === "name")?.value; + if (!name) continue; + defines.push({ + name, + value: define.attrs.find((a) => a.name === "value")?.value ?? "", + line: lineOf(lineMap, define.start), + }); + } + } + + const includesElem = root.children.find((c) => localName(c.name) === "Includes"); + if (includesElem) { + for (const inc of includesElem.children) { + if (localName(inc.name) !== "Include") continue; + const source = inc.attrs.find((a) => a.name === "source")?.value; + if (!source) continue; + const type = inc.attrs.find((a) => a.name === "type")?.value; + includes.push({ + type: + type && INCLUDE_TYPES.has(type) + ? (type as "all" | "instance" | "reference") + : null, + source, + line: lineOf(lineMap, inc.start), + }); + } + } + + for (const el of parse.elements) { + if (localName(el.name) !== "include") continue; + if (el.parent === root) continue; // already handled above + const href = el.attrs.find((a) => a.name === "href")?.value; + if (!href) continue; + nestedXiIncludes.push({ + href, + xpointer: el.attrs.find((a) => a.name === "xpointer")?.value ?? null, + line: lineOf(lineMap, el.start), + }); + } + + return { assets, defines, includes, rootXiIncludes, nestedXiIncludes }; +} + +/** Converts a shallow scan (offsets) into index records (1-based lines). */ +export function recordsFromShallow(scan: ShallowDocument, lineMap: LineMap): IndexRecords { + return { + assets: scan.assets.map((a) => ({ + type: a.name, + id: a.id, + line: lineOf(lineMap, a.idValueStart), + })), + defines: scan.defines.map((d) => ({ + name: d.name, + value: d.value, + line: lineOf(lineMap, d.start), + })), + includes: scan.includes.map((i) => ({ + type: i.type, + source: i.source, + line: lineOf(lineMap, i.start), + })), + rootXiIncludes: scan.rootXiIncludes.map((x) => ({ + href: x.href, + xpointer: x.xpointer, + line: lineOf(lineMap, x.start), + })), + nestedXiIncludes: scan.nestedXiIncludes.map((x) => ({ + href: x.href, + xpointer: x.xpointer, + line: lineOf(lineMap, x.start), + })), + }; +} diff --git a/src/indexer/shallowScan.ts b/src/indexer/shallowScan.ts index d3d7845..9930d54 100644 --- a/src/indexer/shallowScan.ts +++ b/src/indexer/shallowScan.ts @@ -60,7 +60,10 @@ export interface ShallowScanError { export interface ShallowDocument { assets: ShallowAssetRecord[]; includes: ShallowIncludeRecord[]; - xiIncludes: ShallowXiIncludeRecord[]; + /** elements that are direct children of the root. */ + rootXiIncludes: ShallowXiIncludeRecord[]; + /** elements nested anywhere else in the document. */ + nestedXiIncludes: ShallowXiIncludeRecord[]; defines: ShallowDefineRecord[]; errors: ShallowScanError[]; } @@ -81,7 +84,8 @@ export function scanXmlShallow(text: string): ShallowDocument { const errors: ShallowScanError[] = []; const assets: ShallowAssetRecord[] = []; const includes: ShallowIncludeRecord[] = []; - const xiIncludes: ShallowXiIncludeRecord[] = []; + const rootXiIncludes: ShallowXiIncludeRecord[] = []; + const nestedXiIncludes: ShallowXiIncludeRecord[] = []; const defines: ShallowDefineRecord[] = []; // Depth of the currently open element stack. The document root opens at @@ -196,11 +200,13 @@ export function scanXmlShallow(text: string): ShallowDocument { const href = findAttr(inner, base, "href"); const xpointer = findAttr(inner, base, "xpointer"); if (href?.value) { - xiIncludes.push({ + const rec: ShallowXiIncludeRecord = { href: href.value, xpointer: xpointer?.value ?? null, start: lt, - }); + }; + if (depth === 1) rootXiIncludes.push(rec); + else nestedXiIncludes.push(rec); } } if (!selfClosing) depth++; @@ -210,7 +216,7 @@ export function scanXmlShallow(text: string): ShallowDocument { i = gt + 1; } - return { assets, includes, xiIncludes, defines, errors }; + return { assets, includes, rootXiIncludes, nestedXiIncludes, defines, errors }; } /** diff --git a/src/indexer/types.ts b/src/indexer/types.ts index 626e393..cdd4fa2 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -1,8 +1,8 @@ import type { XmlDocument } from "../language/xmlParser"; import type { ManifestInfo } from "./manifestParser"; import type { LineMap } from "../language/xmlParser"; -import type { ShallowDocument } from "./shallowScan"; -import type { DocumentCache, ShallowScanCache } from "./caches"; +import type { IndexRecords } from "./records"; +import type { DocumentCache, IncludeResolveCache, IndexRecordsCache } from "./caches"; export type AssetOrigin = "project" | "sdk" | "manifest"; @@ -71,6 +71,16 @@ export interface IndexStats { shallowScannedFiles: number; /** Shallow scans served from the persistent cache (unchanged files). */ shallowCacheHits: number; + /** Parsed XML files served from the persistent records cache. */ + recordsCacheHits: number; + /** Include/xi:include resolutions served from the resolve cache. */ + resolveCacheHits: number; + /** Include/xi:include resolutions performed during this build. */ + resolveCalls: number; + /** Time spent enumerating Include source candidates (ms). */ + candidatesMs: number; + /** Time spent walking the include graph (ms). */ + walkMs: number; assetCount: number; defineCount: number; manifestFiles: number; @@ -110,8 +120,21 @@ export interface IndexOptions { walker: FileWalker; /** Optional parse-tree cache shared across rebuilds (owned by the workspace). */ documentCache?: DocumentCache; - /** Optional shallow-scan cache shared across rebuilds (owned by the workspace). */ - shallowCache?: ShallowScanCache; + /** Optional index-records cache shared across rebuilds (owned by the workspace). */ + recordsCache?: IndexRecordsCache; + /** Optional include-resolution cache shared across rebuilds (owned by the workspace). */ + resolveCache?: IncludeResolveCache; + /** + * When true, cached documents whose path is not in `changedFiles` are used + * without a per-file stat. The workspace enables this while a file watcher + * invalidates caches for changed paths; a forced reindex passes false. + */ + trustUnchanged?: boolean; + /** + * Normalized paths (see `normKey`) known to have changed since the caches + * were populated. Only consulted when `trustUnchanged` is true. + */ + changedFiles?: ReadonlySet; } export interface FileWalker { @@ -132,9 +155,9 @@ export interface ParsedFile { file: IndexedFile; parse: XmlDocument | null; /** - * Shallow records for large art-asset XML documents (.w3x etc.); null for - * fully parsed XML and for binary files. + * Compact index records (assets/defines/includes/xi:include with lines). + * Present for every indexable XML document; null for binary files. */ - shallow: ShallowDocument | null; + records: IndexRecords | null; lineMap: LineMap | null; } diff --git a/src/workspace.ts b/src/workspace.ts index 997e862..7930791 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { join, dirname } from "node:path"; import { CachedDirectoryWalker } from "./indexer/fileScanner"; import { ModIndexer } from "./indexer/indexer"; -import { DocumentCache, ShallowScanCache } from "./indexer/caches"; +import { DocumentCache, IncludeResolveCache, IndexRecordsCache } from "./indexer/caches"; import type { ModIndex } from "./indexer/types"; import { readSettings, type ExtensionSettings } from "./settings"; @@ -20,13 +20,17 @@ export class ModWorkspace { // only re-reads files whose stat changed (crucial for the ~2.6 GB of .w3x // art assets in a project like Corona). private documentCache = new DocumentCache(); - private shallowCache = new ShallowScanCache(); + private recordsCache = new IndexRecordsCache(); + private resolveCache = new IncludeResolveCache(); + private context: vscode.ExtensionContext; + private watchers: vscode.FileSystemWatcher[] = []; private statusBar: vscode.StatusBarItem; private rebuildTimer: ReturnType | null = null; private building = false; private dirty = false; constructor(context: vscode.ExtensionContext) { + this.context = context; this.settings = readSettings(); this.statusBar = vscode.window.createStatusBarItem( vscode.StatusBarAlignment.Left, @@ -57,11 +61,69 @@ export class ModWorkspace { this.statusBar.hide(); return; } + this.startWatching(); this.statusBar.text = "$(sync~spin) RA3 XML: indexing…"; this.statusBar.show(); await this.rebuild(); } + /** + * Invalidates cached documents for a path (called by the file watcher and + * on document save), so the next rebuild re-reads it instead of trusting + * the cached copy. + */ + invalidate(path: string): void { + if (!path) return; + this.documentCache.invalidate(path); + this.recordsCache.invalidate(path); + } + + /** + * Called when files are created or deleted: include-resolution results + * (which encode file existence) are no longer trustworthy. + */ + invalidateExistence(): void { + this.resolveCache.clear(); + } + + /** + * Watches the project, SDK and extra DATA roots for file changes and + * invalidates the corresponding cache entries. With caches invalidated + * precisely, rebuilds can trust every other cached file and skip per-file + * stats (huge win on mechanical drives; Corona rebuild dropped from ~38s + * to a few seconds). + */ + private startWatching(): void { + if (!this.projectRoot) return; + const roots = new Set([ + this.projectRoot, + this.settings.sdkPath, + ...this.settings.additionalDataSearchPaths, + ]); + for (const root of roots) { + if (!existsSync(root)) continue; + try { + const watcher = vscode.workspace.createFileSystemWatcher( + new vscode.RelativePattern(root, "**/*"), + ); + watcher.onDidCreate((uri) => { + this.invalidate(uri.fsPath); + this.invalidateExistence(); + }); + watcher.onDidChange((uri) => this.invalidate(uri.fsPath)); + watcher.onDidDelete((uri) => { + this.invalidate(uri.fsPath); + this.invalidateExistence(); + }); + this.watchers.push(watcher); + this.context.subscriptions.push(watcher); + } catch { + // The root may be temporarily unavailable (e.g. removable drive); + // indexing still works, just without watcher-based invalidation. + } + } + } + scheduleRebuild(): void { if (!this.projectRoot) return; if (this.rebuildTimer) clearTimeout(this.rebuildTimer); @@ -70,12 +132,13 @@ export class ModWorkspace { }, REBUILD_DEBOUNCE_MS); } - async rebuild(): Promise { + async rebuild(force = false): Promise { if (!this.projectRoot) return; if (this.building) { this.dirty = true; return; } + if (force) this.resolveCache.clear(); this.building = true; this.settings = readSettings(); try { @@ -88,7 +151,11 @@ export class ModWorkspace { additionalDataSearchPaths: this.settings.additionalDataSearchPaths, walker: this.walker, documentCache: this.documentCache, - shallowCache: this.shallowCache, + recordsCache: this.recordsCache, + resolveCache: this.resolveCache, + // Trust cache entries unless the user explicitly asked for a full + // verification (ra3modxml.reindex). + trustUnchanged: !force, }); const started = Date.now(); this.index = await indexer.build(); @@ -121,7 +188,7 @@ export class ModWorkspace { return { file: { path, stat: null }, parse, - shallow: null, + records: null, lineMap: new LineMap(text), }; } diff --git a/test/caches.test.mjs b/test/caches.test.mjs new file mode 100644 index 0000000..9f7b85a --- /dev/null +++ b/test/caches.test.mjs @@ -0,0 +1,73 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + DocumentCache, + IncludeResolveCache, + IndexRecordsCache, +} from "../out/indexer/caches.js"; + +function parsed(path, elements) { + return { + file: { path, stat: { mtimeMs: 1, size: 1 } }, + parse: { root: { name: "r" }, elements: new Array(elements), errors: [] }, + records: null, + lineMap: null, + }; +} + +test("DocumentCache evicts the largest tree when over the element budget", () => { + const cache = new DocumentCache(64, 100); + cache.set(parsed("a.xml", 60)); + cache.set(parsed("b.xml", 60)); + assert.equal(cache.get("a.xml"), undefined, "largest tree evicted first"); + assert.ok(cache.get("b.xml")); +}); + +test("DocumentCache evicts least recently used when over capacity", () => { + const cache = new DocumentCache(2, 1_000_000); + cache.set(parsed("a.xml", 10)); + cache.set(parsed("b.xml", 10)); + cache.set(parsed("c.xml", 10)); + assert.equal(cache.get("a.xml"), undefined); + assert.ok(cache.get("b.xml")); + assert.ok(cache.get("c.xml")); +}); + +test("DocumentCache invalidate frees budget", () => { + const cache = new DocumentCache(64, 100); + cache.set(parsed("a.xml", 60)); + cache.invalidate("a.xml"); + cache.set(parsed("b.xml", 60)); + assert.ok(cache.get("b.xml"), "invalidating a freed its budget share"); + cache.set(parsed("c.xml", 60)); + assert.equal(cache.get("a.xml"), undefined); + assert.ok(cache.get("c.xml")); +}); + +test("IndexRecordsCache stores and invalidates entries", () => { + const cache = new IndexRecordsCache(); + const entry = { + stat: { mtimeMs: 1, size: 1 }, + records: { assets: [], defines: [], includes: [], rootXiIncludes: [], nestedXiIncludes: [] }, + kind: "full", + }; + cache.set("a.xml", entry); + assert.equal(cache.get("a.xml"), entry); + cache.invalidate("a.xml"); + assert.equal(cache.get("a.xml"), undefined); +}); + +test("IncludeResolveCache stores sources and manifest lookups", () => { + const cache = new IncludeResolveCache(); + const key = "dir|DATA:static.xml"; + const result = { path: "C:/sdk/static.xml", prefix: "DATA", raw: "DATA:static.xml" }; + assert.equal(cache.get(key), undefined); + cache.set(key, result); + assert.equal(cache.get(key), result); + assert.equal(cache.getManifest("static.xml"), undefined); + cache.setManifest("static.xml", "C:/sdk/builtmods/static.manifest"); + assert.equal(cache.getManifest("static.xml"), "C:/sdk/builtmods/static.manifest"); + cache.clear(); + assert.equal(cache.get(key), undefined); + assert.equal(cache.getManifest("static.xml"), undefined); +}); diff --git a/test/indexer.test.mjs b/test/indexer.test.mjs index 4b784f1..9cd0933 100644 --- a/test/indexer.test.mjs +++ b/test/indexer.test.mjs @@ -6,7 +6,11 @@ import fs from "node:fs"; import os from "node:os"; import { ModIndexer } from "../out/indexer/indexer.js"; import { CachedDirectoryWalker } from "../out/indexer/fileScanner.js"; -import { DocumentCache, ShallowScanCache } from "../out/indexer/caches.js"; +import { + DocumentCache, + IncludeResolveCache, + IndexRecordsCache, +} from "../out/indexer/caches.js"; import { resolveReferenceTargetsForType } from "../out/indexer/refs.js"; const root = dirname(dirname(fileURLToPath(import.meta.url))); @@ -131,7 +135,8 @@ test("w3x files appear in Include source completion candidates", async () => { test("shallow scans and full parses are cached across rebuilds", async () => { const documentCache = new DocumentCache(); - const shallowCache = new ShallowScanCache(); + const recordsCache = new IndexRecordsCache(); + const resolveCache = new IncludeResolveCache(); const makeIndexer = () => new ModIndexer({ projectDir: project, @@ -141,7 +146,8 @@ test("shallow scans and full parses are cached across rebuilds", async () => { additionalDataSearchPaths: [], walker: new CachedDirectoryWalker(), documentCache, - shallowCache, + recordsCache, + resolveCache, }); const first = await makeIndexer().build(); @@ -150,10 +156,70 @@ test("shallow scans and full parses are cached across rebuilds", async () => { assert.equal(first.stats.shallowScannedFiles, 2, "w3x + sniffed w3d scanned on first build"); assert.equal(second.stats.shallowScannedFiles, 0, "unchanged art assets are not re-scanned"); assert.equal(second.stats.shallowCacheHits, 2); + assert.ok(second.stats.recordsCacheHits > 0, "XML records served from cache"); + assert.ok(second.stats.resolveCacheHits > 0, "include resolutions served from cache"); + assert.equal(second.stats.resolveCalls, 0, "no include re-resolved on a trusted rebuild"); assert.equal(second.stats.assetCount, first.stats.assetCount); assert.equal(second.stats.indexedFiles, first.stats.indexedFiles); }); +test("trusted rebuilds skip unchanged files; invalidation forces re-reads", async () => { + const documentCache = new DocumentCache(); + const recordsCache = new IndexRecordsCache(); + const resolveCache = new IncludeResolveCache(); + const opts = () => ({ + projectDir: project, + sdkDir: sdk, + builtmodsDirs: [join(sdk, "builtmods")], + indexSageXml: true, + additionalDataSearchPaths: [], + walker: new CachedDirectoryWalker(), + documentCache, + recordsCache, + resolveCache, + }); + + // Trusted rebuild: cached files are reused without any per-file stat/read. + const first = await new ModIndexer({ ...opts(), trustUnchanged: true }).build(); + assert.equal(first.stats.shallowScannedFiles, 2); + const second = await new ModIndexer({ ...opts(), trustUnchanged: true }).build(); + assert.equal(second.stats.shallowScannedFiles, 0); + assert.equal(second.stats.shallowCacheHits, 2); + assert.ok(second.stats.recordsCacheHits > 0); + + // Simulate the file watcher / save handler: invalidate one art asset. + // The next trusted rebuild must re-scan exactly that file. + recordsCache.invalidate(join(project, "Data", "Includes", "Models", "Tank_SKN.w3x")); + const third = await new ModIndexer({ ...opts(), trustUnchanged: true }).build(); + assert.equal(third.stats.shallowScannedFiles, 1); + assert.equal(third.stats.shallowCacheHits, 1); + assert.ok( + third.assetsById.get("tank_skn")?.some((d) => d.type === "W3DContainer"), + "re-scanned w3x asset present", + ); + + // Invalidate one XML file: exactly its records are re-extracted. + recordsCache.invalidate(join(project, "Data", "Includes", "Units.xml")); + const fourth = await new ModIndexer({ ...opts(), trustUnchanged: true }).build(); + assert.equal(fourth.stats.recordsCacheHits, third.stats.recordsCacheHits - 1); + assert.ok( + fourth.assetsById.get("testtank")?.some((d) => d.type === "GameObject"), + "re-parsed XML asset present", + ); + + // A forced rebuild (ra3modxml.reindex) verifies stats but still reuses + // content caches for unchanged files. + const forced = await new ModIndexer({ ...opts(), trustUnchanged: false }).build(); + assert.equal(forced.stats.shallowScannedFiles, 0); + assert.equal(forced.stats.shallowCacheHits, 2); +}); + +test("index stats include candidate/walk phase timings", async () => { + const idx = await buildIndex(); + assert.equal(typeof idx.stats.candidatesMs, "number"); + assert.equal(typeof idx.stats.walkMs, "number"); +}); + test("w3x with a UTF-8 BOM is indexed with correct offsets", async (t) => { const tmp = fs.mkdtempSync(join(os.tmpdir(), "ra3modxml-bom-")); t.after(() => fs.rmSync(tmp, { recursive: true, force: true })); diff --git a/test/records.test.mjs b/test/records.test.mjs new file mode 100644 index 0000000..3ece3d5 --- /dev/null +++ b/test/records.test.mjs @@ -0,0 +1,65 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseXml, LineMap } from "../out/language/xmlParser.js"; +import { extractIndexRecords, recordsFromShallow } from "../out/indexer/records.js"; +import { scanXmlShallow } from "../out/indexer/shallowScan.js"; + +test("extractIndexRecords mirrors the walk semantics", () => { + const text = ` + + + + + + + + + + + + + + + + +`; + const lineMap = new LineMap(text); + const records = extractIndexRecords(parseXml(text), lineMap); + assert.deepEqual( + records.assets.map((a) => [a.type, a.id, a.line]), + [ + ["GameObject", "Tank", 10], + ["WeaponTemplate", "TankGun", 11], + ["GameObject", "Tank2", 13], + ], + ); + assert.deepEqual( + records.defines.map((d) => [d.name, d.value]), + [["HP", "100"]], + ); + assert.deepEqual( + records.includes.map((i) => [i.type, i.source]), + [ + ["all", "Units.xml"], + ["instance", "Base.xml"], + ], + ); + assert.deepEqual( + records.rootXiIncludes.map((x) => [x.href, x.line]), + [["DATA:Extra.xml", 12]], + ); + assert.deepEqual( + records.nestedXiIncludes.map((x) => [x.href, x.line]), + [["DATA:Nested.xml", 15]], + ); +}); + +test("recordsFromShallow converts offsets to 1-based lines", () => { + const text = `\n \n`; + const lineMap = new LineMap(text); + const records = recordsFromShallow(scanXmlShallow(text), lineMap); + assert.equal(records.assets.length, 1); + assert.equal(records.assets[0].type, "W3DContainer"); + assert.equal(records.assets[0].id, "A"); + assert.equal(records.assets[0].line, 2); +}); diff --git a/test/shallowScan.test.mjs b/test/shallowScan.test.mjs index 499793e..a461953 100644 --- a/test/shallowScan.test.mjs +++ b/test/shallowScan.test.mjs @@ -33,9 +33,10 @@ test("extracts top-level assets, includes, defines and xi:include", () => { assert.equal(doc.defines.length, 1); assert.equal(doc.defines[0].name, "MODEL_SCALE"); assert.equal(doc.defines[0].value, "1.0"); - assert.equal(doc.xiIncludes.length, 1); - assert.equal(doc.xiIncludes[0].href, "DATA:Includes/Extra.xml"); - assert.match(doc.xiIncludes[0].xpointer, /xpointer\(/); + assert.equal(doc.rootXiIncludes.length, 1); + assert.equal(doc.rootXiIncludes[0].href, "DATA:Includes/Extra.xml"); + assert.match(doc.rootXiIncludes[0].xpointer, /xpointer\(/); + assert.equal(doc.nestedXiIncludes.length, 0); // Recorded offsets must slice the original text back out. const container = doc.assets[0]; @@ -98,3 +99,24 @@ test("nested module payload does not create asset records", () => { ["MESH"], ); }); + +test("distinguishes root-level and nested xi:include", () => { + const doc = scanXmlShallow(` + + + + + + + + +`); + assert.deepEqual( + doc.rootXiIncludes.map((x) => x.href), + ["DATA:Top.xml"], + ); + assert.deepEqual( + doc.nestedXiIncludes.map((x) => x.href), + ["DATA:Nested.xml"], + ); +});