From 4cc07b9abc0f1b23d1c07d13142452226e6927ef Mon Sep 17 00:00:00 2001 From: lanyizi Date: Sun, 2 Aug 2026 14:32:18 +0200 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=20W3X?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + README.md | 7 + debug.log | 1 - docs/analysis-issues.md | 95 ++++++ docs/plan.md | 38 ++- docs/requirements.md | 8 + src/extension.ts | 2 +- src/indexer/caches.ts | 94 ++++++ src/indexer/fileScanner.ts | 17 +- src/indexer/indexer.ts | 318 ++++++++++++++---- src/indexer/shallowScan.ts | 268 +++++++++++++++ src/indexer/types.ts | 15 + src/language/xmlParser.ts | 8 + src/workspace.ts | 11 +- .../Data/Includes/Models/Tank_Damaged.dds | 2 + .../minimod/Data/Includes/Models/Tank_FP.w3d | 4 + .../minimod/Data/Includes/Models/Tank_SKN.w3x | 5 + .../minimod/Data/Includes/VehicleArt.xml | 8 + test/fixtures/minimod/Data/Mod.xml | 1 + test/indexer.test.mjs | 121 +++++++ test/shallowScan.test.mjs | 100 ++++++ test/xmlParser.test.mjs | 7 +- 22 files changed, 1052 insertions(+), 79 deletions(-) delete mode 100644 debug.log create mode 100644 src/indexer/caches.ts create mode 100644 src/indexer/shallowScan.ts create mode 100644 test/fixtures/minimod/Data/Includes/Models/Tank_Damaged.dds create mode 100644 test/fixtures/minimod/Data/Includes/Models/Tank_FP.w3d create mode 100644 test/fixtures/minimod/Data/Includes/Models/Tank_SKN.w3x create mode 100644 test/fixtures/minimod/Data/Includes/VehicleArt.xml create mode 100644 test/shallowScan.test.mjs diff --git a/.gitignore b/.gitignore index 32a58c0..be6ee62 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ out/ dist/ *.vsix OpenSAGE/ +debug.log \ No newline at end of file diff --git a/README.md b/README.md index 2695093..2d9204e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ - **引用导航**：从引用值（`CommandSet="..."`、`Weapon="..."`、`inheritFrom`）跳转到定义（严格按引用类型过滤，候选由 `ra3modxml.definitionMode` 控制：`all` 列出 mod + 原版、`project-only` 优先项目内定义）；`Ctrl+点击` Include / `xi:include href` 打开目标文件；Find All References 搜索整个工作区；文档大纲列出顶层资产与 `$DEFINE`。 - **错误检查**：XML 格式错误、未知元素/属性（`xi:` 等外来命名空间不误报）、顶层资产缺 `id`、重复 ID、未解析引用（含类型不匹配）、Include / 嵌套 `xi:include` 目标找不到、`$DEFINE` 未定义。 - **manifest 支持**：`` 指向的 `static/global/audio.manifest`（SDK `builtmods`）会被解析，manifest 中的原版资产 ID 可用于补全/悬停/导航/诊断。 +- **美术资产（`.w3x`）**：`W3X.xml` / `ART:` include 链中的 `.w3x` 模型文件会被 + 索引（`W3DContainer` / `W3DMesh` / `W3DHierarchy` 等顶层资产），因此 + `Model@Name`、`Hierarchy`、`Mesh` 等引用可以解析、悬停与跳转。超大模型 + （几十 MB 的顶点/三角形数据）采用浅扫描——只提取顶层资产记录、不建 DOM 树， + 结果在 workspace 级缓存并跨重建复用，保存文件触发的重建不会重读未变化的模型文件。 ## 使用 @@ -72,6 +77,8 @@ src/ manifestParser.ts .manifest 二进制解析（移植 OpenSAGE ManifestFile.cs） fileScanner.ts 目录扫描与 Include source 候选 refs.ts 引用目标解析（按引用类型过滤） + shallowScan.ts .w3x 等大体积美术资产顶层浅扫描（纯 TS，不建 DOM） + caches.ts 跨重建持久缓存（DocumentCache / ShallowScanCache） indexer.ts 工作区索引器（后台、缓存、增量重建） features/ completion / hover / navigation / diagnostics / semanticTokens syntaxes/ TextMate 注入语法 diff --git a/debug.log b/debug.log deleted file mode 100644 index 5f1d689..0000000 --- a/debug.log +++ /dev/null @@ -1 +0,0 @@ -[0801/032923.150:ERROR:third_party\crashpad\crashpad\util\win\registration_protocol_win.cc:108] CreateFile: 拒绝访问。 (0x5) diff --git a/docs/analysis-issues.md b/docs/analysis-issues.md index 4c33656..2666076 100644 --- a/docs/analysis-issues.md +++ b/docs/analysis-issues.md @@ -590,3 +590,98 @@ token：文档出现解析错误时，由插件用自己的容错解析树继续 配色，可能与 TextMate XML 配色略有差异——只在打字过程中出现，可接受； - 未实现 `provideDocumentSemanticTokensEdits`（delta 版本），每次全量计算；单文件 解析在 KB 级，开销可忽略。 + +--- + +## 十三、问题分析（第八轮，2026-08-02）：`.w3x` 美术资产未被索引（AUGunship_SKN） + +### 问题 + +AttachTest `Harbinger Gunship\GameObject.xml` 中 `` +报两条错误： + +1. Problems 面板诊断：`Unresolved reference "AUGunship_SKN" (not found in the current index)` + （`unresolved-reference`，来自 `features/diagnostics.ts`）； +2. 悬停提示：`No matching definition of type BaseRenderAssetType in the current index + (may exist in a compiled manifest or vanilla data).`（来自 `features/hover.ts`）。 + +两条是同一缺失定义的两种呈现（诊断 + hover），不是两个独立 bug。 +`W3DContainer:AUGunship_SKN` 确实由 mod 定义——但定义在 +`Harbinger Gunship\W3X\AUGUNSHIP_SKN.w3x` 里，而索引器只解析 `.xml` / `.manifestxml`。 + +### 关键事实（实测） + +1. **`.w3x` 是文本 XML**：文件头即 `` + ``，内容为 + `` + `` 子树。 + 附带 UTF-8 BOM（抽样 292 个 Corona w3x：0 个 UTF-16，40 个带 BOM）。 +2. **w3x 通过 `` 链进索引**：`Mod.xml → … → W3X.xml → W3X/*.w3x`， + 也常用 `ART:xxx.w3x`（SDK 根、项目 `Art` 等搜索路径）。索引器此前只把 w3x 登记为 + 文件（`stream.files`），从不解析，因此其顶层资产不在 `assetsById` 中。 +3. **类型匹配本来是对的**：`Model@Name` 的 refType 是 `BaseRenderAssetType`， + `W3DContainer → BaseRenderAssetType` 可赋值（`isAssignableTo` = true）。缺的只是定义。 +4. **manifest / SageXml 支持早已存在**（第二轮/第三轮），但该资产是 mod 自己的 w3x， + 不在 `builtmods/*.manifest` 也不在 SageXml——hover 的 "may exist in a compiled manifest + or vanilla data" 只是通用兜底文案。 + +### 规模调查（Corona，D: 盘已连接） + +- **3788 个 w3x，共 2.64 GB**；最大 22.8 MB；163 个超过原 4 MB 解析上限，11 个超 10 MB。 +- 大文件结构符合"建模软件导出"模式：顶层是少量固定 W3D 资产 + （`W3DHierarchy` + 若干 `W3DMesh` / `W3DContainer` / `W3DCollisionBox`， + 有的还带 `` 引用 `ART:*.xml`）；体积大头是 `W3DMesh` 内的 + `Vertices/V`、`Normals/N`、`TexCoords/T`、`Triangles/T` 等 `maxOccurs="unbounded"` + 数值元素。例如 21.7 MB 的 `CBRefinery_BLD.W3X` 只有 22 个顶层记录， + Vertices+Triangles 块占约 12 MB（55%）。 +- 全量建 DOM 的代价（实测 6.3 MB Aegis 文件）：407 ms、193,651 个元素、 + **堆内存 +109 MB（约 17 倍文本体积）**；22 MB 文件外推约 1.5 s + ~380 MB/份。 + 浅扫描（只取顶层记录）：6.3 MB ~200 ms、22 MB ~600 ms，保留内存近似为零。 +- **读整个文件不可避免，但建树不是**：浅扫描仍然是线性扫描全文（要知道顶层元素边界 + 就必须扫完），只是不分配子节点对象——所以"事后优化"完全有意义，且是必要项。 + +### 修复 + +1. **新增浅扫描模块** `src/indexer/shallowScan.ts`（纯 TS）： + 单次线性扫描，只提取顶层元素 `name + id`（含精确 offset）、顶层 `` + 的 `Include@type/source`、任意层级 `` 的 `href/xpointer`、`` + 常量；注释 / CDATA / DOCTYPE / PI 整体跳过；属性解析兼容引号内 `>` 与 `/`。 +2. **索引模式三分**：`.xml`/`.manifestxml` 全量解析（4 MB 上限不变）； + `.w3x` 一律浅扫描；未知扩展名先嗅探文件头（512 字节，BOM/空白后以 `<` 开头且无 + NUL 字节 → 按 XML 浅扫描，否则按二进制仅登记）。w3x 自身的 `` 与 + 嵌套 `xi:include` 会继续被 walk（BAB 语义）。 +3. **持久缓存**：`DocumentCache` / `ShallowScanCache` 移到 `src/indexer/caches.ts`， + 由 `ModWorkspace` 持有并传入每次新建的 `ModIndexer`；读取时按 `mtimeMs + size` + 校验，未变化不重读。这是 w3x 方案在 Corona 上可用的前提（否则每次保存都重读 2.6 GB）。 +4. **BOM 剥离**：`xmlParser` 新增 `stripBom()`，全量解析与浅扫描前统一剥离， + 保证第一行偏移与编辑器一致。 +5. **Include source 补全候选**：DATA 目录与项目相对路径候选加入 `.w3x` + （`fileScanner`），`W3X/xxx.w3x`、`ART/xxx.w3x` 可补全。 +6. 索引报告/状态栏新增 `shallowScannedFiles` / `shallowCacheHits` 统计。 + +### 验证 + +- 单元测试 **53 → 63 全绿**：新增 `test/shallowScan.test.mjs`（顶层资产 / Includes / + xi:include / Defines / 引号内 `>` / CDATA / 未闭合标签 / 数值载荷不产生记录）； + indexer 新增 w3x 链、`.w3d` 嗅探、二进制 `.dds` 跳过、跨重建缓存命中、 + BOM 偏移断言；xmlParser 新增 `stripBom` 用例。 +- AttachTest 实机： + - 首次构建 1.2 s，浅扫 62 个文件；`Model@Name=AUGunship_SKN` → + `W3DContainer @ …\W3X\AUGUNSHIP_SKN.w3x:3`；`Hierarchy=AUGunship_SKL`、 + `AUGunship_FP` 同样解析；资产数 35,546 → 35,607。 + - 第二次构建 354 ms，0 次重扫，62 次缓存命中。 +- Corona 实机（D: 盘）： + - 首次构建 241 s：8,976 个文件、**浅扫 4,829 个**、资产 64,868 + （manifest 35,322）、3 个流、183 个 Define。 + - 第二次构建 38 s：0 次重扫、4,829 次缓存命中、资产数一致。 + - `CBREFINERY_BLD` 正确解析到 `W3DHierarchy @ cb/CBRefinery_BLD.w3x:16` 与 + `W3DContainer @ …:776412`。 + +### 边界与后续 + +- 首次建索引较慢（Corona ~4 分钟，机械盘）：读 2.6 GB 是下限，浅扫描本身 ~30-90 s； + 后续可做并行扫描或把 w3x 顶层记录做成独立小缓存文件。 +- 第二次构建仍有 ~38 s（主要是对 ~9k 文件逐文件 stat + 重建 Map，机械盘随机读）； + 后续可做"文件清单 + stat 快照"级缓存。 +- 浅扫描对 w3x 不做 XSD 校验（编辑器特性不注册 w3x 语言）；若用户手动把 `*.w3x` + 关联为 xml，完整解析仍会发生在打开的文档上（VS Code 自身行为）。 +- UTF-16 XML 的 w3x 会被嗅探判定为二进制（文件头含 NUL）；实测生态中不存在， + 如遇可再扩展解码。 diff --git a/docs/plan.md b/docs/plan.md index f8d988f..52485e2 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -85,7 +85,9 @@ src/ manifestParser.ts .manifest 二进制解析 + 类型/ID 推导（纯 TS） fileScanner.ts 目录遍历缓存 + Include source 候选收集 refs.ts 引用目标解析（按 refType / isRef / inheritFrom 过滤，纯 TS） - indexer.ts 工作区索引器（资产/Define/流/manifest 合并，LRU 解析缓存） + shallowScan.ts 大体积美术资产（.w3x 等）顶层浅扫描（纯 TS，不建 DOM） + caches.ts 跨重建持久缓存（DocumentCache / ShallowScanCache） + indexer.ts 工作区索引器（资产/Define/流/manifest/w3x 合并） types.ts 共享类型 features/ completion.ts 补全 provider（元素/属性/值，上下文感知；xs:list 多值按当前段过滤） @@ -109,6 +111,11 @@ test/ 1. **语言激活范围**：不劫持 `*.xml`。通过 `workspaceContains:**/Data/Mod.xml`、`**/*.babproj` 激活；语法高亮为**纯注入** grammar（不声明 `language`，避免覆盖内置 XML 语法）。 2. **索引范围与默认值**：索引“项目 Data + additionalmaps + 沿 include 可达的 SageXml 原版源码”；SDK 路径默认 `C:\Apps\RA3-MODSDK-X`（可配置）。`reference` include 解析为 `builtmods` 下对应 manifest（惰性解析、按文件缓存），manifest 缺失/无效时回退到占位 XML。 + **美术资产（.w3x）**：`` / `ART:` 指向的 `.w3x`（及内容嗅探为 + XML 的未知扩展名文件）按其顶层资产入库（`W3DContainer` / `W3DMesh` / + `W3DHierarchy` / `W3DCollisionBox` 等），使 `Model@Name`、`Hierarchy`、`Mesh` + 等引用可解析；大模型文件**浅扫描**（不建 DOM），结果缓存在 workspace 级、 + 跨重建复用（详见设计决策 14）。 3. **manifest 资产建模**：类型优先用哈希表，未知时从名称前缀推导；可引用 ID 取最后冒号段；类型名统一走大小写规范化（`W3dContainer` ↔ `W3DContainer`），类型匹配严格遵循 XSD 继承链。 4. **上下文感知元素类型**：同名元素按父元素类型解析（`resolveElementType` 沿解析树逐层 `childTypeOf`，失败回退全局映射），保证 `` 等元素的属性/引用判定正确。 5. **引用判定与解析**：`refType` 或 `isRef` 均视为引用；带 `refType` 时严格按类型过滤（同名 ID 不串类型）；`inheritFrom` 按可继承类型过滤。**局部作用域例外**（`isLocalReferenceAttribute`）：`id` 是元素自身的定义点——无 refType 或 refType 与自身类型兼容时不检查、不解析（`RoadObject@id→Road` 这类跨类型 id 引用保留检查）；Poid 类型属性是管线局部引用，全局索引无法判定，不检查、不解析。 @@ -128,6 +135,20 @@ test/ XML 编辑器皆然）；扩展注册 `DocumentSemanticTokensProvider`，仅当解析报错时用 语义 token 覆盖标签名 / 属性名 / 属性值（标准 token 类型 `type` / `property` / `string`，主题自带配色）。合法文件返回空，观感与纯 TextMate 完全一致。 +14. **大体积美术资产浅扫描 + 跨重建持久缓存**（第八轮，2026-08-02）： + - 实测 Corona：3788 个 w3x / 2.64 GB，163 个超过 4 MB，最大 22.8 MB；大文件是 + `W3DHierarchy` + 若干 `W3DMesh`（`Vertices/V`、`Triangles/T` 等 unbounded 数值 + 载荷），顶层记录通常只有几个到二十几个。 + - 全量解析内存放大约 17 倍（6.3 MB 文本 → +109 MB DOM），不可接受；`scanXmlShallow` + 单次线性扫描只提取顶层 `name+id`、``、``、``， + 22 MB 文件 ~600 ms、保留内存≈0。 + - 索引按扩展名三分：`.xml`/`.manifestxml` 全量解析（4 MB 上限不变）；`.w3x` 浅扫描； + 未知扩展名嗅探文件头（`<` 开头、无 NUL）决定按 XML 浅扫描或二进制登记。 + - `DocumentCache` / `ShallowScanCache` 由 `ModWorkspace` 持有，每次重建传入新的 + `ModIndexer`；按 `mtimeMs + size` 校验，未变化不重读。Corona 第二次构建 + w3x 重扫数为 0（4,829 次缓存命中）。 + - 读取整个文件不可避免（顶层边界需要全量扫描），但建 DOM 不是；优化的是 + "不分配子节点对象"与"跨重建不重读"，两者叠加后方案可行。 ## 三、实施步骤 @@ -136,22 +157,25 @@ test/ 3. [x] 生成模型：`schema-model.json`（XSD，295 顶层元素 / 1851 类型）与 `asset-types.json`（79 个 AssetType 哈希）。 4. [x] 纯 TS 核心：include 解析、manifest 解析、XML 解析封装、索引器、引用解析。 5. [x] 功能层：补全、hover、导航、诊断、大纲、高亮 grammar。 -6. [x] 单测（fixture Mod，53 个用例全绿：含 xs:list 枚举、未闭合引号恢复、list 多值 +6. [x] 单测（fixture Mod，63 个用例全绿：含 xs:list 枚举、未闭合引号恢复、list 多值 分段、带 vscode stub 的补全集成、语义 token 兜底）+ 编译 + `vsce package` 打包 （ra3-mod-xml-0.1.0.vsix，约 495KB）。 -7. [x] 在 AttachTest / GenEvoTest / Corona 上做冒烟验证，并按真实项目反馈修复问题（详见 `docs/analysis-issues.md` 六轮分析）。 +7. [x] 在 AttachTest / GenEvoTest / Corona 上做冒烟验证，并按真实项目反馈修复问题（详见 `docs/analysis-issues.md` 八轮分析）。 +8. [x] w3x 美术资产索引（第八轮）：新增 `shallowScan.ts` / `caches.ts`，w3x 与内容嗅探 + XML 走浅扫描，Include source 补全候选加入 w3x，BOM 剥离，缓存跨重建持久化； + AttachTest 报错场景（`AUGunship_SKN`）修复，Corona 首次/二次构建实测验证。 ## 四、验证结果（实测） | 项目 | 规模 | 索引耗时 | 资产数 | 说明 | |---|---|---|---|---| -| AttachTest | 71 文件 | ~0.6s | 35,546（manifest 35,322） | 2 个流（static + mapmetadata） | -| GenEvoTest | 24 文件 | ~1.8s | 35,392（manifest 35,322） | 项目 ID（alliedmcv 等）正确收录 | -| Corona | 3,448 文件（+ 非 XML 资产路径） | ~54.5s | 55,305（manifest 35,322） | 3 个流、183 个 Define、0 诊断 | +| AttachTest | 88 文件 | ~1.2s | 35,607（manifest 35,322，w3x 浅扫 62） | 2 个流（static + mapmetadata）；二次构建 354ms / 62 缓存命中 | +| GenEvoTest | 66 文件 | 首次 ~2.6s / 二次 ~0.4s | 35,502（manifest 35,322，w3x 浅扫 38） | 2 个流、73 个 Define；二次构建 0 重扫 / 38 缓存命中 | +| Corona | 8,976 文件 | 首次 ~241s / 二次 ~38s | 64,868（manifest 35,322，w3x 浅扫 4,829） | 3 个流、183 个 Define、0 诊断；二次构建 0 重扫 / 4,829 缓存命中 | 单元测试覆盖：XML 解析（自闭合/容错/偏移/未闭合引号行尾恢复）、补全上下文（未闭合引号仍为 attribute-value、list 多值分段）、补全集成（vscode stub 下 `LocomotorTemplate@Surfaces` 未闭合引号枚举补全、空格后第二段过滤与替换范围）、语义 token（标签/属性/值范围、合法文档返回空、malformed 返回兜底 token）、include 解析（BAB 顺序、SDK 根优先于 SageXml）、manifest 二进制解析（合成 v5 样本、类型/ID 推导）、索引器（资产/Define/流/缺失 include/嵌套 xi:include）、XSD 模型（上下文类型、`childTypeOf`、大小写规范化、属性级 refType、外来命名空间判定、`xs:list` 枚举继承与 `isList` 标记）、引用过滤（`Weapon="X"` 只跳 `WeaponTemplate`、模块 `id` 定义点、Poid 局部引用、`xi:include` 不校验、`Side="Allies"` 命中 manifest 的 `PlayerTemplate`）。 -> 注：`D:\Mods\CoronaMod` 位于移动硬盘，当前未连接；GenEvoTest / Corona 的回归需在 D: 盘可用时补跑（用户会另行通知）。 +> 注：D: 盘移动硬盘已恢复连接；Corona 已按上述新数据回归（第八轮）。 ## 五、假设与开放问题 diff --git a/docs/requirements.md b/docs/requirements.md index d976966..ca62a5a 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -70,6 +70,14 @@ XML 之间的组织靠 `` 标签，共有三种语义： - `TypeId` 哈希 → 类型名的映射来自 OpenSAGE 的 `AssetType` 枚举（本工作区可提取）； - 资产名与源文件名各自存放在独立的空字符结尾字符串缓冲区中。 +**补充（美术资产 `.w3x` 索引）**： +- `W3X.xml` / `ART:` include 链中的 `.w3x` 是文本 XML（建模工具导出，BAB 同样按 + XML 编译），其顶层资产（`W3DContainer` / `W3DMesh` / `W3DHierarchy` 等）应参与 + 补全、悬停、导航与诊断——`Model@Name`、`Hierarchy`、`Mesh` 等引用依赖这些定义； +- 大模型文件（实测 Corona 最大 22.8 MB，顶点/三角形数据占大头）采用**顶层浅扫描** + （不建 DOM 树），结果在 workspace 级缓存并跨重建复用，避免每次保存都重读整个 + 美术资产目录（Corona 约 2.6 GB）。 + ### P1：非近期目标（本期不做，但预留扩展点） 6. **高效搜索**：Mod 项目巨大（Corona 约 7500 个 XML、38MB）时直接全文搜索很慢，需要一个高效的 XML 内容索引机制。 diff --git a/src/extension.ts b/src/extension.ts index f95c5a7..07e5618 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -135,7 +135,7 @@ export function activate(context: vscode.ExtensionContext): void { void vscode.window.showInformationMessage( `RA3 Mod XML index\n` + `Project: ${s.projectDir}\n` + - `Files: ${s.indexedFiles} (${s.parsedFiles} parsed)\n` + + `Files: ${s.indexedFiles} (${s.parsedFiles} parsed, ${s.shallowScannedFiles} shallow-scanned, ${s.shallowCacheHits} cached)\n` + `Assets: ${s.assetCount} (${s.manifestAssetCount} from ${s.manifestFiles} manifests)\n` + `Defines: ${s.defineCount} · Streams: ${s.streams} · Candidates: ${s.sourceCandidates}\n` + `Indexed in ${(s.elapsedMs / 1000).toFixed(1)}s`, diff --git a/src/indexer/caches.ts b/src/indexer/caches.ts new file mode 100644 index 0000000..010e92f --- /dev/null +++ b/src/indexer/caches.ts @@ -0,0 +1,94 @@ +/** + * Caches shared by the indexer. + * + * The workspace owns one instance of each cache and passes them into every + * ModIndexer, so a rebuild (which creates a fresh indexer) does not re-read + * files whose stat (mtime/size) is unchanged. This is what makes indexing + * large art-asset corpora (Corona: ~3800 .w3x files, 2.6 GB) practical: + * after the first build, save-triggered rebuilds only re-scan files that + * actually changed. + */ + +import { resolve } from "node:path"; +import type { ParsedFile } from "./types"; + +/** Case-insensitive absolute path key. */ +export function normKey(path: string): string { + return resolve(path).toLowerCase(); +} + +/** + * LRU cache for fully parsed XML documents. Parse trees of huge mods can be + * memory-heavy, so only a bounded number of recent documents is retained; + * evicted entries are re-read from disk on demand. + */ +export class DocumentCache { + private map = new Map(); + + constructor(private capacity = 64) {} + + get(path: string): ParsedFile | undefined { + const key = normKey(path); + const hit = this.map.get(key); + if (!hit) return undefined; + this.map.delete(key); + this.map.set(key, hit); + return hit; + } + + set(parsed: ParsedFile): void { + const key = normKey(parsed.file.path); + this.map.delete(key); + this.map.set(key, parsed); + if (this.map.size > this.capacity) { + const oldest = this.map.keys().next().value; + if (oldest !== undefined) this.map.delete(oldest); + } + } + + invalidate(path: string): void { + this.map.delete(normKey(path)); + } + + clear(): void { + this.map.clear(); + } +} + +/** + * Cache for shallow-scanned art-asset documents (.w3x and content-sniffed + * XML files). The retained records are tiny (top-level assets, includes, + * defines) compared to a full DOM, so the capacity can be much larger. + */ +export class ShallowScanCache { + private map = new Map(); + + constructor(private capacity = 8192) {} + + get(path: string): ParsedFile | undefined { + const key = normKey(path); + const hit = this.map.get(key); + if (!hit) return undefined; + this.map.delete(key); + this.map.set(key, hit); + return hit; + } + + set(parsed: ParsedFile): void { + const key = normKey(parsed.file.path); + this.map.delete(key); + this.map.set(key, parsed); + if (this.map.size > this.capacity) { + const oldest = this.map.keys().next().value; + if (oldest !== undefined) this.map.delete(oldest); + } + } + + invalidate(path: string): void { + this.map.delete(normKey(path)); + } + + clear(): void { + this.map.clear(); + } +} diff --git a/src/indexer/fileScanner.ts b/src/indexer/fileScanner.ts index 6b5fca0..7fc430d 100644 --- a/src/indexer/fileScanner.ts +++ b/src/indexer/fileScanner.ts @@ -1,7 +1,18 @@ import { readdir, stat } from "node:fs/promises"; -import { join, relative, resolve } from "node:path"; +import { extname, join, relative, resolve } from "node:path"; import type { FileWalker, SourceCandidate } from "./types"; +/** + * Extensions offered as Include/@source candidates in DATA directories: + * regular XML plus art-asset XML (.w3x) that mods include via W3X.xml hubs. + * ART/AUDIO directories already list every file. + */ +const DATA_CANDIDATE_EXTENSIONS = new Set([".xml", ".w3x"]); + +function isDataCandidate(path: string): boolean { + return DATA_CANDIDATE_EXTENSIONS.has(extname(path).toLowerCase()); +} + /** * Recursive file list walker with a simple directory-mtime cache. Used to * enumerate candidate files for Include/@source completion. Directory scans @@ -79,13 +90,13 @@ export async function collectSourceCandidates( for (const dir of dataDirs) { const files = await walker.listFiles(dir); for (const f of files) { - if (!f.toLowerCase().endsWith(".xml")) continue; + if (!isDataCandidate(f)) continue; add(f, dir, "DATA"); } if (samePath(dir, projectDataDir)) { // Also offer project-relative paths (what Mod.xml itself uses). for (const f of files) { - if (f.toLowerCase().endsWith(".xml")) add(f, projectDataDir, null); + if (isDataCandidate(f)) add(f, projectDataDir, null); } } } diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 3212d31..81b3012 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -11,9 +11,15 @@ * outside the extension. */ -import { readFile, readdir, stat } from "node:fs/promises"; +import { open, readFile, readdir, stat } from "node:fs/promises"; import { basename, dirname, extname, join, resolve } from "node:path"; -import { LineMap, parseXml, type XmlDocument, type XmlElement } from "../language/xmlParser"; +import { + LineMap, + parseXml, + stripBom, + type XmlDocument, + type XmlElement, +} from "../language/xmlParser"; import { buildSearchPaths, manifestPathForReference, @@ -28,6 +34,8 @@ import { } from "./manifestParser"; import { canonicalTypeName } from "../model/schemaModel"; import { collectSourceCandidates } from "./fileScanner"; +import { DocumentCache, ShallowScanCache, normKey } from "./caches"; +import { scanXmlShallow } from "./shallowScan"; import type { AssetDef, DefineDef, @@ -42,54 +50,27 @@ import type { const MAX_DEPTH = 300; /** Files above this size are never parsed (safety against binary blobs). */ const MAX_PARSE_BYTES = 4 * 1024 * 1024; -/** Only these extensions are treated as XML documents. */ -const XML_EXTENSIONS = new Set([".xml", ".manifestxml"]); - -function normKey(path: string): string { - return resolve(path).toLowerCase(); -} - /** - * LRU cache for parsed documents. The parse trees of huge mods can be - * memory-heavy, so only a bounded number of recent documents is retained; - * evicted entries are re-read from disk on demand. + * Fully parsed XML documents. `.xml` / `.manifestxml` files are small enough + * that a full DOM is affordable. */ -export class DocumentCache { - private map = new Map(); +const FULL_XML_EXTENSIONS = new Set([".xml", ".manifestxml"]); +/** + * XML documents whose top-level structure is all the index needs (art-asset + * files exported by modeling tools, e.g. .w3x). They are shallow-scanned so + * multi-megabyte vertex/triangle payloads never become a DOM. + */ +const SHALLOW_XML_EXTENSIONS = new Set([".w3x"]); +/** Bytes peeked when deciding whether an unknown extension is XML text. */ +const SNIFF_BYTES = 512; - constructor(private capacity = 64) {} - - get(path: string): ParsedFile | undefined { - const key = normKey(path); - const hit = this.map.get(key); - if (!hit) return undefined; - this.map.delete(key); - this.map.set(key, hit); - return hit; - } - - set(parsed: ParsedFile): void { - const key = normKey(parsed.file.path); - this.map.delete(key); - this.map.set(key, parsed); - if (this.map.size > this.capacity) { - const oldest = this.map.keys().next().value; - if (oldest !== undefined) this.map.delete(oldest); - } - } - - invalidate(path: string): void { - this.map.delete(normKey(path)); - } - - clear(): void { - this.map.clear(); - } -} +type XmlMode = "full" | "shallow" | "binary"; export class ModIndexer { private searchPaths: SearchPaths; - private docs = new DocumentCache(); + private docs: DocumentCache; + private shallowDocs: ShallowScanCache; + private scanCounters = { shallowScannedFiles: 0, shallowCacheHits: 0 }; private assets = new Map>(); private assetsById = new Map(); private defines = new Map(); @@ -106,42 +87,124 @@ export class ModIndexer { this.searchPaths = buildSearchPaths(opts.sdkDir, opts.projectDir, { DATA: opts.additionalDataSearchPaths, }); + // Caches may be owned by the workspace so they survive rebuilds. + this.docs = opts.documentCache ?? new DocumentCache(); + this.shallowDocs = opts.shallowCache ?? new ShallowScanCache(); } - /** Re-reads and caches a document; null when unreadable. */ + /** + * Returns a document for indexing/navigation: + * - `.xml` / `.manifestxml` files are fully parsed (bounded by + * MAX_PARSE_BYTES); + * - `.w3x` (and unknown-extension files whose content looks like XML) are + * shallow-scanned, so huge model files never become a DOM; + * - everything else is registered as a file but never parsed. + * + * Cached entries are reused when the file stat is unchanged, which lets a + * workspace-owned cache survive rebuilds. + */ async readDocument(path: string): Promise { - const hit = this.docs.get(path); - if (hit) return hit; + const key = normKey(path); + const mode = await this.detectXmlMode(path); + if (mode === "shallow") return this.scanShallow(path); + if (mode === "binary") { + try { + const st = await stat(path); + const file: IndexedFile = { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }; + this.files.set(key, file); + return { file, parse: null, shallow: null, lineMap: null }; + } catch { + const file: IndexedFile = { path: resolve(path), stat: null }; + this.files.set(key, file); + return { file, parse: null, shallow: null, lineMap: null }; + } + } try { - const [st, text] = await Promise.all([stat(path), readFile(path, "utf8")]); + const st = await stat(path); + const hit = this.docs.get(key); + if ( + hit?.file.stat && + hit.file.stat.mtimeMs === st.mtimeMs && + hit.file.stat.size === st.size + ) { + this.files.set(key, hit.file); + return hit; + } if (st.size > MAX_PARSE_BYTES) { const file: IndexedFile = { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }; - const parsed: ParsedFile = { file, parse: null, lineMap: null }; + const parsed: ParsedFile = { file, parse: null, shallow: null, lineMap: null }; this.docs.set(parsed); - this.files.set(normKey(parsed.file.path), file); + this.files.set(key, file); return parsed; } + const text = stripBom(await readFile(path, "utf8")); const parse = parseXml(text); const parsed: ParsedFile = { file: { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }, parse, + shallow: null, lineMap: new LineMap(text), }; this.docs.set(parsed); - this.files.set(normKey(parsed.file.path), parsed.file); + this.files.set(key, parsed.file); return parsed; } catch { const parsed: ParsedFile = { file: { path: resolve(path), stat: null }, parse: null, + shallow: null, lineMap: null, }; this.docs.set(parsed); - this.files.set(normKey(parsed.file.path), parsed.file); + this.files.set(key, parsed.file); return parsed; } } + /** + * Shallow-scans a large art-asset XML document (no DOM built) and caches + * the top-level records. Cache hits are counted separately so tests and + * the index report can verify that rebuilds skip unchanged files. + */ + private async scanShallow(path: string): Promise { + const key = normKey(path); + try { + const st = await stat(path); + const hit = this.shallowDocs.get(key); + if ( + hit?.file.stat && + hit.file.stat.mtimeMs === st.mtimeMs && + hit.file.stat.size === st.size + ) { + this.scanCounters.shallowCacheHits++; + this.files.set(key, hit.file); + return hit; + } + const text = stripBom(await readFile(path, "utf8")); + const shallow = scanXmlShallow(text); + const parsed: ParsedFile = { + file: { path: resolve(path), stat: { mtimeMs: st.mtimeMs, size: st.size } }, + parse: null, + shallow, + lineMap: new LineMap(text), + }; + this.shallowDocs.set(parsed); + this.files.set(key, parsed.file); + this.scanCounters.shallowScannedFiles++; + return parsed; + } catch { + return null; + } + } + + /** Decides how a resolved include target should be consumed. */ + private async detectXmlMode(path: string): Promise { + const ext = extname(path).toLowerCase(); + if (FULL_XML_EXTENSIONS.has(ext)) return "full"; + if (SHALLOW_XML_EXTENSIONS.has(ext)) return "shallow"; + return (await looksLikeXml(path)) ? "shallow" : "binary"; + } + /** Returns the cached parse if present (does not read from disk). */ cachedDocument(path: string): ParsedFile | undefined { return this.docs.get(path); @@ -253,6 +316,8 @@ export class ModIndexer { parsedFiles: [...this.files.values()].filter( (f) => f.stat != null && f.stat.size <= MAX_PARSE_BYTES, ).length, + shallowScannedFiles: this.scanCounters.shallowScannedFiles, + shallowCacheHits: this.scanCounters.shallowCacheHits, assetCount: [...this.assets.values()].reduce((sum, byId) => sum + byId.size, 0), defineCount: this.defines.size, manifestFiles: this.manifests.size, @@ -294,11 +359,15 @@ export class ModIndexer { stream.files.add(key); - // Binary assets (w3x/dds/...) are referenced but never parsed. - if (!isXmlPath(path)) return; - + // readDocument handles every mode: full XML parse, shallow scan for + // art-asset XML (.w3x / content-sniffed), or binary registration only. const parsed = await this.readDocument(path); - if (!parsed?.parse?.root) return; + if (!parsed) return; + if (parsed.shallow) { + await this.walkShallow(parsed, stream, depth, mode === "instance"); + return; + } + if (!parsed.parse?.root) return; const root = parsed.parse.root; for (const child of root.children) { @@ -366,12 +435,12 @@ export class ModIndexer { const manifestPath = manifestPathForReference(source, this.opts.builtmodsDirs); if (manifestPath) { const loaded = await this.loadManifest(manifestPath, stream.name); - if (!loaded && isXmlPath(resolved.path)) { + if (!loaded) { // The manifest could not be parsed (missing/invalid): fall back - // to the placeholder XML so its content is still available. + // to the placeholder target so its content is still available. await this.walk(resolved.path, "instance", stream, depth + 1); } - } else if (isXmlPath(resolved.path)) { + } else { // reference to a real XML file: treat its assets as available await this.walk(resolved.path, "instance", stream, depth + 1); } @@ -400,7 +469,95 @@ export class ModIndexer { continue; } stream.files.add(normKey(resolved.path)); - if (isXmlPath(resolved.path)) { + if ((await this.detectXmlMode(resolved.path)) !== "binary") { + await this.walk(resolved.path, "all", stream, depth + 1); + } + } + } + + /** + * Consumes a shallow-scanned art-asset document: top-level assets, + * top-level , and nested targets. + */ + private async walkShallow( + parsed: ParsedFile, + stream: StreamInfo, + depth: number, + viaInstance: boolean, + ): Promise { + const scan = parsed.shallow; + if (!scan) return; + const file = parsed.file.path; + + for (const asset of scan.assets) { + this.addAsset({ + type: asset.name, + id: asset.id, + file, + line: lineOf(parsed, asset.idValueStart), + origin: this.originOf(file), + stream: stream.name, + viaInstance, + }); + } + + for (const define of scan.defines) { + const entry: DefineDef = { + name: define.name, + value: define.value, + file, + line: lineOf(parsed, define.start), + origin: this.originOf(file), + }; + const arr = this.defines.get(define.name.toLowerCase()); + if (arr) arr.push(entry); + else this.defines.set(define.name.toLowerCase(), [entry]); + } + + for (const inc of scan.includes) { + const resolved = resolveSource(inc.source, dirname(file), this.searchPaths); + if (!resolved.path) { + this.diagnostics.push({ + file, + line: lineOf(parsed, inc.start), + message: `Include target not found: ${inc.source}`, + severity: "warning", + code: "include-not-found", + }); + continue; + } + if (inc.type === "all" || inc.type === "instance") { + await this.walk( + resolved.path, + inc.type === "all" ? "all" : "instance", + stream, + depth + 1, + ); + } else if (inc.type === "reference") { + const manifestPath = manifestPathForReference(inc.source, this.opts.builtmodsDirs); + if (manifestPath) { + const loaded = await this.loadManifest(manifestPath, stream.name); + if (!loaded) await this.walk(resolved.path, "instance", stream, depth + 1); + } else { + await this.walk(resolved.path, "instance", stream, depth + 1); + } + } + } + + for (const xi of scan.xiIncludes) { + const resolved = resolveSource(xi.href, dirname(file), this.searchPaths); + if (!resolved.path) { + this.diagnostics.push({ + file, + line: lineOf(parsed, xi.start), + message: `xi:include target not found: ${xi.href}`, + severity: "warning", + code: "include-not-found", + }); + continue; + } + stream.files.add(normKey(resolved.path)); + if ((await this.detectXmlMode(resolved.path)) !== "binary") { await this.walk(resolved.path, "all", stream, depth + 1); } } @@ -425,9 +582,17 @@ export class ModIndexer { }); return; } - if (!isXmlPath(resolved.path)) return; + if ((await this.detectXmlMode(resolved.path)) === "binary") return; const target = await this.readDocument(resolved.path); - if (!target?.parse?.root) return; + if (!target) return; + if (target.shallow) { + // Shallow-scanned targets have no tree to select xpointer children + // from; index their top-level content as a whole. + stream.files.add(normKey(target.file.path)); + await this.walk(target.file.path, "all", stream, depth + 1); + return; + } + if (!target.parse?.root) return; const xpointer = xi.attrs.find((a) => a.name === "xpointer")?.value ?? ""; let candidates: XmlElement[]; @@ -551,9 +716,32 @@ function lineOf(parsed: ParsedFile, offset: number): number { return parsed.lineMap.positionAt(offset).line + 1; } -function isXmlPath(path: string): boolean { - const ext = extname(path).toLowerCase(); - return XML_EXTENSIONS.has(ext); +/** + * Content sniffing for include targets with unknown extensions (e.g. art + * formats beyond .w3x): a small header that starts with "<" after an + * optional UTF-8 BOM / whitespace and contains no NUL bytes is treated as + * XML text; anything else is a binary asset (registered, never parsed). + */ +async function looksLikeXml(path: string): Promise { + try { + const fh = await open(path, "r"); + try { + const buf = Buffer.alloc(SNIFF_BYTES); + const { bytesRead } = await fh.read(buf, 0, SNIFF_BYTES, 0); + const head = buf.subarray(0, bytesRead); + if (head.includes(0)) return false; + let i = 0; + if (head[0] === 0xef && head[1] === 0xbb && head[2] === 0xbf) i = 3; + while (i < head.length && (head[i] === 0x20 || head[i] === 0x09 || head[i] === 0x0a || head[i] === 0x0d)) { + i++; + } + return i < head.length && head[i] === 0x3c; // "<" + } finally { + await fh.close(); + } + } catch { + return false; + } } async function findCaseInsensitiveDir(dir: string): Promise { diff --git a/src/indexer/shallowScan.ts b/src/indexer/shallowScan.ts new file mode 100644 index 0000000..d3d7845 --- /dev/null +++ b/src/indexer/shallowScan.ts @@ -0,0 +1,268 @@ +/** + * Shallow XML scanner for large art-asset documents (e.g. .w3x). + * + * Full XML parsing builds an element tree whose memory footprint is roughly + * 17x the source text (measured on Corona model files). Model exports like + * contain hundreds of thousands of tiny numeric elements + * (Vertices/V, Triangles/T, ...) that the extension never needs: the index + * only consumes top-level asset definitions (name + id), top-level + * , nested targets and constants. + * + * This scanner performs a single linear pass without building child nodes, + * so multi-megabyte model files can be indexed in linear time with ~zero + * retained memory. + * + * Pure TypeScript: no vscode dependency, reusable outside the extension. + */ + +export interface ShallowAssetRecord { + /** Top-level element name, e.g. "W3DContainer". */ + name: string; + /** Value of the id attribute. */ + id: string; + /** Offset of the first id value character. */ + idValueStart: number; + /** Offset one past the last id value character. */ + idValueEnd: number; + /** Offset of the element's "<". */ + start: number; + /** Offset one past the ">" of the start tag. */ + startTagEnd: number; +} + +export interface ShallowIncludeRecord { + /** "all" | "instance" | "reference", or null when absent/unknown. */ + type: "all" | "instance" | "reference" | null; + source: string; + /** Offset of the element. */ + start: number; +} + +export interface ShallowXiIncludeRecord { + href: string; + xpointer: string | null; + /** Offset of the element. */ + start: number; +} + +export interface ShallowDefineRecord { + name: string; + value: string; + /** Offset of the element. */ + start: number; +} + +export interface ShallowScanError { + message: string; + offset: number; +} + +export interface ShallowDocument { + assets: ShallowAssetRecord[]; + includes: ShallowIncludeRecord[]; + xiIncludes: ShallowXiIncludeRecord[]; + defines: ShallowDefineRecord[]; + errors: ShallowScanError[]; +} + +interface AttrHit { + value: string; + valueStart: number; + valueEnd: number; +} + +/** + * Matches a complete XML tag while respecting quoted attribute values, so a + * ">" or "/" inside a value never terminates the tag early. + */ +const TAG_RE = /<(?:"[^"]*"|'[^']*'|[^'"<>])*>/g; + +export function scanXmlShallow(text: string): ShallowDocument { + const errors: ShallowScanError[] = []; + const assets: ShallowAssetRecord[] = []; + const includes: ShallowIncludeRecord[] = []; + const xiIncludes: ShallowXiIncludeRecord[] = []; + const defines: ShallowDefineRecord[] = []; + + // Depth of the currently open element stack. The document root opens at + // depth 0 -> 1, so its direct children open when depth === 1. + let depth = 0; + let inIncludes = false; + let inDefines = false; + let i = 0; + + while (i < text.length) { + const lt = text.indexOf("<", i); + if (lt < 0) break; + + // Non-element constructs: comments, CDATA, DOCTYPE and processing + // instructions are skipped whole so their content never looks like tags. + if (text.startsWith("", lt + 4); + if (close < 0) { + errors.push({ message: "Unterminated comment", offset: lt }); + break; + } + i = close + 3; + continue; + } + if (text.startsWith("", lt + 9); + if (close < 0) { + errors.push({ message: "Unterminated CDATA section", offset: lt }); + break; + } + i = close + 3; + continue; + } + if (text.startsWith("", lt + 2); + if (close < 0) { + errors.push({ message: "Unterminated DOCTYPE", offset: lt }); + break; + } + i = close + 1; + continue; + } + if (text.startsWith("", lt + 2); + if (close < 0) { + errors.push({ message: "Unterminated processing instruction", offset: lt }); + break; + } + i = close + 2; + continue; + } + + TAG_RE.lastIndex = lt; + const m = TAG_RE.exec(text); + if (!m) { + errors.push({ message: "Unterminated tag", offset: lt }); + break; + } + const tag = m[0]; + const gt = m.index + tag.length - 1; + const inner = tag.slice(1, -1); + const closing = inner.startsWith("/"); + const selfClosing = !closing && /\/\s*$/.test(inner); + const body = closing ? inner.slice(1) : inner; + let nameEnd = 0; + while (nameEnd < body.length && !/[\s/>]/.test(body[nameEnd])) nameEnd++; + const name = body.slice(0, nameEnd); + const base = lt + 1; + + if (!closing) { + // Top-level elements (direct children of the root). + if (depth === 1) { + inIncludes = name === "Includes"; + inDefines = name === "Defines"; + const idAttr = findAttr(inner, base, "id"); + if (idAttr && idAttr.value) { + assets.push({ + name, + id: idAttr.value, + idValueStart: idAttr.valueStart, + idValueEnd: idAttr.valueEnd, + start: lt, + startTagEnd: gt + 1, + }); + } + } + // entries inside the top-level block. + if (inIncludes && depth === 2 && name === "Include") { + const type = findAttr(inner, base, "type"); + const source = findAttr(inner, base, "source"); + if (source?.value) { + includes.push({ + type: + type?.value === "all" || type?.value === "instance" || type?.value === "reference" + ? type.value + : null, + source: source.value, + start: lt, + }); + } + } + // entries inside the top-level block. + if (inDefines && depth === 2 && name === "Define") { + const nameAttr = findAttr(inner, base, "name"); + const valueAttr = findAttr(inner, base, "value"); + if (nameAttr?.value) { + defines.push({ name: nameAttr.value, value: valueAttr?.value ?? "", start: lt }); + } + } + // Nested (or any *:include) anywhere in the document. + if (localName(name) === "include") { + const href = findAttr(inner, base, "href"); + const xpointer = findAttr(inner, base, "xpointer"); + if (href?.value) { + xiIncludes.push({ + href: href.value, + xpointer: xpointer?.value ?? null, + start: lt, + }); + } + } + if (!selfClosing) depth++; + } else { + depth--; + } + i = gt + 1; + } + + return { assets, includes, xiIncludes, defines, errors }; +} + +/** + * Scans the attributes of a tag (its inner text, without "<" and ">") for + * the first occurrence of `want` and returns its value with absolute offsets + * (base = offset one past the "<"). + */ +function findAttr(inner: string, base: number, want: string): AttrHit | null { + let i = 0; + while (i < inner.length) { + while (i < inner.length && /\s/.test(inner[i])) i++; + // Self-closing marker or end of tag: no more attributes. + if (i >= inner.length || inner[i] === "/" || inner[i] === ">") break; + const nameStart = i; + while (i < inner.length && !/[\s=/>]/.test(inner[i])) i++; + const name = inner.slice(nameStart, i); + while (i < inner.length && /\s/.test(inner[i])) i++; + if (inner[i] === "=") { + i++; + while (i < inner.length && /\s/.test(inner[i])) i++; + const q = inner[i]; + if (q === '"' || q === "'") { + const vStart = i + 1; + const vEnd = inner.indexOf(q, vStart); + if (vEnd < 0) { + return name === want ? { value: "", valueStart: -1, valueEnd: -1 } : null; + } + if (name === want) { + return { + value: inner.slice(vStart, vEnd), + valueStart: base + vStart, + valueEnd: base + vEnd, + }; + } + i = vEnd + 1; + continue; + } + // Unquoted value (tolerated). + const vs = i; + while (i < inner.length && !/[\s>]/.test(inner[i])) i++; + if (name === want) { + return { value: inner.slice(vs, i), valueStart: base + vs, valueEnd: base + i }; + } + continue; + } + // Attribute without a value (rare but tolerated). + if (name === want) return { value: "", valueStart: -1, valueEnd: -1 }; + } + return null; +} + +function localName(name: string): string { + const idx = name.lastIndexOf(":"); + return idx >= 0 ? name.slice(idx + 1) : name; +} diff --git a/src/indexer/types.ts b/src/indexer/types.ts index f8ab2d6..626e393 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -1,6 +1,8 @@ import type { XmlDocument } from "../language/xmlParser"; import type { ManifestInfo } from "./manifestParser"; import type { LineMap } from "../language/xmlParser"; +import type { ShallowDocument } from "./shallowScan"; +import type { DocumentCache, ShallowScanCache } from "./caches"; export type AssetOrigin = "project" | "sdk" | "manifest"; @@ -65,6 +67,10 @@ export interface IndexStats { sdkDir: string; indexedFiles: number; parsedFiles: number; + /** Art-asset documents indexed via shallow scan (no DOM tree). */ + shallowScannedFiles: number; + /** Shallow scans served from the persistent cache (unchanged files). */ + shallowCacheHits: number; assetCount: number; defineCount: number; manifestFiles: number; @@ -102,6 +108,10 @@ export interface IndexOptions { additionalDataSearchPaths: string[]; /** Directory walker used to enumerate files for source completion. */ walker: FileWalker; + /** Optional parse-tree cache shared across rebuilds (owned by the workspace). */ + documentCache?: DocumentCache; + /** Optional shallow-scan cache shared across rebuilds (owned by the workspace). */ + shallowCache?: ShallowScanCache; } export interface FileWalker { @@ -121,5 +131,10 @@ export interface ParseCache { export interface ParsedFile { file: IndexedFile; parse: XmlDocument | null; + /** + * Shallow records for large art-asset XML documents (.w3x etc.); null for + * fully parsed XML and for binary files. + */ + shallow: ShallowDocument | null; lineMap: LineMap | null; } diff --git a/src/language/xmlParser.ts b/src/language/xmlParser.ts index d745ef5..8d5f74f 100644 --- a/src/language/xmlParser.ts +++ b/src/language/xmlParser.ts @@ -66,6 +66,14 @@ export interface Position { character: number; } +/** + * Removes a leading UTF-8 byte-order mark (U+FEFF) so source offsets match + * the text as editors expose it (VS Code strips the BOM from document text). + */ +export function stripBom(text: string): string { + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; +} + /** Precomputes line start offsets for offset <-> position conversion. */ export class LineMap { private lineStarts: number[] = [0]; diff --git a/src/workspace.ts b/src/workspace.ts index 2b3b04b..997e862 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -3,6 +3,7 @@ import { existsSync } from "node:fs"; import { join, dirname } from "node:path"; import { CachedDirectoryWalker } from "./indexer/fileScanner"; import { ModIndexer } from "./indexer/indexer"; +import { DocumentCache, ShallowScanCache } from "./indexer/caches"; import type { ModIndex } from "./indexer/types"; import { readSettings, type ExtensionSettings } from "./settings"; @@ -15,6 +16,11 @@ export class ModWorkspace { settings: ExtensionSettings; private walker = new CachedDirectoryWalker(); + // Caches owned here survive rebuilds: a fresh ModIndexer reuses them and + // only re-reads files whose stat changed (crucial for the ~2.6 GB of .w3x + // art assets in a project like Corona). + private documentCache = new DocumentCache(); + private shallowCache = new ShallowScanCache(); private statusBar: vscode.StatusBarItem; private rebuildTimer: ReturnType | null = null; private building = false; @@ -81,6 +87,8 @@ export class ModWorkspace { indexSageXml: this.settings.indexSageXml, additionalDataSearchPaths: this.settings.additionalDataSearchPaths, walker: this.walker, + documentCache: this.documentCache, + shallowCache: this.shallowCache, }); const started = Date.now(); this.index = await indexer.build(); @@ -90,7 +98,7 @@ export class ModWorkspace { this.statusBar.text = `$(symbol-misc) RA3 XML: ${formatCount(s.assetCount)} assets`; this.statusBar.tooltip = `${s.projectDir}\n` + - `${s.indexedFiles} files indexed (${secs}s)\n` + + `${s.indexedFiles} files indexed (${s.parsedFiles} parsed, ${s.shallowScannedFiles} art assets shallow-scanned, ${secs}s)\n` + `${s.assetCount} assets (${s.manifestAssetCount} from ${s.manifestFiles} manifests)\n` + `${s.defineCount} defines, ${s.streams} streams, ${s.sourceCandidates} include candidates`; } catch (err) { @@ -113,6 +121,7 @@ export class ModWorkspace { return { file: { path, stat: null }, parse, + shallow: null, lineMap: new LineMap(text), }; } diff --git a/test/fixtures/minimod/Data/Includes/Models/Tank_Damaged.dds b/test/fixtures/minimod/Data/Includes/Models/Tank_Damaged.dds new file mode 100644 index 0000000..8248346 --- /dev/null +++ b/test/fixtures/minimod/Data/Includes/Models/Tank_Damaged.dds @@ -0,0 +1,2 @@ +This is a fake DDS payload used to verify that binary assets included from +an XML hub are registered as files but never parsed as XML. diff --git a/test/fixtures/minimod/Data/Includes/Models/Tank_FP.w3d b/test/fixtures/minimod/Data/Includes/Models/Tank_FP.w3d new file mode 100644 index 0000000..2116741 --- /dev/null +++ b/test/fixtures/minimod/Data/Includes/Models/Tank_FP.w3d @@ -0,0 +1,4 @@ + + + + diff --git a/test/fixtures/minimod/Data/Includes/Models/Tank_SKN.w3x b/test/fixtures/minimod/Data/Includes/Models/Tank_SKN.w3x new file mode 100644 index 0000000..922a7d5 --- /dev/null +++ b/test/fixtures/minimod/Data/Includes/Models/Tank_SKN.w3x @@ -0,0 +1,5 @@ + + + + + diff --git a/test/fixtures/minimod/Data/Includes/VehicleArt.xml b/test/fixtures/minimod/Data/Includes/VehicleArt.xml new file mode 100644 index 0000000..f643e1a --- /dev/null +++ b/test/fixtures/minimod/Data/Includes/VehicleArt.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/test/fixtures/minimod/Data/Mod.xml b/test/fixtures/minimod/Data/Mod.xml index a852aa8..3148b93 100644 --- a/test/fixtures/minimod/Data/Mod.xml +++ b/test/fixtures/minimod/Data/Mod.xml @@ -9,5 +9,6 @@ + diff --git a/test/indexer.test.mjs b/test/indexer.test.mjs index 4efc578..4b784f1 100644 --- a/test/indexer.test.mjs +++ b/test/indexer.test.mjs @@ -2,8 +2,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; +import fs from "node:fs"; +import os from "node:os"; import { ModIndexer } from "../out/indexer/indexer.js"; import { CachedDirectoryWalker } from "../out/indexer/fileScanner.js"; +import { DocumentCache, ShallowScanCache } from "../out/indexer/caches.js"; +import { resolveReferenceTargetsForType } from "../out/indexer/refs.js"; const root = dirname(dirname(fileURLToPath(import.meta.url))); const project = join(root, "test", "fixtures", "minimod"); @@ -70,3 +74,120 @@ test("provides include source candidates", async () => { assert.ok(xml.some((c) => c.source === "Includes/Units.xml")); assert.ok(xml.some((c) => c.source === "DATA:static.xml")); }); + +test("indexes art-asset XML (.w3x / sniffed .w3d) via shallow scan and skips binary", async () => { + const idx = await buildIndex(); + + // The .w3x hub chain: Mod.xml -> VehicleArt.xml -> Models/Tank_SKN.w3x. + const skn = idx.assetsById.get("tank_skn"); + assert.ok(skn?.some((d) => d.type === "W3DContainer"), "W3DContainer from .w3x indexed"); + assert.ok( + idx.assetsById.get("tank_skl")?.some((d) => d.type === "W3DHierarchy"), + "W3DHierarchy from .w3x indexed", + ); + const container = skn.find((d) => d.type === "W3DContainer"); + assert.match(container.file, /Tank_SKN\.w3x$/); + assert.ok(container.line > 0, "definition line recorded from shallow scan"); + + // Unknown extension with XML content is sniffed and indexed. + assert.ok( + idx.assetsById.get("tank_fp")?.some((d) => d.type === "W3DMesh"), + "unknown-extension XML (.w3d) sniffed and indexed", + ); + + // Binary content is registered as a file but never parsed. + assert.equal(idx.assetsById.get("tank_damaged"), undefined, "binary asset never indexed"); + assert.ok( + idx.files.has( + join(project, "Data", "Includes", "Models", "Tank_Damaged.dds").toLowerCase(), + ), + "binary include target registered as a file", + ); + + // The reported Harbinger scenario: (refType + // BaseRenderAssetType) must resolve to the W3DContainer defined in the w3x. + const targets = resolveReferenceTargetsForType( + idx, + "ScriptedModelDrawModel", + "Name", + "Tank_SKN", + ); + assert.equal(targets.length, 1); + assert.equal(targets[0].def.type, "W3DContainer"); + assert.match(targets[0].def.file, /Tank_SKN\.w3x$/); +}); + +test("w3x files appear in Include source completion candidates", async () => { + const idx = await buildIndex(); + assert.ok( + idx.sourceCandidates.some((c) => c.source === "Includes/Models/Tank_SKN.w3x"), + "project-relative w3x candidate", + ); + assert.ok( + idx.sourceCandidates.some((c) => c.source === "DATA:Includes/Models/Tank_SKN.w3x"), + "DATA: w3x candidate", + ); +}); + +test("shallow scans and full parses are cached across rebuilds", async () => { + const documentCache = new DocumentCache(); + const shallowCache = new ShallowScanCache(); + const makeIndexer = () => + new ModIndexer({ + projectDir: project, + sdkDir: sdk, + builtmodsDirs: [join(sdk, "builtmods")], + indexSageXml: true, + additionalDataSearchPaths: [], + walker: new CachedDirectoryWalker(), + documentCache, + shallowCache, + }); + + const first = await makeIndexer().build(); + const second = await makeIndexer().build(); + + assert.equal(first.stats.shallowScannedFiles, 2, "w3x + sniffed w3d scanned on first build"); + assert.equal(second.stats.shallowScannedFiles, 0, "unchanged art assets are not re-scanned"); + assert.equal(second.stats.shallowCacheHits, 2); + assert.equal(second.stats.assetCount, first.stats.assetCount); + assert.equal(second.stats.indexedFiles, first.stats.indexedFiles); +}); + +test("w3x with a UTF-8 BOM is indexed with correct offsets", async (t) => { + const tmp = fs.mkdtempSync(join(os.tmpdir(), "ra3modxml-bom-")); + t.after(() => fs.rmSync(tmp, { recursive: true, force: true })); + const projectDir = join(tmp, "project"); + fs.mkdirSync(join(projectDir, "Data", "Models"), { recursive: true }); + fs.writeFileSync( + join(projectDir, "Data", "Mod.xml"), + ` + + + + +`, + "utf8", + ); + fs.writeFileSync( + join(projectDir, "Data", "Models", "Tank_BOM.w3x"), + "\uFEFF\n" + + "\n" + + " \n" + + "\n", + "utf8", + ); + + const idx = await new ModIndexer({ + projectDir, + sdkDir: sdk, + builtmodsDirs: [join(sdk, "builtmods")], + indexSageXml: false, + additionalDataSearchPaths: [], + walker: new CachedDirectoryWalker(), + }).build(); + + const def = idx.assetsById.get("tank_bom")?.find((d) => d.type === "W3DContainer"); + assert.ok(def, "BOM-prefixed w3x asset indexed"); + assert.equal(def.line, 3, "id line is correct despite the BOM"); +}); diff --git a/test/shallowScan.test.mjs b/test/shallowScan.test.mjs new file mode 100644 index 0000000..499793e --- /dev/null +++ b/test/shallowScan.test.mjs @@ -0,0 +1,100 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { scanXmlShallow } from "../out/indexer/shallowScan.js"; + +test("extracts top-level assets, includes, defines and xi:include", () => { + const text = ` + + + + + + + + + + + Model_SKN.GUN + + + + +`; + const doc = scanXmlShallow(text); + assert.equal(doc.errors.length, 0); + assert.deepEqual( + doc.assets.map((a) => `${a.name}:${a.id}`), + ["W3DContainer:Model_SKN", "W3DMesh:Model_SKN.GUN"], + ); + assert.equal(doc.includes.length, 2); + assert.equal(doc.includes[0].type, "all"); + assert.equal(doc.includes[0].source, "Art/Model_SKN.w3x"); + assert.equal(doc.includes[1].type, "reference"); + assert.equal(doc.defines.length, 1); + assert.equal(doc.defines[0].name, "MODEL_SCALE"); + assert.equal(doc.defines[0].value, "1.0"); + assert.equal(doc.xiIncludes.length, 1); + assert.equal(doc.xiIncludes[0].href, "DATA:Includes/Extra.xml"); + assert.match(doc.xiIncludes[0].xpointer, /xpointer\(/); + + // Recorded offsets must slice the original text back out. + const container = doc.assets[0]; + assert.equal(text.slice(container.idValueStart, container.idValueEnd), "Model_SKN"); + assert.equal(text.slice(container.start, container.startTagEnd).startsWith(" and / inside quoted values, self-closing tags and comments", () => { + const text = ` + + + + +`; + const doc = scanXmlShallow(text); + assert.equal(doc.errors.length, 0); + assert.deepEqual( + doc.assets.map((a) => a.id), + ["Weird", "X"], + ); +}); + +test("skips CDATA and processing instructions", () => { + const text = ` + + ]]> + + +`; + const doc = scanXmlShallow(text); + assert.equal(doc.errors.length, 0); + assert.deepEqual( + doc.assets.map((a) => a.id), + ["REAL"], + ); +}); + +test("reports unterminated tags and keeps partial results", () => { + const doc = scanXmlShallow('= 1); + assert.deepEqual( + doc.assets.map((a) => a.id), + ["A"], + ); +}); + +test("nested module payload does not create asset records", () => { + // Only top-level elements with an id are assets; the hundreds of thousands + // of numeric V/T elements inside a mesh are not. + const doc = scanXmlShallow(` + + + + +`); + assert.deepEqual( + doc.assets.map((a) => a.id), + ["MESH"], + ); +}); diff --git a/test/xmlParser.test.mjs b/test/xmlParser.test.mjs index af37847..64450fb 100644 --- a/test/xmlParser.test.mjs +++ b/test/xmlParser.test.mjs @@ -1,6 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { parseXml, findElementAt } from "../out/language/xmlParser.js"; +import { parseXml, findElementAt, stripBom } from "../out/language/xmlParser.js"; + +test("stripBom removes a leading UTF-8 byte-order mark", () => { + assert.equal(stripBom("\uFEFF"), ""); + assert.equal(stripBom(""), ""); +}); test("parses elements, attributes and positions", () => { const text = `\n\t\n`;