Fix APT CLI roundtrip visual and script preservation

Preserve edittext channels, alpha/color transforms, inactive placement fields, zero ratio/clipping and null ClipAction presence. Keep unlabelled flag bits, quoted/escaped script strings, export declarations and correctly typed CONST and pushdata values. Validate signed integers and write byte-aligned UTF-8. Add 23 public CLI regressions and record the native 174-resource batch plus bounded retests: four discovered pages are repaired, while the sound placeholder and native float formatting remain explicit accuracy limits. Rebuild the 1.0.1 Windows executable and leave unrelated SDK changes untouched.
2026-10-13 02:40:37 +02:00
parent 5511354783
commit e0b8dedc04
18 changed files with 5111 additions and 115 deletions
@@ -87,7 +87,14 @@ function build(input, output, opts) {
 function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  U.resetLabels();

-  const $ = cheerio.load(humanXml, { xmlMode: true, decodeEntities: false });
+  const $ = cheerio.load(humanXml, { xmlMode: true, decodeEntities: true });
+  $('aptdata').children().each(function () {
+    const character = $(this);
+    if (character.attr('comment') === 'not implemented yet!') {
+      throw new Error('cc3tools did not export the ' + this.tagName + ' character payload (code ' +
+        character.attr('code') + '); rebuilding this resource cannot preserve it');
+    }
+  });
  const ActionNames = oldStyle ? C.ActionNames2 : C.ActionNames;
  let isbad = false;

@@ -147,12 +154,12 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  // Sort the collection
  Array.prototype.sort.call(actionConstants, sortf);

-  const constmap = { str: {}, num: {} };
+  const constmap = { str: Object.create(null), num: Object.create(null) };
  actionConstants.each(function () {
    if ($(this).is('[str]')) {
      constmap.str[$(this).attr('str')] = 1;
    } else if ($(this).is('[num]')) {
-      constmap.num[$(this).attr('num')] = 1;
+      constmap.num[constantInteger($(this).attr('num'))] = 1;
    }
  });

@@ -180,8 +187,8 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {

  let err = false;
  actionContainers.each(function () {
-    const localstr = {};
-    const localnum = {};
+    const localstr = Object.create(null);
+    const localnum = Object.create(null);
    const localconst = [];
    let ci = 0;
    const myactions = $(this).find(C.CONST_ACTION_SELECTOR);
@@ -233,6 +240,9 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  // ──────────────────────────────────────────
  // 3. Write .const binary file to tmpdir
  // ──────────────────────────────────────────
+  for (const value of consttable) {
+    if (typeof value === 'string' && value.includes('\0')) throw new Error('String constants cannot contain NUL');
+  }
  const cf = fs.openSync(tmpdir + baseName + '.const', 'w');
  const constbuf = Buffer.alloc(0x20);
  constbuf.asciiWrite('Apt constant file\x1A\0\0');
@@ -242,7 +252,6 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  fs.writeSync(cf, constbuf, 0, 0x20);

  const tmpbuf = Buffer.alloc(8);
-  const strbuf = Buffer.alloc(512);
  let strpos = 0x20 + 8 * consttable.length;

  for (let i = 0; i < consttable.length; i++) {
@@ -250,14 +259,14 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    if (typeof cv === 'string') {
      tmpbuf.writeUInt32LE(1);
      tmpbuf.writeUInt32LE(strpos, 4);
-      strbuf.asciiWrite(cv + '\0');
-      fs.writeSync(cf, strbuf, 0, cv.length + 1, strpos);
-      strpos += cv.length + 1;
+      const strbuf = Buffer.from(cv + '\0', 'utf8');
+      fs.writeSync(cf, strbuf, 0, strbuf.length, strpos);
+      strpos += strbuf.length;
      strpos = Math.ceil(strpos / 4) * 4;
    } else {
      console.warn('Warning: integer constant (' + cv + ') stored in const table');
      tmpbuf.writeUInt32LE(7);
-      tmpbuf.writeUInt32LE(cv, 4);
+      tmpbuf.writeUInt32LE(constantInteger(cv) >>> 0, 4);
    }
    fs.writeSync(cf, tmpbuf, 0, 8, 0x20 + i * 8);
  }
@@ -417,13 +426,19 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
      $v.removeAttr('rotm00 rotm01 rotm10 rotm11 tx ty');
    }
    // unknown (packed int) → rgbaadd (only if non-zero)
-    const unk = $v.attr('unknown');
+    const unk = $v.is('placeobject') ? $v.attr('unknown') : undefined;
    if (unk !== undefined && parseInt(unk, 10) !== 0) {
      const u = parseInt(unk, 10);
      $v.attr('rgbaadd', (u & 0xff) + ',' + ((u >> 8) & 0xff) + ',' +
        ((u >> 16) & 0xff) + ',' + ((u >> 24) & 0xff));
      $v.removeAttr('unknown');
    }
+    // Expanded cc3tools fields are the colour multiplier, not absent/default data.
+    if ($v.is('placeobject') && $v.attr('rgbamul') === undefined &&
+        ['red', 'green', 'blue', 'alpha'].some(k => $v.attr(k) !== undefined)) {
+      $v.attr('rgbamul', ['red', 'green', 'blue', 'alpha'].map(k => $v.attr(k) ?? '255').join(','));
+      $v.removeAttr('red green blue alpha');
+    }
  });

  // ──────────────────────────────────────────
@@ -460,13 +475,13 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    } else {
      flags |= C.PlaceObjectFlags.HasColorTransform;
    }
-    if ($v.attr('ratio') === undefined || $v.attr('ratio') === '0') {
+    if ($v.attr('ratio') === undefined) {
      $v.attr('ratio', '0');
      flags &= ~C.PlaceObjectFlags.HasRatio;
    } else {
      flags |= C.PlaceObjectFlags.HasRatio;
    }
-    if ($v.attr('clipdepth') === '-1' || $v.attr('clipdepth') === undefined) {
+    if ($v.attr('clipdepth') === undefined) {
      $v.attr('clipdepth', '-1');
      flags &= ~C.PlaceObjectFlags.HasClipDepth;
    } else {
@@ -477,9 +492,9 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    } else {
      flags |= C.PlaceObjectFlags.HasName;
    }
-    if ($v.find('clipactions').length === 0) {
-      flags &= ~C.PlaceObjectFlags.HasClipAction;
-    } else {
+    // An active bit with a null clip-action pointer is valid stock data.
+    // Keep that bit without creating an array; clear it explicitly in input flags.
+    if ($v.find('clipactions').length !== 0) {
      flags |= C.PlaceObjectFlags.HasClipAction;
    }
    $v.attr('flags', flags);
@@ -490,9 +505,9 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  // ──────────────────────────────────────────
  let newmanifest = "<?xml version='1.0' encoding='utf-8'?>\n" +
    '<AssetDeclaration xmlns="uri:ea.com:eala:asset" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n' +
-    '<AptAptData id="' + baseName + '_apt" File="' + baseName + '.apt" />\n' +
-    '<AptConstData id="' + baseName + '_const" File="' + baseName + '.const" />\n' +
-    '<AptDatData id="' + baseName + '_dat" File="' + baseName + '.dat" />\n';
+    '<AptAptData id="' + xmlAttribute(baseName + '_apt') + '" File="' + xmlAttribute(baseName + '.apt') + '" />\n' +
+    '<AptConstData id="' + xmlAttribute(baseName + '_const') + '" File="' + xmlAttribute(baseName + '.const') + '" />\n' +
+    '<AptDatData id="' + xmlAttribute(baseName + '_dat') + '" File="' + xmlAttribute(baseName + '.dat') + '" />\n';

  let newdat = '; Created by AptXMLEditor.\n';
  const texcache = {};
@@ -545,7 +560,7 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {

    const rupathFull = (outDir + rupath).toLowerCase();
    rucache[rupathFull] = text2;
-    newmanifest += '<AptGeometryData id="' + baseName + '_' + sid + '" File="' + rupath + '" AptID="' + sid + '"/>\n';
+    newmanifest += '<AptGeometryData id="' + xmlAttribute(baseName + '_' + sid) + '" File="' + xmlAttribute(rupath) + '" AptID="' + sid + '"/>\n';
    sp.attr('geometry', sid);
    ru.remove();
  });
@@ -566,7 +581,7 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
        newid = texid;
      }
      texcache[img] = newid;
-      newmanifest += '<Texture id="apt_' + baseName + '_' + newid + '" File="' + img1.split(',')[0] + '" OutputFormat="A8R8G8B8" GenerateMipMaps="false" AllowAutomaticResize="false"/>\n';
+      newmanifest += '<Texture id="' + xmlAttribute('apt_' + baseName + '_' + newid) + '" File="' + xmlAttribute(img1.split(',')[0]) + '" OutputFormat="A8R8G8B8" GenerateMipMaps="false" AllowAutomaticResize="false"/>\n';
    }

    if (img.indexOf(',') > 0) {
@@ -637,20 +652,16 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    }
  });

-  // Swap red/blue for edittext
-  $('edittext').each(function () {
-    const $v = $(this);
-    const r = $v.attr('red');
-    const b = $v.attr('blue');
-    $v.attr('red', b);
-    $v.attr('blue', r);
-  });
+  // edittext channels already use the same order as cc3tools expand output.

  // Remove end/dummy markers
  $('end,dummy').remove();

  // Serialize
  let output = $.xml();
+  // Literal attribute whitespace is normalised by XML readers; entities preserve it.
+  output = output.replace(/(\s[\w:.-]+=")([^"]*)"/g, (match, prefix, value) =>
+    prefix + value.replace(/\r/g, '&#13;').replace(/\n/g, '&#10;').replace(/\t/g, '&#9;') + '"');
  output = output.replace(/^[\t]+$/gm, '').replace(/[\r\n]+/g, '\n');

  return {
@@ -826,7 +837,8 @@ function stringtoaction(s, ActionNames, $) {
  }

  // Parse arguments based on action type
-  const args = argStr ? parseArgs(argStr) : [];
+  const operands = argStr ? parseArgs(argStr) : [];
+  const args = operands.map(operand => operand.value);

  // Branch instructions: instr labelname
  if (aid === C.ACTION_BRANCHALWAYS || aid === C.ACTION_BRANCHIFTRUE || aid === C.EA_BRANCHIFFALSE) {
@@ -859,13 +871,12 @@ function stringtoaction(s, ActionNames, $) {
    const a = U.el('pushdata', $);
    a.attr('action', aid);
    for (let i = 0; i < args.length; i++) {
-      const c = U.el('constant', $);
-      const numVal = parseInt(args[i], 10);
-      if (isNaN(numVal) || args[i] === '') {
+      const c = U.el('data', $);
+      if (operands[i].quoted || args[i] === '' || !isNumber(args[i])) {
        c.attr('str', args[i] || '');
        c.attr('val', 0);
      } else {
-        c.attr('num', numVal);
+        c.attr('num', constantInteger(args[i]));
        c.attr('val', 0);
      }
      a.append(c);
@@ -894,14 +905,13 @@ function stringtoaction(s, ActionNames, $) {
    a.attr('action', aid);
    // Check if the original argument was quoted (indicates a string constant)
    // e.g. pushconst "1" is a string, pushconst 1 is a number
-    if (argStr && argStr[0] === '"') {
+    if (operands[0]?.quoted) {
      a.attr('str', args[0] || '');
    } else {
-      const numVal = parseInt(args[0], 10);
-      if (isNaN(numVal) || args[0] === '') {
+      if (args[0] === '' || !isNumber(args[0])) {
        a.attr('str', args[0] || '');
      } else {
-        a.attr('num', numVal);
+        a.attr('num', constantInteger(args[0]));
      }
    }
    return a;
@@ -919,14 +929,13 @@ function stringtoaction(s, ActionNames, $) {
  if (aid === C.EA_PUSHWORDCONSTANT) {
    const a = U.el('short', $);
    a.attr('action', aid);
-    if (argStr && argStr[0] === '"') {
+    if (operands[0]?.quoted) {
      a.attr('str', args[0] || '');
    } else {
-      const numVal = parseInt(args[0], 10);
-      if (isNaN(numVal) || args[0] === '') {
+      if (args[0] === '' || !isNumber(args[0])) {
        a.attr('str', args[0] || '');
      } else {
-        a.attr('num', numVal);
+        a.attr('num', constantInteger(args[0]));
      }
    }
    return a;
@@ -970,33 +979,56 @@ function stringtoaction(s, ActionNames, $) {
 // ─── Parse argument string ───
 function parseArgs(argStr) {
  const args = [];
-  let current = '';
-  let inQuote = false;
-
-  for (let i = 0; i < argStr.length; i++) {
-    const c = argStr[i];
-    if (c === '"') {
-      inQuote = !inQuote;
-      current += c;
-    } else if (c === ',' && !inQuote) {
-      args.push(current.trim());
-      current = '';
+  let i = 0;
+  while (i < argStr.length) {
+    while (/\s/.test(argStr[i] || '') && i < argStr.length) i++;
+    const quoted = argStr[i] === '"';
+    let value = '';
+    if (quoted) {
+      i++;
+      let closed = false;
+      while (i < argStr.length) {
+        const c = argStr[i++];
+        if (c === '"') { closed = true; break; }
+        if (c === '\\' && i < argStr.length) {
+          const escaped = argStr[i++];
+          const escapes = { '"': '"', '\\': '\\', n: '\n', r: '\r', t: '\t' };
+          value += Object.hasOwn(escapes, escaped) ? escapes[escaped] : '\\' + escaped;
+        } else {
+          value += c;
+        }
+      }
+      if (!closed) throw new Error('Unterminated quoted argument');
+      while (/\s/.test(argStr[i] || '') && i < argStr.length) i++;
+      if (i < argStr.length && argStr[i] !== ',') throw new Error('Unexpected text after quoted argument');
    } else {
-      current += c;
+      while (i < argStr.length && argStr[i] !== ',') value += argStr[i++];
+      value = value.trim();
+    }
+    args.push({ value, quoted });
+    if (argStr[i] === ',') {
+      i++;
+      if (i === argStr.length) args.push({ value: '', quoted: false });
    }
  }
-  if (current.trim()) {
-    args.push(current.trim());
-  }
+  return args;
+}

-  // Strip surrounding quotes from each argument (matching original eval() behavior)
-  return args.map(s => {
-    const t = s.trim();
-    if (t.length >= 2 && t[0] === '"' && t[t.length - 1] === '"') {
-      return t.slice(1, -1);
-    }
-    return t;
-  });
+function isNumber(value) {
+  return typeof value === 'string' && /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i.test(value);
+}
+
+function constantInteger(value) {
+  const number = Number(value);
+  if (!Number.isInteger(number) || number < -0x80000000 || number > 0x7fffffff) {
+    throw new Error('Constant pool numbers must be signed 32-bit integers: ' + value);
+  }
+  return number;
+}
+
+function xmlAttribute(value) {
+  return String(value).replace(/&/g, '&amp;').replace(/"/g, '&quot;')
+    .replace(/</g, '&lt;').replace(/>/g, '&gt;');
 }

 // ─── bad() — create error marker element ───