Fix APT CLI roundtrip visual and script preservation

Preserve edittext channels, alpha/color transforms, inactive placement fields, zero ratio/clipping and null ClipAction presence. Keep unlabelled flag bits, quoted/escaped script strings, export declarations and correctly typed CONST and pushdata values. Validate signed integers and write byte-aligned UTF-8. Add 23 public CLI regressions and record the native 174-resource batch plus bounded retests: four discovered pages are repaired, while the sound placeholder and native float formatting remain explicit accuracy limits. Rebuild the 1.0.1 Windows executable and leave unrelated SDK changes untouched.
2026-10-13 02:40:37 +02:00
parent 5511354783
commit e0b8dedc04
18 changed files with 5111 additions and 115 deletions
@@ -87,7 +87,14 @@ function build(input, output, opts) {
 function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  U.resetLabels();

-  const $ = cheerio.load(humanXml, { xmlMode: true, decodeEntities: false });
+  const $ = cheerio.load(humanXml, { xmlMode: true, decodeEntities: true });
+  $('aptdata').children().each(function () {
+    const character = $(this);
+    if (character.attr('comment') === 'not implemented yet!') {
+      throw new Error('cc3tools did not export the ' + this.tagName + ' character payload (code ' +
+        character.attr('code') + '); rebuilding this resource cannot preserve it');
+    }
+  });
  const ActionNames = oldStyle ? C.ActionNames2 : C.ActionNames;
  let isbad = false;

@@ -147,12 +154,12 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  // Sort the collection
  Array.prototype.sort.call(actionConstants, sortf);

-  const constmap = { str: {}, num: {} };
+  const constmap = { str: Object.create(null), num: Object.create(null) };
  actionConstants.each(function () {
    if ($(this).is('[str]')) {
      constmap.str[$(this).attr('str')] = 1;
    } else if ($(this).is('[num]')) {
-      constmap.num[$(this).attr('num')] = 1;
+      constmap.num[constantInteger($(this).attr('num'))] = 1;
    }
  });

@@ -180,8 +187,8 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {

  let err = false;
  actionContainers.each(function () {
-    const localstr = {};
-    const localnum = {};
+    const localstr = Object.create(null);
+    const localnum = Object.create(null);
    const localconst = [];
    let ci = 0;
    const myactions = $(this).find(C.CONST_ACTION_SELECTOR);
@@ -233,6 +240,9 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  // ──────────────────────────────────────────
  // 3. Write .const binary file to tmpdir
  // ──────────────────────────────────────────
+  for (const value of consttable) {
+    if (typeof value === 'string' && value.includes('\0')) throw new Error('String constants cannot contain NUL');
+  }
  const cf = fs.openSync(tmpdir + baseName + '.const', 'w');
  const constbuf = Buffer.alloc(0x20);
  constbuf.asciiWrite('Apt constant file\x1A\0\0');
@@ -242,7 +252,6 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  fs.writeSync(cf, constbuf, 0, 0x20);

  const tmpbuf = Buffer.alloc(8);
-  const strbuf = Buffer.alloc(512);
  let strpos = 0x20 + 8 * consttable.length;

  for (let i = 0; i < consttable.length; i++) {
@@ -250,14 +259,14 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    if (typeof cv === 'string') {
      tmpbuf.writeUInt32LE(1);
      tmpbuf.writeUInt32LE(strpos, 4);
-      strbuf.asciiWrite(cv + '\0');
-      fs.writeSync(cf, strbuf, 0, cv.length + 1, strpos);
-      strpos += cv.length + 1;
+      const strbuf = Buffer.from(cv + '\0', 'utf8');
+      fs.writeSync(cf, strbuf, 0, strbuf.length, strpos);
+      strpos += strbuf.length;
      strpos = Math.ceil(strpos / 4) * 4;
    } else {
      console.warn('Warning: integer constant (' + cv + ') stored in const table');
      tmpbuf.writeUInt32LE(7);
-      tmpbuf.writeUInt32LE(cv, 4);
+      tmpbuf.writeUInt32LE(constantInteger(cv) >>> 0, 4);
    }
    fs.writeSync(cf, tmpbuf, 0, 8, 0x20 + i * 8);
  }
@@ -417,13 +426,19 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
      $v.removeAttr('rotm00 rotm01 rotm10 rotm11 tx ty');
    }
    // unknown (packed int) → rgbaadd (only if non-zero)
-    const unk = $v.attr('unknown');
+    const unk = $v.is('placeobject') ? $v.attr('unknown') : undefined;
    if (unk !== undefined && parseInt(unk, 10) !== 0) {
      const u = parseInt(unk, 10);
      $v.attr('rgbaadd', (u & 0xff) + ',' + ((u >> 8) & 0xff) + ',' +
        ((u >> 16) & 0xff) + ',' + ((u >> 24) & 0xff));
      $v.removeAttr('unknown');
    }
+    // Expanded cc3tools fields are the colour multiplier, not absent/default data.
+    if ($v.is('placeobject') && $v.attr('rgbamul') === undefined &&
+        ['red', 'green', 'blue', 'alpha'].some(k => $v.attr(k) !== undefined)) {
+      $v.attr('rgbamul', ['red', 'green', 'blue', 'alpha'].map(k => $v.attr(k) ?? '255').join(','));
+      $v.removeAttr('red green blue alpha');
+    }
  });

  // ──────────────────────────────────────────
@@ -460,13 +475,13 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    } else {
      flags |= C.PlaceObjectFlags.HasColorTransform;
    }
-    if ($v.attr('ratio') === undefined || $v.attr('ratio') === '0') {
+    if ($v.attr('ratio') === undefined) {
      $v.attr('ratio', '0');
      flags &= ~C.PlaceObjectFlags.HasRatio;
    } else {
      flags |= C.PlaceObjectFlags.HasRatio;
    }
-    if ($v.attr('clipdepth') === '-1' || $v.attr('clipdepth') === undefined) {
+    if ($v.attr('clipdepth') === undefined) {
      $v.attr('clipdepth', '-1');
      flags &= ~C.PlaceObjectFlags.HasClipDepth;
    } else {
@@ -477,9 +492,9 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    } else {
      flags |= C.PlaceObjectFlags.HasName;
    }
-    if ($v.find('clipactions').length === 0) {
-      flags &= ~C.PlaceObjectFlags.HasClipAction;
-    } else {
+    // An active bit with a null clip-action pointer is valid stock data.
+    // Keep that bit without creating an array; clear it explicitly in input flags.
+    if ($v.find('clipactions').length !== 0) {
      flags |= C.PlaceObjectFlags.HasClipAction;
    }
    $v.attr('flags', flags);
@@ -490,9 +505,9 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
  // ──────────────────────────────────────────
  let newmanifest = "<?xml version='1.0' encoding='utf-8'?>\n" +
    '<AssetDeclaration xmlns="uri:ea.com:eala:asset" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n' +
-    '<AptAptData id="' + baseName + '_apt" File="' + baseName + '.apt" />\n' +
-    '<AptConstData id="' + baseName + '_const" File="' + baseName + '.const" />\n' +
-    '<AptDatData id="' + baseName + '_dat" File="' + baseName + '.dat" />\n';
+    '<AptAptData id="' + xmlAttribute(baseName + '_apt') + '" File="' + xmlAttribute(baseName + '.apt') + '" />\n' +
+    '<AptConstData id="' + xmlAttribute(baseName + '_const') + '" File="' + xmlAttribute(baseName + '.const') + '" />\n' +
+    '<AptDatData id="' + xmlAttribute(baseName + '_dat') + '" File="' + xmlAttribute(baseName + '.dat') + '" />\n';

  let newdat = '; Created by AptXMLEditor.\n';
  const texcache = {};
@@ -545,7 +560,7 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {

    const rupathFull = (outDir + rupath).toLowerCase();
    rucache[rupathFull] = text2;
-    newmanifest += '<AptGeometryData id="' + baseName + '_' + sid + '" File="' + rupath + '" AptID="' + sid + '"/>\n';
+    newmanifest += '<AptGeometryData id="' + xmlAttribute(baseName + '_' + sid) + '" File="' + xmlAttribute(rupath) + '" AptID="' + sid + '"/>\n';
    sp.attr('geometry', sid);
    ru.remove();
  });
@@ -566,7 +581,7 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
        newid = texid;
      }
      texcache[img] = newid;
-      newmanifest += '<Texture id="apt_' + baseName + '_' + newid + '" File="' + img1.split(',')[0] + '" OutputFormat="A8R8G8B8" GenerateMipMaps="false" AllowAutomaticResize="false"/>\n';
+      newmanifest += '<Texture id="' + xmlAttribute('apt_' + baseName + '_' + newid) + '" File="' + xmlAttribute(img1.split(',')[0]) + '" OutputFormat="A8R8G8B8" GenerateMipMaps="false" AllowAutomaticResize="false"/>\n';
    }

    if (img.indexOf(',') > 0) {
@@ -637,20 +652,16 @@ function xmlcleanhint(humanXml, baseName, outDir, tmpdir, oldStyle) {
    }
  });

-  // Swap red/blue for edittext
-  $('edittext').each(function () {
-    const $v = $(this);
-    const r = $v.attr('red');
-    const b = $v.attr('blue');
-    $v.attr('red', b);
-    $v.attr('blue', r);
-  });
+  // edittext channels already use the same order as cc3tools expand output.

  // Remove end/dummy markers
  $('end,dummy').remove();

  // Serialize
  let output = $.xml();
+  // Literal attribute whitespace is normalised by XML readers; entities preserve it.
+  output = output.replace(/(\s[\w:.-]+=")([^"]*)"/g, (match, prefix, value) =>
+    prefix + value.replace(/\r/g, '&#13;').replace(/\n/g, '&#10;').replace(/\t/g, '&#9;') + '"');
  output = output.replace(/^[\t]+$/gm, '').replace(/[\r\n]+/g, '\n');

  return {
@@ -826,7 +837,8 @@ function stringtoaction(s, ActionNames, $) {
  }

  // Parse arguments based on action type
-  const args = argStr ? parseArgs(argStr) : [];
+  const operands = argStr ? parseArgs(argStr) : [];
+  const args = operands.map(operand => operand.value);

  // Branch instructions: instr labelname
  if (aid === C.ACTION_BRANCHALWAYS || aid === C.ACTION_BRANCHIFTRUE || aid === C.EA_BRANCHIFFALSE) {
@@ -859,13 +871,12 @@ function stringtoaction(s, ActionNames, $) {
    const a = U.el('pushdata', $);
    a.attr('action', aid);
    for (let i = 0; i < args.length; i++) {
-      const c = U.el('constant', $);
-      const numVal = parseInt(args[i], 10);
-      if (isNaN(numVal) || args[i] === '') {
+      const c = U.el('data', $);
+      if (operands[i].quoted || args[i] === '' || !isNumber(args[i])) {
        c.attr('str', args[i] || '');
        c.attr('val', 0);
      } else {
-        c.attr('num', numVal);
+        c.attr('num', constantInteger(args[i]));
        c.attr('val', 0);
      }
      a.append(c);
@@ -894,14 +905,13 @@ function stringtoaction(s, ActionNames, $) {
    a.attr('action', aid);
    // Check if the original argument was quoted (indicates a string constant)
    // e.g. pushconst "1" is a string, pushconst 1 is a number
-    if (argStr && argStr[0] === '"') {
+    if (operands[0]?.quoted) {
      a.attr('str', args[0] || '');
    } else {
-      const numVal = parseInt(args[0], 10);
-      if (isNaN(numVal) || args[0] === '') {
+      if (args[0] === '' || !isNumber(args[0])) {
        a.attr('str', args[0] || '');
      } else {
-        a.attr('num', numVal);
+        a.attr('num', constantInteger(args[0]));
      }
    }
    return a;
@@ -919,14 +929,13 @@ function stringtoaction(s, ActionNames, $) {
  if (aid === C.EA_PUSHWORDCONSTANT) {
    const a = U.el('short', $);
    a.attr('action', aid);
-    if (argStr && argStr[0] === '"') {
+    if (operands[0]?.quoted) {
      a.attr('str', args[0] || '');
    } else {
-      const numVal = parseInt(args[0], 10);
-      if (isNaN(numVal) || args[0] === '') {
+      if (args[0] === '' || !isNumber(args[0])) {
        a.attr('str', args[0] || '');
      } else {
-        a.attr('num', numVal);
+        a.attr('num', constantInteger(args[0]));
      }
    }
    return a;
@@ -970,33 +979,56 @@ function stringtoaction(s, ActionNames, $) {
 // ─── Parse argument string ───
 function parseArgs(argStr) {
  const args = [];
-  let current = '';
-  let inQuote = false;
-
-  for (let i = 0; i < argStr.length; i++) {
-    const c = argStr[i];
-    if (c === '"') {
-      inQuote = !inQuote;
-      current += c;
-    } else if (c === ',' && !inQuote) {
-      args.push(current.trim());
-      current = '';
+  let i = 0;
+  while (i < argStr.length) {
+    while (/\s/.test(argStr[i] || '') && i < argStr.length) i++;
+    const quoted = argStr[i] === '"';
+    let value = '';
+    if (quoted) {
+      i++;
+      let closed = false;
+      while (i < argStr.length) {
+        const c = argStr[i++];
+        if (c === '"') { closed = true; break; }
+        if (c === '\\' && i < argStr.length) {
+          const escaped = argStr[i++];
+          const escapes = { '"': '"', '\\': '\\', n: '\n', r: '\r', t: '\t' };
+          value += Object.hasOwn(escapes, escaped) ? escapes[escaped] : '\\' + escaped;
+        } else {
+          value += c;
+        }
+      }
+      if (!closed) throw new Error('Unterminated quoted argument');
+      while (/\s/.test(argStr[i] || '') && i < argStr.length) i++;
+      if (i < argStr.length && argStr[i] !== ',') throw new Error('Unexpected text after quoted argument');
    } else {
-      current += c;
+      while (i < argStr.length && argStr[i] !== ',') value += argStr[i++];
+      value = value.trim();
+    }
+    args.push({ value, quoted });
+    if (argStr[i] === ',') {
+      i++;
+      if (i === argStr.length) args.push({ value: '', quoted: false });
    }
  }
-  if (current.trim()) {
-    args.push(current.trim());
-  }
+  return args;
+}

-  // Strip surrounding quotes from each argument (matching original eval() behavior)
-  return args.map(s => {
-    const t = s.trim();
-    if (t.length >= 2 && t[0] === '"' && t[t.length - 1] === '"') {
-      return t.slice(1, -1);
-    }
-    return t;
-  });
+function isNumber(value) {
+  return typeof value === 'string' && /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i.test(value);
+}
+
+function constantInteger(value) {
+  const number = Number(value);
+  if (!Number.isInteger(number) || number < -0x80000000 || number > 0x7fffffff) {
+    throw new Error('Constant pool numbers must be signed 32-bit integers: ' + value);
+  }
+  return number;
+}
+
+function xmlAttribute(value) {
+  return String(value).replace(/&/g, '&amp;').replace(/"/g, '&quot;')
+    .replace(/</g, '&lt;').replace(/>/g, '&gt;');
 }

 // ─── bad() — create error marker element ───
@@ -26,7 +26,7 @@ function expand(input, output, opts) {
    const rawXml = U.readRawXml(tmpdir, baseName);

    // 4. Parse manifest if available (for texture name resolution)
-    const manifest = manifestData ? cheerio.load(manifestData, { xmlMode: true, decodeEntities: false }) : null;
+    const manifest = manifestData ? cheerio.load(manifestData, { xmlMode: true, decodeEntities: true }) : null;

    // 5. Parse .dat if available
    const dats = parseDat(datData);
@@ -69,20 +69,11 @@ function parseDat(datData) {
 function xmladdhint(rawXml, baseName, aptName, manifest, dats, filedir, oldStyle) {
  U.resetLabels();

-  const $ = cheerio.load(rawXml, { xmlMode: true, decodeEntities: false });
+  const $ = cheerio.load(rawXml, { xmlMode: true, decodeEntities: true });
  const ActionNames = oldStyle ? C.ActionNames2 : C.ActionNames;
  const xmldoc = $;

-  // Remove duplicated exports
-  const exp = {};
-  $('export').each(function () {
-    const nm = $(this).attr('name');
-    if (exp[nm]) {
-      $(this).remove();
-    } else {
-      exp[nm] = true;
-    }
-  });
+  // Preserve export order and repeated names, including Object prototype names.

  // Calculate aligned positions for each action container
  const actionContainers = $('initaction, buttonaction, action, clipaction');
@@ -179,8 +170,8 @@ function xmladdhint(rawXml, baseName, aptName, manifest, dats, filedir, oldStyle
      }
    });
    astr += U.indentstr(v);
-    // Replace CDATA with text node (entity-encoded)
-    v.empty().text(astr);
+    // Parse an XML CDATA node so serialization does not entity-encode assembly text.
+    v.empty().append('<![CDATA[' + astr.replace(/\]\]>/g, ']]]]><![CDATA[>') + ']]>');
  });

  // Resolve flags to symbolic names
@@ -268,10 +259,10 @@ function xmladdhint(rawXml, baseName, aptName, manifest, dats, filedir, oldStyle
      $v.removeAttr('character');
    }
    if (flags.indexOf('HasMatrix') < 0) {
-      $v.removeAttr('matrix');
+      $v.removeAttr('matrix rotm00 rotm01 rotm10 rotm11 tx ty');
    }
    if (flags.indexOf('HasColorTransform') < 0) {
-      $v.removeAttr('rgbamul rgbaadd');
+      $v.removeAttr('rgbamul rgbaadd red green blue alpha unknown');
    }
    if (flags.indexOf('HasRatio') < 0) {
      $v.removeAttr('ratio');
@@ -287,12 +278,7 @@ function xmladdhint(rawXml, baseName, aptName, manifest, dats, filedir, oldStyle
    }
  });

-  // Clean up: remove extra attributes from the final output
-  $('edittext').each(function () {
-    const $v = $(this);
-    // Override for xmladdhint: store original color info
-    // (the original code swaps red/blue for edittext display - preserve that)
-  });
+  // Keep edittext channels in cc3tools' order; build uses the same order.

  // Clean internal attributes
  $('*').each(function () {
@@ -302,29 +288,11 @@ function xmladdhint(rawXml, baseName, aptName, manifest, dats, filedir, oldStyle

  // Serialize
  let output = $.xml();
-  // Wrap action container text content in CDATA sections
-  // This prevents cheerio from entity-encoding & in the action assembly text
-  output = output.replace(
-    /<(initaction|buttonaction|action|clipaction)(\s[^>]*)?>([\s\S]*?)<\/\1>/g,
-    (match, tag, attrs, content) => {
-      const trimmed = content.trim();
-      if (!trimmed) return match;
-      return '<' + tag + (attrs || '') + '><![CDATA[' + content + ']]></' + tag + '>';
-    }
-  );
  // Clean up excessive whitespace
  output = output.replace(/^[\t]+$/gm, '').replace(/[\r\n]+/g, '\n');
  return output;
 }

-// ─── Decode XML entities in attribute values ───
-// With decodeEntities: false in cheerio load, attribute values preserve XML entities
-// (e.g. &amp; stays as &amp;). We need to decode them for human-readable output.
-function decodeAttrEntities(s) {
-  if (typeof s !== 'string') return s;
-  return s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'");
-}
-
 // ─── actiontostring(): XML action node → text assembly ───
 function actiontostring(a, ActionNames, $) {
  const action = parseInt(a.attr('action'), 10);
@@ -347,7 +315,7 @@ function actiontostring(a, ActionNames, $) {
      action === C.EA_CALLNAMEDFUNCTION || action === C.EA_CALLNAMEDMETHODPOP ||
      action === C.EA_CALLNAMEDMETHOD || action === C.EA_PUSHVALUEOFVAR ||
      action === C.EA_GETNAMEDMEMBER || action === C.EA_PUSHWORDCONSTANT) {
-    return aname + ' ' + (a.attr('str') !== undefined ? U.mkstr(decodeAttrEntities(a.attr('str'))) : a.attr('num'));
+    return aname + ' ' + (a.attr('str') !== undefined ? U.mkstr(a.attr('str')) : a.attr('num'));
  }

  // goto: ActionGotoFrame pos
@@ -357,7 +325,7 @@ function actiontostring(a, ActionNames, $) {

  // geturl
  if (a[0].tagName === 'geturl') {
-    return aname + ' ' + U.mkstr(decodeAttrEntities(a.attr('str1'))) + ', ' + U.mkstr(decodeAttrEntities(a.attr('str2')));
+    return aname + ' ' + U.mkstr(a.attr('str1')) + ', ' + U.mkstr(a.attr('str2'));
  }

  // definefunction
@@ -408,7 +376,7 @@ function actiontostring(a, ActionNames, $) {

  // string instruction: pushstr "hello"
  if (a[0].tagName === 'string') {
-    return aname + ' ' + U.mkstr(decodeAttrEntities(a.attr('str')));
+    return aname + ' ' + U.mkstr(a.attr('str'));
  }

  // pushdata: pushdata val1, val2, ...
@@ -416,7 +384,7 @@ function actiontostring(a, ActionNames, $) {
    const vals = [];
    a.children().each(function () {
      const v = $(this);
-      vals.push(v.attr('str') !== undefined ? U.mkstr(decodeAttrEntities(v.attr('str'))) : v.attr('num'));
+      vals.push(v.attr('str') !== undefined ? U.mkstr(v.attr('str')) : v.attr('num'));
    });
    return aname + ' ' + vals.join(', ');
  }
@@ -35,7 +35,8 @@ function valuetoflags(v, e) {
    }
  }
  if (v0) {
-    throw new Error('unknown bit flags ' + v0 + ' in ' + v);
+    // Keep unlabelled converter bits explicit, without assigning guessed meanings.
+    keys.push('0x' + (v0 >>> 0).toString(16));
  }
  return keys.join('|');
 }
@@ -48,6 +49,8 @@ function flagstovalue(f, e) {
    const k = e[parts[i]];
    if (k !== undefined) {
      v |= k;
+    } else if (/^0x[0-9a-f]{1,8}$/i.test(parts[i])) {
+      v |= Number(parts[i]);
    } else {
      throw new Error('invalid bit flag: ' + parts[i]);
    }
@@ -60,8 +63,10 @@ function encodeXML(s) {
  const a = new Array(s.length);
  for (let i = 0; i < s.length; i++) {
    const c = s[i];
-    if (c === '\r') a[i] = '\\r';
+    if (c === '\\') a[i] = '\\\\';
+    else if (c === '\r') a[i] = '\\r';
    else if (c === '\n') a[i] = '\\n';
+    else if (c === '\t') a[i] = '\\t';
    else if (c === '"') a[i] = '\\"';
    else a[i] = c;
  }