From 23f8be394c297b55b3a8e9bc208ee7dfb39adeed Mon Sep 17 00:00:00 2001 From: EnderTheCoder Date: Tue, 29 Sep 2026 23:57:23 +0800 Subject: [PATCH] feat(assets): read retail assets through the vendored libra3assets library Vendor libra3assets (C++26 modules: BIG4, RefPack, BinaryAsset, CSF, CkMp map) under third_party/ and add the ra3_assets target. ra3.fs, ra3.map and ra3.terrain become thin adapters over it: - ra3.fs delegates RefPack and the BIG4 index/payload reads (index-only, payloads read on demand). - ra3.map decodes ObjectsList and CSF via map_document/csf_table; starts come from player_starts(), replacing the off-by-one whole-buffer scan. - ra3.terrain takes the CkMp container and HeightMapData from map_document, with BlendTileData (not modelled by the library) parsed from the chunk payload. Adds an opt-in real-asset check (OPENRA3_TEST_ASSETS) plus the library's own unit suite as ra3assets_unit. --- CMakeLists.txt | 39 +- src/fs/ra3.fs.cppm | 248 ++---- src/map/ra3.map.cppm | 288 ++----- src/terrain/ra3.terrain.cppm | 142 ++-- tests/ra3_tests.cpp | 95 +++ third_party/libra3assets/README.md | 134 +++ third_party/libra3assets/src/assets.cppm | 65 ++ third_party/libra3assets/src/big.cppm | 274 ++++++ third_party/libra3assets/src/binary.cppm | 396 +++++++++ third_party/libra3assets/src/bytes.cppm | 220 +++++ third_party/libra3assets/src/csf.cppm | 180 ++++ third_party/libra3assets/src/error.cppm | 56 ++ third_party/libra3assets/src/map.cppm | 778 ++++++++++++++++++ third_party/libra3assets/src/refpack.cppm | 284 +++++++ .../libra3assets/tests/test_assets.cpp | 310 +++++++ third_party/libra3assets/tests/test_main.cpp | 5 + third_party/libra3assets/tests/test_main.hpp | 83 ++ 17 files changed, 3078 insertions(+), 519 deletions(-) create mode 100644 third_party/libra3assets/README.md create mode 100644 third_party/libra3assets/src/assets.cppm create mode 100644 third_party/libra3assets/src/big.cppm create mode 100644 third_party/libra3assets/src/binary.cppm create mode 100644 third_party/libra3assets/src/bytes.cppm create mode 100644 third_party/libra3assets/src/csf.cppm create mode 100644 third_party/libra3assets/src/error.cppm create mode 100644 third_party/libra3assets/src/map.cppm create mode 100644 third_party/libra3assets/src/refpack.cppm create mode 100644 third_party/libra3assets/tests/test_assets.cpp create mode 100644 third_party/libra3assets/tests/test_main.cpp create mode 100644 third_party/libra3assets/tests/test_main.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a8b4dbf..8ee7b3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,6 +196,25 @@ target_sources(ra3_enderlog PUBLIC FILE_SET CXX_MODULES FILES third_party/libend target_compile_features(ra3_enderlog PUBLIC cxx_std_26) openra3_target_defaults(ra3_enderlog) +# --- RA3 built-in asset containers (vendored libra3assets) -------------------- +# Reader/writer for the files the retail game ships: `BIG4` archives, the EA +# RefPack codec, SAGE `.csf` string tables and `.map` (`CkMp`) containers, plus +# the compiled `BinaryAsset` streams. A sibling of libenderlog: C++26 modules +# (`import ra3.assets;`) importing the standard library. It owns the on-disk +# format knowledge, so `ra3.fs` / `ra3.map` / `ra3.terrain` become thin adapters. +add_library(ra3_assets STATIC) +target_sources(ra3_assets PUBLIC FILE_SET CXX_MODULES FILES + third_party/libra3assets/src/assets.cppm + third_party/libra3assets/src/error.cppm + third_party/libra3assets/src/bytes.cppm + third_party/libra3assets/src/refpack.cppm + third_party/libra3assets/src/big.cppm + third_party/libra3assets/src/binary.cppm + third_party/libra3assets/src/csf.cppm + third_party/libra3assets/src/map.cppm) +target_compile_features(ra3_assets PUBLIC cxx_std_26) +openra3_target_defaults(ra3_assets) + # --- engine core ------------------------------------------------------------- add_library(ra3_core STATIC) target_sources(ra3_core PUBLIC FILE_SET CXX_MODULES FILES src/core/ra3.core.cppm) @@ -225,16 +244,16 @@ target_sources(ra3_game PUBLIC FILE_SET CXX_MODULES FILES src/game/ra3.game.cppm target_link_libraries(ra3_game PUBLIC ra3_core ra3_logic) openra3_target_defaults(ra3_game) -# --- filesystem: BIG4 + RefPack --------------------------------------------- +# --- filesystem: BIG4 + RefPack (adapter over libra3assets) ------------------- add_library(ra3_fs STATIC) target_sources(ra3_fs PUBLIC FILE_SET CXX_MODULES FILES src/fs/ra3.fs.cppm) -target_link_libraries(ra3_fs PUBLIC ra3_core) +target_link_libraries(ra3_fs PUBLIC ra3_core ra3_assets) openra3_target_defaults(ra3_fs) # --- map discovery/loading --------------------------------------------------- add_library(ra3_map STATIC) target_sources(ra3_map PUBLIC FILE_SET CXX_MODULES FILES src/map/ra3.map.cppm) -target_link_libraries(ra3_map PUBLIC ra3_core ra3_fs) +target_link_libraries(ra3_map PUBLIC ra3_core ra3_fs ra3_assets) openra3_target_defaults(ra3_map) # --- minimal skirmish simulation -------------------------------------------- @@ -252,7 +271,7 @@ openra3_target_defaults(ra3_render) # --- real map terrain (HeightMapData / BlendTileData) ------------------------ add_library(ra3_terrain STATIC) target_sources(ra3_terrain PUBLIC FILE_SET CXX_MODULES FILES src/terrain/ra3.terrain.cppm) -target_link_libraries(ra3_terrain PUBLIC ra3_core ra3_fs ra3_render) +target_link_libraries(ra3_terrain PUBLIC ra3_core ra3_fs ra3_render ra3_assets) openra3_target_defaults(ra3_terrain) # --- map static art (compiled W3D meshes) ------------------------------------ @@ -580,3 +599,15 @@ add_executable(ra3_tests tests/ra3_tests.cpp) target_link_libraries(ra3_tests PRIVATE ra3) openra3_target_defaults(ra3_tests) add_test(NAME ra3_tests COMMAND ra3_tests) + +# libra3assets' own dependency-free unit tests (RefPack + BIG4 + BinaryAsset + +# CSF + CkMp round-trips), built against the vendored copy so the upstream +# suite keeps guarding the readers OpenRA3 now relies on. +add_executable(ra3assets_unit + third_party/libra3assets/tests/test_main.cpp + third_party/libra3assets/tests/test_assets.cpp) +target_include_directories(ra3assets_unit PRIVATE third_party/libra3assets/tests) +target_link_libraries(ra3assets_unit PRIVATE ra3_assets) +target_compile_features(ra3assets_unit PRIVATE cxx_std_26) +openra3_target_defaults(ra3assets_unit) +add_test(NAME ra3assets_unit COMMAND ra3assets_unit) diff --git a/src/fs/ra3.fs.cppm b/src/fs/ra3.fs.cppm index 3847cf2..4091071 100644 --- a/src/fs/ra3.fs.cppm +++ b/src/fs/ra3.fs.cppm @@ -3,13 +3,16 @@ export module ra3.fs; import std; export import ra3.core; +import ra3.assets; /** * Reading of the retail game's on-disk assets. * * Red Alert 3 ships its data in `BIG4` archives under `\Data`, with - * individual payloads compressed by EA's RefPack codec. This module implements - * the container and codec so OpenRA3 can read a user's own installation. + * individual payloads compressed by EA's RefPack codec. The container and codec + * are implemented by the vendored `libra3assets` (`ra3.assets`, a sibling of + * `libenderlog`); this module is the thin adapter OpenRA3's loader talks to, so + * the format knowledge lives in one place. * * No game data is ever written into the repository; callers point the loader at * their local install (`--game-dir` / `RA3_GAME_DIR`, default `C:\Red Alert 3`). @@ -42,11 +45,21 @@ export namespace ra3::fs { uint32 size = 0; }; - /** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */ - [[nodiscard]] inline auto is_refpack(std::span data) -> bool { - return data.size() >= 2 && (data[0] & refpack_mask) == 0x10U && data[1] == refpack_magic2; + namespace detail { + /** View a `uint8` range as bytes, the currency of `libra3assets`. */ + [[nodiscard]] inline auto as_bytes(std::span data) -> std::span { return std::as_bytes(data); } + + /** Copy a `libra3assets` byte buffer into OpenRA3's `uint8` vector. */ + [[nodiscard]] inline auto to_u8(std::vector bytes) -> std::vector { + std::vector out(bytes.size()); + if (!bytes.empty()) std::memcpy(out.data(), bytes.data(), bytes.size()); + return out; + } } + /** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */ + [[nodiscard]] inline auto is_refpack(std::span data) -> bool { return ra3::assets::is_refpack(detail::as_bytes(data)); } + /** * Decompress an EA RefPack stream. * @@ -55,80 +68,17 @@ export namespace ra3::fs { * @throws refpack_error if the stream is malformed or the length disagrees. */ [[nodiscard]] inline auto refpack_decompress(std::span data) -> std::vector { - if (!is_refpack(data)) throw refpack_error("not a RefPack stream"); - - usize pos = 0; - const auto header = data[pos++]; - const bool large_files = (header & 0x80U) != 0; - const bool compressed_size_present = (header & 0x01U) != 0; - pos++; // 0xFB - - const usize size_bytes = large_files ? 4U : 3U; - auto read_size = [&]() -> uint32 { - uint32 value = 0; - for (usize i = 0; i < size_bytes; ++i) { - if (pos >= data.size()) throw refpack_error("truncated RefPack size field"); - value = (value << 8) | data[pos++]; - } - return value; - }; - - if (compressed_size_present) (void)read_size(); - const auto out_len = read_size(); - - std::vector out; - out.reserve(out_len); - - auto copy_literals = [&](usize count) { - if (pos + count > data.size()) throw refpack_error("truncated RefPack literals"); - out.insert(out.end(), data.begin() + static_cast(pos), data.begin() + static_cast(pos + count)); - pos += count; - }; - - auto copy_reference = [&](usize length, usize distance) { - if (distance == 0 || distance > out.size()) throw refpack_error("RefPack back-reference out of range"); - usize start = out.size() - distance; - for (usize i = 0; i < length; ++i) out.push_back(out[start + i]); - }; - - while (pos < data.size()) { - const auto cmd = data[pos++]; - if ((cmd & 0x80U) == 0) { // 2-byte command - if (pos >= data.size()) throw refpack_error("truncated 2-byte command"); - const auto b2 = data[pos++]; - copy_literals(cmd & 0x03U); - copy_reference(((cmd & 0x1CU) >> 2) + 3, ((cmd & 0x60U) << 3) + b2 + 1); - } else if ((cmd & 0x40U) == 0) { // 3-byte command - if (pos + 1 >= data.size()) throw refpack_error("truncated 3-byte command"); - const auto b2 = data[pos]; - const auto b3 = data[pos + 1]; - pos += 2; - copy_literals((b2 & 0xC0U) >> 6); - copy_reference((cmd & 0x3FU) + 4, ((b2 & 0x3FU) << 8) + b3 + 1); - } else if ((cmd & 0x20U) == 0) { // 4-byte command - if (pos + 2 >= data.size()) throw refpack_error("truncated 4-byte command"); - const auto b2 = data[pos]; - const auto b3 = data[pos + 1]; - const auto b4 = data[pos + 2]; - pos += 3; - copy_literals(cmd & 0x03U); - copy_reference(((cmd & 0x0CU) << 6) + b4 + 5, ((cmd & 0x10U) << 12) + (b2 << 8) + b3 + 1); - } else if (cmd < 0xFCU) { // long literal run - copy_literals(((cmd & 0x1FU) + 1) << 2); - } else { // stop - copy_literals(cmd & 0x03U); - break; - } + try { + return detail::to_u8(ra3::assets::refpack_decompress(detail::as_bytes(data))); + } catch (const ra3::assets::refpack_error &error) { + throw refpack_error(error.what()); } - - if (out.size() != out_len) throw refpack_error("RefPack length mismatch"); - return out; } /** Decompress `data` when it is RefPack, otherwise copy it unchanged. */ [[nodiscard]] inline auto maybe_decompress(std::span data) -> std::vector { - if (is_refpack(data)) return refpack_decompress(data); - return {data.begin(), data.end()}; + if (!is_refpack(data)) return {data.begin(), data.end()}; + return refpack_decompress(data); } /** @@ -137,23 +87,11 @@ export namespace ra3::fs { * @throws refpack_error if the stream is malformed. */ [[nodiscard]] inline auto refpack_output_size(std::span data) -> uint32 { - if (!is_refpack(data)) throw refpack_error("not a RefPack stream"); - usize pos = 0; - const auto header = data[pos++]; - const bool large_files = (header & 0x80U) != 0; - const bool compressed_size_present = (header & 0x01U) != 0; - pos++; // 0xFB - const usize size_bytes = large_files ? 4U : 3U; - auto read_size = [&]() -> uint32 { - uint32 value = 0; - for (usize i = 0; i < size_bytes; ++i) { - if (pos >= data.size()) throw refpack_error("truncated RefPack size field"); - value = (value << 8) | data[pos++]; - } - return value; - }; - if (compressed_size_present) (void)read_size(); - return read_size(); + try { + return ra3::assets::refpack_output_size(detail::as_bytes(data)); + } catch (const ra3::assets::refpack_error &error) { + throw refpack_error(error.what()); + } } [[nodiscard]] constexpr auto read_be32(const uint8 *p) -> uint32 { @@ -165,81 +103,32 @@ export namespace ra3::fs { } /** - * A parsed `BIG4` archive. Only the index is held in memory; payloads are - * read from disk on demand so multi-hundred-megabyte archives stay cheap. + * A parsed `BIG4` archive. + * + * The index and payloads are held by a `ra3::assets::big_archive`; this + * adapter exposes the OpenRA3-facing surface (`big_entry`, `uint8` buffers) + * over it. Payloads are RefPack-decompressed on request. */ class big_archive { public: /** - * Parse the index of a `BIG4` archive. + * Parse a `BIG4` archive. * * @param path Archive path. * @throws archive_error if the file is missing, not `BIG4`, or truncated. */ [[nodiscard]] static auto open(const std::filesystem::path &path) -> big_archive { - big_archive archive; - archive.path_ = path; - std::error_code ec; - archive.file_size_ = static_cast(std::filesystem::file_size(path, ec)); - if (ec) throw archive_error("cannot stat archive: " + path.string()); - - std::ifstream in(path, std::ios::binary); - if (!in) throw archive_error("cannot open archive: " + path.string()); - - std::array header{}; - in.read(reinterpret_cast(header.data()), static_cast(header.size())); - if (!in || std::memcmp(header.data(), big_magic.data(), big_magic.size()) != 0) throw archive_error("not a BIG4 archive: " + path.string()); - - const auto count = read_be32(header.data() + 8); - - // Read the variable-length index, growing the window until parsed. - usize window = std::min(archive.file_size_, std::max(1U << 20U, static_cast(count) * 256U)); - std::vector index; - for (;;) { - index.resize(window); - in.clear(); - in.seekg(0); - in.read(reinterpret_cast(index.data()), static_cast(window)); - const auto got = static_cast(in.gcount()); - index.resize(got); - - archive.entries_.clear(); - archive.entries_.reserve(count); - usize pos = 16; - bool complete = true; - for (uint32 i = 0; i < count; ++i) { - if (pos + 8 > index.size()) { - complete = false; - break; - } - big_entry entry; - entry.offset = read_be32(index.data() + pos); - entry.size = read_be32(index.data() + pos + 4); - pos += 8; - const auto *begin = reinterpret_cast(index.data() + pos); - const auto *end = reinterpret_cast(std::memchr(begin, '\0', index.size() - pos)); - if (end == nullptr) { - complete = false; - break; - } - entry.name.assign(begin, end); - pos += static_cast(end - begin) + 1U; - archive.entries_.push_back(std::move(entry)); - } - if (complete) break; - if (window >= archive.file_size_) throw archive_error("truncated BIG4 index: " + path.string()); - window = std::min(archive.file_size_, window * 2U); + try { + return big_archive{ra3::assets::big_archive::open(path), path}; + } catch (const ra3::assets::asset_error &error) { + throw archive_error(error.what()); } - - archive.index_.reserve(archive.entries_.size()); - for (usize i = 0; i < archive.entries_.size(); ++i) archive.index_.emplace(archive.entries_[i].name, i); - return archive; } [[nodiscard]] auto path() const -> const std::filesystem::path & { return path_; } [[nodiscard]] auto entries() const -> const std::vector & { return entries_; } [[nodiscard]] auto size() const -> usize { return entries_.size(); } - [[nodiscard]] auto contains(std::string_view name) const -> bool { return index_.contains(std::string{name}); } + [[nodiscard]] auto contains(std::string_view name) const -> bool { return archive_.contains(name); } /** Entry names whose path contains `needle`, in index order. */ [[nodiscard]] auto find(std::string_view needle) const -> std::vector { @@ -258,42 +147,33 @@ export namespace ra3::fs { * @throws archive_error if the entry is missing or unreadable. */ [[nodiscard]] auto read(std::string_view name, bool decompress = true) const -> std::vector { - const auto it = index_.find(std::string{name}); - if (it == index_.end()) throw archive_error("no such entry: " + std::string{name}); - const auto &entry = entries_[it->second]; - - std::ifstream in(path_, std::ios::binary); - if (!in) throw archive_error("cannot open archive: " + path_.string()); - in.seekg(static_cast(entry.offset)); - std::vector raw(entry.size); - in.read(reinterpret_cast(raw.data()), static_cast(raw.size())); - if (!in) throw archive_error("short read for entry: " + entry.name); - return decompress ? maybe_decompress(raw) : raw; + try { + return detail::to_u8(archive_.read(name, decompress)); + } catch (const ra3::assets::refpack_error &error) { + throw refpack_error(error.what()); + } catch (const ra3::assets::asset_error &error) { + throw archive_error(error.what()); + } } /** Read the first `count` stored bytes of an entry (no decompression). */ [[nodiscard]] auto read_prefix(std::string_view name, usize count) const -> std::vector { - const auto it = index_.find(std::string{name}); - if (it == index_.end()) throw archive_error("no such entry: " + std::string{name}); - const auto &entry = entries_[it->second]; - - std::ifstream in(path_, std::ios::binary); - if (!in) throw archive_error("cannot open archive: " + path_.string()); - in.seekg(static_cast(entry.offset)); - const auto want = std::min(count, static_cast(entry.size)); - std::vector raw(want); - in.read(reinterpret_cast(raw.data()), static_cast(raw.size())); - raw.resize(static_cast(in.gcount())); - return raw; + try { + return detail::to_u8(archive_.read_prefix(name, count)); + } catch (const ra3::assets::asset_error &error) { + throw archive_error(error.what()); + } } private: - big_archive() = default; + big_archive(ra3::assets::big_archive archive, std::filesystem::path path) : archive_(std::move(archive)), path_(std::move(path)) { + entries_.reserve(archive_.size()); + for (const auto &entry: archive_.entries()) entries_.push_back({entry.name, entry.offset, entry.size}); + } + ra3::assets::big_archive archive_; std::filesystem::path path_; - usize file_size_ = 0; std::vector entries_; - std::unordered_map index_; }; /** @@ -306,18 +186,6 @@ export namespace ra3::fs { * @return The install root, or `std::nullopt` when none is found. */ [[nodiscard]] inline auto find_game_dir(const std::optional &explicit_dir = std::nullopt) -> std::optional { - auto qualifies = [](const std::filesystem::path &candidate) { - std::error_code ec; - return !candidate.empty() && std::filesystem::is_directory(candidate / "Data", ec); - }; - - if (explicit_dir && qualifies(*explicit_dir)) return explicit_dir; - if (const auto *env = std::getenv("RA3_GAME_DIR"); env != nullptr && *env != '\0') { - const std::filesystem::path candidate{env}; - if (qualifies(candidate)) return candidate; - } - const std::filesystem::path default_dir{"C:/Red Alert 3"}; - if (qualifies(default_dir)) return default_dir; - return std::nullopt; + return ra3::assets::find_game_dir(explicit_dir); } } diff --git a/src/map/ra3.map.cppm b/src/map/ra3.map.cppm index 5924cbb..38a2b09 100644 --- a/src/map/ra3.map.cppm +++ b/src/map/ra3.map.cppm @@ -4,6 +4,7 @@ import std; export import ra3.core; export import ra3.fs; +import ra3.assets; /** * Red Alert 3 map discovery and loading. @@ -50,6 +51,9 @@ export namespace ra3::map { }; namespace detail { + /** View a `uint8` range as bytes, the currency of `libra3assets`. */ + [[nodiscard]] inline auto as_bytes(std::span data) -> std::span { return std::as_bytes(data); } + [[nodiscard]] inline auto split_path(std::string_view path) -> std::vector { std::vector parts; usize start = 0; @@ -82,54 +86,6 @@ export namespace ra3::map { return file; } - [[nodiscard]] inline auto find_bytes(std::span haystack, std::string_view needle) -> usize { - if (needle.empty()) return 0; - const auto *needle_begin = reinterpret_cast(needle.data()); - const auto it = std::search(haystack.begin(), haystack.end(), needle_begin, needle_begin + needle.size()); - return it == haystack.end() ? static_cast(-1) : static_cast(it - haystack.begin()); - } - - [[nodiscard]] inline auto read_le_f32(const uint8 *p) -> real { - const auto bits = fs::read_le32(p); - real value = 0.0F; - std::memcpy(&value, &bits, sizeof(value)); - return value; - } - - /** First plausible `(x, y, 0)` float triple at or after `from`. */ - [[nodiscard]] inline auto scan_triple(std::span data, usize from, usize window) -> std::optional { - const auto end = std::min(data.size(), from + window); - for (usize p = from; p + 12U <= end; ++p) { - const auto x = read_le_f32(data.data() + p); - const auto y = read_le_f32(data.data() + p + 4U); - const auto z = read_le_f32(data.data() + p + 8U); - if (z == 0.0F && std::isfinite(x) && std::isfinite(y) && x > 0.0F && x < 20000.0F && y > 0.0F && y < 20000.0F) { - return start_position{x, y, z}; - } - } - return std::nullopt; - } - - /** Extract `Player_1_Start` .. `Player_N_Start` waypoint coordinates. */ - [[nodiscard]] inline auto extract_start_positions(std::span ckmp, int max_players = 8) -> std::vector { - std::vector result; - for (int n = 1; n <= max_players; ++n) { - const auto needle = std::string{"Player_"} + std::to_string(n) + "_Start"; - usize cursor = 0; - while (cursor < ckmp.size()) { - const auto at = find_bytes(ckmp.subspan(cursor), needle); - if (at == static_cast(-1)) break; - const auto abs = cursor + at; - if (const auto triple = scan_triple(ckmp, abs + needle.size(), 128); triple) { - result.push_back(*triple); - break; - } - cursor = abs + 1; - } - } - return result; - } - /** Number of distinct integer positions (used to reject degenerate sets). */ [[nodiscard]] inline auto distinct_positions(const std::vector &starts) -> usize { std::vector> seen; @@ -196,9 +152,24 @@ export namespace ra3::map { return fs::maybe_decompress(payload); } - /** Recover `Player_N_Start` waypoints from raw, uncompressed `CkMp` bytes. */ + /** + * Recover `Player_N_Start` waypoints from raw, uncompressed `CkMp` bytes. + * + * `libra3assets` decodes the `ObjectsList` `*Waypoints/Waypoint` objects and + * returns the position of each `waypointName == "Player__Start"`. This + * replaces the previous whole-buffer heuristic, which scanned for the string + * and then took the first plausible float triple *after* it — off by one + * (the position precedes the name in each object) and prone to matching + * arbitrary bytes. + */ [[nodiscard]] inline auto starts_from_ckmp(std::span ckmp) -> std::vector { - auto starts = detail::extract_start_positions(ckmp); + std::vector starts; + try { + const auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp)); + for (const auto &start: document.player_starts()) starts.push_back({start.position.x, start.position.y, start.position.z}); + } catch (const std::exception &) { + return {}; + } if (detail::distinct_positions(starts) < 2U) starts.clear(); return starts; } @@ -221,117 +192,34 @@ export namespace ra3::map { uint32 road_type = 0U; ///< SAGE `RoadType` flags; 0 for a non-road object. }; - namespace detail { - struct ckmp_chunk { - std::string name; - uint16 version = 0; - usize offset = 0; - usize size = 0; - }; - - /** Parse the `CkMp` chunk tree and its `index -> name` table. */ - [[nodiscard]] inline auto parse_ckmp_chunks(std::span data) -> std::pair, std::vector> { - if (data.size() < 8U || std::memcmp(data.data(), "CkMp", 4) != 0) return {}; - usize pos = 4; - const auto read_u32 = [&](usize at) { - return static_cast(data[at]) | (static_cast(data[at + 1U]) << 8U) | (static_cast(data[at + 2U]) << 16U) | - (static_cast(data[at + 3U]) << 24U); - }; - const auto read_u16 = [&](usize at) { return static_cast(static_cast(data[at]) | (static_cast(data[at + 1U]) << 8U)); }; - const auto count = read_u32(pos); - pos += 4; - std::vector names(count + 1U); - for (uint32 i = count; i >= 1U && pos < data.size(); --i) { - const auto len = data[pos++]; - if (pos + len + 4U > data.size()) break; - names[i] = std::string{reinterpret_cast(data.data() + pos), len}; - pos += len + 4U; - } - std::vector chunks; - while (pos + 10U <= data.size()) { - const auto index = read_u32(pos); - const auto version = read_u16(pos + 4U); - const auto size = read_u32(pos + 6U); - pos += 10U; - if (index >= names.size() || pos + size > data.size()) break; - chunks.push_back({names[index], version, pos, size}); - pos += size; - } - return {std::move(names), std::move(chunks)}; - } - } - /** * Every object the map places (the `ObjectsList` chunk). * - * Layout (OpenSAGE `Data/Map/{ObjectsList,MapObject,AssetProperty}.cs`): - * the chunk is a list of nested `Object` assets, each a `Coord3D`, a Z - * `angle`, a `RoadType`, a `u16`-prefixed type-name and a property list - * whose keys index the shared name table. + * Decoded by `libra3assets` (`ra3.assets`): the map is parsed as a + * `map_document`, whose `ObjectsList` accessor walks the nested `Object` + * assets (a `Coord3D`, a Z `angle`, a `RoadType`, a `u16`-prefixed + * type-name and a property list keyed into the shared name table). * - * @return The objects in chunk order; empty when the map has no object list. + * @return The objects in chunk order; empty when the map is malformed or has + * no object list. */ [[nodiscard]] inline auto parse_objects(std::span ckmp) -> std::vector { - const auto [names, chunks] = detail::parse_ckmp_chunks(ckmp); - (void) names; std::vector objects; - const auto read_u16 = [&](usize at) { return static_cast(static_cast(ckmp[at]) | (static_cast(ckmp[at + 1U]) << 8U)); }; - const auto read_u32 = [&](usize at) { - return static_cast(ckmp[at]) | (static_cast(ckmp[at + 1U]) << 8U) | (static_cast(ckmp[at + 2U]) << 16U) | - (static_cast(ckmp[at + 3U]) << 24U); - }; - const auto read_f32 = [&](usize at) { - const auto bits = read_u32(at); - real value = 0.0F; - std::memcpy(&value, &bits, sizeof(value)); - return value; - }; - - for (const auto &chunk: chunks) { - if (chunk.name != "ObjectsList") continue; - usize p = chunk.offset; - const auto end = chunk.offset + chunk.size; - while (p + 6U <= end) { - p += 4U; // asset index (always `Object`) - p += 2U; // asset version - const auto asset_size = read_u32(p); - p += 4U; - if (asset_size < 24U || p + asset_size > end) break; - const auto asset_end = p + asset_size; - - map_object object; - object.x = read_f32(p); - object.y = read_f32(p + 4U); - object.z = read_f32(p + 8U); - object.angle = read_f32(p + 12U); - object.road_type = read_u32(p + 16U); - p += 20U; // Coord3D + angle + road type - const auto name_len = read_u16(p); - p += 2U; - if (p + name_len > asset_end) break; - object.type = std::string{reinterpret_cast(ckmp.data() + p), name_len}; - p += name_len; - - const auto property_count = static_cast(read_u16(p)); - p += 2U; - for (uint32 i = 0; i < property_count && p + 4U <= asset_end; ++i) { - const auto type = ckmp[p++]; - p += 3U; // property name index (u24) into the shared name table - usize value_size = 0; - if (type == 0U) { - value_size = 1U; // boolean - } else if (type == 1U || type == 2U) { - value_size = 4U; // integer / real - } else if (p + 2U <= asset_end) { - const auto len = read_u16(p); - value_size = 2U + static_cast(len) * ((type == 4U) ? 2U : 1U); // ascii/unicode/unknown - } - p += value_size; - if (p > asset_end) break; - } - objects.push_back(std::move(object)); - p = asset_end; + try { + const auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp)); + for (const auto &object: document.objects()) { + map_object out; + out.type = object.type_name; + out.x = object.position.x; + out.y = object.position.y; + out.z = object.position.z; + out.angle = object.angle; + out.scale = 1.0F; + out.road_type = static_cast(object.road); + objects.push_back(std::move(out)); } + } catch (const std::exception &) { + return {}; } return objects; } @@ -351,96 +239,30 @@ export namespace ra3::map { } }; - namespace detail { - /** - * Decode one SAGE `.csf` string value. - * - * RA3's `gamestrings.csf` stores each UTF-16 code unit with the low byte - * XORed by `0xFF` (the high byte is the padding `0xFF`); undoing that - * yields the plain ASCII/UTF-8 text. - */ - [[nodiscard]] inline auto decode_csf_string(std::span raw) -> std::string { - std::string value; - for (usize i = 0; i + 1U < raw.size(); i += 2U) { - const auto ch = static_cast(static_cast(raw[i] ^ 0xFFU)); - if (ch == '\0') break; - value.push_back(ch); - } - return value; - } - } - /** * Parse `MAP:` display names out of a SAGE `.csf` string table. * * The map list UI reads its labels from the install's `data\gamestrings.csf` * under the key `MAP:` (e.g. `MAP:MAP_MP_2_FEASEL4` is - * "Battlebase Beta"). Values are byte-XORed with `0xFF`. + * "Battlebase Beta"). Decoding is `libra3assets`' `csf_table` (each UTF-16 + * code unit is bit-inverted; the table lookup is case-insensitive). */ [[nodiscard]] inline auto parse_map_names(std::span csf) -> map_name_table { map_name_table table; - if (csf.size() < 24U || std::memcmp(csf.data(), " FSC", 4) != 0) return table; - - const auto read = [&](usize p) -> uint32 { - return static_cast(csf[p]) | (static_cast(csf[p + 1U]) << 8U) | (static_cast(csf[p + 2U]) << 16U) | - (static_cast(csf[p + 3U]) << 24U); - }; - - usize pos = 24; // header: magic + version + label/string counts + 8 reserved bytes - while (pos + 4U <= csf.size()) { - if (std::memcmp(csf.data() + pos, " LBL", 4) != 0) { - ++pos; - continue; - } - pos += 4; - if (pos + 4U > csf.size()) break; - const auto count = read(pos); - pos += 4; - std::vector labels; - labels.reserve(count); - bool ok = true; - for (uint32 i = 0; i < count; ++i) { - if (pos + 4U > csf.size()) { - ok = false; - break; - } - const auto len = read(pos); - pos += 4; - if (pos + len > csf.size()) { - ok = false; - break; - } - labels.emplace_back(reinterpret_cast(csf.data() + pos), len); - pos += len; - } - if (!ok) break; - for (const auto &label: labels) { - if (pos + 4U > csf.size() || std::memcmp(csf.data() + pos, " RTS", 4) != 0) { - ok = false; - break; - } - pos += 4; - if (pos + 4U > csf.size()) { - ok = false; - break; - } - const auto chars = read(pos); - pos += 4; - const auto bytes = static_cast(chars) * 2U; - if (pos + bytes > csf.size()) { - ok = false; - break; - } - auto value = detail::decode_csf_string(csf.subspan(pos, bytes)); - pos += bytes; + try { + const auto strings = ra3::assets::csf_table::parse(detail::as_bytes(csf)); + for (const auto &entry: strings.entries()) { constexpr std::string_view prefix = "MAP:"; - if (label.size() > prefix.size() && label.compare(0, prefix.size(), prefix) == 0) { - auto id = label.substr(prefix.size()); - std::transform(id.begin(), id.end(), id.begin(), [](unsigned char ch) { return static_cast(std::tolower(ch)); }); - if (!value.empty()) table.names.try_emplace(std::move(id), std::move(value)); - } + if (entry.label.size() <= prefix.size() || entry.label.compare(0, prefix.size(), prefix) != 0) continue; + auto id = entry.label.substr(prefix.size()); + std::transform(id.begin(), id.end(), id.begin(), [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + auto value = ra3::assets::utf16_to_utf8(entry.value()); + // CSF strings are NUL-terminated; the terminator is not part of the text. + if (const auto nul = value.find('\0'); nul != std::string::npos) value.resize(nul); + if (!value.empty()) table.names.try_emplace(std::move(id), std::move(value)); } - if (!ok) break; + } catch (const std::exception &) { + return {}; } return table; } diff --git a/src/terrain/ra3.terrain.cppm b/src/terrain/ra3.terrain.cppm index 525d040..5546c85 100644 --- a/src/terrain/ra3.terrain.cppm +++ b/src/terrain/ra3.terrain.cppm @@ -6,6 +6,7 @@ export import ra3.core; export import ra3.fs; export import ra3.render; export import ra3.models; +import ra3.assets; /** * The map's real terrain, read from the compiled `CkMp` chunk tree. @@ -91,82 +92,24 @@ export namespace ra3::terrain { }; namespace detail { + /** View a `uint8` range as bytes, the currency of `libra3assets`. */ + [[nodiscard]] inline auto as_bytes(std::span data) -> std::span { return std::as_bytes(data); } + + /** View a `libra3assets` byte range as OpenRA3's `uint8`. */ + [[nodiscard]] inline auto as_u8(std::span data) -> std::span { + return {reinterpret_cast(data.data()), data.size()}; + } + [[nodiscard]] inline auto read_u16(const uint8 *p) -> uint16 { return static_cast(p[0]) | (static_cast(p[1]) << 8U); } [[nodiscard]] inline auto read_u32(const uint8 *p) -> uint32 { return static_cast(p[0]) | (static_cast(p[1]) << 8U) | (static_cast(p[2]) << 16U) | (static_cast(p[3]) << 24U); } - struct chunk { - std::string name; - uint16 version = 0; - usize offset = 0; - usize size = 0; - }; - - /** - * Parse the `CkMp` chunk tree into a flat chunk list. - * - * Layout: `"CkMp"`, `u32 assetCount`, then `assetCount` entries of - * `{ u8 nameLen, name, u32 index }` (index descending from the count), - * then `{ u32 index, u16 version, u32 dataSize, data[dataSize] }` per - * chunk until the end. - */ - [[nodiscard]] inline auto parse_chunks(std::span data) -> std::vector { - if (data.size() < 8U || std::memcmp(data.data(), "CkMp", 4) != 0) throw terrain_error("not a CkMp map"); - usize pos = 4; - const auto count = read_u32(data.data() + pos); - pos += 4; - - std::vector names(count + 1U); - for (uint32 i = count; i >= 1U; --i) { - if (pos >= data.size()) throw terrain_error("truncated asset-name table"); - const auto len = data[pos++]; - if (pos + len + 4U > data.size()) throw terrain_error("truncated asset-name"); - names[i] = std::string{reinterpret_cast(data.data() + pos), len}; - pos += len; - pos += 4; // asset index (== i) - } - - std::vector chunks; - while (pos + 10U <= data.size()) { - const auto index = read_u32(data.data() + pos); - const auto version = read_u16(data.data() + pos + 4U); - const auto size = read_u32(data.data() + pos + 6U); - pos += 10; - if (index >= names.size() || pos + size > data.size()) throw terrain_error("bad chunk header"); - chunks.push_back({names[index], version, pos, size}); - pos += size; - } - return chunks; - } - - [[nodiscard]] inline auto find(const std::vector &chunks, std::string_view name) -> const chunk * { - for (const auto &c: chunks) { - if (c.name == name) return &c; - } - return nullptr; - } - - inline auto parse_heightmap(std::span data, const chunk &c, map_data &out) -> void { - usize p = c.offset; - out.width = read_u32(data.data() + p); - out.height = read_u32(data.data() + p + 4U); - out.border_width = read_u32(data.data() + p + 8U); - const auto border_count = read_u32(data.data() + p + 12U); - p += 16; - p += static_cast(border_count) * (c.version >= 6U ? 16U : 8U); - p += 4U; // area - - if (out.width == 0U || out.height == 0U) throw terrain_error("empty heightmap"); - const auto area = static_cast(out.width) * out.height; - out.elevations.resize(area); - for (usize i = 0; i < area; ++i) { - if (p >= data.size()) throw terrain_error("truncated heightmap"); - out.elevations[i] = c.version >= 5U ? read_u16(data.data() + p) : data[p]; - p += c.version >= 5U ? 2U : 1U; - } - } + // The `CkMp` container and `HeightMapData` chunk are modelled by + // `libra3assets` (`ra3.assets`); `parse_map` reads them through a + // `map_document`. Only `BlendTileData` (which the library does not type + // yet) is parsed here, over the chunk payload the document exposes. /** Where the texture table ends and how many `BlendDescription`s follow. */ struct texture_table_info { @@ -246,20 +189,20 @@ export namespace ra3::terrain { } } - inline auto parse_blend(std::span data, const chunk &c, map_data &out) -> void { + inline auto parse_blend(std::span payload, uint16 version, map_data &out) -> void { const auto area = static_cast(out.width) * out.height; - usize p = c.offset; - const auto num_tiles = read_u32(data.data() + p); + usize p = 0; + const auto num_tiles = read_u32(payload.data() + p); p += 4; if (num_tiles != area) throw terrain_error("BlendTileData tile count mismatch"); out.tiles.resize(area); - for (usize i = 0; i < area; ++i) out.tiles[i] = read_u16(data.data() + p + i * 2U); + for (usize i = 0; i < area; ++i) out.tiles[i] = read_u16(payload.data() + p + i * 2U); p += area * 2U; - const auto bits = (c.version >= 14U && c.version < 24U) ? 32U : 16U; + const auto bits = (version >= 14U && version < 24U) ? 32U : 16U; const auto word = bits / 8U; const auto read_index = [&](usize off) -> uint16 { - return word == 4U ? static_cast(read_u32(data.data() + off)) : read_u16(data.data() + off); + return word == 4U ? static_cast(read_u32(payload.data() + off)) : read_u16(payload.data() + off); }; out.blends.resize(area); for (usize i = 0; i < area; ++i) out.blends[i] = read_index(p + i * word); @@ -269,33 +212,48 @@ export namespace ra3::terrain { p += area * word; p += area * word; // CliffTextures (not rendered yet) - const auto chunk_end = c.offset + c.size; - const auto table = parse_textures(data, p, chunk_end, out); - parse_blend_descriptions(data, table, chunk_end, out); + const auto chunk_end = payload.size(); + const auto table = parse_textures(payload, p, chunk_end, out); + parse_blend_descriptions(payload, table, chunk_end, out); } } /** * Parse the terrain chunks out of a `CkMp` (uncompressed) map payload. * + * The container and `HeightMapData` come from `libra3assets` + * (`map_document`); `BlendTileData` (which the library does not model yet) + * is decoded here from the chunk payload the document exposes. + * * @throws terrain_error if the chunk tree or terrain chunks are malformed. */ [[nodiscard]] inline auto parse_map(std::span ckmp) -> map_data { - const auto chunks = detail::parse_chunks(ckmp); - const auto *heightmap = detail::find(chunks, "HeightMapData"); - const auto *blend = detail::find(chunks, "BlendTileData"); - if (heightmap == nullptr || blend == nullptr) throw terrain_error("map has no terrain chunks"); + try { + auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp)); - map_data out; - detail::parse_heightmap(ckmp, *heightmap, out); - detail::parse_blend(ckmp, *blend, out); - if (const auto *water = detail::find(chunks, "GlobalWaterSettings"); water != nullptr && water->size >= 8U) { - out.has_water = detail::read_u32(ckmp.data() + water->offset) != 0U; - const auto bits = detail::read_u32(ckmp.data() + water->offset + 4U); - std::memcpy(&out.water_plane_z, &bits, sizeof(out.water_plane_z)); + map_data out; + const auto height = document.height_map(); + if (!height) throw terrain_error("map has no HeightMapData chunk"); + out.width = height->width; + out.height = height->height; + out.border_width = height->border_width; + out.elevations.assign(height->elevations.begin(), height->elevations.end()); + + const auto *blend = document.find_chunk("BlendTileData"); + if (blend == nullptr) throw terrain_error("map has no BlendTileData chunk"); + detail::parse_blend(detail::as_u8(blend->payload), blend->version, out); + + if (const auto *water = document.find_chunk("GlobalWaterSettings"); water != nullptr && water->payload.size() >= 8U) { + const auto *bytes = reinterpret_cast(water->payload.data()); + out.has_water = detail::read_u32(bytes) != 0U; + const auto bits = detail::read_u32(bytes + 4U); + std::memcpy(&out.water_plane_z, &bits, sizeof(out.water_plane_z)); + } + out.valid = true; + return out; + } catch (const ra3::assets::asset_error &error) { + throw terrain_error(error.what()); } - out.valid = true; - return out; } /** The decoded terrain textures, parallel to `map_data::textures`. */ diff --git a/tests/ra3_tests.cpp b/tests/ra3_tests.cpp index 95938ba..1f819e3 100644 --- a/tests/ra3_tests.cpp +++ b/tests/ra3_tests.cpp @@ -1,5 +1,6 @@ import std; import ra3; +import ra3.assets; namespace { int failures = 0; @@ -300,6 +301,100 @@ auto main() -> int { check(models::decode_dds(std::vector{1U, 2U, 3U}).empty(), "a non-DDS payload yields no image"); + // Real-asset checks (opt-in): set OPENRA3_TEST_ASSETS to an extracted assets + // directory. They run the libra3assets-backed readers over the retail data and + // compare the map names against the golden `maps/map_names.tsv` written by the + // previous hand-rolled parser. + if (const char *assets_env = std::getenv("OPENRA3_TEST_ASSETS"); assets_env != nullptr && *assets_env != '\0') { + const std::filesystem::path assets{assets_env}; + const auto read_all = [](const std::filesystem::path &path) { + std::ifstream in(path, std::ios::binary); + return std::vector{std::istreambuf_iterator(in), std::istreambuf_iterator()}; + }; + + const auto golden = assets / "maps" / "map_names.tsv"; + // The golden table came from the newest `Lang-English.big`; mirror that. + const auto pick_english_csf = [](const std::filesystem::path &raw) -> std::filesystem::path { + std::filesystem::path best; + int best_version = -1; + std::error_code ec; + for (const auto &entry: std::filesystem::directory_iterator(raw, ec)) { + if (!entry.is_directory()) continue; + const auto name = entry.path().filename().string(); + if (name.rfind("Lang-English", 0) != 0) continue; + int version = 0; + try { + version = std::stoi(name.substr(12)); + } catch (const std::exception &) { + continue; + } + if (version > best_version) { + best_version = version; + best = entry.path() / "data" / "gamestrings.csf"; + } + } + if (!best.empty() && std::filesystem::exists(best)) return best; + return raw / "English" / "data" / "gamestrings.csf"; + }; + const auto csf_path = pick_english_csf(assets / "raw"); + if (std::filesystem::exists(golden) && std::filesystem::exists(csf_path)) { + const auto table = map::parse_map_names(read_all(csf_path)); + std::ifstream in(golden, std::ios::binary); + std::string line; + std::size_t matched = 0; + std::size_t mismatched = 0; + while (std::getline(in, line)) { + const auto tab = line.find('\t'); + if (tab == std::string::npos) continue; + const auto id = line.substr(0, tab); + const auto expected = line.substr(tab + 1U); + if (table.lookup(id) == expected) { + ++matched; + } else { + ++mismatched; + if (mismatched <= 5U) std::printf(" name mismatch %s: golden '%s' new '%s'\n", id.c_str(), expected.c_str(), table.lookup(id).c_str()); + } + } + std::printf(" real CSF: %zu names, golden %zu matched, %zu mismatched\n", table.names.size(), matched, mismatched); + check(mismatched == 0U, "map names reproduce the golden map_names.tsv"); + } + + std::size_t maps_ok = 0; + std::size_t maps_bad = 0; + std::size_t library_starts = 0; + std::size_t no_starts = 0; + std::size_t roundtrip_bad = 0; + const auto to_bytes = [](std::span data) { + return std::span{reinterpret_cast(data.data()), data.size()}; + }; + for (const auto &entry: std::filesystem::directory_iterator(assets / "maps")) { + if (!entry.is_regular_file() || entry.path().extension() != ".map") continue; + try { + const auto ckmp = map::to_ckmp(read_all(entry.path())); + const auto parsed = terrain::parse_map(ckmp); + const auto starts = map::starts_from_ckmp(ckmp); + const auto document = ra3::assets::map_document::parse(to_bytes(ckmp)); + if (document.player_starts().size() >= 2U) ++library_starts; else ++no_starts; + if (!std::ranges::equal(document.to_ckmp(), to_bytes(ckmp))) ++roundtrip_bad; + if (parsed.width > 0U && parsed.height > 0U && parsed.elevations.size() == static_cast(parsed.width) * parsed.height && starts.size() >= 2U) { + ++maps_ok; + } else { + ++maps_bad; + if (maps_bad <= 5U) + std::printf(" map oddity %s: %ux%u starts %zu\n", entry.path().filename().string().c_str(), parsed.width, parsed.height, starts.size()); + } + } catch (const std::exception &error) { + ++maps_bad; + if (maps_bad <= 5U) std::printf(" map FAIL %s: %s\n", entry.path().filename().string().c_str(), error.what()); + } + } + std::printf(" real maps: %zu ok, %zu bad; player_starts resolved %zu, unresolved %zu; round-trip mismatched %zu\n", maps_ok, maps_bad, library_starts, + no_starts, roundtrip_bad); + check(maps_bad == 0U, "every extracted map parses terrain + starts through libra3assets"); + check(no_starts == 0U, "libra3assets player_starts() resolves every retail map's start waypoints"); + check(roundtrip_bad == 0U, "libra3assets re-serialises every retail map byte-for-byte"); + } + if (failures == 0) { std::puts("ra3_tests: OK"); } diff --git a/third_party/libra3assets/README.md b/third_party/libra3assets/README.md new file mode 100644 index 0000000..48cedc1 --- /dev/null +++ b/third_party/libra3assets/README.md @@ -0,0 +1,134 @@ +# libra3assets + +A dependency-free C++26 library that reads **and writes** the resource files +Red Alert 3 ships, so tooling (map editors in particular) can work with the +retail assets directly. No engine, no game install required to build - only to +feed it data. + +It is a sub-project of this repository, a sibling of +[`libra3replay`](../libra3replay/README.md), and follows the same build +conventions (C++26 modules, `import std;`, GCC 16, CMake 4 + Ninja). + +## What it handles + +| Module | Format | Read | Write | +| ------------------ | ------------------------------------------------------------------- | :--: | :---: | +| `ra3.assets:big` | `BIG4` archives (`Data\*.big`) | yes | yes | +| `ra3.assets:refpack` | EA RefPack codec (`10 FB`) | yes | yes | +| `ra3.assets:binary` | compiled `BinaryAsset` streams (`.bin` + `.manifest`, `cdata`) | yes | - | +| `ra3.assets:csf` | SAGE `.csf` string tables (`gamestrings.csf`) | yes | yes | +| `ra3.assets:map` | SAGE `.map` containers (`CkMp`) incl. `HeightMapData`/objects | yes | yes | +| `ra3.assets:bytes` | bounds-checked little-/big-endian readers and writers | yes | yes | + +The **map** module is the centrepiece for a map editor. A `.map` is modelled as +an ordered list of named, versioned `CkMp` chunks; chunks the library does not +type (e.g. `BlendTileData`, `SidesList`) are preserved byte-for-byte and the +asset-name table keeps its original indices, so an edit/ serialise cycle is +lossless. Typed accessors cover the terrain grid (`HeightMapData`), every placed +object (`ObjectsList`, including the `*Waypoints/Waypoint` objects that carry +`Player_N_Start`), `MPPositionList`, `WorldInfo` and `WaypointsList`. + +The **binary** module parses the compiled `BinaryAsset` streams that +BinaryAssetBuilder produces (`data\static.bin`, `data\global.bin`, ...): the +manifest index, each asset's instance slice, its relocation/import sidecars and +its `cdata` blob, plus the hash used to name assets. + +## Build + +```bash +cmake -S libra3assets -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++-16 +cmake --build build +ctest --test-dir build --output-on-failure +``` + +The library is a set of C++20/26 modules (`ra3.assets` plus the `:error`, +`:bytes`, `:refpack`, `:big`, `:binary`, `:csf` and `:map` partitions) that +imports the standard library (`import std;`). That needs **CMake 4.0+ with the +Ninja generator** and a compiler whose standard library ships a `std` module - +**GCC 16** in practice. The tests need nothing but the library itself. + +> Building with the distro GCC 16: pass `-DCMAKE_CXX_COMPILER=g++-16`. + +## Use + +```cpp +import std; +import ra3.assets; + +using namespace ra3::assets; + +// --- a map, for a map editor ------------------------------------------------- +auto document = map_document::open("map_mp_2_black1b.map"); + +if (const auto height = document.height_map()) { + std::println("terrain {}x{}", height->width, height->height); + auto grid = *height; // copy, then edit + grid.set(10, 10, grid.at(10, 10) + 1); // raise a cell + document.set_height_map(std::move(grid)); // re-encode the chunk +} + +for (const auto &start: document.player_starts()) + std::println("Player {} at ({}, {})", start.index, start.position.x, start.position.y); + +for (const auto &object: document.objects()) + if (object.type_name == "*Waypoints/Waypoint") + std::println("{} -> {}", object.property("waypointName")->as_ascii(), object.position.y); + +// Lossless: unknown chunks and the name table survive the round-trip. +write_file("edited.map", document.serialize(/* compress = */ true)); + +// --- a BIG4 archive ---------------------------------------------------------- +const auto archive = big_archive::open("Data/GlobalStream.big"); +std::println("{} entries", archive.size()); +for (const auto *entry: archive.find("audio")) + std::println("{} ({} bytes)", entry->name, entry->size); + +// --- a compiled BinaryAsset stream ------------------------------------------ +const auto stream = binary_stream_from_big(archive, binary_stream::global, /* need_data = */ false); +for (const auto &[type, count]: stream.type_counts()) + std::println("{:6} {}", count, type); +``` + +The API deliberately avoids integer IDs for anything selectable: chunk kinds are +`chunk_kind` (e.g. `chunk_kind::height_map_data`), streams are `binary_stream` +(`binary_stream::global`), property types are `property_type`, and an +`asset_property` carries a `std::variant`. + +## Command-line tool + +`ra3assets-cli` is built alongside the library: + +``` +ra3assets-cli big list [match] +ra3assets-cli big extract [match] +ra3assets-cli binary list [type] +ra3assets-cli binary types [static|global|locale|static_l|static_m] +ra3assets-cli binary cat +ra3assets-cli map info +ra3assets-cli map starts +ra3assets-cli map repack [--compress] +ra3assets-cli csf [label] +ra3assets-cli hash ... +``` + +## Notes and limits + +- A `BIG4` archive opened from disk keeps only its index in memory and reads + payloads on demand, so enumerating the ~700 MB retail `StaticStream.big` costs + a few megabytes; pass `need_data = false` to `binary_stream_from_big` when you + only need the manifest. A parsed `.map` document is held in full - that is what + its typed accessors edit. +- `set_height_map` keeps the on-disk `HeightMapData` version (RA3 uses 6, i.e. + 16-bit elevations). Older 8-bit maps serialise back as 8-bit. +- RefPack compression is a plain greedy LZ77 matcher. It is lossless against the + bundled decoder and produces streams the game's decoder accepts, but it is + slightly less dense than the retail compressor (≈3% on a typical map). +- `BlendTileData` (terrain texture blending) is preserved but not yet + type-modelled; a map editor should treat the chunk as opaque for now. The + planned follow-up covers `BlendTileData` and `SidesList`. +- Only `BIG4` is supported (RA3's format); older `BIGF` archives are not. + +The on-disk layouts were reverse engineered from the shipped files and +cross-checked against [`ra3tools`](../ra3tools/), OpenRA3's `ra3.fs`/`ra3.map` +modules and the OpenSAGE re-implementation (`reference/OpenSAGE`). diff --git a/third_party/libra3assets/src/assets.cppm b/third_party/libra3assets/src/assets.cppm new file mode 100644 index 0000000..1dd0f6d --- /dev/null +++ b/third_party/libra3assets/src/assets.cppm @@ -0,0 +1,65 @@ +/** + * libra3assets - read (and write) Red Alert 3's built-in resource files. + * + * The primary module interface re-exports every partition: + * + * - `:error` - the `asset_error` exception family + * - `:bytes` - bounds-checked little-/big-endian readers and writers + * - `:refpack` - EA's RefPack codec (decompress and compress) + * - `:big` - `BIG4` archives (`Data\*.big`), read and write + * - `:binary` - compiled `BinaryAsset` streams (`*.bin` + `.manifest`) + * - `:csf` - SAGE `.csf` string tables + * - `:map` - SAGE `.map` containers (`CkMp`), read and write + * + * Nothing here reads game data on its own: callers point the library at their + * own installation (`RA3_GAME_DIR` / `C:\Red Alert 3`). + */ +export module ra3.assets; + +export import :error; +export import :bytes; +export import :refpack; +export import :big; +export import :binary; +export import :csf; +export import :map; + +import std; + +export namespace ra3::assets { + + /** + * Locate a Red Alert 3 installation. + * + * Resolution order: the explicit argument, then `$RA3_GAME_DIR`, then the + * default `C:\Red Alert 3`. A directory qualifies only if it has a `Data` + * subdirectory. + */ + [[nodiscard]] inline auto find_game_dir(const std::optional &explicit_dir = std::nullopt) -> std::optional { + const auto qualifies = [](const std::filesystem::path &candidate) { + std::error_code ec; + return !candidate.empty() && std::filesystem::is_directory(candidate / "Data", ec); + }; + + if (explicit_dir && qualifies(*explicit_dir)) return explicit_dir; + if (const auto *env = std::getenv("RA3_GAME_DIR"); env != nullptr && *env != '\0') { + const std::filesystem::path candidate{env}; + if (qualifies(candidate)) return candidate; + } + const std::filesystem::path default_dir{"C:/Red Alert 3"}; + if (qualifies(default_dir)) return default_dir; + return std::nullopt; + } + + /** Every `Data\*.big` archive in an installation, sorted by name. */ + [[nodiscard]] inline auto list_archives(const std::filesystem::path &game_dir) -> std::vector { + std::vector archives; + std::error_code ec; + for (const auto &entry: std::filesystem::directory_iterator(game_dir / "Data", ec)) { + if (entry.is_regular_file() && entry.path().extension() == ".big") archives.push_back(entry.path()); + } + std::ranges::sort(archives); + return archives; + } + +} // namespace ra3::assets diff --git a/third_party/libra3assets/src/big.cppm b/third_party/libra3assets/src/big.cppm new file mode 100644 index 0000000..2c28c63 --- /dev/null +++ b/third_party/libra3assets/src/big.cppm @@ -0,0 +1,274 @@ +/** + * `BIG4` archives - Red Alert 3's on-disk asset container. + * + * Retail RA3 ships its data as `Data\*.big`: a 16-byte header, a variable- + * length entry index and the 64-byte-aligned payloads. The count and the entry + * offsets/sizes are **big-endian** (the header size is little-endian, matching + * the retail files); each payload may itself be RefPack-compressed, which is + * detected from its own magic rather than flagged in the index. + * + * Both directions are implemented so a map editor can repack an archive. + */ +export module ra3.assets:big; + +import std; + +import :bytes; +import :error; +import :refpack; + +export namespace ra3::assets { + + /** One file inside a `BIG4` archive. */ + struct big_entry { + std::string name; + std::uint32_t offset = 0; + std::uint32_t size = 0; + }; + + namespace detail { + [[nodiscard]] inline auto align_up(std::uint64_t value, std::uint64_t alignment) -> std::uint64_t { + return (value + alignment - 1U) / alignment * alignment; + } + + [[nodiscard]] inline constexpr auto bswap32(std::uint32_t value) -> std::uint32_t { + return ((value & 0x000000FFU) << 24U) | ((value & 0x0000FF00U) << 8U) | ((value & 0x00FF0000U) >> 8U) | ((value & 0xFF000000U) >> 24U); + } + } + + /** + * A parsed `BIG4` archive. + * + * Opening from disk parses only the index; payloads are read (and + * RefPack-decompressed) on demand, so a multi-hundred-megabyte archive costs + * a few megabytes to enumerate. An archive built from an in-memory image + * (`from_bytes`) slices its payloads out of that buffer instead. + */ + class big_archive { + public: + /** Read and parse an archive from disk, keeping only the index in memory. */ + [[nodiscard]] static auto open(const std::filesystem::path &path) -> big_archive { + big_archive archive; + archive.path_ = path; + + std::error_code ec; + archive.file_size_ = static_cast(std::filesystem::file_size(path, ec)); + if (ec) throw big_error("cannot stat archive: " + path.string()); + + std::ifstream in(path, std::ios::binary); + if (!in) throw big_error("cannot open archive: " + path.string()); + + std::array header{}; + in.read(reinterpret_cast(header.data()), static_cast(header.size())); + if (!in || std::string_view{reinterpret_cast(header.data()), 4} != "BIG4") throw big_error("not a BIG4 archive: " + path.string()); + + // The index is a dense variable-length table from offset 16; read a + // window and grow it until every entry parses. Payloads are never + // touched, so the memory cost stays at the index size. + const auto size = archive.file_size_; + std::uint64_t window = std::min(size, 1U << 20U); + std::vector buffer; + for (;;) { + buffer.resize(static_cast(window)); + in.clear(); + in.seekg(0); + in.read(reinterpret_cast(buffer.data()), static_cast(window)); + const auto got = in.gcount(); + buffer.resize(got > 0 ? static_cast(got) : 0U); + + std::vector entries; + if (big_archive::parse_index(buffer, entries)) { + archive.entries_ = std::move(entries); + break; + } + if (window >= size) throw big_error("truncated BIG4 index: " + path.string()); + window = std::min(size, window * 2U); + } + archive.reindex(); + return archive; + } + + /** Parse an in-memory archive image; payloads are sliced from it. */ + [[nodiscard]] static auto from_bytes(std::vector bytes) -> big_archive { + big_archive archive; + archive.bytes_ = std::move(bytes); + archive.in_memory_ = true; + if (!big_archive::parse_index(archive.bytes_, archive.entries_)) throw big_error("not a BIG4 archive"); + archive.reindex(); + return archive; + } + + [[nodiscard]] auto path() const -> const std::filesystem::path & { return this->path_; } + [[nodiscard]] auto entries() const -> const std::vector & { return this->entries_; } + [[nodiscard]] auto size() const -> std::size_t { return this->entries_.size(); } + [[nodiscard]] auto contains(std::string_view name) const -> bool { return this->index_.contains(std::string{name}); } + + /** The entry with exactly this name, or `nullptr`. */ + [[nodiscard]] auto find_exact(std::string_view name) const -> const big_entry * { + const auto it = this->index_.find(std::string{name}); + return it == this->index_.end() ? nullptr : &this->entries_[it->second]; + } + + /** Every entry whose name contains `needle`, in index order. */ + [[nodiscard]] auto find(std::string_view needle) const -> std::vector { + std::vector matches; + for (const auto &entry: this->entries_) { + if (entry.name.find(needle) != std::string::npos) matches.push_back(&entry); + } + return matches; + } + + /** Read an entry's payload; RefPack-decompress it when `decompress`. */ + [[nodiscard]] auto read(std::string_view name, bool decompress = true) const -> std::vector { + const auto *entry = this->find_exact(name); + if (entry == nullptr) throw big_error("no such entry: " + std::string{name}); + return this->read(*entry, decompress); + } + + [[nodiscard]] auto read(const big_entry &entry, bool decompress = true) const -> std::vector { + const auto raw = this->stored(entry.offset, entry.size); + return decompress ? maybe_decompress(raw) : raw; + } + + /** The first `count` stored bytes of an entry (no decompression). */ + [[nodiscard]] auto read_prefix(std::string_view name, std::size_t count) const -> std::vector { + const auto *entry = this->find_exact(name); + if (entry == nullptr) throw big_error("no such entry: " + std::string{name}); + return this->stored(entry->offset, static_cast(std::min(count, entry->size))); + } + + private: + big_archive() = default; + + /** Parse a `BIG4` index from `data`; false when it is not fully present. */ + [[nodiscard]] static auto parse_index(std::span data, std::vector &out) -> bool { + if (data.size() < 16U) return false; + byte_reader reader{data}; + if (reader.read_ascii(4) != "BIG4") return false; + (void) reader.read_u32(); // total file size (little-endian); implied by the buffer + const auto count = reader.read_be_u32(); + (void) reader.read_be_u32(); // index size + // Each entry is at least `u32 offset + u32 size + NUL name` (9 bytes), + // so a count that cannot fit bounds both the reserve and the window. + if (count > (data.size() - 16U) / 9U) return false; + out.clear(); + out.reserve(count); + for (std::uint32_t i = 0; i < count; ++i) { + if (reader.remaining() < 8U) return false; + big_entry entry; + entry.offset = reader.read_be_u32(); + entry.size = reader.read_be_u32(); + // The NUL-terminated name may run past the current window. + std::string name; + bool terminated = false; + while (reader.remaining() > 0U) { + const auto ch = reader.read_u8(); + if (ch == 0U) { + terminated = true; + break; + } + name.push_back(static_cast(ch)); + } + if (!terminated) return false; + entry.name = std::move(name); + out.push_back(std::move(entry)); + } + return true; + } + + /** Read `count` stored bytes at `offset`, from memory or from disk. */ + [[nodiscard]] auto stored(std::uint32_t offset, std::uint32_t count) const -> std::vector { + if (this->in_memory_) { + if (static_cast(offset) + count > this->bytes_.size()) throw big_error("entry extends past end of archive"); + const auto *begin = this->bytes_.data() + offset; + return std::vector{begin, begin + count}; + } + if (static_cast(offset) + count > this->file_size_) throw big_error("entry extends past end of archive"); + std::ifstream in(this->path_, std::ios::binary); + if (!in) throw big_error("cannot open archive: " + this->path_.string()); + in.seekg(static_cast(offset)); + std::vector raw(count); + if (count > 0U) in.read(reinterpret_cast(raw.data()), static_cast(count)); + if (!in) throw big_error("short read from archive: " + this->path_.string()); + return raw; + } + + auto reindex() -> void { + this->index_.clear(); + for (std::size_t i = 0; i < this->entries_.size(); ++i) this->index_.emplace(this->entries_[i].name, i); + } + + std::filesystem::path path_; + std::vector bytes_; ///< In-memory image (from_bytes); empty when opened from disk. + bool in_memory_ = false; + std::uint64_t file_size_ = 0; + std::vector entries_; + std::unordered_map index_; + }; + + /** + * Builds a `BIG4` archive. + * + * Entry payloads are written 64-byte aligned (as the retail archives are), + * and the header's index size reproduces the retail convention + * (`last-entry end + 8`). `add(..., compress = true)` RefPack-encodes the + * payload when that is smaller, otherwise stores it verbatim. + */ + class big_writer { + public: + auto add(std::string name, std::vector payload, bool compress = false) -> void { + if (compress) { + auto packed = refpack_compress(payload); + if (packed.size() < payload.size()) payload = std::move(packed); + } + this->items_.push_back({std::move(name), std::move(payload)}); + } + + /** Serialise the archive into a fresh byte image. */ + [[nodiscard]] auto write() const -> std::vector { + std::uint64_t table_end = 16U; + for (const auto &item: this->items_) table_end += 8U + item.name.size() + 1U; + const auto index_size = table_end + 8U; + const auto data_start = detail::align_up(index_size, 64U); + + std::vector offsets; + offsets.reserve(this->items_.size()); + std::uint64_t cursor = data_start; + for (const auto &item: this->items_) { + offsets.push_back(cursor); + cursor = detail::align_up(cursor + item.payload.size(), 64U); + } + + byte_writer writer; + writer.write_ascii("BIG4"); + writer.write_u32(static_cast(cursor)); // total file size (little-endian) + writer.write_u32(detail::bswap32(static_cast(this->items_.size()))); + writer.write_u32(detail::bswap32(static_cast(index_size))); + for (std::size_t i = 0; i < this->items_.size(); ++i) { + writer.write_u32(detail::bswap32(static_cast(offsets[i]))); + writer.write_u32(detail::bswap32(static_cast(this->items_[i].payload.size()))); + writer.write_ascii(this->items_[i].name); + writer.write_u8(0); + } + while (writer.size() < data_start) writer.write_u8(0); + for (std::size_t i = 0; i < this->items_.size(); ++i) { + writer.write_bytes(this->items_[i].payload); + if (i + 1U < this->items_.size()) { + while (writer.size() < offsets[i + 1U]) writer.write_u8(0); + } + } + return writer.take(); + } + + private: + struct pending_item { + std::string name; + std::vector payload; + }; + + std::vector items_; + }; + +} // namespace ra3::assets + + diff --git a/third_party/libra3assets/src/binary.cppm b/third_party/libra3assets/src/binary.cppm new file mode 100644 index 0000000..be62e77 --- /dev/null +++ b/third_party/libra3assets/src/binary.cppm @@ -0,0 +1,396 @@ +/** + * Compiled `BinaryAsset` streams (`Data\*.bin` + `.manifest`). + * + * Retail RA3 does not ship its gameplay/art assets as source files: the + * BinaryAssetBuilder compiles every XML/`.w3x` into a `.bin` *instance stream* + * plus a `.manifest` index (`data\static.bin`, `data\global.bin`, ...). This + * partition parses that index, exposes each asset's instance slice, its + * relocation/import sidecars and its custom-data (`cdata`) blob, and implements + * the Bob-Jenkins-style hash the builder names assets with. + * + * Asset names are `"Type:Instance"` (e.g. `W3DMesh:ABAIRFIELD`); `Type` is + * case-sensitive and `Instance` is lowercased before hashing. + */ +export module ra3.assets:binary; + +import std; + +import :big; +import :bytes; +import :error; +import :refpack; + +export namespace ra3::assets { + + /** The compiled instance streams a retail install ships. */ + enum class binary_stream { + static_data, ///< `data\static.bin` - the main art/gameplay stream + global, ///< `data\global.bin` - sounds, AI, UI + locale, ///< `data\locale.bin` - localized on-demand textures + static_low, ///< `data\static_l.bin` - low-detail model LODs + static_medium, ///< `data\static_m.bin` - medium-detail model LODs + }; + + /** The `data\.manifest` stem for a stream. */ + [[nodiscard]] inline auto to_string(binary_stream stream) -> std::string_view { + switch (stream) { + case binary_stream::static_data: return "static"; + case binary_stream::global: return "global"; + case binary_stream::locale: return "locale"; + case binary_stream::static_low: return "static_l"; + case binary_stream::static_medium: return "static_m"; + } + return "static"; + } + + /** Resolve a stream name (e.g. `"global"`), or `std::nullopt`. */ + [[nodiscard]] inline auto binary_stream_from_name(std::string_view name) -> std::optional { + for (const auto candidate: {binary_stream::static_data, binary_stream::global, binary_stream::locale, binary_stream::static_low, binary_stream::static_medium}) { + if (to_string(candidate) == name) return candidate; + } + return std::nullopt; + } + + /** Streams in the order `binary_stream_from_big` prefers when none is given. */ + inline constexpr std::array binary_stream_order{binary_stream::static_data, binary_stream::global, binary_stream::locale, binary_stream::static_low, + binary_stream::static_medium}; + + /** Types compiled from a `.w3x` model source. */ + inline constexpr std::array w3d_types{"W3DMesh", "W3DHierarchy", "W3DAnimation", "W3DContainer", "W3DCollisionBox"}; + + /** + * The hash BinaryAssetBuilder names assets with. + * + * A Bob-Jenkins `lookup3`-style mix. `seed` is the running value; the public + * overloads seed it with the length (`hash_string`). + */ + [[nodiscard]] inline auto fast_hash(std::span data, std::uint32_t seed = 0) -> std::uint32_t { + const auto length = data.size(); + if (length == 0U) return 0x1337C0DEU; + const auto at16 = [&](std::size_t pos) -> std::uint32_t { + return static_cast(std::to_integer(data[pos])) | + (static_cast(std::to_integer(data[pos + 1U])) << 8U); + }; + std::uint32_t h = seed; + std::size_t pos = 0; + const auto extra = length & 3U; + for (std::size_t i = 0; i < (length >> 2U); ++i) { + h += at16(pos); + h ^= (at16(pos + 2U) ^ (h << 5U)) << 11U; + h += h >> 11U; + pos += 4U; + } + if (extra == 1U) { + h += std::to_integer(data[pos]); + h ^= h << 10U; + h += h >> 1U; + } else if (extra == 2U) { + h += at16(pos); + h ^= h << 11U; + h += h >> 17U; + } else if (extra == 3U) { + h += at16(pos); + h ^= h << 16U; + h ^= static_cast(std::to_integer(data[pos + 2U])) << 18U; + h += h >> 11U; + } + h ^= h << 3U; + h += h >> 5U; + h ^= h << 2U; + h += h >> 15U; + h ^= h << 10U; + return h; + } + + /** + * Hash an asset type (case-sensitive, the `TypeId`) or an instance name + * (`case_sensitive = false`, the `InstanceId`). + */ + [[nodiscard]] inline auto hash_string(std::string_view text, bool case_sensitive = true) -> std::uint32_t { + std::string buffer{text}; + if (!case_sensitive) { + std::ranges::transform(buffer, buffer.begin(), [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + } + const auto *bytes = reinterpret_cast(buffer.data()); + return fast_hash(std::span{bytes, buffer.size()}, static_cast(buffer.size())); + } + + /** One asset in a compiled stream. */ + struct binary_asset { + std::uint32_t index = 0; + std::uint32_t type_id = 0; + std::uint32_t instance_id = 0; + std::uint32_t type_hash = 0; + std::uint32_t instance_hash = 0; + std::vector> references; ///< (typeId, instanceId) pairs + std::string name; ///< "Type:Instance" + std::string source; ///< the `.w3x`/XML it was compiled from + std::uint32_t instance_size = 0; + std::uint32_t relocation_size = 0; + std::uint32_t imports_size = 0; + bool tokenized = false; + std::uint32_t instance_offset = 4; ///< into the decompressed `.bin` (after its 4-byte checksum) + std::uint32_t relocation_offset = 4; ///< into the `.relo` + std::uint32_t imports_offset = 4; ///< into the `.imp` + + [[nodiscard]] auto type_name() const -> std::string_view { + const auto colon = this->name.find(':'); + return std::string_view{this->name}.substr(0, colon); + } + + [[nodiscard]] auto instance_name() const -> std::string_view { + const auto colon = this->name.find(':'); + return colon == std::string::npos ? std::string_view{this->name} : std::string_view{this->name}.substr(colon + 1U); + } + + /** + * The `cdata` blob name for a custom-data asset (`AudioFile`, + * `OnDemandTexture`, ...): + * `data\\cdata\....cdata`. + */ + [[nodiscard]] auto cdata_name(std::string_view stream) const -> std::string { + return std::format("data\\{}\\cdata\\{:08x}.{:08x}.{:08x}.{:08x}.cdata", stream, this->type_id, this->type_hash, this->instance_id, + this->instance_hash); + } + }; + + /** A callback that resolves a `cdata` blob name to its decompressed bytes. */ + using cdata_source = std::function>(const std::string &name)>; + + /** + * A parsed `.manifest` (+ optional `.bin`/`.relo`/`.imp`). + * + * Only the manifest is mandatory; `need_data = false` when constructing + * skips the (up to hundreds of megabytes) instance stream, which is all + * listing the assets requires. + */ + class binary_container { + public: + /** Parse a manifest, its instance stream and its optional fixup streams. */ + [[nodiscard]] static auto from_bytes(std::vector manifest, std::vector data = {}, std::vector relocation = {}, + std::vector imports = {}, std::string stream = "static") -> binary_container { + binary_container container; + container.stream_ = std::move(stream); + container.manifest_ = std::move(manifest); + container.data_ = std::move(data); + container.relocation_ = std::move(relocation); + container.imports_ = std::move(imports); + container.parse(); + return container; + } + + [[nodiscard]] auto stream() const -> const std::string & { return this->stream_; } + [[nodiscard]] auto version() const -> std::uint16_t { return this->version_; } + [[nodiscard]] auto is_linked() const -> bool { return this->is_linked_; } + [[nodiscard]] auto assets() const -> const std::vector & { return this->assets_; } + [[nodiscard]] auto size() const -> std::size_t { return this->assets_.size(); } + + /** Count of assets per `Type` (the part before `:`). */ + [[nodiscard]] auto type_counts() const -> std::map { + std::map counts; + for (const auto &asset: this->assets_) ++counts[std::string{asset.type_name()}]; + return counts; + } + + /** Map `(typeId << 32 | instanceId)` to the owning asset. */ + [[nodiscard]] auto asset_index() const -> const std::unordered_map & { return this->asset_index_; } + + /** Every asset, in manifest order, whose `Type` matches (case-insensitive). */ + [[nodiscard]] auto of_type(std::string_view type) const -> std::vector { + std::vector matches; + for (const auto &asset: this->assets_) { + if (std::ranges::equal(asset.type_name(), type, [](char a, char b) { return std::tolower(static_cast(a)) == std::tolower(static_cast(b)); })) + matches.push_back(&asset); + } + return matches; + } + + /** Resolve `"Type:Instance"` (case-sensitive, then insensitive) or `"#index"`. */ + [[nodiscard]] auto find(std::string_view selector) const -> const binary_asset & { + if (selector.starts_with('#')) { + const auto index = static_cast(std::stoul(std::string{selector.substr(1)})); + if (index >= this->assets_.size()) throw binary_error("asset index out of range: " + std::string{selector}); + return this->assets_[index]; + } + for (const auto &asset: this->assets_) { + if (asset.name == selector) return asset; + } + for (const auto &asset: this->assets_) { + if (std::ranges::equal(asset.name, selector, [](char a, char b) { return std::tolower(static_cast(a)) == std::tolower(static_cast(b)); })) + return asset; + } + throw binary_error("no such asset: " + std::string{selector}); + } + + auto set_cdata_source(cdata_source source) -> void { this->cdata_ = std::move(source); } + + [[nodiscard]] auto read_instance(const binary_asset &asset) const -> std::span { + if (asset.instance_size == 0U) return {}; + if (static_cast(asset.instance_offset) + asset.instance_size > this->data_.size()) throw binary_error("instance extends past stream: " + asset.name); + return std::span{this->data_}.subspan(asset.instance_offset, asset.instance_size); + } + + [[nodiscard]] auto read_relocation(const binary_asset &asset) const -> std::span { + if (asset.relocation_size == 0U) return {}; + if (static_cast(asset.relocation_offset) + asset.relocation_size > this->relocation_.size()) throw binary_error("relocation extends past stream: " + asset.name); + return std::span{this->relocation_}.subspan(asset.relocation_offset, asset.relocation_size); + } + + [[nodiscard]] auto read_imports(const binary_asset &asset) const -> std::span { + if (asset.imports_size == 0U) return {}; + if (static_cast(asset.imports_offset) + asset.imports_size > this->imports_.size()) throw binary_error("imports extend past stream: " + asset.name); + return std::span{this->imports_}.subspan(asset.imports_offset, asset.imports_size); + } + + /** The asset's `cdata` blob, or `std::nullopt` when it has none. */ + [[nodiscard]] auto read_cdata(const binary_asset &asset) const -> std::optional> { + if (!this->cdata_) return std::nullopt; + return this->cdata_(asset.cdata_name(this->stream_)); + } + + /** The asset's real payload: its `cdata` blob when present, else its instance. */ + [[nodiscard]] auto read_payload(const binary_asset &asset) const -> std::vector { + if (auto cdata = this->read_cdata(asset)) return *cdata; + const auto instance = this->read_instance(asset); + return {instance.begin(), instance.end()}; + } + + private: + binary_container() = default; + + auto parse() -> void { + byte_reader reader{this->manifest_}; + const auto is_big_endian = reader.read_u8(); + this->is_linked_ = reader.read_bool(); + this->version_ = reader.read_u16(); + if (is_big_endian != 0U) throw binary_error("big-endian manifests are not supported"); + if (this->version_ != 5U && this->version_ != 6U) throw binary_error("unsupported manifest version " + std::to_string(this->version_)); + (void) reader.read_u32(); // stream checksum + (void) reader.read_u32(); // all-types hash + const auto count = reader.read_u32(); + (void) reader.read_u32(); // total instance data size + (void) reader.read_u32(); // max instance chunk size + (void) reader.read_u32(); // max relocation chunk size + (void) reader.read_u32(); // max imports chunk size + const auto reference_buffer_size = reader.read_u32(); + const auto reference_name_buffer_size = reader.read_u32(); + const auto name_buffer_size = reader.read_u32(); + const auto source_buffer_size = reader.read_u32(); + + constexpr std::size_t header_size = 48U; + constexpr std::size_t entry_size = 48U; + if (this->manifest_.size() < header_size + static_cast(count) * entry_size) throw binary_error("manifest entry table is truncated"); + + const auto entries_off = header_size; + const auto references_off = entries_off + static_cast(count) * entry_size; + const auto reference_names_off = references_off + reference_buffer_size; + const auto names_off = reference_names_off + reference_name_buffer_size; + const auto sources_off = names_off + name_buffer_size; + if (sources_off + source_buffer_size > this->manifest_.size()) throw binary_error("manifest string buffers are truncated"); + + const auto cstr = [&](std::size_t base, std::int32_t offset) -> std::string { + if (offset < 0) return {}; + const auto start = base + static_cast(offset); + if (start >= this->manifest_.size()) return {}; + std::size_t end = start; + while (end < this->manifest_.size() && std::to_integer(this->manifest_[end]) != 0U) ++end; + return std::string{reinterpret_cast(this->manifest_.data() + start), end - start}; + }; + + this->assets_.reserve(count); + std::uint32_t instance_offset = 4; + std::uint32_t relocation_offset = 4; + std::uint32_t imports_offset = 4; + for (std::uint32_t i = 0; i < count; ++i) { + byte_reader entry{std::span{this->manifest_}.subspan(entries_off + static_cast(i) * entry_size, entry_size)}; + binary_asset asset; + asset.index = i; + asset.type_id = entry.read_u32(); + asset.instance_id = entry.read_u32(); + asset.type_hash = entry.read_u32(); + asset.instance_hash = entry.read_u32(); + const auto reference_offset = entry.read_i32(); + const auto reference_count = entry.read_i32(); + const auto name_offset = entry.read_i32(); + const auto source_offset = entry.read_i32(); + asset.instance_size = entry.read_u32(); + asset.relocation_size = entry.read_u32(); + asset.imports_size = entry.read_u32(); + asset.tokenized = entry.read_u32() != 0U; + asset.name = cstr(names_off, name_offset); + asset.source = cstr(sources_off, source_offset); + asset.instance_offset = instance_offset; + asset.relocation_offset = relocation_offset; + asset.imports_offset = imports_offset; + instance_offset += asset.instance_size; + relocation_offset += asset.relocation_size; + imports_offset += asset.imports_size; + + for (std::int32_t j = 0; j < reference_count; ++j) { + byte_reader ref{std::span{this->manifest_}.subspan(references_off + static_cast(reference_offset) + + static_cast(j) * 8U, + 8U)}; + asset.references.emplace_back(ref.read_u32(), ref.read_u32()); + } + this->asset_index_.emplace((static_cast(asset.type_id) << 32U) | asset.instance_id, this->assets_.size()); + this->assets_.push_back(std::move(asset)); + } + } + + std::string stream_ = "static"; + std::vector manifest_; + std::vector data_; + std::vector relocation_; + std::vector imports_; + std::uint16_t version_ = 0; + bool is_linked_ = false; + std::vector assets_; + std::unordered_map asset_index_; + cdata_source cdata_; + }; + + namespace detail { + [[nodiscard]] inline auto pick_manifest(std::string_view stream) -> std::string { return std::format("data\\{}.manifest", stream); } + } + + /** + * Load one BinaryAsset stream out of a `BIG4` archive. + * + * `stream` selects the `.manifest` (default: the first of + * `binary_stream_order` found). When `need_data` is false only the manifest + * is read, which is enough to enumerate assets without decompressing the + * multi-hundred-megabyte instance stream. + * + * The returned container keeps a `cdata` reader that borrows `archive`; keep + * the archive alive for as long as the container is used. + */ + [[nodiscard]] inline auto binary_stream_from_big(const big_archive &archive, std::optional stream = std::nullopt, bool need_data = true) -> binary_container { + std::optional chosen = stream; + if (!chosen) { + for (const auto candidate: binary_stream_order) { + if (archive.contains(detail::pick_manifest(to_string(candidate)))) { + chosen = candidate; + break; + } + } + } + if (!chosen) throw binary_error("archive has no known .manifest entry"); + + const auto manifest_name = detail::pick_manifest(to_string(*chosen)); + const auto base = manifest_name.substr(0, manifest_name.size() - std::string_view{".manifest"}.size()); + const auto manifest = maybe_decompress(archive.read(manifest_name, false)); + std::vector data; + if (need_data) data = maybe_decompress(archive.read(base + ".bin", false)); + const auto relocation = maybe_decompress(archive.read(base + ".relo", false)); + const auto imports = maybe_decompress(archive.read(base + ".imp", false)); + + auto container = binary_container::from_bytes(std::move(manifest), std::move(data), std::move(relocation), std::move(imports), std::string{to_string(*chosen)}); + container.set_cdata_source([&archive](const std::string &name) -> std::optional> { + const auto *entry = archive.find_exact(name); + if (entry == nullptr) return std::nullopt; + return maybe_decompress(archive.read(*entry, false)); + }); + return container; + } + +} // namespace ra3::assets diff --git a/third_party/libra3assets/src/bytes.cppm b/third_party/libra3assets/src/bytes.cppm new file mode 100644 index 0000000..fd47a96 --- /dev/null +++ b/third_party/libra3assets/src/bytes.cppm @@ -0,0 +1,220 @@ +/** + * Bounds-checked little-/big-endian byte readers and writers. + * + * Red Alert 3's asset formats are little-endian, with one notable exception: + * the `BIG4` index stores its counts, offsets and sizes big-endian. Both byte + * orders are provided here. Every read is bounds-checked and throws + * `format_error` rather than reading past the end of the buffer. + */ +export module ra3.assets:bytes; + +import std; + +import :error; + +export namespace ra3::assets { + + /** A cursor over a read-only byte range with checked accessors. */ + class byte_reader { + public: + explicit byte_reader(std::span data) + : data_(data) { + } + + [[nodiscard]] auto size() const -> std::size_t { return this->data_.size(); } + [[nodiscard]] auto position() const -> std::size_t { return this->pos_; } + [[nodiscard]] auto remaining() const -> std::size_t { return this->data_.size() - this->pos_; } + [[nodiscard]] auto empty() const -> bool { return this->pos_ >= this->data_.size(); } + + auto seek(std::size_t at) -> void { + if (at > this->data_.size()) throw format_error("seek past end of buffer"); + this->pos_ = at; + } + + auto skip(std::size_t count) -> void { + this->require(count); + this->pos_ += count; + } + + [[nodiscard]] auto read_u8() -> std::uint8_t { + this->require(1); + return std::to_integer(this->data_[this->pos_++]); + } + + [[nodiscard]] auto read_bool() -> bool { return this->read_u8() != 0U; } + + [[nodiscard]] auto read_u16() -> std::uint16_t { + return static_cast(this->read_le(2)); + } + + [[nodiscard]] auto read_u24() -> std::uint32_t { return this->read_le(3); } + + [[nodiscard]] auto read_u32() -> std::uint32_t { return this->read_le(4); } + + [[nodiscard]] auto read_u64() -> std::uint64_t { return this->read_le(8); } + + [[nodiscard]] auto read_i16() -> std::int16_t { return static_cast(this->read_u16()); } + + [[nodiscard]] auto read_i32() -> std::int32_t { return static_cast(this->read_u32()); } + + [[nodiscard]] auto read_f32() -> float { return std::bit_cast(this->read_u32()); } + + [[nodiscard]] auto read_be_u16() -> std::uint16_t { return static_cast(this->read_be(2)); } + [[nodiscard]] auto read_be_u32() -> std::uint32_t { return this->read_be(4); } + + /** A view of the next `count` bytes; the cursor advances past them. */ + [[nodiscard]] auto read_bytes(std::size_t count) -> std::span { + this->require(count); + const auto view = this->data_.subspan(this->pos_, count); + this->pos_ += count; + return view; + } + + /** `count` bytes decoded as Latin-1 (one byte per character). */ + [[nodiscard]] auto read_ascii(std::size_t count) -> std::string { + const auto view = this->read_bytes(count); + return std::string{reinterpret_cast(view.data()), view.size()}; + } + + /** A `u16`-length-prefixed ASCII string. */ + [[nodiscard]] auto read_u16_prefixed_ascii() -> std::string { return this->read_ascii(this->read_u16()); } + + /** A `u16`-length-prefixed string, decoded as UTF-8 (ASCII-compatible). */ + [[nodiscard]] auto read_u16_prefixed_ascii_as_utf8() -> std::string { return this->read_u16_prefixed_ascii(); } + + /** A `u16`-length-prefixed UTF-16LE string (length counted in code units). */ + [[nodiscard]] auto read_u16_prefixed_utf16() -> std::u16string { + const auto count = this->read_u16(); + this->require(static_cast(count) * 2U); + std::u16string text(count, u'\0'); + for (std::uint16_t i = 0; i < count; ++i) text[i] = static_cast(this->read_u16()); + return text; + } + + /** A NUL-terminated ASCII string; the cursor stops just past the NUL. */ + [[nodiscard]] auto read_cstring() -> std::string { + std::string text; + while (this->pos_ < this->data_.size()) { + const auto ch = static_cast(this->read_u8()); + if (ch == '\0') break; + text.push_back(ch); + } + return text; + } + + private: + auto require(std::size_t count) const -> void { + if (this->pos_ + count > this->data_.size()) throw format_error("unexpected end of buffer"); + } + + [[nodiscard]] auto read_le(std::size_t width) -> std::uint64_t { + this->require(width); + std::uint64_t value = 0; + for (std::size_t i = 0; i < width; ++i) value |= static_cast(std::to_integer(this->data_[this->pos_ + i])) << (8U * i); + this->pos_ += width; + return value; + } + + [[nodiscard]] auto read_be(std::size_t width) -> std::uint64_t { + this->require(width); + std::uint64_t value = 0; + for (std::size_t i = 0; i < width; ++i) value = (value << 8U) | static_cast(std::to_integer(this->data_[this->pos_ + i])); + this->pos_ += width; + return value; + } + + std::span data_; + std::size_t pos_ = 0; + }; + + /** An append-only little-endian byte buffer. */ + class byte_writer { + public: + [[nodiscard]] auto size() const -> std::size_t { return this->buffer_.size(); } + [[nodiscard]] auto data() const -> std::span { return this->buffer_; } + [[nodiscard]] auto take() -> std::vector { return std::move(this->buffer_); } + + auto write_u8(std::uint8_t value) -> void { this->buffer_.push_back(static_cast(value)); } + + auto write_bool(bool value) -> void { this->write_u8(value ? 1U : 0U); } + + auto write_u16(std::uint16_t value) -> void { this->write_le(value, 2); } + auto write_u24(std::uint32_t value) -> void { this->write_le(value, 3); } + auto write_u32(std::uint32_t value) -> void { this->write_le(value, 4); } + auto write_u64(std::uint64_t value) -> void { this->write_le(value, 8); } + auto write_i16(std::int16_t value) -> void { this->write_u16(static_cast(value)); } + auto write_i32(std::int32_t value) -> void { this->write_u32(static_cast(value)); } + auto write_f32(float value) -> void { this->write_u32(std::bit_cast(value)); } + + auto write_bytes(std::span bytes) -> void { this->buffer_.insert(this->buffer_.end(), bytes.begin(), bytes.end()); } + + auto write_ascii(std::string_view text) -> void { + for (const auto ch: text) this->buffer_.push_back(static_cast(static_cast(ch))); + } + + auto write_u16_prefixed_ascii(std::string_view text) -> void { + this->write_u16(static_cast(text.size())); + this->write_ascii(text); + } + + auto write_u16_prefixed_utf16(std::u16string_view text) -> void { + this->write_u16(static_cast(text.size())); + for (const auto unit: text) { + this->write_u16(static_cast(unit)); + } + } + + /** Overwrite a previously written `u32` (used to backpatch sizes). */ + auto patch_u32(std::size_t offset, std::uint32_t value) -> void { + if (offset + 4U > this->buffer_.size()) throw format_error("patch offset past end of buffer"); + for (std::size_t i = 0; i < 4U; ++i) this->buffer_[offset + i] = static_cast((value >> (8U * i)) & 0xFFU); + } + + private: + auto write_le(std::uint64_t value, std::size_t width) -> void { + for (std::size_t i = 0; i < width; ++i) this->buffer_.push_back(static_cast((value >> (8U * i)) & 0xFFU)); + } + + std::vector buffer_; + }; + + /** Convenience: encode a UTF-8 string (ASCII subset) as UTF-16LE code units. */ + [[nodiscard]] inline auto to_utf16(std::string_view text) -> std::u16string { + std::u16string out; + out.reserve(text.size()); + for (const auto ch: text) out.push_back(static_cast(static_cast(ch))); + return out; + } + + /** Convenience: narrow a UTF-16 string that is known to be within Latin-1. */ + [[nodiscard]] inline auto to_ascii(std::u16string_view text) -> std::string { + std::string out; + out.reserve(text.size()); + for (const auto unit: text) out.push_back(static_cast(unit & 0xFFU)); + return out; + } + + /** Read a whole file as bytes. */ + [[nodiscard]] inline auto read_file(const std::filesystem::path &path) -> std::vector { + std::ifstream in(path, std::ios::binary); + if (!in) throw format_error("cannot open file: " + path.string()); + in.seekg(0, std::ios::end); + const auto end = in.tellg(); + if (end < 0) throw format_error("cannot size file: " + path.string()); + std::vector bytes(static_cast(end)); + in.seekg(0, std::ios::beg); + if (!bytes.empty()) in.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; + } + + /** Write bytes to a file, creating parent directories as needed. */ + inline auto write_file(const std::filesystem::path &path, std::span bytes) -> void { + std::error_code ec; + if (!path.parent_path().empty()) std::filesystem::create_directories(path.parent_path(), ec); + std::ofstream out(path, std::ios::binary); + if (!out) throw format_error("cannot write file: " + path.string()); + out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (!out) throw format_error("cannot write file: " + path.string()); + } + +} // namespace ra3::assets diff --git a/third_party/libra3assets/src/csf.cppm b/third_party/libra3assets/src/csf.cppm new file mode 100644 index 0000000..503bc99 --- /dev/null +++ b/third_party/libra3assets/src/csf.cppm @@ -0,0 +1,180 @@ +/** + * SAGE `.csf` string tables (`data\gamestrings.csf`). + * + * The compressed unit/label localization table Red Alert 3 ships per language + * in `Lang-*.big` / `English.big`. Each value is a UTF-16 string whose every + * code unit is bit-inverted (`code ^ 0xFFFF`); labels are plain ASCII and are + * matched case-insensitively. + */ +export module ra3.assets:csf; + +import std; + +import :big; +import :bytes; +import :error; +import :refpack; + +export namespace ra3::assets { + + /** Magic of the label block (`" LBL"` little-endian). */ + inline constexpr std::uint32_t csf_label_flag = 0x4C424C20U; + + /** Magic of a value block (`" RTS"` little-endian). */ + inline constexpr std::uint32_t csf_value_flag = 0x53545220U; + + /** One label and its (usually single) string value. */ + struct csf_entry { + std::string label; + std::vector values; + + /** The concatenation of every value block, the way the game renders it. */ + [[nodiscard]] auto value() const -> std::u16string { + std::u16string joined; + for (const auto &chunk: this->values) joined += chunk; + return joined; + } + }; + + /** Decode a UTF-16 string (with surrogate pairs) to UTF-8. */ + [[nodiscard]] inline auto utf16_to_utf8(std::u16string_view text) -> std::string { + std::string out; + for (std::size_t i = 0; i < text.size(); ++i) { + auto code = static_cast(text[i]); + if (code >= 0xD800U && code <= 0xDBFFU && i + 1U < text.size()) { + const auto low = static_cast(text[i + 1U]); + if (low >= 0xDC00U && low <= 0xDFFFU) { + code = 0x10000U + ((code - 0xD800U) << 10U) + (low - 0xDC00U); + ++i; + } + } + if (code < 0x80U) { + out.push_back(static_cast(code)); + } else if (code < 0x800U) { + out.push_back(static_cast(0xC0U | (code >> 6U))); + out.push_back(static_cast(0x80U | (code & 0x3FU))); + } else if (code < 0x10000U) { + out.push_back(static_cast(0xE0U | (code >> 12U))); + out.push_back(static_cast(0x80U | ((code >> 6U) & 0x3FU))); + out.push_back(static_cast(0x80U | (code & 0x3FU))); + } else { + out.push_back(static_cast(0xF0U | (code >> 18U))); + out.push_back(static_cast(0x80U | ((code >> 12U) & 0x3FU))); + out.push_back(static_cast(0x80U | ((code >> 6U) & 0x3FU))); + out.push_back(static_cast(0x80U | (code & 0x3FU))); + } + } + return out; + } + + /** + * A parsed `.csf` table. + * + * Labels are stored verbatim; lookup upper-cases the key because the engine + * treats them case-insensitively. + */ + class csf_table { + public: + /** Parse a `.csf` image. */ + [[nodiscard]] static auto parse(std::span bytes) -> csf_table { + byte_reader reader{bytes}; + const auto magic = reader.read_ascii(4); + if (magic != "CSF " && magic != " FSC") throw csf_error("not a CSF file"); + csf_table table; + table.version_ = reader.read_u32(); + const auto num_labels = reader.read_u32(); + (void) reader.read_u32(); // number of value blocks + (void) reader.read_u32(); // reserved + (void) reader.read_u32(); // reserved + table.entries_.reserve(num_labels); + for (std::uint32_t i = 0; i < num_labels; ++i) { + (void) reader.read_u32(); // label flag (" LBL") + const auto value_count = reader.read_u32(); + const auto label_length = reader.read_u32(); + csf_entry entry; + entry.label = reader.read_ascii(label_length); + entry.values.reserve(value_count); + for (std::uint32_t j = 0; j < value_count; ++j) { + (void) reader.read_u32(); // value flag (" RTS") + const auto char_count = reader.read_u32(); + std::u16string value(char_count, u'\0'); + for (std::uint32_t k = 0; k < char_count; ++k) value[k] = static_cast(reader.read_u16() ^ 0xFFFFU); + entry.values.push_back(std::move(value)); + } + table.entries_.push_back(std::move(entry)); + } + table.reindex(); + return table; + } + + /** Read and parse a `.csf` file from disk. */ + [[nodiscard]] static auto open(const std::filesystem::path &path) -> csf_table { + return csf_table::parse(read_file(path)); + } + + /** Load `data\gamestrings.csf` out of an archive. */ + [[nodiscard]] static auto from_big(const big_archive &archive, std::string_view entry = "data\\gamestrings.csf") -> csf_table { + return csf_table::parse(maybe_decompress(archive.read(entry, false))); + } + + /** Serialise the table back to a `.csf` image. */ + [[nodiscard]] auto write() const -> std::vector { + byte_writer writer; + writer.write_ascii(" FSC"); + writer.write_u32(this->version_); + writer.write_u32(static_cast(this->entries_.size())); + std::uint32_t value_count = 0; + for (const auto &entry: this->entries_) value_count += static_cast(entry.values.size()); + writer.write_u32(value_count); + writer.write_u32(0); + writer.write_u32(0); + for (const auto &entry: this->entries_) { + writer.write_u32(csf_label_flag); + writer.write_u32(static_cast(entry.values.size())); + writer.write_u32(static_cast(entry.label.size())); + writer.write_ascii(entry.label); + for (const auto &value: entry.values) { + writer.write_u32(csf_value_flag); + writer.write_u32(static_cast(value.size())); + for (const auto unit: value) writer.write_u16(static_cast(unit) ^ 0xFFFFU); + } + } + return writer.take(); + } + + [[nodiscard]] auto version() const -> std::uint32_t { return this->version_; } + [[nodiscard]] auto entries() const -> const std::vector & { return this->entries_; } + [[nodiscard]] auto size() const -> std::size_t { return this->entries_.size(); } + + /** The entry for `label`, or `nullptr`; the match is case-insensitive. */ + [[nodiscard]] auto find(std::string_view label) const -> const csf_entry * { + auto key = std::string{label}; + std::ranges::transform(key, key.begin(), [](unsigned char ch) { return static_cast(std::toupper(ch)); }); + const auto it = this->index_.find(key); + return it == this->index_.end() ? nullptr : &this->entries_[it->second]; + } + + /** The UTF-8 value for `label`, or `fallback` when absent. */ + [[nodiscard]] auto lookup(std::string_view label, std::string fallback = {}) const -> std::string { + const auto *entry = this->find(label); + if (entry == nullptr) return fallback; + auto text = utf16_to_utf8(entry->value()); + return text.empty() ? std::move(fallback) : text; + } + + private: + auto reindex() -> void { + this->index_.clear(); + for (std::size_t i = 0; i < this->entries_.size(); ++i) { + auto key = this->entries_[i].label; + std::ranges::transform(key, key.begin(), [](unsigned char ch) { return static_cast(std::toupper(ch)); }); + this->index_.try_emplace(std::move(key), i); + } + } + + std::uint32_t version_ = 3; + std::vector entries_; + std::unordered_map index_; + }; + +} // namespace ra3::assets diff --git a/third_party/libra3assets/src/error.cppm b/third_party/libra3assets/src/error.cppm new file mode 100644 index 0000000..aa8539e --- /dev/null +++ b/third_party/libra3assets/src/error.cppm @@ -0,0 +1,56 @@ +/** + * Exception hierarchy of libra3assets. + * + * Every failure the library reports derives from `asset_error` (itself a + * `std::runtime_error`), so a caller can catch the whole family with one + * handler and still discriminate by the concrete type when it matters. + */ +export module ra3.assets:error; + +import std; + +export namespace ra3::assets { + + /** Base class of every failure libra3assets reports. */ + class asset_error: public std::runtime_error { + public: + using std::runtime_error::runtime_error; + }; + + /** A byte range is not a well-formed Red Alert 3 asset. */ + class format_error: public asset_error { + public: + using asset_error::asset_error; + }; + + /** An EA RefPack stream is malformed or its length disagrees. */ + class refpack_error: public asset_error { + public: + using asset_error::asset_error; + }; + + /** A `BIG4` archive is malformed, truncated or missing an entry. */ + class big_error: public asset_error { + public: + using asset_error::asset_error; + }; + + /** A compiled `BinaryAsset` manifest/stream is malformed. */ + class binary_error: public asset_error { + public: + using asset_error::asset_error; + }; + + /** A SAGE `.csf` string table is malformed. */ + class csf_error: public asset_error { + public: + using asset_error::asset_error; + }; + + /** A SAGE `.map` container or one of its chunks is malformed. */ + class map_error: public asset_error { + public: + using asset_error::asset_error; + }; + +} // namespace ra3::assets diff --git a/third_party/libra3assets/src/map.cppm b/third_party/libra3assets/src/map.cppm new file mode 100644 index 0000000..9c46ad6 --- /dev/null +++ b/third_party/libra3assets/src/map.cppm @@ -0,0 +1,778 @@ +/** + * SAGE `.map` containers - the core of the map editor. + * + * A `.map` is a `CkMp` chunk tree. On disk it may be wrapped twice: inside a + * `BIG4` the payload is one RefPack stream; the `.map` file it yields is either + * a bare `CkMp` tree or an `EAR\0`-wrapped RefPack stream around one. This + * partition unwraps all three shapes and models the tree as an ordered list of + * named, versioned chunks. + * + * The container round-trips losslessly: chunks this library does not model are + * preserved byte-for-byte, and the asset-name table keeps its original indices + * so unparsed chunks (which embed name indices) stay valid. On top of the + * container sit typed accessors for the chunks a map editor needs: + * `HeightMapData` (the terrain grid), `ObjectsList` (every placed prop, building + * and `*Waypoints/Waypoint`), `MPPositionList`, `WorldInfo` and `WaypointsList`. + */ +export module ra3.assets:map; + +import std; + +import :big; +import :bytes; +import :error; +import :refpack; + +export namespace ra3::assets { + + /** A world-space coordinate. */ + struct coord3d { + float x = 0.0F; + float y = 0.0F; + float z = 0.0F; + }; + + /** + * A map object's `RoadType` bit flags. + * + * Zero for most objects; set on objects the terrain road system connects + * (`Start`/`End`/`BridgeStart`/`BridgeEnd`/`Angled`/`TightCurve`/`EndCap`). + */ + enum class road_type : std::uint32_t { + none = 0, + start = 2, + end = 4, + angled = 8, + bridge_start = 16, + bridge_end = 32, + tight_curve = 64, + end_cap = 128, + unknown3 = 256, + unknown4 = 512, + }; + + [[nodiscard]] constexpr auto operator|(road_type a, road_type b) -> road_type { + return static_cast(static_cast(a) | static_cast(b)); + } + [[nodiscard]] constexpr auto operator&(road_type a, road_type b) -> road_type { + return static_cast(static_cast(a) & static_cast(b)); + } + + /** True when `value` has every bit of `flag` set. */ + [[nodiscard]] constexpr auto has_flag(road_type value, road_type flag) -> bool { + return (static_cast(value) & static_cast(flag)) != 0U; + } + + /** The `CkMp` chunk types SAGE defines, plus `unknown` for anything else. */ + enum class chunk_kind { + asset_list, + global_version, + height_map_data, + blend_tile_data, + world_info, + mp_position_list, + sides_list, + library_map_lists, + teams, + player_scripts_list, + build_lists, + objects_list, + polygon_triggers, + trigger_areas, + global_water_settings, + fog_settings, + mission_hot_spots, + mission_objectives, + standing_water_areas, + river_areas, + standing_wave_areas, + global_lighting, + post_effects_chunk, + environment_data, + named_cameras, + camera_animation_list, + castle_templates, + waypoints_list, + skybox_settings, + unknown, + }; + + /** The canonical chunk name (the `unknown` kind has no name). */ + [[nodiscard]] inline auto to_string(chunk_kind kind) -> std::string_view { + switch (kind) { + case chunk_kind::asset_list: return "AssetList"; + case chunk_kind::global_version: return "GlobalVersion"; + case chunk_kind::height_map_data: return "HeightMapData"; + case chunk_kind::blend_tile_data: return "BlendTileData"; + case chunk_kind::world_info: return "WorldInfo"; + case chunk_kind::mp_position_list: return "MPPositionList"; + case chunk_kind::sides_list: return "SidesList"; + case chunk_kind::library_map_lists: return "LibraryMapLists"; + case chunk_kind::teams: return "Teams"; + case chunk_kind::player_scripts_list: return "PlayerScriptsList"; + case chunk_kind::build_lists: return "BuildLists"; + case chunk_kind::objects_list: return "ObjectsList"; + case chunk_kind::polygon_triggers: return "PolygonTriggers"; + case chunk_kind::trigger_areas: return "TriggerAreas"; + case chunk_kind::global_water_settings: return "GlobalWaterSettings"; + case chunk_kind::fog_settings: return "FogSettings"; + case chunk_kind::mission_hot_spots: return "MissionHotSpots"; + case chunk_kind::mission_objectives: return "MissionObjectives"; + case chunk_kind::standing_water_areas: return "StandingWaterAreas"; + case chunk_kind::river_areas: return "RiverAreas"; + case chunk_kind::standing_wave_areas: return "StandingWaveAreas"; + case chunk_kind::global_lighting: return "GlobalLighting"; + case chunk_kind::post_effects_chunk: return "PostEffectsChunk"; + case chunk_kind::environment_data: return "EnvironmentData"; + case chunk_kind::named_cameras: return "NamedCameras"; + case chunk_kind::camera_animation_list: return "CameraAnimationList"; + case chunk_kind::castle_templates: return "CastleTemplates"; + case chunk_kind::waypoints_list: return "WaypointsList"; + case chunk_kind::skybox_settings: return "SkyboxSettings"; + case chunk_kind::unknown: break; + } + return {}; + } + + /** Classify a chunk name. */ + [[nodiscard]] inline auto chunk_kind_of(std::string_view name) -> chunk_kind { + for (int i = 0; i < static_cast(chunk_kind::unknown); ++i) { + const auto kind = static_cast(i); + if (to_string(kind) == name) return kind; + } + return chunk_kind::unknown; + } + + /** The map's asset-name table: `index -> name`, indices preserved on write. */ + class name_table { + public: + [[nodiscard]] auto name(std::uint32_t index) const -> std::string_view { + if (index >= this->names_.size()) throw map_error("asset name index out of range: " + std::to_string(index)); + return this->names_[index]; + } + + /** The existing index of `name`, or a freshly appended one. */ + [[nodiscard]] auto get_or_create(std::string_view name) -> std::uint32_t { + for (std::uint32_t i = 1; i < this->names_.size(); ++i) { + if (this->names_[i] == name) return i; + } + this->names_.emplace_back(name); + return static_cast(this->names_.size() - 1U); + } + + [[nodiscard]] auto entries() const -> const std::vector & { return this->names_; } + [[nodiscard]] auto size() const -> std::size_t { return this->names_.empty() ? 0U : this->names_.size() - 1U; } + + /** Resize the table (used while parsing); index 0 stays the unused slot. */ + auto resize(std::size_t count) -> void { this->names_.resize(count); } + + /** Assign the name at `index` (used while parsing). */ + auto set_name(std::uint32_t index, std::string name) -> void { this->names_.at(index) = std::move(name); } + + private: + std::vector names_{std::string{}}; // index 0 is unused + }; + + /** One `CkMp` chunk: a named, versioned, opaque payload. */ + struct map_chunk { + std::string name; + chunk_kind kind = chunk_kind::unknown; + std::uint32_t asset_index = 0; + std::uint16_t version = 0; + std::vector payload; + }; + + /** A `HeightMapData` border rectangle. */ + struct height_map_border { + std::uint32_t corner1_x = 0; + std::uint32_t corner1_y = 0; + std::uint32_t x = 0; + std::uint32_t y = 0; + }; + + /** The parsed `HeightMapData` chunk: the terrain elevation grid. */ + struct height_map_data { + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t border_width = 0; + std::vector borders; + std::vector elevations; ///< `width * height`, row-major (y outer, x inner) + std::uint16_t version = 6; + + /** Metres per elevation unit (uint16 grids scale by 0.0390625). */ + [[nodiscard]] auto vertical_scale() const -> float { return this->version >= 5U ? 0.0390625F : 0.625F; } + + [[nodiscard]] auto at(std::uint32_t x, std::uint32_t y) const -> std::uint16_t { return this->elevations.at(static_cast(y) * this->width + x); } + auto set(std::uint32_t x, std::uint32_t y, std::uint16_t value) -> void { this->elevations.at(static_cast(y) * this->width + x) = value; } + + /** Parse a `HeightMapData` chunk payload. */ + [[nodiscard]] static auto parse(std::uint16_t version, std::span payload) -> height_map_data { + byte_reader reader{payload}; + height_map_data data; + data.version = version; + data.width = reader.read_u32(); + data.height = reader.read_u32(); + data.border_width = reader.read_u32(); + const auto border_count = reader.read_u32(); + data.borders.reserve(border_count); + for (std::uint32_t i = 0; i < border_count; ++i) { + height_map_border border; + if (version >= 6U) { + border.corner1_x = reader.read_u32(); + border.corner1_y = reader.read_u32(); + } + border.x = reader.read_u32(); + border.y = reader.read_u32(); + data.borders.push_back(border); + } + const auto area = reader.read_u32(); + const auto expected = static_cast(data.width) * data.height; + if (area != expected) throw map_error("HeightMapData area does not match width * height"); + data.elevations.resize(static_cast(expected)); + for (std::uint32_t y = 0; y < data.height; ++y) { + for (std::uint32_t x = 0; x < data.width; ++x) { + data.elevations[static_cast(y) * data.width + x] = version >= 5U ? reader.read_u16() : reader.read_u8(); + } + } + return data; + } + + /** Serialise the chunk payload (without the 10-byte chunk header). */ + [[nodiscard]] auto serialize() const -> std::vector { + byte_writer writer; + writer.write_u32(this->width); + writer.write_u32(this->height); + writer.write_u32(this->border_width); + writer.write_u32(static_cast(this->borders.size())); + for (const auto &border: this->borders) { + if (this->version >= 6U) { + writer.write_u32(border.corner1_x); + writer.write_u32(border.corner1_y); + } + writer.write_u32(border.x); + writer.write_u32(border.y); + } + writer.write_u32(this->width * this->height); + for (const auto elevation: this->elevations) { + if (this->version >= 5U) { + writer.write_u16(elevation); + } else { + writer.write_u8(static_cast(elevation)); + } + } + return writer.take(); + } + }; + + /** The value type of an asset property. */ + enum class property_type : std::uint8_t { + boolean = 0, + integer = 1, + real_number = 2, + ascii_string = 3, + unicode_string = 4, + unknown = 5, + }; + + /** The payload of an `asset_property`; the active alternative tracks `property_type`. */ + using property_value = std::variant; + + /** One key/value pair carried by a map object or the world info. */ + struct asset_property { + std::string name; + property_type type = property_type::integer; + property_value value{}; ///< bool, int32, float, ASCII/UTF-8 string, or UTF-16 string + + [[nodiscard]] static auto boolean(std::string name, bool value) -> asset_property { + return {std::move(name), property_type::boolean, value}; + } + [[nodiscard]] static auto integer(std::string name, std::int32_t value) -> asset_property { + return {std::move(name), property_type::integer, value}; + } + [[nodiscard]] static auto real(std::string name, float value) -> asset_property { + return {std::move(name), property_type::real_number, value}; + } + [[nodiscard]] static auto text(std::string name, std::string value) -> asset_property { + return {std::move(name), property_type::ascii_string, std::move(value)}; + } + [[nodiscard]] static auto wide_text(std::string name, std::u16string value) -> asset_property { + return {std::move(name), property_type::unicode_string, std::move(value)}; + } + + [[nodiscard]] auto as_bool() const -> bool { + const auto *held = std::get_if(&this->value); + return held != nullptr && *held; + } + [[nodiscard]] auto as_int() const -> std::int32_t { + const auto *held = std::get_if(&this->value); + return held != nullptr ? *held : 0; + } + [[nodiscard]] auto as_real() const -> float { + const auto *held = std::get_if(&this->value); + return held != nullptr ? *held : 0.0F; + } + [[nodiscard]] auto as_ascii() const -> std::string_view { + const auto *held = std::get_if(&this->value); + return held != nullptr ? std::string_view{*held} : std::string_view{}; + } + [[nodiscard]] auto as_unicode() const -> const std::u16string * { return std::get_if(&this->value); } + }; + + /** One object placed on the map (building, prop, waypoint, ...). */ + struct map_object { + std::string type_name; ///< the SAGE `ThingTemplate` name, e.g. `*Waypoints/Waypoint` + coord3d position; + float angle = 0.0F; + road_type road = road_type::none; ///< road connectivity flags (0 for most objects) + std::uint16_t version = 1; + std::vector properties; + + [[nodiscard]] auto property(std::string_view key) const -> const asset_property * { + for (const auto &entry: this->properties) { + if (entry.name == key) return &entry; + } + return nullptr; + } + }; + + /** A player start position recovered from a `Player_N_Start` waypoint. */ + struct player_start { + int index = 0; ///< the `N` in `Player_N_Start` (1-based) + coord3d position; + }; + + /** One `MPPositionInfo` entry. */ + struct mp_position { + bool is_human = false; + bool is_computer = false; + bool load_ai_script = false; + std::uint32_t team = 0; + std::vector side_restrictions; + std::uint16_t version = 1; + }; + + /** One `WaypointPath` (a link between two waypoints by id). */ + struct waypoint_path { + std::int32_t start_id = 0; + std::int32_t end_id = 0; + }; + + namespace detail { + /** True when `data` begins with the ASCII tag (avoids `memcmp` with a literal). */ + [[nodiscard]] inline auto starts_with(std::span data, std::string_view tag) -> bool { + if (data.size() < tag.size()) return false; + for (std::size_t i = 0; i < tag.size(); ++i) { + if (std::to_integer(data[i]) != static_cast(tag[i])) return false; + } + return true; + } + + [[nodiscard]] inline auto read_property(byte_reader &reader, const name_table &names) -> asset_property { + asset_property property; + property.type = static_cast(reader.read_u8()); + property.name = names.name(reader.read_u24()); + switch (property.type) { + case property_type::boolean: property.value = reader.read_bool(); break; + case property_type::integer: property.value = reader.read_i32(); break; + case property_type::real_number: property.value = reader.read_f32(); break; + case property_type::ascii_string: + case property_type::unknown: property.value = reader.read_u16_prefixed_ascii(); break; + case property_type::unicode_string: property.value = reader.read_u16_prefixed_utf16(); break; + } + return property; + } + + inline auto write_property(byte_writer &writer, const asset_property &property, name_table &names) -> void { + writer.write_u8(static_cast(property.type)); + writer.write_u24(names.get_or_create(property.name)); + switch (property.type) { + case property_type::boolean: writer.write_bool(property.as_bool()); break; + case property_type::integer: writer.write_i32(property.as_int()); break; + case property_type::real_number: writer.write_f32(property.as_real()); break; + case property_type::ascii_string: + case property_type::unknown: writer.write_u16_prefixed_ascii(property.as_ascii()); break; + case property_type::unicode_string: { + const auto *text = property.as_unicode(); + writer.write_u16_prefixed_utf16(text != nullptr ? *text : std::u16string{}); + break; + } + } + } + + [[nodiscard]] inline auto parse_property_list(byte_reader &reader, const name_table &names) -> std::vector { + const auto count = reader.read_u16(); + std::vector properties; + properties.reserve(count); + for (std::uint16_t i = 0; i < count; ++i) properties.push_back(read_property(reader, names)); + return properties; + } + + inline auto write_property_list(byte_writer &writer, const std::vector &properties, name_table &names) -> void { + writer.write_u16(static_cast(properties.size())); + for (const auto &property: properties) write_property(writer, property, names); + } + + /** Split `Player__Start` into `n`, or 0 when the name does not match. */ + [[nodiscard]] inline auto parse_player_start_name(std::string_view name) -> int { + constexpr std::string_view prefix = "Player_"; + constexpr std::string_view suffix = "_Start"; + if (!name.starts_with(prefix) || !name.ends_with(suffix)) return 0; + const auto digits = name.substr(prefix.size(), name.size() - prefix.size() - suffix.size()); + if (digits.empty() || !std::ranges::all_of(digits, [](char ch) { return ch >= '0' && ch <= '9'; })) return 0; + return std::stoi(std::string{digits}); + } + } + + /** + * A parsed `.map`: the asset-name table plus the ordered chunk list. + */ + class map_document { + public: + /** Unwrap (BIG payload / `EAR\0` / bare RefPack / bare `CkMp`) and parse. */ + [[nodiscard]] static auto parse(std::span file) -> map_document { + auto data = std::vector{file.begin(), file.end()}; + if (is_refpack(data)) data = refpack_decompress(data); // a BIG payload + if (detail::starts_with(data, std::string_view{"EAR\0", 4})) { + const auto *body = reinterpret_cast(data.data()) + 8; + std::span payload{body, data.size() - 8U}; + data = maybe_decompress(payload); + } else if (is_refpack(data)) { + data = refpack_decompress(data); + } + return map_document::from_ckmp(data); + } + + [[nodiscard]] static auto open(const std::filesystem::path &path) -> map_document { + return map_document::parse(read_file(path)); + } + + [[nodiscard]] auto names() const -> const name_table & { return this->names_; } + [[nodiscard]] auto names() -> name_table & { return this->names_; } + [[nodiscard]] auto chunks() const -> const std::vector & { return this->chunks_; } + [[nodiscard]] auto chunks() -> std::vector & { return this->chunks_; } + + [[nodiscard]] auto find_chunk(std::string_view name) const -> const map_chunk * { + for (const auto &chunk: this->chunks_) { + if (chunk.name == name) return &chunk; + } + return nullptr; + } + + [[nodiscard]] auto find_chunk(std::string_view name) -> map_chunk * { + for (auto &chunk: this->chunks_) { + if (chunk.name == name) return &chunk; + } + return nullptr; + } + + [[nodiscard]] auto has_chunk(std::string_view name) const -> bool { return this->find_chunk(name) != nullptr; } + + /** The chunk of a known kind, or `nullptr`. */ + [[nodiscard]] auto find_chunk(chunk_kind kind) const -> const map_chunk * { + for (const auto &chunk: this->chunks_) { + if (chunk.kind == kind) return &chunk; + } + return nullptr; + } + + [[nodiscard]] auto find_chunk(chunk_kind kind) -> map_chunk * { + for (auto &chunk: this->chunks_) { + if (chunk.kind == kind) return &chunk; + } + return nullptr; + } + + [[nodiscard]] auto has_chunk(chunk_kind kind) const -> bool { return this->find_chunk(kind) != nullptr; } + + // -- typed chunks --------------------------------------------------- + + /** The terrain grid, or `std::nullopt` when the map has no `HeightMapData`. */ + [[nodiscard]] auto height_map() const -> std::optional { + const auto *chunk = this->find_chunk("HeightMapData"); + if (chunk == nullptr) return std::nullopt; + return height_map_data::parse(chunk->version, chunk->payload); + } + + /** Replace (or create) the `HeightMapData` chunk from a grid. */ + auto set_height_map(height_map_data data) -> void { + auto *chunk = this->find_chunk("HeightMapData"); + if (chunk == nullptr) { + chunk = &this->chunks_.emplace_back(); + chunk->name = "HeightMapData"; + chunk->kind = chunk_kind::height_map_data; + chunk->asset_index = this->names_.get_or_create("HeightMapData"); + chunk->version = data.version; + } else { + data.version = chunk->version; // keep the file's on-disk version + } + chunk->payload = data.serialize(); + } + + /** Every object the map places. */ + [[nodiscard]] auto objects() const -> std::vector { + const auto *chunk = this->find_chunk("ObjectsList"); + if (chunk == nullptr) return {}; + return map_document::decode_objects(this->names_, chunk->payload); + } + + /** Replace (or create) the `ObjectsList` chunk. */ + auto set_objects(const std::vector &objects) -> void { + auto *chunk = this->find_chunk("ObjectsList"); + if (chunk == nullptr) { + chunk = &this->chunks_.emplace_back(); + chunk->name = "ObjectsList"; + chunk->kind = chunk_kind::objects_list; + chunk->asset_index = this->names_.get_or_create("ObjectsList"); + chunk->version = 1; + } + chunk->payload = map_document::encode_objects(this->names_, objects); + } + + /** The `Player_N_Start` waypoints, ordered by index. */ + [[nodiscard]] auto player_starts() const -> std::vector { + std::vector starts; + for (const auto &object: this->objects()) { + if (object.type_name != "*Waypoints/Waypoint") continue; + const auto *name_property = object.property("waypointName"); + if (name_property == nullptr) continue; + const auto index = detail::parse_player_start_name(name_property->as_ascii()); + if (index == 0) continue; + starts.push_back({index, object.position}); + } + std::ranges::sort(starts, {}, &player_start::index); + return starts; + } + + [[nodiscard]] auto mp_positions() const -> std::vector { + const auto *chunk = this->find_chunk("MPPositionList"); + if (chunk == nullptr) return {}; + return map_document::decode_mp_positions(chunk->payload); + } + + auto set_mp_positions(const std::vector &positions) -> void { + auto *chunk = this->find_chunk("MPPositionList"); + if (chunk == nullptr) { + chunk = &this->chunks_.emplace_back(); + chunk->name = "MPPositionList"; + chunk->kind = chunk_kind::mp_position_list; + chunk->asset_index = this->names_.get_or_create("MPPositionList"); + chunk->version = 1; + } + chunk->payload = map_document::encode_mp_positions(this->names_, positions); + } + + /** The `WorldInfo` property list. */ + [[nodiscard]] auto world_info() const -> std::vector { + const auto *chunk = this->find_chunk("WorldInfo"); + if (chunk == nullptr) return {}; + byte_reader reader{chunk->payload}; + return detail::parse_property_list(reader, this->names_); + } + + auto set_world_info(const std::vector &properties) -> void { + auto *chunk = this->find_chunk("WorldInfo"); + if (chunk == nullptr) { + chunk = &this->chunks_.emplace_back(); + chunk->name = "WorldInfo"; + chunk->kind = chunk_kind::world_info; + chunk->asset_index = this->names_.get_or_create("WorldInfo"); + chunk->version = 1; + } + byte_writer writer; + detail::write_property_list(writer, properties, this->names_); + chunk->payload = writer.take(); + } + + /** The waypoint paths (`WaypointsList`). */ + [[nodiscard]] auto waypoint_paths() const -> std::vector { + const auto *chunk = this->find_chunk("WaypointsList"); + if (chunk == nullptr) return {}; + byte_reader reader{chunk->payload}; + const auto count = reader.read_u32(); + std::vector paths; + paths.reserve(count); + for (std::uint32_t i = 0; i < count; ++i) paths.push_back({reader.read_i32(), reader.read_i32()}); + return paths; + } + + // -- serialisation -------------------------------------------------- + + /** The bare `CkMp` byte image. */ + [[nodiscard]] auto to_ckmp() const -> std::vector { + byte_writer writer; + writer.write_ascii("CkMp"); + const auto count = static_cast(this->names_.size()); + writer.write_u32(count); + for (std::uint32_t i = count; i >= 1U; --i) { + const auto &name = this->names_.entries()[i]; + writer.write_u8(static_cast(name.size())); + writer.write_ascii(name); + writer.write_u32(i); + } + for (const auto &chunk: this->chunks_) { + writer.write_u32(chunk.asset_index); + writer.write_u16(chunk.version); + writer.write_u32(static_cast(chunk.payload.size())); + writer.write_bytes(chunk.payload); + } + return writer.take(); + } + + /** + * A standalone `.map` image. + * + * When `compress` is true the result is `EAR\0` + `u32` decompressed size + * + a RefPack stream (the retail compression). When false it is a bare + * `CkMp` tree. To store a map inside a `BIG4`, pass the *uncompressed* + * image and let `big_writer::add(..., compress = true)` wrap it once. + */ + [[nodiscard]] auto serialize(bool compress = false) const -> std::vector { + auto ckmp = this->to_ckmp(); + if (!compress) return ckmp; + byte_writer writer; + writer.write_ascii(std::string_view{"EAR\0", 4}); + writer.write_u32(static_cast(ckmp.size())); + const auto packed = refpack_compress(ckmp); + writer.write_bytes(packed); + return writer.take(); + } + + /** True when the name table carries `name` (top-level chunk, property, ...). */ + [[nodiscard]] auto has_name(std::string_view name) const -> bool { + for (std::uint32_t i = 1; i < this->names_.entries().size(); ++i) { + if (this->names_.entries()[i] == name) return true; + } + return false; + } + + private: + map_document() = default; + + [[nodiscard]] static auto from_ckmp(std::span data) -> map_document { + if (data.size() < 8U || !detail::starts_with(data, "CkMp")) throw map_error("not a CkMp map"); + byte_reader reader{data}; + (void) reader.read_ascii(4); + const auto count = reader.read_u32(); + map_document document; + document.names_.resize(count + 1U); + for (std::uint32_t i = count; i >= 1U; --i) { + const auto length = reader.read_u8(); + auto name = reader.read_ascii(length); + const auto index = reader.read_u32(); + if (index != i) throw map_error("asset name table index mismatch"); + document.names_.set_name(i, std::move(name)); + } + while (reader.remaining() >= 10U) { + const auto index = reader.read_u32(); + const auto version = reader.read_u16(); + const auto size = reader.read_u32(); + if (index >= document.names_.entries().size() || reader.remaining() < size) throw map_error("truncated CkMp chunk"); + map_chunk chunk; + chunk.asset_index = index; + chunk.name = std::string{document.names_.entries()[index]}; + chunk.kind = chunk_kind_of(chunk.name); + chunk.version = version; + const auto payload = reader.read_bytes(size); + chunk.payload.assign(payload.begin(), payload.end()); + document.chunks_.push_back(std::move(chunk)); + } + return document; + } + + [[nodiscard]] static auto decode_objects(const name_table &names, std::span payload) -> std::vector { + byte_reader reader{payload}; + std::vector objects; + while (reader.remaining() >= 10U) { + (void) reader.read_u32(); // nested asset index (always "Object") + const auto version = reader.read_u16(); + const auto size = reader.read_u32(); + if (reader.remaining() < size) throw map_error("truncated Object asset"); + const auto body = reader.read_bytes(size); + byte_reader body_reader{body}; + map_object object; + object.version = version; + object.position.x = body_reader.read_f32(); + object.position.y = body_reader.read_f32(); + object.position.z = body_reader.read_f32(); + object.angle = body_reader.read_f32(); + object.road = static_cast(body_reader.read_u32()); + object.type_name = body_reader.read_ascii(body_reader.read_u16()); + object.properties = detail::parse_property_list(body_reader, names); + objects.push_back(std::move(object)); + } + return objects; + } + + [[nodiscard]] static auto encode_objects(name_table &names, const std::vector &objects) -> std::vector { + const auto object_index = names.get_or_create("Object"); + byte_writer writer; + for (const auto &object: objects) { + byte_writer body; + body.write_f32(object.position.x); + body.write_f32(object.position.y); + body.write_f32(object.position.z); + body.write_f32(object.angle); + body.write_u32(static_cast(object.road)); + body.write_u16(static_cast(object.type_name.size())); + body.write_ascii(object.type_name); + detail::write_property_list(body, object.properties, names); + writer.write_u32(object_index); + writer.write_u16(object.version); + writer.write_u32(static_cast(body.size())); + writer.write_bytes(body.data()); + } + return writer.take(); + } + + [[nodiscard]] static auto decode_mp_positions(std::span payload) -> std::vector { + byte_reader reader{payload}; + std::vector positions; + while (reader.remaining() >= 10U) { + (void) reader.read_u32(); // nested asset index (always "MPPositionInfo") + const auto version = reader.read_u16(); + const auto size = reader.read_u32(); + if (reader.remaining() < size) throw map_error("truncated MPPositionInfo asset"); + const auto body = reader.read_bytes(size); + byte_reader body_reader{body}; + mp_position position; + position.version = version; + position.is_human = body_reader.read_bool(); + position.is_computer = body_reader.read_bool(); + if (version > 0U) position.load_ai_script = body_reader.read_bool(); + position.team = body_reader.read_u32(); + if (version > 0U) { + const auto count = body_reader.read_u32(); + position.side_restrictions.reserve(count); + for (std::uint32_t i = 0; i < count; ++i) position.side_restrictions.push_back(body_reader.read_u16_prefixed_ascii()); + } + positions.push_back(std::move(position)); + } + return positions; + } + + [[nodiscard]] static auto encode_mp_positions(name_table &names, const std::vector &positions) -> std::vector { + const auto info_index = names.get_or_create("MPPositionInfo"); + byte_writer writer; + for (const auto &position: positions) { + byte_writer body; + body.write_bool(position.is_human); + body.write_bool(position.is_computer); + if (position.version > 0U) body.write_bool(position.load_ai_script); + body.write_u32(position.team); + if (position.version > 0U) { + body.write_u32(static_cast(position.side_restrictions.size())); + for (const auto &side: position.side_restrictions) body.write_u16_prefixed_ascii(side); + } + writer.write_u32(info_index); + writer.write_u16(position.version); + writer.write_u32(static_cast(body.size())); + writer.write_bytes(body.data()); + } + return writer.take(); + } + + name_table names_; + std::vector chunks_; + }; + +} // namespace ra3::assets diff --git a/third_party/libra3assets/src/refpack.cppm b/third_party/libra3assets/src/refpack.cppm new file mode 100644 index 0000000..aa6414e --- /dev/null +++ b/third_party/libra3assets/src/refpack.cppm @@ -0,0 +1,284 @@ +/** + * EA's RefPack compression codec. + * + * Red Alert 3 compresses individual `BIG4` payloads and every SAGE `.map` with + * RefPack (the `10 FB` stream shared across EA titles). This partition decodes + * and encodes that stream. The decoder is a port of the reference used across + * the project (`ra3tools/ra3_big.py`, `OpenRA3`'s `ra3.fs`); the encoder is a + * greedy LZ77 matcher that emits only canonical tokens, so anything it produces + * is readable by the same decoder (and by the game). + */ +export module ra3.assets:refpack; + +import std; + +import :bytes; +import :error; + +export namespace ra3::assets { + + /** Max back-reference distance (the 17-bit RefPack window). */ + inline constexpr std::size_t refpack_window = 1U << 17U; + + /** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */ + [[nodiscard]] inline auto is_refpack(std::span data) -> bool { + return data.size() >= 2U && (std::to_integer(data[0]) & 0x3EU) == 0x10U && std::to_integer(data[1]) == 0xFBU; + } + + /** Read the declared output size from a RefPack header without decompressing. */ + [[nodiscard]] inline auto refpack_output_size(std::span data) -> std::uint32_t { + if (!is_refpack(data)) throw refpack_error("not a RefPack stream"); + std::size_t pos = 0; + const auto header = std::to_integer(data[pos++]); + const bool large_files = (header & 0x80U) != 0U; + const bool compressed_size_present = (header & 0x01U) != 0U; + pos++; // 0xFB + const std::size_t size_bytes = large_files ? 4U : 3U; + const auto read_size = [&]() -> std::uint32_t { + std::uint32_t value = 0; + for (std::size_t i = 0; i < size_bytes; ++i) { + if (pos >= data.size()) throw refpack_error("truncated RefPack size field"); + value = (value << 8U) | std::to_integer(data[pos++]); + } + return value; + }; + if (compressed_size_present) (void) read_size(); + return read_size(); + } + + /** + * Decompress an EA RefPack stream. + * + * @throws refpack_error if the stream is malformed or the length disagrees. + */ + [[nodiscard]] inline auto refpack_decompress(std::span data) -> std::vector { + if (!is_refpack(data)) throw refpack_error("not a RefPack stream"); + + std::size_t pos = 0; + const auto header = std::to_integer(data[pos++]); + const bool large_files = (header & 0x80U) != 0U; + const bool compressed_size_present = (header & 0x01U) != 0U; + pos++; // 0xFB + + const std::size_t size_bytes = large_files ? 4U : 3U; + const auto read_size = [&]() -> std::uint32_t { + std::uint32_t value = 0; + for (std::size_t i = 0; i < size_bytes; ++i) { + if (pos >= data.size()) throw refpack_error("truncated RefPack size field"); + value = (value << 8U) | std::to_integer(data[pos++]); + } + return value; + }; + + if (compressed_size_present) (void) read_size(); + const auto out_len = read_size(); + + std::vector out; + out.reserve(out_len); // guarantees no reallocation, so overlapping reads stay valid + + const auto copy_literals = [&](std::size_t count) { + if (pos + count > data.size()) throw refpack_error("truncated RefPack literals"); + out.insert(out.end(), data.begin() + static_cast(pos), data.begin() + static_cast(pos + count)); + pos += count; + }; + + // A back-reference may overlap its own output (an RLE run): out[start + i] + // is read one byte at a time, so the pattern repeats correctly. + const auto copy_reference = [&](std::size_t length, std::size_t distance) { + if (distance == 0U || distance > out.size()) throw refpack_error("RefPack back-reference out of range"); + const auto start = out.size() - distance; + if (out.size() + length > out_len) throw refpack_error("RefPack output overrun"); + for (std::size_t i = 0; i < length; ++i) out.push_back(out[start + i]); + }; + + while (pos < data.size()) { + const auto cmd = std::to_integer(data[pos++]); + if ((cmd & 0x80U) == 0U) { // 2-byte command + if (pos >= data.size()) throw refpack_error("truncated 2-byte command"); + const auto b2 = std::to_integer(data[pos++]); + copy_literals(cmd & 0x03U); + copy_reference(((cmd & 0x1CU) >> 2U) + 3U, ((cmd & 0x60U) << 3U) + b2 + 1U); + } else if ((cmd & 0x40U) == 0U) { // 3-byte command + if (pos + 1U >= data.size()) throw refpack_error("truncated 3-byte command"); + const auto b2 = std::to_integer(data[pos]); + const auto b3 = std::to_integer(data[pos + 1U]); + pos += 2U; + copy_literals((b2 & 0xC0U) >> 6U); + copy_reference((cmd & 0x3FU) + 4U, ((b2 & 0x3FU) << 8U) + b3 + 1U); + } else if ((cmd & 0x20U) == 0U) { // 4-byte command + if (pos + 2U >= data.size()) throw refpack_error("truncated 4-byte command"); + const auto b2 = std::to_integer(data[pos]); + const auto b3 = std::to_integer(data[pos + 1U]); + const auto b4 = std::to_integer(data[pos + 2U]); + pos += 3U; + copy_literals(cmd & 0x03U); + copy_reference(((cmd & 0x0CU) << 6U) + b4 + 5U, ((cmd & 0x10U) << 12U) + (static_cast(b2) << 8U) + b3 + 1U); + } else if (cmd < 0xFCU) { // long literal run + copy_literals((static_cast(cmd & 0x1FU) + 1U) << 2U); + } else { // stop + copy_literals(cmd & 0x03U); + break; + } + } + + if (out.size() != out_len) throw refpack_error("RefPack length mismatch"); + return out; + } + + /** Decompress `data` when it is RefPack, otherwise copy it unchanged. */ + [[nodiscard]] inline auto maybe_decompress(std::span data) -> std::vector { + if (is_refpack(data)) return refpack_decompress(data); + return {data.begin(), data.end()}; + } + + namespace detail { + /** True when `(length, distance)` maps to one of the three canonical tokens. */ + [[nodiscard]] inline constexpr auto refpack_match_encodable(std::size_t length, std::size_t distance) -> bool { + if (length >= 3U && length <= 10U && distance >= 1U && distance <= 1024U) return true; + if (length >= 4U && length <= 67U && distance >= 1U && distance <= 16384U) return true; + if (length >= 5U && length <= 1028U && distance >= 1U && distance <= 131072U) return true; + return false; + } + + /** Greedy RefPack encoder state. */ + class refpack_encoder { + public: + explicit refpack_encoder(std::span input) + : input_(input), chain_(input.size(), -1) { + } + + [[nodiscard]] auto run() -> std::vector { + const auto n = this->input_.size(); + const bool large = n >= (1U << 24U); + this->out_.write_u8(static_cast(0x10U | (large ? 0x80U : 0x00U))); + this->out_.write_u8(0xFBU); + const std::size_t size_bytes = large ? 4U : 3U; + for (std::size_t i = size_bytes; i-- > 0U;) this->out_.write_u8(static_cast((n >> (8U * i)) & 0xFFU)); + + std::size_t pos = 0; + std::size_t literals_start = 0; + while (pos < n) { + std::size_t best_len = 0; + std::size_t best_dist = 0; + if (pos + 2U < n) this->find_match(pos, best_len, best_dist); + if (best_len >= 3U) { + const auto pending = pos - literals_start; + const auto carry = pending % 4U; // 0..3 literals ride with the token + const auto run = pending - carry; // always a multiple of 4 + this->emit_literal_run(literals_start, run); + this->emit_match(literals_start + run, carry, best_len, best_dist); + for (std::size_t i = pos; i < pos + best_len; ++i) this->insert(i); + pos += best_len; + literals_start = pos; + } else { + this->insert(pos); + ++pos; + } + } + + const auto pending = n - literals_start; + const auto carry = pending % 4U; + this->emit_literal_run(literals_start, pending - carry); + this->out_.write_u8(static_cast(0xFCU | carry)); + this->write_literals(literals_start + (pending - carry), carry); + return this->out_.take(); + } + + private: + void insert(std::size_t pos) { + if (pos + 2U >= this->input_.size()) return; + const auto bucket = this->hash3(pos); + this->chain_[pos] = this->head_[bucket]; + this->head_[bucket] = static_cast(pos); + } + + void find_match(std::size_t pos, std::size_t &best_len, std::size_t &best_dist) const { + const auto n = this->input_.size(); + const auto max_len = std::min(1028U, n - pos); + auto candidate = this->head_[this->hash3(pos)]; + int depth = 0; + while (candidate >= 0 && depth < 64) { + const auto c = static_cast(candidate); + const auto distance = pos - c; + if (distance > refpack_window) break; // the chain only walks backwards + std::size_t length = 0; + while (length < max_len && this->input_[c + length] == this->input_[pos + length]) ++length; + if (length > best_len && refpack_match_encodable(length, distance)) { + best_len = length; + best_dist = distance; + if (length == max_len) break; + } + candidate = this->chain_[c]; + ++depth; + } + } + + void emit_literal_run(std::size_t offset, std::size_t count) { + std::size_t remaining = count; + std::size_t at = offset; + while (remaining >= 4U) { + std::size_t step = std::min(112U, remaining); + step -= step % 4U; + if (step < 4U) step = 4U; + this->out_.write_u8(static_cast(0xE0U | ((step / 4U) - 1U))); + this->write_literals(at, step); + at += step; + remaining -= step; + } + } + + void emit_match(std::size_t literal_offset, std::size_t literal_count, std::size_t length, std::size_t distance) { + const auto d = static_cast(distance - 1U); + if (length >= 3U && length <= 10U && distance <= 1024U) { // 2-byte token + const auto cmd = static_cast((((d >> 8U) & 0x03U) << 5U) | (static_cast(length - 3U) << 2U) | + static_cast(literal_count)); + this->out_.write_u8(cmd); + this->out_.write_u8(static_cast(d & 0xFFU)); + } else if (length >= 4U && length <= 67U && distance <= 16384U) { // 3-byte token + this->out_.write_u8(static_cast(0x80U | (length - 4U))); + this->out_.write_u8(static_cast((literal_count << 6U) | ((d >> 8U) & 0x3FU))); + this->out_.write_u8(static_cast(d & 0xFFU)); + } else if (length >= 5U && length <= 1028U && distance <= 131072U) { // 4-byte token + const auto l = static_cast(length - 5U); + const auto cmd = static_cast(0xC0U | (((l >> 8U) & 0x03U) << 2U) | ((d >> 12U) & 0x10U) | + static_cast(literal_count)); + this->out_.write_u8(cmd); + this->out_.write_u8(static_cast((d >> 8U) & 0xFFU)); + this->out_.write_u8(static_cast(d & 0xFFU)); + this->out_.write_u8(static_cast(l & 0xFFU)); + } else { + throw refpack_error("internal error: unencodable RefPack match"); + } + this->write_literals(literal_offset, literal_count); + } + + void write_literals(std::size_t offset, std::size_t count) { + for (std::size_t i = 0; i < count; ++i) this->out_.write_u8(std::to_integer(this->input_[offset + i])); + } + + [[nodiscard]] auto hash3(std::size_t pos) const -> std::size_t { + const auto a = static_cast(std::to_integer(this->input_[pos])); + const auto b = static_cast(std::to_integer(this->input_[pos + 1U])); + const auto c = static_cast(std::to_integer(this->input_[pos + 2U])); + return ((a | (b << 8U) | (c << 16U)) * 2654435761U) >> (32U - 16U); + } + + std::span input_; + std::vector chain_; + std::vector head_ = std::vector(1U << 16U, -1); + byte_writer out_; + }; + } + + /** + * Compress a byte range into a canonical RefPack stream. + * + * The encoder is a plain greedy LZ77: it never emits a token form the + * decoder above cannot read, so `refpack_decompress(refpack_compress(x))` + * is lossless for every input. + */ + [[nodiscard]] inline auto refpack_compress(std::span input) -> std::vector { + return detail::refpack_encoder{input}.run(); + } + +} // namespace ra3::assets diff --git a/third_party/libra3assets/tests/test_assets.cpp b/third_party/libra3assets/tests/test_assets.cpp new file mode 100644 index 0000000..46c4076 --- /dev/null +++ b/third_party/libra3assets/tests/test_assets.cpp @@ -0,0 +1,310 @@ +#include "test_main.hpp" + +import std; +import ra3.assets; + +using namespace ra3::assets; + +namespace { + + auto bytes_of(std::string_view text) -> std::vector { + std::vector out; + out.reserve(text.size()); + for (const auto ch: text) out.push_back(static_cast(static_cast(ch))); + return out; + } + + auto text_of(std::span bytes) -> std::string { + return std::string{reinterpret_cast(bytes.data()), bytes.size()}; + } + + /** A deterministic pseudo-random buffer (no external RNG needed). */ + auto pseudo_random(std::size_t size, std::uint32_t seed) -> std::vector { + std::vector out; + out.reserve(size); + std::uint32_t state = seed; + for (std::size_t i = 0; i < size; ++i) { + state = state * 1664525U + 1013904223U; + out.push_back(static_cast((state >> 16U) & 0xFFU)); + } + return out; + } + + auto make_empty_ckmp(const std::vector &names) -> std::vector { + byte_writer writer; + writer.write_ascii("CkMp"); + writer.write_u32(static_cast(names.size())); + for (std::uint32_t i = static_cast(names.size()); i >= 1U; --i) { + writer.write_u8(static_cast(names[i - 1].size())); + writer.write_ascii(names[i - 1]); + writer.write_u32(i); + } + return writer.take(); + } + + auto make_csf() -> std::vector { + byte_writer writer; + writer.write_ascii(" FSC"); + writer.write_u32(3); + writer.write_u32(2); // labels + writer.write_u32(2); // value blocks + writer.write_u32(0); + writer.write_u32(0); + writer.write_u32(csf_label_flag); + writer.write_u32(1); + writer.write_u32(3); + writer.write_ascii("ABC"); + writer.write_u32(csf_value_flag); + writer.write_u32(5); + for (const auto ch: std::string_view{"Hello"}) writer.write_u16(static_cast(ch) ^ 0xFFFFU); + writer.write_u32(csf_label_flag); + writer.write_u32(1); + writer.write_u32(3); + writer.write_ascii("UNI"); + writer.write_u32(csf_value_flag); + writer.write_u32(1); + writer.write_u16(0x4E2DU ^ 0xFFFFU); // U+4E2D (CJK) + return writer.take(); + } + +} // namespace + +TEST(refpack_roundtrip_small) { + const std::vector samples{"", "A", "AB", "ABC", "ABCD", "abcabcabc", std::string(300, 'z')}; + for (const auto &sample: samples) { + const auto input = bytes_of(sample); + const auto packed = refpack_compress(input); + const auto restored = refpack_decompress(packed); + CHECK_EQ(restored.size(), input.size()); + CHECK(std::ranges::equal(restored, input)); + } +} + +TEST(refpack_roundtrip_large_and_repetitive) { + std::vector input; + for (int i = 0; i < 2000; ++i) { + for (const auto ch: std::string_view{"lorem ipsum dolor sit amet "}) input.push_back(static_cast(ch)); + } + const auto random = pseudo_random(50000, 0x1234ABCDU); + input.insert(input.end(), random.begin(), random.end()); + const auto packed = refpack_compress(input); + CHECK(packed.size() < input.size()); // the repeated prefix must compress + CHECK(std::ranges::equal(refpack_decompress(packed), input)); +} + +TEST(refpack_decode_known_stream) { + // "abcd" literals, then a back-reference (distance 4, length 3) -> "abcdabc". + const std::vector stream{ + std::byte{0x10}, std::byte{0xFB}, std::byte{0x00}, std::byte{0x00}, std::byte{0x07}, std::byte{0xE0}, + std::byte{'a'}, std::byte{'b'}, std::byte{'c'}, std::byte{'d'}, std::byte{0x00}, std::byte{0x03}, + std::byte{0xFC}, + }; + CHECK(is_refpack(stream)); + CHECK_EQ(text_of(refpack_decompress(stream)), "abcdabc"); + CHECK_EQ(refpack_output_size(stream), 7U); +} + +TEST(big_roundtrip) { + big_writer writer; + writer.add("data\\a.txt", bytes_of("hello world")); + writer.add("data\\b.bin", bytes_of(std::string(1000, 'x')), true); + const auto image = writer.write(); + + CHECK(image.size() >= 16U); + const std::string magic{reinterpret_cast(image.data()), 4}; + CHECK_EQ(magic, "BIG4"); + + const auto archive = big_archive::from_bytes(image); + CHECK_EQ(archive.size(), 2U); + CHECK(archive.contains("data\\a.txt")); + CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello world"); + CHECK_EQ(archive.read("data\\b.bin").size(), 1000U); + CHECK_EQ(archive.find("data\\").size(), 2U); + // The first payload starts 64-byte aligned, as the retail archives do. + CHECK_EQ(archive.entries()[0].offset % 64U, 0U); +} + +TEST(big_open_from_disk_is_lazy) { + big_writer writer; + writer.add("data\\a.txt", bytes_of("hello disk")); + writer.add("data\\b.bin", bytes_of(std::string(4096, 'q')), true); + const auto image = writer.write(); + + const auto path = std::filesystem::temp_directory_path() / "libra3assets_big_lazy.big"; + write_file(path, image); + + const auto archive = big_archive::open(path); + CHECK_EQ(archive.size(), 2U); + CHECK(archive.path() == path); + CHECK(archive.contains("data\\a.txt")); + CHECK_EQ(archive.entries()[0].offset % 64U, 0U); + CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello disk"); + CHECK_EQ(archive.read("data\\b.bin").size(), 4096U); + CHECK_EQ(text_of(archive.read_prefix("data\\a.txt", 5)), "hello"); + CHECK(archive.read_prefix("data\\b.bin", 8U).size() == 8U); + std::error_code ec; + std::filesystem::remove(path, ec); +} + +TEST(binary_manifest_and_hash) { + CHECK_EQ(hash_string("W3DMesh"), 0xC2B1A262U); + CHECK_EQ(hash_string("ABAIRFIELD", false), 0x2B479BD3U); + CHECK(hash_string("Texture") != hash_string("Texture", false)); + + const std::string name = "W3DMesh:TEST"; + const std::string source = "art:test.w3x"; + byte_writer manifest; + manifest.write_u8(0); // not big-endian + manifest.write_u8(0); // not linked + manifest.write_u16(5); // version + manifest.write_u32(0); // checksum + manifest.write_u32(0); // all-types hash + manifest.write_u32(1); // count + manifest.write_u32(12); + manifest.write_u32(0); + manifest.write_u32(0); + manifest.write_u32(0); + manifest.write_u32(0); // reference buffer + manifest.write_u32(0); // reference-name buffer + manifest.write_u32(static_cast(name.size() + 1)); + manifest.write_u32(static_cast(source.size() + 1)); + manifest.write_u32(hash_string("W3DMesh")); + manifest.write_u32(hash_string("TEST", false)); + manifest.write_u32(hash_string("W3DMesh")); + manifest.write_u32(hash_string("TEST", false)); + manifest.write_i32(0); // reference offset + manifest.write_i32(0); // reference count + manifest.write_i32(0); // name offset + manifest.write_i32(0); // source offset + manifest.write_u32(8); // instance size + manifest.write_u32(0); // relocation size + manifest.write_u32(0); // imports size + manifest.write_u32(0); // tokenized + manifest.write_ascii(name); + manifest.write_u8(0); + manifest.write_ascii(source); + manifest.write_u8(0); + + byte_writer data; + data.write_u32(0); // stream checksum + data.write_ascii("PAYLOAD!"); + + auto container = binary_container::from_bytes(manifest.take(), data.take()); + CHECK_EQ(container.size(), 1U); + const auto &asset = container.assets()[0]; + CHECK_EQ(asset.name, "W3DMesh:TEST"); + CHECK_EQ(asset.type_name(), "W3DMesh"); + CHECK_EQ(asset.instance_name(), "TEST"); + CHECK_EQ(asset.source, "art:test.w3x"); + CHECK_EQ(text_of(container.read_instance(asset)), "PAYLOAD!"); + CHECK_EQ(container.find("#0").name, "W3DMesh:TEST"); + CHECK_EQ(container.of_type("w3dmesh").size(), 1U); + CHECK_EQ(container.type_counts().at("W3DMesh"), 1U); + CHECK(container.read_relocation(asset).empty()); + + container.set_cdata_source([&asset](const std::string &cdata_name) -> std::optional> { + if (cdata_name == asset.cdata_name("static")) return bytes_of("CDATA"); + return std::nullopt; + }); + CHECK(container.read_cdata(asset).has_value()); + CHECK_EQ(text_of(container.read_payload(asset)), "CDATA"); +} + +TEST(csf_roundtrip) { + auto table = csf_table::parse(make_csf()); + CHECK_EQ(table.size(), 2U); + CHECK_EQ(table.version(), 3U); + CHECK_EQ(table.lookup("abc"), "Hello"); // lookup is case-insensitive + CHECK_EQ(table.lookup("ABC"), "Hello"); + CHECK_EQ(table.lookup("uni"), "\xE4\xB8\xAD"); + CHECK(table.find("missing") == nullptr); + + const auto rewritten = table.write(); + const auto reparsed = csf_table::parse(rewritten); + CHECK_EQ(reparsed.size(), 2U); + CHECK_EQ(reparsed.lookup("abc"), "Hello"); + CHECK_EQ(reparsed.lookup("uni"), "\xE4\xB8\xAD"); +} + +TEST(csf_decode_utf16_surrogate) { + const std::u16string emoji = u"\U0001F600"; + CHECK_EQ(utf16_to_utf8(emoji), "\xF0\x9F\x98\x80"); +} + +TEST(map_roundtrip) { + const std::vector names{"HeightMapData", "ObjectsList", "Object", "waypointName", "MPPositionList", "MPPositionInfo", "WorldInfo", "waypointID"}; + auto document = map_document::parse(make_empty_ckmp(names)); + CHECK_EQ(document.names().size(), names.size()); + + height_map_data height; + height.width = 2; + height.height = 2; + height.version = 6; + height.elevations = {1, 2, 3, 4}; + document.set_height_map(height); + + map_object waypoint; + waypoint.type_name = "*Waypoints/Waypoint"; + waypoint.position = {100.0F, 200.0F, 0.0F}; + waypoint.properties.push_back(asset_property::text("waypointName", "Player_1_Start")); + waypoint.properties.push_back(asset_property::integer("waypointID", 0)); + document.set_objects({waypoint}); + + mp_position position; + position.is_human = true; + position.team = 1; + document.set_mp_positions({position}); + + const auto ckmp = document.to_ckmp(); + const auto reparsed = map_document::parse(ckmp); + + CHECK(reparsed.has_chunk(chunk_kind::height_map_data)); + CHECK(reparsed.has_chunk(chunk_kind::objects_list)); + CHECK(reparsed.has_chunk(chunk_kind::mp_position_list)); + CHECK_EQ(reparsed.find_chunk("HeightMapData")->kind, chunk_kind::height_map_data); + CHECK_EQ(chunk_kind_of("BlendTileData"), chunk_kind::blend_tile_data); + CHECK_EQ(chunk_kind_of("NotAChunk"), chunk_kind::unknown); + + const auto restored = reparsed.height_map(); + CHECK(restored.has_value()); + CHECK_EQ(restored->width, 2U); + CHECK_EQ(restored->at(1, 1), 4U); + CHECK_EQ(restored->vertical_scale(), 0.0390625F); + + const auto objects = reparsed.objects(); + CHECK_EQ(objects.size(), 1U); + CHECK_EQ(objects[0].type_name, "*Waypoints/Waypoint"); + CHECK_EQ(objects[0].property("waypointID")->as_int(), 0); + + const auto starts = reparsed.player_starts(); + CHECK_EQ(starts.size(), 1U); + CHECK_EQ(starts[0].index, 1); + CHECK_EQ(starts[0].position.x, 100.0F); + + const auto positions = reparsed.mp_positions(); + CHECK_EQ(positions.size(), 1U); + CHECK(positions[0].is_human); + CHECK_EQ(positions[0].team, 1U); +} + +TEST(map_compressed_and_big_payload) { + const std::vector names{"HeightMapData"}; + auto document = map_document::parse(make_empty_ckmp(names)); + height_map_data height; + height.width = 1; + height.height = 1; + height.version = 6; + height.elevations = {7}; + document.set_height_map(height); + + // EAR\0 + RefPack round-trip. + const auto compressed = document.serialize(true); + const auto from_compressed = map_document::parse(compressed); + CHECK(from_compressed.height_map().has_value()); + CHECK_EQ(from_compressed.height_map()->at(0, 0), 7U); + + // A BIG payload is one more RefPack layer over the map file. + const auto big_payload = refpack_compress(document.to_ckmp()); + const auto from_big = map_document::parse(big_payload); + CHECK_EQ(from_big.height_map()->at(0, 0), 7U); +} diff --git a/third_party/libra3assets/tests/test_main.cpp b/third_party/libra3assets/tests/test_main.cpp new file mode 100644 index 0000000..b090fb1 --- /dev/null +++ b/third_party/libra3assets/tests/test_main.cpp @@ -0,0 +1,5 @@ +#include "test_main.hpp" + +int main() { + return ra3test::run_all(); +} diff --git a/third_party/libra3assets/tests/test_main.hpp b/third_party/libra3assets/tests/test_main.hpp new file mode 100644 index 0000000..1b73c9f --- /dev/null +++ b/third_party/libra3assets/tests/test_main.hpp @@ -0,0 +1,83 @@ +#ifndef RA3TEST_HPP +#define RA3TEST_HPP + +// A tiny dependency-free test harness (registry + CHECK macros). Each unit-test +// executable compiles test_main.cpp and links the sources under test. + +#include +#include +#include +#include +#include + +namespace ra3test { + + struct test_case { + std::string name; + std::function fn; + }; + + inline auto registry() -> std::vector & { + static std::vector tests; + return tests; + } + + inline auto failure_count() -> int & { + static int count = 0; + return count; + } + + struct registrar { + registrar(std::string name, std::function fn) { registry().push_back({std::move(name), std::move(fn)}); } + }; + + inline auto report_failure(const std::string &expr, const std::string &file, int line) -> void { + ++failure_count(); + std::cerr << " FAIL " << file << ':' << line << " " << expr << '\n'; + } + + inline auto check(bool condition, const std::string &expr, const std::string &file, int line) -> void { + if (!condition) report_failure(expr, file, line); + } + + inline auto run_all() -> int { + int passed = 0; + for (auto &test: registry()) { + const int before = failure_count(); + std::cout << "[ RUN ] " << test.name << '\n'; + try { + test.fn(); + } catch (const std::exception &exc) { + report_failure(std::string("uncaught exception: ") + exc.what(), __FILE__, __LINE__); + } + if (failure_count() == before) { + ++passed; + std::cout << "[ OK ] " << test.name << '\n'; + } + } + std::cout << "\n" << passed << '/' << registry().size() << " tests passed, " << failure_count() << " failure(s)\n"; + return failure_count() == 0 ? 0 : 1; + } + +} // namespace ra3test + +#define TEST(name) \ + static void name(); \ + static ::ra3test::registrar ra3test_reg_##name(#name, name); \ + static void name() + +#define CHECK(cond) ::ra3test::check((cond), #cond, __FILE__, __LINE__) +#define CHECK_EQ(a, b) ::ra3test::check_eq((a), (b), #a, #b, __FILE__, __LINE__) + +namespace ra3test { + template + auto check_eq(const A &a, const B &b, const std::string &ea, const std::string &eb, const std::string &file, int line) -> void { + if (!(a == b)) { + std::ostringstream os; + os << ea << " == " << eb; + report_failure(os.str(), file, line); + } + } +} + +#endif // RA3TEST_HPP