EnderTheCoder/OpenRA3
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(assets): read retail assets through the vendored libra3assets library

Browse Source 
Vendor libra3assets (C++26 modules: BIG4, RefPack, BinaryAsset, CSF, CkMp map)
under third_party/ and add the ra3_assets target. ra3.fs, ra3.map and ra3.terrain
become thin adapters over it:

- ra3.fs delegates RefPack and the BIG4 index/payload reads (index-only, payloads
  read on demand).
- ra3.map decodes ObjectsList and CSF via map_document/csf_table; starts come from
  player_starts(), replacing the off-by-one whole-buffer scan.
- ra3.terrain takes the CkMp container and HeightMapData from map_document, with
  BlendTileData (not modelled by the library) parsed from the chunk payload.

Adds an opt-in real-asset check (OPENRA3_TEST_ASSETS) plus the library's own unit
suite as ra3assets_unit.
This commit is contained in:
EnderTheCoder
2026-09-29 23:58:13 +08:00
parent 8047e80ed6
commit 23f8be394c
17 changed files with 3078 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+35 -4
View File
@@ -196,6 +196,25 @@ target_sources(ra3_enderlog PUBLIC FILE_SET CXX_MODULES FILES third_party/libend
target_compile_features(ra3_enderlog PUBLIC cxx_std_26)
openra3_target_defaults(ra3_enderlog)
# --- RA3 built-in asset containers (vendored libra3assets) --------------------
# Reader/writer for the files the retail game ships: `BIG4` archives, the EA
# RefPack codec, SAGE `.csf` string tables and `.map` (`CkMp`) containers, plus
# the compiled `BinaryAsset` streams. A sibling of libenderlog: C++26 modules
# (`import ra3.assets;`) importing the standard library. It owns the on-disk
# format knowledge, so `ra3.fs` / `ra3.map` / `ra3.terrain` become thin adapters.
add_library(ra3_assets STATIC)
target_sources(ra3_assets PUBLIC FILE_SET CXX_MODULES FILES
third_party/libra3assets/src/assets.cppm
third_party/libra3assets/src/error.cppm
third_party/libra3assets/src/bytes.cppm
third_party/libra3assets/src/refpack.cppm
third_party/libra3assets/src/big.cppm
third_party/libra3assets/src/binary.cppm
third_party/libra3assets/src/csf.cppm
third_party/libra3assets/src/map.cppm)
target_compile_features(ra3_assets PUBLIC cxx_std_26)
openra3_target_defaults(ra3_assets)
# --- engine core -------------------------------------------------------------
add_library(ra3_core STATIC)
target_sources(ra3_core PUBLIC FILE_SET CXX_MODULES FILES src/core/ra3.core.cppm)
@@ -225,16 +244,16 @@ target_sources(ra3_game PUBLIC FILE_SET CXX_MODULES FILES src/game/ra3.game.cppm
target_link_libraries(ra3_game PUBLIC ra3_core ra3_logic)
openra3_target_defaults(ra3_game)
# --- filesystem: BIG4 + RefPack ---------------------------------------------
# --- filesystem: BIG4 + RefPack (adapter over libra3assets) -------------------
add_library(ra3_fs STATIC)
target_sources(ra3_fs PUBLIC FILE_SET CXX_MODULES FILES src/fs/ra3.fs.cppm)
target_link_libraries(ra3_fs PUBLIC ra3_core)
target_link_libraries(ra3_fs PUBLIC ra3_core ra3_assets)
openra3_target_defaults(ra3_fs)
# --- map discovery/loading ---------------------------------------------------
add_library(ra3_map STATIC)
target_sources(ra3_map PUBLIC FILE_SET CXX_MODULES FILES src/map/ra3.map.cppm)
target_link_libraries(ra3_map PUBLIC ra3_core ra3_fs)
target_link_libraries(ra3_map PUBLIC ra3_core ra3_fs ra3_assets)
openra3_target_defaults(ra3_map)
# --- minimal skirmish simulation --------------------------------------------
@@ -252,7 +271,7 @@ openra3_target_defaults(ra3_render)
# --- real map terrain (HeightMapData / BlendTileData) ------------------------
add_library(ra3_terrain STATIC)
target_sources(ra3_terrain PUBLIC FILE_SET CXX_MODULES FILES src/terrain/ra3.terrain.cppm)
target_link_libraries(ra3_terrain PUBLIC ra3_core ra3_fs ra3_render)
target_link_libraries(ra3_terrain PUBLIC ra3_core ra3_fs ra3_render ra3_assets)
openra3_target_defaults(ra3_terrain)
# --- map static art (compiled W3D meshes) ------------------------------------
@@ -580,3 +599,15 @@ add_executable(ra3_tests tests/ra3_tests.cpp)
target_link_libraries(ra3_tests PRIVATE ra3)
openra3_target_defaults(ra3_tests)
add_test(NAME ra3_tests COMMAND ra3_tests)
# libra3assets' own dependency-free unit tests (RefPack + BIG4 + BinaryAsset +
# CSF + CkMp round-trips), built against the vendored copy so the upstream
# suite keeps guarding the readers OpenRA3 now relies on.
add_executable(ra3assets_unit
third_party/libra3assets/tests/test_main.cpp
third_party/libra3assets/tests/test_assets.cpp)
target_include_directories(ra3assets_unit PRIVATE third_party/libra3assets/tests)
target_link_libraries(ra3assets_unit PRIVATE ra3_assets)
target_compile_features(ra3assets_unit PRIVATE cxx_std_26)
openra3_target_defaults(ra3assets_unit)
add_test(NAME ra3assets_unit COMMAND ra3assets_unit)
src/fs/ra3.fs.cppm
+58 -190
View File
@@ -3,13 +3,16 @@ export module ra3.fs;
import std;
export import ra3.core;
import ra3.assets;
/**
* Reading of the retail game's on-disk assets.
*
* Red Alert 3 ships its data in `BIG4` archives under `<install>\Data`, with
* individual payloads compressed by EA's RefPack codec. This module implements
* the container and codec so OpenRA3 can read a user's own installation.
* individual payloads compressed by EA's RefPack codec. The container and codec
* are implemented by the vendored `libra3assets` (`ra3.assets`, a sibling of
* `libenderlog`); this module is the thin adapter OpenRA3's loader talks to, so
* the format knowledge lives in one place.
*
* No game data is ever written into the repository; callers point the loader at
* their local install (`--game-dir` / `RA3_GAME_DIR`, default `C:\Red Alert 3`).
@@ -42,11 +45,21 @@ export namespace ra3::fs {
uint32 size = 0;
};
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const uint8> data) -> bool {
return data.size() >= 2 && (data[0] & refpack_mask) == 0x10U && data[1] == refpack_magic2;
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
/** Copy a `libra3assets` byte buffer into OpenRA3's `uint8` vector. */
[[nodiscard]] inline auto to_u8(std::vector<std::byte> bytes) -> std::vector<uint8> {
std::vector<uint8> out(bytes.size());
if (!bytes.empty()) std::memcpy(out.data(), bytes.data(), bytes.size());
return out;
}
}
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const uint8> data) -> bool { return ra3::assets::is_refpack(detail::as_bytes(data)); }
/**
* Decompress an EA RefPack stream.
*
@@ -55,80 +68,17 @@ export namespace ra3::fs {
* @throws refpack_error if the stream is malformed or the length disagrees.
*/
[[nodiscard]] inline auto refpack_decompress(std::span<const uint8> data) -> std::vector<uint8> {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
usize pos = 0;
const auto header = data[pos++];
const bool large_files = (header & 0x80U) != 0;
const bool compressed_size_present = (header & 0x01U) != 0;
pos++; // 0xFB
const usize size_bytes = large_files ? 4U : 3U;
auto read_size = [&]() -> uint32 {
uint32 value = 0;
for (usize i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8) | data[pos++];
}
return value;
};
if (compressed_size_present) (void)read_size();
const auto out_len = read_size();
std::vector<uint8> out;
out.reserve(out_len);
auto copy_literals = [&](usize count) {
if (pos + count > data.size()) throw refpack_error("truncated RefPack literals");
out.insert(out.end(), data.begin() + static_cast<std::ptrdiff_t>(pos), data.begin() + static_cast<std::ptrdiff_t>(pos + count));
pos += count;
};
auto copy_reference = [&](usize length, usize distance) {
if (distance == 0 || distance > out.size()) throw refpack_error("RefPack back-reference out of range");
usize start = out.size() - distance;
for (usize i = 0; i < length; ++i) out.push_back(out[start + i]);
};
while (pos < data.size()) {
const auto cmd = data[pos++];
if ((cmd & 0x80U) == 0) { // 2-byte command
if (pos >= data.size()) throw refpack_error("truncated 2-byte command");
const auto b2 = data[pos++];
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x1CU) >> 2) + 3, ((cmd & 0x60U) << 3) + b2 + 1);
} else if ((cmd & 0x40U) == 0) { // 3-byte command
if (pos + 1 >= data.size()) throw refpack_error("truncated 3-byte command");
const auto b2 = data[pos];
const auto b3 = data[pos + 1];
pos += 2;
copy_literals((b2 & 0xC0U) >> 6);
copy_reference((cmd & 0x3FU) + 4, ((b2 & 0x3FU) << 8) + b3 + 1);
} else if ((cmd & 0x20U) == 0) { // 4-byte command
if (pos + 2 >= data.size()) throw refpack_error("truncated 4-byte command");
const auto b2 = data[pos];
const auto b3 = data[pos + 1];
const auto b4 = data[pos + 2];
pos += 3;
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x0CU) << 6) + b4 + 5, ((cmd & 0x10U) << 12) + (b2 << 8) + b3 + 1);
} else if (cmd < 0xFCU) { // long literal run
copy_literals(((cmd & 0x1FU) + 1) << 2);
} else { // stop
copy_literals(cmd & 0x03U);
break;
}
try {
return detail::to_u8(ra3::assets::refpack_decompress(detail::as_bytes(data)));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
}
if (out.size() != out_len) throw refpack_error("RefPack length mismatch");
return out;
}
/** Decompress `data` when it is RefPack, otherwise copy it unchanged. */
[[nodiscard]] inline auto maybe_decompress(std::span<const uint8> data) -> std::vector<uint8> {
if (is_refpack(data)) return refpack_decompress(data);
return {data.begin(), data.end()};
if (!is_refpack(data)) return {data.begin(), data.end()};
return refpack_decompress(data);
}
/**
@@ -137,23 +87,11 @@ export namespace ra3::fs {
* @throws refpack_error if the stream is malformed.
*/
[[nodiscard]] inline auto refpack_output_size(std::span<const uint8> data) -> uint32 {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
usize pos = 0;
const auto header = data[pos++];
const bool large_files = (header & 0x80U) != 0;
const bool compressed_size_present = (header & 0x01U) != 0;
pos++; // 0xFB
const usize size_bytes = large_files ? 4U : 3U;
auto read_size = [&]() -> uint32 {
uint32 value = 0;
for (usize i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8) | data[pos++];
}
return value;
};
if (compressed_size_present) (void)read_size();
return read_size();
try {
return ra3::assets::refpack_output_size(detail::as_bytes(data));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
}
}
[[nodiscard]] constexpr auto read_be32(const uint8 *p) -> uint32 {
@@ -165,81 +103,32 @@ export namespace ra3::fs {
}
/**
* A parsed `BIG4` archive. Only the index is held in memory; payloads are
* read from disk on demand so multi-hundred-megabyte archives stay cheap.
* A parsed `BIG4` archive.
*
* The index and payloads are held by a `ra3::assets::big_archive`; this
* adapter exposes the OpenRA3-facing surface (`big_entry`, `uint8` buffers)
* over it. Payloads are RefPack-decompressed on request.
*/
class big_archive {
public:
/**
* Parse the index of a `BIG4` archive.
* Parse a `BIG4` archive.
*
* @param path Archive path.
* @throws archive_error if the file is missing, not `BIG4`, or truncated.
*/
[[nodiscard]] static auto open(const std::filesystem::path &path) -> big_archive {
big_archive archive;
archive.path_ = path;
std::error_code ec;
archive.file_size_ = static_cast<usize>(std::filesystem::file_size(path, ec));
if (ec) throw archive_error("cannot stat archive: " + path.string());
std::ifstream in(path, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path.string());
std::array<uint8, 16> header{};
in.read(reinterpret_cast<char *>(header.data()), static_cast<std::streamsize>(header.size()));
if (!in || std::memcmp(header.data(), big_magic.data(), big_magic.size()) != 0) throw archive_error("not a BIG4 archive: " + path.string());
const auto count = read_be32(header.data() + 8);
// Read the variable-length index, growing the window until parsed.
usize window = std::min(archive.file_size_, std::max<usize>(1U << 20U, static_cast<usize>(count) * 256U));
std::vector<uint8> index;
for (;;) {
index.resize(window);
in.clear();
in.seekg(0);
in.read(reinterpret_cast<char *>(index.data()), static_cast<std::streamsize>(window));
const auto got = static_cast<usize>(in.gcount());
index.resize(got);
archive.entries_.clear();
archive.entries_.reserve(count);
usize pos = 16;
bool complete = true;
for (uint32 i = 0; i < count; ++i) {
if (pos + 8 > index.size()) {
complete = false;
break;
}
big_entry entry;
entry.offset = read_be32(index.data() + pos);
entry.size = read_be32(index.data() + pos + 4);
pos += 8;
const auto *begin = reinterpret_cast<const char *>(index.data() + pos);
const auto *end = reinterpret_cast<const char *>(std::memchr(begin, '\0', index.size() - pos));
if (end == nullptr) {
complete = false;
break;
}
entry.name.assign(begin, end);
pos += static_cast<usize>(end - begin) + 1U;
archive.entries_.push_back(std::move(entry));
}
if (complete) break;
if (window >= archive.file_size_) throw archive_error("truncated BIG4 index: " + path.string());
window = std::min(archive.file_size_, window * 2U);
try {
return big_archive{ra3::assets::big_archive::open(path), path};
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
archive.index_.reserve(archive.entries_.size());
for (usize i = 0; i < archive.entries_.size(); ++i) archive.index_.emplace(archive.entries_[i].name, i);
return archive;
}
[[nodiscard]] auto path() const -> const std::filesystem::path & { return path_; }
[[nodiscard]] auto entries() const -> const std::vector<big_entry> & { return entries_; }
[[nodiscard]] auto size() const -> usize { return entries_.size(); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return index_.contains(std::string{name}); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return archive_.contains(name); }
/** Entry names whose path contains `needle`, in index order. */
[[nodiscard]] auto find(std::string_view needle) const -> std::vector<const big_entry *> {
@@ -258,42 +147,33 @@ export namespace ra3::fs {
* @throws archive_error if the entry is missing or unreadable.
*/
[[nodiscard]] auto read(std::string_view name, bool decompress = true) const -> std::vector<uint8> {
const auto it = index_.find(std::string{name});
if (it == index_.end()) throw archive_error("no such entry: " + std::string{name});
const auto &entry = entries_[it->second];
std::ifstream in(path_, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path_.string());
in.seekg(static_cast<std::streamoff>(entry.offset));
std::vector<uint8> raw(entry.size);
in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(raw.size()));
if (!in) throw archive_error("short read for entry: " + entry.name);
return decompress ? maybe_decompress(raw) : raw;
try {
return detail::to_u8(archive_.read(name, decompress));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
}
/** Read the first `count` stored bytes of an entry (no decompression). */
[[nodiscard]] auto read_prefix(std::string_view name, usize count) const -> std::vector<uint8> {
const auto it = index_.find(std::string{name});
if (it == index_.end()) throw archive_error("no such entry: " + std::string{name});
const auto &entry = entries_[it->second];
std::ifstream in(path_, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path_.string());
in.seekg(static_cast<std::streamoff>(entry.offset));
const auto want = std::min(count, static_cast<usize>(entry.size));
std::vector<uint8> raw(want);
in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(raw.size()));
raw.resize(static_cast<usize>(in.gcount()));
return raw;
try {
return detail::to_u8(archive_.read_prefix(name, count));
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
}
private:
big_archive() = default;
big_archive(ra3::assets::big_archive archive, std::filesystem::path path) : archive_(std::move(archive)), path_(std::move(path)) {
entries_.reserve(archive_.size());
for (const auto &entry: archive_.entries()) entries_.push_back({entry.name, entry.offset, entry.size});
}
ra3::assets::big_archive archive_;
std::filesystem::path path_;
usize file_size_ = 0;
std::vector<big_entry> entries_;
std::unordered_map<std::string, usize> index_;
};
/**
@@ -306,18 +186,6 @@ export namespace ra3::fs {
* @return The install root, or `std::nullopt` when none is found.
*/
[[nodiscard]] inline auto find_game_dir(const std::optional<std::filesystem::path> &explicit_dir = std::nullopt) -> std::optional<std::filesystem::path> {
auto qualifies = [](const std::filesystem::path &candidate) {
std::error_code ec;
return !candidate.empty() && std::filesystem::is_directory(candidate / "Data", ec);
};
if (explicit_dir && qualifies(*explicit_dir)) return explicit_dir;
if (const auto *env = std::getenv("RA3_GAME_DIR"); env != nullptr && *env != '\0') {
const std::filesystem::path candidate{env};
if (qualifies(candidate)) return candidate;
}
const std::filesystem::path default_dir{"C:/Red Alert 3"};
if (qualifies(default_dir)) return default_dir;
return std::nullopt;
return ra3::assets::find_game_dir(explicit_dir);
}
}
src/map/ra3.map.cppm
+55 -233
View File
@@ -4,6 +4,7 @@ import std;
export import ra3.core;
export import ra3.fs;
import ra3.assets;
/**
* Red Alert 3 map discovery and loading.
@@ -50,6 +51,9 @@ export namespace ra3::map {
};
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
[[nodiscard]] inline auto split_path(std::string_view path) -> std::vector<std::string> {
std::vector<std::string> parts;
usize start = 0;
@@ -82,54 +86,6 @@ export namespace ra3::map {
return file;
}
[[nodiscard]] inline auto find_bytes(std::span<const uint8> haystack, std::string_view needle) -> usize {
if (needle.empty()) return 0;
const auto *needle_begin = reinterpret_cast<const uint8 *>(needle.data());
const auto it = std::search(haystack.begin(), haystack.end(), needle_begin, needle_begin + needle.size());
return it == haystack.end() ? static_cast<usize>(-1) : static_cast<usize>(it - haystack.begin());
}
[[nodiscard]] inline auto read_le_f32(const uint8 *p) -> real {
const auto bits = fs::read_le32(p);
real value = 0.0F;
std::memcpy(&value, &bits, sizeof(value));
return value;
}
/** First plausible `(x, y, 0)` float triple at or after `from`. */
[[nodiscard]] inline auto scan_triple(std::span<const uint8> data, usize from, usize window) -> std::optional<start_position> {
const auto end = std::min(data.size(), from + window);
for (usize p = from; p + 12U <= end; ++p) {
const auto x = read_le_f32(data.data() + p);
const auto y = read_le_f32(data.data() + p + 4U);
const auto z = read_le_f32(data.data() + p + 8U);
if (z == 0.0F && std::isfinite(x) && std::isfinite(y) && x > 0.0F && x < 20000.0F && y > 0.0F && y < 20000.0F) {
return start_position{x, y, z};
}
}
return std::nullopt;
}
/** Extract `Player_1_Start` .. `Player_N_Start` waypoint coordinates. */
[[nodiscard]] inline auto extract_start_positions(std::span<const uint8> ckmp, int max_players = 8) -> std::vector<start_position> {
std::vector<start_position> result;
for (int n = 1; n <= max_players; ++n) {
const auto needle = std::string{"Player_"} + std::to_string(n) + "_Start";
usize cursor = 0;
while (cursor < ckmp.size()) {
const auto at = find_bytes(ckmp.subspan(cursor), needle);
if (at == static_cast<usize>(-1)) break;
const auto abs = cursor + at;
if (const auto triple = scan_triple(ckmp, abs + needle.size(), 128); triple) {
result.push_back(*triple);
break;
}
cursor = abs + 1;
}
}
return result;
}
/** Number of distinct integer positions (used to reject degenerate sets). */
[[nodiscard]] inline auto distinct_positions(const std::vector<start_position> &starts) -> usize {
std::vector<std::pair<int, int>> seen;
@@ -196,9 +152,24 @@ export namespace ra3::map {
return fs::maybe_decompress(payload);
}
/** Recover `Player_N_Start` waypoints from raw, uncompressed `CkMp` bytes. */
/**
* Recover `Player_N_Start` waypoints from raw, uncompressed `CkMp` bytes.
*
* `libra3assets` decodes the `ObjectsList` `*Waypoints/Waypoint` objects and
* returns the position of each `waypointName == "Player_<n>_Start"`. This
* replaces the previous whole-buffer heuristic, which scanned for the string
* and then took the first plausible float triple *after* it — off by one
* (the position precedes the name in each object) and prone to matching
* arbitrary bytes.
*/
[[nodiscard]] inline auto starts_from_ckmp(std::span<const uint8> ckmp) -> std::vector<start_position> {
auto starts = detail::extract_start_positions(ckmp);
std::vector<start_position> starts;
try {
const auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp));
for (const auto &start: document.player_starts()) starts.push_back({start.position.x, start.position.y, start.position.z});
} catch (const std::exception &) {
return {};
}
if (detail::distinct_positions(starts) < 2U) starts.clear();
return starts;
}
@@ -221,117 +192,34 @@ export namespace ra3::map {
uint32 road_type = 0U; ///< SAGE `RoadType` flags; 0 for a non-road object.
};
namespace detail {
struct ckmp_chunk {
std::string name;
uint16 version = 0;
usize offset = 0;
usize size = 0;
};
/** Parse the `CkMp` chunk tree and its `index -> name` table. */
[[nodiscard]] inline auto parse_ckmp_chunks(std::span<const uint8> data) -> std::pair<std::vector<std::string>, std::vector<ckmp_chunk>> {
if (data.size() < 8U || std::memcmp(data.data(), "CkMp", 4) != 0) return {};
usize pos = 4;
const auto read_u32 = [&](usize at) {
return static_cast<uint32>(data[at]) | (static_cast<uint32>(data[at + 1U]) << 8U) | (static_cast<uint32>(data[at + 2U]) << 16U) |
(static_cast<uint32>(data[at + 3U]) << 24U);
};
const auto read_u16 = [&](usize at) { return static_cast<uint16>(static_cast<uint32>(data[at]) | (static_cast<uint32>(data[at + 1U]) << 8U)); };
const auto count = read_u32(pos);
pos += 4;
std::vector<std::string> names(count + 1U);
for (uint32 i = count; i >= 1U && pos < data.size(); --i) {
const auto len = data[pos++];
if (pos + len + 4U > data.size()) break;
names[i] = std::string{reinterpret_cast<const char *>(data.data() + pos), len};
pos += len + 4U;
}
std::vector<ckmp_chunk> chunks;
while (pos + 10U <= data.size()) {
const auto index = read_u32(pos);
const auto version = read_u16(pos + 4U);
const auto size = read_u32(pos + 6U);
pos += 10U;
if (index >= names.size() || pos + size > data.size()) break;
chunks.push_back({names[index], version, pos, size});
pos += size;
}
return {std::move(names), std::move(chunks)};
}
}
/**
* Every object the map places (the `ObjectsList` chunk).
*
* Layout (OpenSAGE `Data/Map/{ObjectsList,MapObject,AssetProperty}.cs`):
* the chunk is a list of nested `Object` assets, each a `Coord3D`, a Z
* `angle`, a `RoadType`, a `u16`-prefixed type-name and a property list
* whose keys index the shared name table.
* Decoded by `libra3assets` (`ra3.assets`): the map is parsed as a
* `map_document`, whose `ObjectsList` accessor walks the nested `Object`
* assets (a `Coord3D`, a Z `angle`, a `RoadType`, a `u16`-prefixed
* type-name and a property list keyed into the shared name table).
*
* @return The objects in chunk order; empty when the map has no object list.
* @return The objects in chunk order; empty when the map is malformed or has
* no object list.
*/
[[nodiscard]] inline auto parse_objects(std::span<const uint8> ckmp) -> std::vector<map_object> {
const auto [names, chunks] = detail::parse_ckmp_chunks(ckmp);
(void) names;
std::vector<map_object> objects;
const auto read_u16 = [&](usize at) { return static_cast<uint16>(static_cast<uint32>(ckmp[at]) | (static_cast<uint32>(ckmp[at + 1U]) << 8U)); };
const auto read_u32 = [&](usize at) {
return static_cast<uint32>(ckmp[at]) | (static_cast<uint32>(ckmp[at + 1U]) << 8U) | (static_cast<uint32>(ckmp[at + 2U]) << 16U) |
(static_cast<uint32>(ckmp[at + 3U]) << 24U);
};
const auto read_f32 = [&](usize at) {
const auto bits = read_u32(at);
real value = 0.0F;
std::memcpy(&value, &bits, sizeof(value));
return value;
};
for (const auto &chunk: chunks) {
if (chunk.name != "ObjectsList") continue;
usize p = chunk.offset;
const auto end = chunk.offset + chunk.size;
while (p + 6U <= end) {
p += 4U; // asset index (always `Object`)
p += 2U; // asset version
const auto asset_size = read_u32(p);
p += 4U;
if (asset_size < 24U || p + asset_size > end) break;
const auto asset_end = p + asset_size;
map_object object;
object.x = read_f32(p);
object.y = read_f32(p + 4U);
object.z = read_f32(p + 8U);
object.angle = read_f32(p + 12U);
object.road_type = read_u32(p + 16U);
p += 20U; // Coord3D + angle + road type
const auto name_len = read_u16(p);
p += 2U;
if (p + name_len > asset_end) break;
object.type = std::string{reinterpret_cast<const char *>(ckmp.data() + p), name_len};
p += name_len;
const auto property_count = static_cast<uint32>(read_u16(p));
p += 2U;
for (uint32 i = 0; i < property_count && p + 4U <= asset_end; ++i) {
const auto type = ckmp[p++];
p += 3U; // property name index (u24) into the shared name table
usize value_size = 0;
if (type == 0U) {
value_size = 1U; // boolean
} else if (type == 1U || type == 2U) {
value_size = 4U; // integer / real
} else if (p + 2U <= asset_end) {
const auto len = read_u16(p);
value_size = 2U + static_cast<usize>(len) * ((type == 4U) ? 2U : 1U); // ascii/unicode/unknown
}
p += value_size;
if (p > asset_end) break;
}
objects.push_back(std::move(object));
p = asset_end;
try {
const auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp));
for (const auto &object: document.objects()) {
map_object out;
out.type = object.type_name;
out.x = object.position.x;
out.y = object.position.y;
out.z = object.position.z;
out.angle = object.angle;
out.scale = 1.0F;
out.road_type = static_cast<uint32>(object.road);
objects.push_back(std::move(out));
}
} catch (const std::exception &) {
return {};
}
return objects;
}
@@ -351,96 +239,30 @@ export namespace ra3::map {
}
};
namespace detail {
/**
* Decode one SAGE `.csf` string value.
*
* RA3's `gamestrings.csf` stores each UTF-16 code unit with the low byte
* XORed by `0xFF` (the high byte is the padding `0xFF`); undoing that
* yields the plain ASCII/UTF-8 text.
*/
[[nodiscard]] inline auto decode_csf_string(std::span<const uint8> raw) -> std::string {
std::string value;
for (usize i = 0; i + 1U < raw.size(); i += 2U) {
const auto ch = static_cast<char>(static_cast<uint8>(raw[i] ^ 0xFFU));
if (ch == '\0') break;
value.push_back(ch);
}
return value;
}
}
/**
* Parse `MAP:<id>` display names out of a SAGE `.csf` string table.
*
* The map list UI reads its labels from the install's `data\gamestrings.csf`
* under the key `MAP:<UPPERCASE_ID>` (e.g. `MAP:MAP_MP_2_FEASEL4` is
* "Battlebase Beta"). Values are byte-XORed with `0xFF`.
* "Battlebase Beta"). Decoding is `libra3assets`' `csf_table` (each UTF-16
* code unit is bit-inverted; the table lookup is case-insensitive).
*/
[[nodiscard]] inline auto parse_map_names(std::span<const uint8> csf) -> map_name_table {
map_name_table table;
if (csf.size() < 24U || std::memcmp(csf.data(), " FSC", 4) != 0) return table;
const auto read = [&](usize p) -> uint32 {
return static_cast<uint32>(csf[p]) | (static_cast<uint32>(csf[p + 1U]) << 8U) | (static_cast<uint32>(csf[p + 2U]) << 16U) |
(static_cast<uint32>(csf[p + 3U]) << 24U);
};
usize pos = 24; // header: magic + version + label/string counts + 8 reserved bytes
while (pos + 4U <= csf.size()) {
if (std::memcmp(csf.data() + pos, " LBL", 4) != 0) {
++pos;
continue;
}
pos += 4;
if (pos + 4U > csf.size()) break;
const auto count = read(pos);
pos += 4;
std::vector<std::string> labels;
labels.reserve(count);
bool ok = true;
for (uint32 i = 0; i < count; ++i) {
if (pos + 4U > csf.size()) {
ok = false;
break;
}
const auto len = read(pos);
pos += 4;
if (pos + len > csf.size()) {
ok = false;
break;
}
labels.emplace_back(reinterpret_cast<const char *>(csf.data() + pos), len);
pos += len;
}
if (!ok) break;
for (const auto &label: labels) {
if (pos + 4U > csf.size() || std::memcmp(csf.data() + pos, " RTS", 4) != 0) {
ok = false;
break;
}
pos += 4;
if (pos + 4U > csf.size()) {
ok = false;
break;
}
const auto chars = read(pos);
pos += 4;
const auto bytes = static_cast<usize>(chars) * 2U;
if (pos + bytes > csf.size()) {
ok = false;
break;
}
auto value = detail::decode_csf_string(csf.subspan(pos, bytes));
pos += bytes;
try {
const auto strings = ra3::assets::csf_table::parse(detail::as_bytes(csf));
for (const auto &entry: strings.entries()) {
constexpr std::string_view prefix = "MAP:";
if (label.size() > prefix.size() && label.compare(0, prefix.size(), prefix) == 0) {
auto id = label.substr(prefix.size());
std::transform(id.begin(), id.end(), id.begin(), [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
if (!value.empty()) table.names.try_emplace(std::move(id), std::move(value));
}
if (entry.label.size() <= prefix.size() || entry.label.compare(0, prefix.size(), prefix) != 0) continue;
auto id = entry.label.substr(prefix.size());
std::transform(id.begin(), id.end(), id.begin(), [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
auto value = ra3::assets::utf16_to_utf8(entry.value());
// CSF strings are NUL-terminated; the terminator is not part of the text.
if (const auto nul = value.find('\0'); nul != std::string::npos) value.resize(nul);
if (!value.empty()) table.names.try_emplace(std::move(id), std::move(value));
}
if (!ok) break;
} catch (const std::exception &) {
return {};
}
return table;
}
src/terrain/ra3.terrain.cppm
+50 -92
View File
@@ -6,6 +6,7 @@ export import ra3.core;
export import ra3.fs;
export import ra3.render;
export import ra3.models;
import ra3.assets;
/**
* The map's real terrain, read from the compiled `CkMp` chunk tree.
@@ -91,82 +92,24 @@ export namespace ra3::terrain {
};
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
/** View a `libra3assets` byte range as OpenRA3's `uint8`. */
[[nodiscard]] inline auto as_u8(std::span<const std::byte> data) -> std::span<const uint8> {
return {reinterpret_cast<const uint8 *>(data.data()), data.size()};
}
[[nodiscard]] inline auto read_u16(const uint8 *p) -> uint16 { return static_cast<uint16>(p[0]) | (static_cast<uint16>(p[1]) << 8U); }
[[nodiscard]] inline auto read_u32(const uint8 *p) -> uint32 {
return static_cast<uint32>(p[0]) | (static_cast<uint32>(p[1]) << 8U) | (static_cast<uint32>(p[2]) << 16U) |
(static_cast<uint32>(p[3]) << 24U);
}
struct chunk {
std::string name;
uint16 version = 0;
usize offset = 0;
usize size = 0;
};
/**
* Parse the `CkMp` chunk tree into a flat chunk list.
*
* Layout: `"CkMp"`, `u32 assetCount`, then `assetCount` entries of
* `{ u8 nameLen, name, u32 index }` (index descending from the count),
* then `{ u32 index, u16 version, u32 dataSize, data[dataSize] }` per
* chunk until the end.
*/
[[nodiscard]] inline auto parse_chunks(std::span<const uint8> data) -> std::vector<chunk> {
if (data.size() < 8U || std::memcmp(data.data(), "CkMp", 4) != 0) throw terrain_error("not a CkMp map");
usize pos = 4;
const auto count = read_u32(data.data() + pos);
pos += 4;
std::vector<std::string> names(count + 1U);
for (uint32 i = count; i >= 1U; --i) {
if (pos >= data.size()) throw terrain_error("truncated asset-name table");
const auto len = data[pos++];
if (pos + len + 4U > data.size()) throw terrain_error("truncated asset-name");
names[i] = std::string{reinterpret_cast<const char *>(data.data() + pos), len};
pos += len;
pos += 4; // asset index (== i)
}
std::vector<chunk> chunks;
while (pos + 10U <= data.size()) {
const auto index = read_u32(data.data() + pos);
const auto version = read_u16(data.data() + pos + 4U);
const auto size = read_u32(data.data() + pos + 6U);
pos += 10;
if (index >= names.size() || pos + size > data.size()) throw terrain_error("bad chunk header");
chunks.push_back({names[index], version, pos, size});
pos += size;
}
return chunks;
}
[[nodiscard]] inline auto find(const std::vector<chunk> &chunks, std::string_view name) -> const chunk * {
for (const auto &c: chunks) {
if (c.name == name) return &c;
}
return nullptr;
}
inline auto parse_heightmap(std::span<const uint8> data, const chunk &c, map_data &out) -> void {
usize p = c.offset;
out.width = read_u32(data.data() + p);
out.height = read_u32(data.data() + p + 4U);
out.border_width = read_u32(data.data() + p + 8U);
const auto border_count = read_u32(data.data() + p + 12U);
p += 16;
p += static_cast<usize>(border_count) * (c.version >= 6U ? 16U : 8U);
p += 4U; // area
if (out.width == 0U || out.height == 0U) throw terrain_error("empty heightmap");
const auto area = static_cast<usize>(out.width) * out.height;
out.elevations.resize(area);
for (usize i = 0; i < area; ++i) {
if (p >= data.size()) throw terrain_error("truncated heightmap");
out.elevations[i] = c.version >= 5U ? read_u16(data.data() + p) : data[p];
p += c.version >= 5U ? 2U : 1U;
}
}
// The `CkMp` container and `HeightMapData` chunk are modelled by
// `libra3assets` (`ra3.assets`); `parse_map` reads them through a
// `map_document`. Only `BlendTileData` (which the library does not type
// yet) is parsed here, over the chunk payload the document exposes.
/** Where the texture table ends and how many `BlendDescription`s follow. */
struct texture_table_info {
@@ -246,20 +189,20 @@ export namespace ra3::terrain {
}
}
inline auto parse_blend(std::span<const uint8> data, const chunk &c, map_data &out) -> void {
inline auto parse_blend(std::span<const uint8> payload, uint16 version, map_data &out) -> void {
const auto area = static_cast<usize>(out.width) * out.height;
usize p = c.offset;
const auto num_tiles = read_u32(data.data() + p);
usize p = 0;
const auto num_tiles = read_u32(payload.data() + p);
p += 4;
if (num_tiles != area) throw terrain_error("BlendTileData tile count mismatch");
out.tiles.resize(area);
for (usize i = 0; i < area; ++i) out.tiles[i] = read_u16(data.data() + p + i * 2U);
for (usize i = 0; i < area; ++i) out.tiles[i] = read_u16(payload.data() + p + i * 2U);
p += area * 2U;
const auto bits = (c.version >= 14U && c.version < 24U) ? 32U : 16U;
const auto bits = (version >= 14U && version < 24U) ? 32U : 16U;
const auto word = bits / 8U;
const auto read_index = [&](usize off) -> uint16 {
return word == 4U ? static_cast<uint16>(read_u32(data.data() + off)) : read_u16(data.data() + off);
return word == 4U ? static_cast<uint16>(read_u32(payload.data() + off)) : read_u16(payload.data() + off);
};
out.blends.resize(area);
for (usize i = 0; i < area; ++i) out.blends[i] = read_index(p + i * word);
@@ -269,33 +212,48 @@ export namespace ra3::terrain {
p += area * word;
p += area * word; // CliffTextures (not rendered yet)
const auto chunk_end = c.offset + c.size;
const auto table = parse_textures(data, p, chunk_end, out);
parse_blend_descriptions(data, table, chunk_end, out);
const auto chunk_end = payload.size();
const auto table = parse_textures(payload, p, chunk_end, out);
parse_blend_descriptions(payload, table, chunk_end, out);
}
}
/**
* Parse the terrain chunks out of a `CkMp` (uncompressed) map payload.
*
* The container and `HeightMapData` come from `libra3assets`
* (`map_document`); `BlendTileData` (which the library does not model yet)
* is decoded here from the chunk payload the document exposes.
*
* @throws terrain_error if the chunk tree or terrain chunks are malformed.
*/
[[nodiscard]] inline auto parse_map(std::span<const uint8> ckmp) -> map_data {
const auto chunks = detail::parse_chunks(ckmp);
const auto *heightmap = detail::find(chunks, "HeightMapData");
const auto *blend = detail::find(chunks, "BlendTileData");
if (heightmap == nullptr || blend == nullptr) throw terrain_error("map has no terrain chunks");
try {
auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp));
map_data out;
detail::parse_heightmap(ckmp, *heightmap, out);
detail::parse_blend(ckmp, *blend, out);
if (const auto *water = detail::find(chunks, "GlobalWaterSettings"); water != nullptr && water->size >= 8U) {
out.has_water = detail::read_u32(ckmp.data() + water->offset) != 0U;
const auto bits = detail::read_u32(ckmp.data() + water->offset + 4U);
std::memcpy(&out.water_plane_z, &bits, sizeof(out.water_plane_z));
map_data out;
const auto height = document.height_map();
if (!height) throw terrain_error("map has no HeightMapData chunk");
out.width = height->width;
out.height = height->height;
out.border_width = height->border_width;
out.elevations.assign(height->elevations.begin(), height->elevations.end());
const auto *blend = document.find_chunk("BlendTileData");
if (blend == nullptr) throw terrain_error("map has no BlendTileData chunk");
detail::parse_blend(detail::as_u8(blend->payload), blend->version, out);
if (const auto *water = document.find_chunk("GlobalWaterSettings"); water != nullptr && water->payload.size() >= 8U) {
const auto *bytes = reinterpret_cast<const uint8 *>(water->payload.data());
out.has_water = detail::read_u32(bytes) != 0U;
const auto bits = detail::read_u32(bytes + 4U);
std::memcpy(&out.water_plane_z, &bits, sizeof(out.water_plane_z));
}
out.valid = true;
return out;
} catch (const ra3::assets::asset_error &error) {
throw terrain_error(error.what());
}
out.valid = true;
return out;
}
/** The decoded terrain textures, parallel to `map_data::textures`. */
tests/ra3_tests.cpp
+95
View File
@@ -1,5 +1,6 @@
import std;
import ra3;
import ra3.assets;
namespace {
int failures = 0;
@@ -300,6 +301,100 @@ auto main() -> int {
check(models::decode_dds(std::vector<core::uint8>{1U, 2U, 3U}).empty(), "a non-DDS payload yields no image");
// Real-asset checks (opt-in): set OPENRA3_TEST_ASSETS to an extracted assets
// directory. They run the libra3assets-backed readers over the retail data and
// compare the map names against the golden `maps/map_names.tsv` written by the
// previous hand-rolled parser.
if (const char *assets_env = std::getenv("OPENRA3_TEST_ASSETS"); assets_env != nullptr && *assets_env != '\0') {
const std::filesystem::path assets{assets_env};
const auto read_all = [](const std::filesystem::path &path) {
std::ifstream in(path, std::ios::binary);
return std::vector<core::uint8>{std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};
};
const auto golden = assets / "maps" / "map_names.tsv";
// The golden table came from the newest `Lang-English<N>.big`; mirror that.
const auto pick_english_csf = [](const std::filesystem::path &raw) -> std::filesystem::path {
std::filesystem::path best;
int best_version = -1;
std::error_code ec;
for (const auto &entry: std::filesystem::directory_iterator(raw, ec)) {
if (!entry.is_directory()) continue;
const auto name = entry.path().filename().string();
if (name.rfind("Lang-English", 0) != 0) continue;
int version = 0;
try {
version = std::stoi(name.substr(12));
} catch (const std::exception &) {
continue;
}
if (version > best_version) {
best_version = version;
best = entry.path() / "data" / "gamestrings.csf";
}
}
if (!best.empty() && std::filesystem::exists(best)) return best;
return raw / "English" / "data" / "gamestrings.csf";
};
const auto csf_path = pick_english_csf(assets / "raw");
if (std::filesystem::exists(golden) && std::filesystem::exists(csf_path)) {
const auto table = map::parse_map_names(read_all(csf_path));
std::ifstream in(golden, std::ios::binary);
std::string line;
std::size_t matched = 0;
std::size_t mismatched = 0;
while (std::getline(in, line)) {
const auto tab = line.find('\t');
if (tab == std::string::npos) continue;
const auto id = line.substr(0, tab);
const auto expected = line.substr(tab + 1U);
if (table.lookup(id) == expected) {
++matched;
} else {
++mismatched;
if (mismatched <= 5U) std::printf(" name mismatch %s: golden '%s' new '%s'\n", id.c_str(), expected.c_str(), table.lookup(id).c_str());
}
}
std::printf(" real CSF: %zu names, golden %zu matched, %zu mismatched\n", table.names.size(), matched, mismatched);
check(mismatched == 0U, "map names reproduce the golden map_names.tsv");
}
std::size_t maps_ok = 0;
std::size_t maps_bad = 0;
std::size_t library_starts = 0;
std::size_t no_starts = 0;
std::size_t roundtrip_bad = 0;
const auto to_bytes = [](std::span<const core::uint8> data) {
return std::span<const std::byte>{reinterpret_cast<const std::byte *>(data.data()), data.size()};
};
for (const auto &entry: std::filesystem::directory_iterator(assets / "maps")) {
if (!entry.is_regular_file() || entry.path().extension() != ".map") continue;
try {
const auto ckmp = map::to_ckmp(read_all(entry.path()));
const auto parsed = terrain::parse_map(ckmp);
const auto starts = map::starts_from_ckmp(ckmp);
const auto document = ra3::assets::map_document::parse(to_bytes(ckmp));
if (document.player_starts().size() >= 2U) ++library_starts; else ++no_starts;
if (!std::ranges::equal(document.to_ckmp(), to_bytes(ckmp))) ++roundtrip_bad;
if (parsed.width > 0U && parsed.height > 0U && parsed.elevations.size() == static_cast<std::size_t>(parsed.width) * parsed.height && starts.size() >= 2U) {
++maps_ok;
} else {
++maps_bad;
if (maps_bad <= 5U)
std::printf(" map oddity %s: %ux%u starts %zu\n", entry.path().filename().string().c_str(), parsed.width, parsed.height, starts.size());
}
} catch (const std::exception &error) {
++maps_bad;
if (maps_bad <= 5U) std::printf(" map FAIL %s: %s\n", entry.path().filename().string().c_str(), error.what());
}
}
std::printf(" real maps: %zu ok, %zu bad; player_starts resolved %zu, unresolved %zu; round-trip mismatched %zu\n", maps_ok, maps_bad, library_starts,
no_starts, roundtrip_bad);
check(maps_bad == 0U, "every extracted map parses terrain + starts through libra3assets");
check(no_starts == 0U, "libra3assets player_starts() resolves every retail map's start waypoints");
check(roundtrip_bad == 0U, "libra3assets re-serialises every retail map byte-for-byte");
}
if (failures == 0) {
std::puts("ra3_tests: OK");
}
third_party/libra3assets/README.md
Vendored
+134
View File
@@ -0,0 +1,134 @@
# libra3assets
A dependency-free C++26 library that reads **and writes** the resource files
Red Alert 3 ships, so tooling (map editors in particular) can work with the
retail assets directly. No engine, no game install required to build - only to
feed it data.
It is a sub-project of this repository, a sibling of
[`libra3replay`](../libra3replay/README.md), and follows the same build
conventions (C++26 modules, `import std;`, GCC 16, CMake 4 + Ninja).
## What it handles
| Module | Format | Read | Write |
| ------------------ | ------------------------------------------------------------------- | :--: | :---: |
| `ra3.assets:big` | `BIG4` archives (`Data\*.big`) | yes | yes |
| `ra3.assets:refpack` | EA RefPack codec (`10 FB`) | yes | yes |
| `ra3.assets:binary` | compiled `BinaryAsset` streams (`.bin` + `.manifest`, `cdata`) | yes | - |
| `ra3.assets:csf` | SAGE `.csf` string tables (`gamestrings.csf`) | yes | yes |
| `ra3.assets:map` | SAGE `.map` containers (`CkMp`) incl. `HeightMapData`/objects | yes | yes |
| `ra3.assets:bytes` | bounds-checked little-/big-endian readers and writers | yes | yes |
The **map** module is the centrepiece for a map editor. A `.map` is modelled as
an ordered list of named, versioned `CkMp` chunks; chunks the library does not
type (e.g. `BlendTileData`, `SidesList`) are preserved byte-for-byte and the
asset-name table keeps its original indices, so an edit/ serialise cycle is
lossless. Typed accessors cover the terrain grid (`HeightMapData`), every placed
object (`ObjectsList`, including the `*Waypoints/Waypoint` objects that carry
`Player_N_Start`), `MPPositionList`, `WorldInfo` and `WaypointsList`.
The **binary** module parses the compiled `BinaryAsset` streams that
BinaryAssetBuilder produces (`data\static.bin`, `data\global.bin`, ...): the
manifest index, each asset's instance slice, its relocation/import sidecars and
its `cdata` blob, plus the hash used to name assets.
## Build
```bash
cmake -S libra3assets -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++-16
cmake --build build
ctest --test-dir build --output-on-failure
```
The library is a set of C++20/26 modules (`ra3.assets` plus the `:error`,
`:bytes`, `:refpack`, `:big`, `:binary`, `:csf` and `:map` partitions) that
imports the standard library (`import std;`). That needs **CMake 4.0+ with the
Ninja generator** and a compiler whose standard library ships a `std` module -
**GCC 16** in practice. The tests need nothing but the library itself.
> Building with the distro GCC 16: pass `-DCMAKE_CXX_COMPILER=g++-16`.
## Use
```cpp
import std;
import ra3.assets;
using namespace ra3::assets;
// --- a map, for a map editor -------------------------------------------------
auto document = map_document::open("map_mp_2_black1b.map");
if (const auto height = document.height_map()) {
std::println("terrain {}x{}", height->width, height->height);
auto grid = *height; // copy, then edit
grid.set(10, 10, grid.at(10, 10) + 1); // raise a cell
document.set_height_map(std::move(grid)); // re-encode the chunk
}
for (const auto &start: document.player_starts())
std::println("Player {} at ({}, {})", start.index, start.position.x, start.position.y);
for (const auto &object: document.objects())
if (object.type_name == "*Waypoints/Waypoint")
std::println("{} -> {}", object.property("waypointName")->as_ascii(), object.position.y);
// Lossless: unknown chunks and the name table survive the round-trip.
write_file("edited.map", document.serialize(/* compress = */ true));
// --- a BIG4 archive ----------------------------------------------------------
const auto archive = big_archive::open("Data/GlobalStream.big");
std::println("{} entries", archive.size());
for (const auto *entry: archive.find("audio"))
std::println("{} ({} bytes)", entry->name, entry->size);
// --- a compiled BinaryAsset stream ------------------------------------------
const auto stream = binary_stream_from_big(archive, binary_stream::global, /* need_data = */ false);
for (const auto &[type, count]: stream.type_counts())
std::println("{:6} {}", count, type);
```
The API deliberately avoids integer IDs for anything selectable: chunk kinds are
`chunk_kind` (e.g. `chunk_kind::height_map_data`), streams are `binary_stream`
(`binary_stream::global`), property types are `property_type`, and an
`asset_property` carries a `std::variant<bool, std::int32_t, float, std::string,
std::u16string>`.
## Command-line tool
`ra3assets-cli` is built alongside the library:
```
ra3assets-cli big list <archive.big> [match]
ra3assets-cli big extract <archive.big> <out-dir> [match]
ra3assets-cli binary list <path> [type]
ra3assets-cli binary types <path> [static|global|locale|static_l|static_m]
ra3assets-cli binary cat <path> <Type:Instance|#index> <out-file>
ra3assets-cli map info <map-file>
ra3assets-cli map starts <map-file>
ra3assets-cli map repack <map-file> <out-file> [--compress]
ra3assets-cli csf <csf-or-game-dir-or-big> [label]
ra3assets-cli hash <text>...
```
## Notes and limits
- A `BIG4` archive opened from disk keeps only its index in memory and reads
payloads on demand, so enumerating the ~700 MB retail `StaticStream.big` costs
a few megabytes; pass `need_data = false` to `binary_stream_from_big` when you
only need the manifest. A parsed `.map` document is held in full - that is what
its typed accessors edit.
- `set_height_map` keeps the on-disk `HeightMapData` version (RA3 uses 6, i.e.
16-bit elevations). Older 8-bit maps serialise back as 8-bit.
- RefPack compression is a plain greedy LZ77 matcher. It is lossless against the
bundled decoder and produces streams the game's decoder accepts, but it is
slightly less dense than the retail compressor (≈3% on a typical map).
- `BlendTileData` (terrain texture blending) is preserved but not yet
type-modelled; a map editor should treat the chunk as opaque for now. The
planned follow-up covers `BlendTileData` and `SidesList`.
- Only `BIG4` is supported (RA3's format); older `BIGF` archives are not.
The on-disk layouts were reverse engineered from the shipped files and
cross-checked against [`ra3tools`](../ra3tools/), OpenRA3's `ra3.fs`/`ra3.map`
modules and the OpenSAGE re-implementation (`reference/OpenSAGE`).
third_party/libra3assets/src/assets.cppm
Vendored
+65
View File
@@ -0,0 +1,65 @@
/**
* libra3assets - read (and write) Red Alert 3's built-in resource files.
*
* The primary module interface re-exports every partition:
*
* - `:error` - the `asset_error` exception family
* - `:bytes` - bounds-checked little-/big-endian readers and writers
* - `:refpack` - EA's RefPack codec (decompress and compress)
* - `:big` - `BIG4` archives (`Data\*.big`), read and write
* - `:binary` - compiled `BinaryAsset` streams (`*.bin` + `.manifest`)
* - `:csf` - SAGE `.csf` string tables
* - `:map` - SAGE `.map` containers (`CkMp`), read and write
*
* Nothing here reads game data on its own: callers point the library at their
* own installation (`RA3_GAME_DIR` / `C:\Red Alert 3`).
*/
export module ra3.assets;
export import :error;
export import :bytes;
export import :refpack;
export import :big;
export import :binary;
export import :csf;
export import :map;
import std;
export namespace ra3::assets {
/**
* Locate a Red Alert 3 installation.
*
* Resolution order: the explicit argument, then `$RA3_GAME_DIR`, then the
* default `C:\Red Alert 3`. A directory qualifies only if it has a `Data`
* subdirectory.
*/
[[nodiscard]] inline auto find_game_dir(const std::optional<std::filesystem::path> &explicit_dir = std::nullopt) -> std::optional<std::filesystem::path> {
const auto qualifies = [](const std::filesystem::path &candidate) {
std::error_code ec;
return !candidate.empty() && std::filesystem::is_directory(candidate / "Data", ec);
};
if (explicit_dir && qualifies(*explicit_dir)) return explicit_dir;
if (const auto *env = std::getenv("RA3_GAME_DIR"); env != nullptr && *env != '\0') {
const std::filesystem::path candidate{env};
if (qualifies(candidate)) return candidate;
}
const std::filesystem::path default_dir{"C:/Red Alert 3"};
if (qualifies(default_dir)) return default_dir;
return std::nullopt;
}
/** Every `Data\*.big` archive in an installation, sorted by name. */
[[nodiscard]] inline auto list_archives(const std::filesystem::path &game_dir) -> std::vector<std::filesystem::path> {
std::vector<std::filesystem::path> archives;
std::error_code ec;
for (const auto &entry: std::filesystem::directory_iterator(game_dir / "Data", ec)) {
if (entry.is_regular_file() && entry.path().extension() == ".big") archives.push_back(entry.path());
}
std::ranges::sort(archives);
return archives;
}
} // namespace ra3::assets
third_party/libra3assets/src/big.cppm
Vendored
+274
View File
@@ -0,0 +1,274 @@
/**
* `BIG4` archives - Red Alert 3's on-disk asset container.
*
* Retail RA3 ships its data as `Data\*.big`: a 16-byte header, a variable-
* length entry index and the 64-byte-aligned payloads. The count and the entry
* offsets/sizes are **big-endian** (the header size is little-endian, matching
* the retail files); each payload may itself be RefPack-compressed, which is
* detected from its own magic rather than flagged in the index.
*
* Both directions are implemented so a map editor can repack an archive.
*/
export module ra3.assets:big;
import std;
import :bytes;
import :error;
import :refpack;
export namespace ra3::assets {
/** One file inside a `BIG4` archive. */
struct big_entry {
std::string name;
std::uint32_t offset = 0;
std::uint32_t size = 0;
};
namespace detail {
[[nodiscard]] inline auto align_up(std::uint64_t value, std::uint64_t alignment) -> std::uint64_t {
return (value + alignment - 1U) / alignment * alignment;
}
[[nodiscard]] inline constexpr auto bswap32(std::uint32_t value) -> std::uint32_t {
return ((value & 0x000000FFU) << 24U) | ((value & 0x0000FF00U) << 8U) | ((value & 0x00FF0000U) >> 8U) | ((value & 0xFF000000U) >> 24U);
}
}
/**
* A parsed `BIG4` archive.
*
* Opening from disk parses only the index; payloads are read (and
* RefPack-decompressed) on demand, so a multi-hundred-megabyte archive costs
* a few megabytes to enumerate. An archive built from an in-memory image
* (`from_bytes`) slices its payloads out of that buffer instead.
*/
class big_archive {
public:
/** Read and parse an archive from disk, keeping only the index in memory. */
[[nodiscard]] static auto open(const std::filesystem::path &path) -> big_archive {
big_archive archive;
archive.path_ = path;
std::error_code ec;
archive.file_size_ = static_cast<std::uint64_t>(std::filesystem::file_size(path, ec));
if (ec) throw big_error("cannot stat archive: " + path.string());
std::ifstream in(path, std::ios::binary);
if (!in) throw big_error("cannot open archive: " + path.string());
std::array<std::byte, 16> header{};
in.read(reinterpret_cast<char *>(header.data()), static_cast<std::streamsize>(header.size()));
if (!in || std::string_view{reinterpret_cast<const char *>(header.data()), 4} != "BIG4") throw big_error("not a BIG4 archive: " + path.string());
// The index is a dense variable-length table from offset 16; read a
// window and grow it until every entry parses. Payloads are never
// touched, so the memory cost stays at the index size.
const auto size = archive.file_size_;
std::uint64_t window = std::min<std::uint64_t>(size, 1U << 20U);
std::vector<std::byte> buffer;
for (;;) {
buffer.resize(static_cast<std::size_t>(window));
in.clear();
in.seekg(0);
in.read(reinterpret_cast<char *>(buffer.data()), static_cast<std::streamsize>(window));
const auto got = in.gcount();
buffer.resize(got > 0 ? static_cast<std::size_t>(got) : 0U);
std::vector<big_entry> entries;
if (big_archive::parse_index(buffer, entries)) {
archive.entries_ = std::move(entries);
break;
}
if (window >= size) throw big_error("truncated BIG4 index: " + path.string());
window = std::min<std::uint64_t>(size, window * 2U);
}
archive.reindex();
return archive;
}
/** Parse an in-memory archive image; payloads are sliced from it. */
[[nodiscard]] static auto from_bytes(std::vector<std::byte> bytes) -> big_archive {
big_archive archive;
archive.bytes_ = std::move(bytes);
archive.in_memory_ = true;
if (!big_archive::parse_index(archive.bytes_, archive.entries_)) throw big_error("not a BIG4 archive");
archive.reindex();
return archive;
}
[[nodiscard]] auto path() const -> const std::filesystem::path & { return this->path_; }
[[nodiscard]] auto entries() const -> const std::vector<big_entry> & { return this->entries_; }
[[nodiscard]] auto size() const -> std::size_t { return this->entries_.size(); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return this->index_.contains(std::string{name}); }
/** The entry with exactly this name, or `nullptr`. */
[[nodiscard]] auto find_exact(std::string_view name) const -> const big_entry * {
const auto it = this->index_.find(std::string{name});
return it == this->index_.end() ? nullptr : &this->entries_[it->second];
}
/** Every entry whose name contains `needle`, in index order. */
[[nodiscard]] auto find(std::string_view needle) const -> std::vector<const big_entry *> {
std::vector<const big_entry *> matches;
for (const auto &entry: this->entries_) {
if (entry.name.find(needle) != std::string::npos) matches.push_back(&entry);
}
return matches;
}
/** Read an entry's payload; RefPack-decompress it when `decompress`. */
[[nodiscard]] auto read(std::string_view name, bool decompress = true) const -> std::vector<std::byte> {
const auto *entry = this->find_exact(name);
if (entry == nullptr) throw big_error("no such entry: " + std::string{name});
return this->read(*entry, decompress);
}
[[nodiscard]] auto read(const big_entry &entry, bool decompress = true) const -> std::vector<std::byte> {
const auto raw = this->stored(entry.offset, entry.size);
return decompress ? maybe_decompress(raw) : raw;
}
/** The first `count` stored bytes of an entry (no decompression). */
[[nodiscard]] auto read_prefix(std::string_view name, std::size_t count) const -> std::vector<std::byte> {
const auto *entry = this->find_exact(name);
if (entry == nullptr) throw big_error("no such entry: " + std::string{name});
return this->stored(entry->offset, static_cast<std::uint32_t>(std::min<std::size_t>(count, entry->size)));
}
private:
big_archive() = default;
/** Parse a `BIG4` index from `data`; false when it is not fully present. */
[[nodiscard]] static auto parse_index(std::span<const std::byte> data, std::vector<big_entry> &out) -> bool {
if (data.size() < 16U) return false;
byte_reader reader{data};
if (reader.read_ascii(4) != "BIG4") return false;
(void) reader.read_u32(); // total file size (little-endian); implied by the buffer
const auto count = reader.read_be_u32();
(void) reader.read_be_u32(); // index size
// Each entry is at least `u32 offset + u32 size + NUL name` (9 bytes),
// so a count that cannot fit bounds both the reserve and the window.
if (count > (data.size() - 16U) / 9U) return false;
out.clear();
out.reserve(count);
for (std::uint32_t i = 0; i < count; ++i) {
if (reader.remaining() < 8U) return false;
big_entry entry;
entry.offset = reader.read_be_u32();
entry.size = reader.read_be_u32();
// The NUL-terminated name may run past the current window.
std::string name;
bool terminated = false;
while (reader.remaining() > 0U) {
const auto ch = reader.read_u8();
if (ch == 0U) {
terminated = true;
break;
}
name.push_back(static_cast<char>(ch));
}
if (!terminated) return false;
entry.name = std::move(name);
out.push_back(std::move(entry));
}
return true;
}
/** Read `count` stored bytes at `offset`, from memory or from disk. */
[[nodiscard]] auto stored(std::uint32_t offset, std::uint32_t count) const -> std::vector<std::byte> {
if (this->in_memory_) {
if (static_cast<std::uint64_t>(offset) + count > this->bytes_.size()) throw big_error("entry extends past end of archive");
const auto *begin = this->bytes_.data() + offset;
return std::vector<std::byte>{begin, begin + count};
}
if (static_cast<std::uint64_t>(offset) + count > this->file_size_) throw big_error("entry extends past end of archive");
std::ifstream in(this->path_, std::ios::binary);
if (!in) throw big_error("cannot open archive: " + this->path_.string());
in.seekg(static_cast<std::streamoff>(offset));
std::vector<std::byte> raw(count);
if (count > 0U) in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(count));
if (!in) throw big_error("short read from archive: " + this->path_.string());
return raw;
}
auto reindex() -> void {
this->index_.clear();
for (std::size_t i = 0; i < this->entries_.size(); ++i) this->index_.emplace(this->entries_[i].name, i);
}
std::filesystem::path path_;
std::vector<std::byte> bytes_; ///< In-memory image (from_bytes); empty when opened from disk.
bool in_memory_ = false;
std::uint64_t file_size_ = 0;
std::vector<big_entry> entries_;
std::unordered_map<std::string, std::size_t> index_;
};
/**
* Builds a `BIG4` archive.
*
* Entry payloads are written 64-byte aligned (as the retail archives are),
* and the header's index size reproduces the retail convention
* (`last-entry end + 8`). `add(..., compress = true)` RefPack-encodes the
* payload when that is smaller, otherwise stores it verbatim.
*/
class big_writer {
public:
auto add(std::string name, std::vector<std::byte> payload, bool compress = false) -> void {
if (compress) {
auto packed = refpack_compress(payload);
if (packed.size() < payload.size()) payload = std::move(packed);
}
this->items_.push_back({std::move(name), std::move(payload)});
}
/** Serialise the archive into a fresh byte image. */
[[nodiscard]] auto write() const -> std::vector<std::byte> {
std::uint64_t table_end = 16U;
for (const auto &item: this->items_) table_end += 8U + item.name.size() + 1U;
const auto index_size = table_end + 8U;
const auto data_start = detail::align_up(index_size, 64U);
std::vector<std::uint64_t> offsets;
offsets.reserve(this->items_.size());
std::uint64_t cursor = data_start;
for (const auto &item: this->items_) {
offsets.push_back(cursor);
cursor = detail::align_up(cursor + item.payload.size(), 64U);
}
byte_writer writer;
writer.write_ascii("BIG4");
writer.write_u32(static_cast<std::uint32_t>(cursor)); // total file size (little-endian)
writer.write_u32(detail::bswap32(static_cast<std::uint32_t>(this->items_.size())));
writer.write_u32(detail::bswap32(static_cast<std::uint32_t>(index_size)));
for (std::size_t i = 0; i < this->items_.size(); ++i) {
writer.write_u32(detail::bswap32(static_cast<std::uint32_t>(offsets[i])));
writer.write_u32(detail::bswap32(static_cast<std::uint32_t>(this->items_[i].payload.size())));
writer.write_ascii(this->items_[i].name);
writer.write_u8(0);
}
while (writer.size() < data_start) writer.write_u8(0);
for (std::size_t i = 0; i < this->items_.size(); ++i) {
writer.write_bytes(this->items_[i].payload);
if (i + 1U < this->items_.size()) {
while (writer.size() < offsets[i + 1U]) writer.write_u8(0);
}
}
return writer.take();
}
private:
struct pending_item {
std::string name;
std::vector<std::byte> payload;
};
std::vector<pending_item> items_;
};
} // namespace ra3::assets
third_party/libra3assets/src/binary.cppm
Vendored
+396
View File
@@ -0,0 +1,396 @@
/**
* Compiled `BinaryAsset` streams (`Data\*.bin` + `.manifest`).
*
* Retail RA3 does not ship its gameplay/art assets as source files: the
* BinaryAssetBuilder compiles every XML/`.w3x` into a `.bin` *instance stream*
* plus a `.manifest` index (`data\static.bin`, `data\global.bin`, ...). This
* partition parses that index, exposes each asset's instance slice, its
* relocation/import sidecars and its custom-data (`cdata`) blob, and implements
* the Bob-Jenkins-style hash the builder names assets with.
*
* Asset names are `"Type:Instance"` (e.g. `W3DMesh:ABAIRFIELD`); `Type` is
* case-sensitive and `Instance` is lowercased before hashing.
*/
export module ra3.assets:binary;
import std;
import :big;
import :bytes;
import :error;
import :refpack;
export namespace ra3::assets {
/** The compiled instance streams a retail install ships. */
enum class binary_stream {
static_data, ///< `data\static.bin` - the main art/gameplay stream
global, ///< `data\global.bin` - sounds, AI, UI
locale, ///< `data\locale.bin` - localized on-demand textures
static_low, ///< `data\static_l.bin` - low-detail model LODs
static_medium, ///< `data\static_m.bin` - medium-detail model LODs
};
/** The `data\<name>.manifest` stem for a stream. */
[[nodiscard]] inline auto to_string(binary_stream stream) -> std::string_view {
switch (stream) {
case binary_stream::static_data: return "static";
case binary_stream::global: return "global";
case binary_stream::locale: return "locale";
case binary_stream::static_low: return "static_l";
case binary_stream::static_medium: return "static_m";
}
return "static";
}
/** Resolve a stream name (e.g. `"global"`), or `std::nullopt`. */
[[nodiscard]] inline auto binary_stream_from_name(std::string_view name) -> std::optional<binary_stream> {
for (const auto candidate: {binary_stream::static_data, binary_stream::global, binary_stream::locale, binary_stream::static_low, binary_stream::static_medium}) {
if (to_string(candidate) == name) return candidate;
}
return std::nullopt;
}
/** Streams in the order `binary_stream_from_big` prefers when none is given. */
inline constexpr std::array<binary_stream, 5> binary_stream_order{binary_stream::static_data, binary_stream::global, binary_stream::locale, binary_stream::static_low,
binary_stream::static_medium};
/** Types compiled from a `.w3x` model source. */
inline constexpr std::array<std::string_view, 5> w3d_types{"W3DMesh", "W3DHierarchy", "W3DAnimation", "W3DContainer", "W3DCollisionBox"};
/**
* The hash BinaryAssetBuilder names assets with.
*
* A Bob-Jenkins `lookup3`-style mix. `seed` is the running value; the public
* overloads seed it with the length (`hash_string`).
*/
[[nodiscard]] inline auto fast_hash(std::span<const std::byte> data, std::uint32_t seed = 0) -> std::uint32_t {
const auto length = data.size();
if (length == 0U) return 0x1337C0DEU;
const auto at16 = [&](std::size_t pos) -> std::uint32_t {
return static_cast<std::uint32_t>(std::to_integer<std::uint8_t>(data[pos])) |
(static_cast<std::uint32_t>(std::to_integer<std::uint8_t>(data[pos + 1U])) << 8U);
};
std::uint32_t h = seed;
std::size_t pos = 0;
const auto extra = length & 3U;
for (std::size_t i = 0; i < (length >> 2U); ++i) {
h += at16(pos);
h ^= (at16(pos + 2U) ^ (h << 5U)) << 11U;
h += h >> 11U;
pos += 4U;
}
if (extra == 1U) {
h += std::to_integer<std::uint8_t>(data[pos]);
h ^= h << 10U;
h += h >> 1U;
} else if (extra == 2U) {
h += at16(pos);
h ^= h << 11U;
h += h >> 17U;
} else if (extra == 3U) {
h += at16(pos);
h ^= h << 16U;
h ^= static_cast<std::uint32_t>(std::to_integer<std::uint8_t>(data[pos + 2U])) << 18U;
h += h >> 11U;
}
h ^= h << 3U;
h += h >> 5U;
h ^= h << 2U;
h += h >> 15U;
h ^= h << 10U;
return h;
}
/**
* Hash an asset type (case-sensitive, the `TypeId`) or an instance name
* (`case_sensitive = false`, the `InstanceId`).
*/
[[nodiscard]] inline auto hash_string(std::string_view text, bool case_sensitive = true) -> std::uint32_t {
std::string buffer{text};
if (!case_sensitive) {
std::ranges::transform(buffer, buffer.begin(), [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
}
const auto *bytes = reinterpret_cast<const std::byte *>(buffer.data());
return fast_hash(std::span<const std::byte>{bytes, buffer.size()}, static_cast<std::uint32_t>(buffer.size()));
}
/** One asset in a compiled stream. */
struct binary_asset {
std::uint32_t index = 0;
std::uint32_t type_id = 0;
std::uint32_t instance_id = 0;
std::uint32_t type_hash = 0;
std::uint32_t instance_hash = 0;
std::vector<std::pair<std::uint32_t, std::uint32_t>> references; ///< (typeId, instanceId) pairs
std::string name; ///< "Type:Instance"
std::string source; ///< the `.w3x`/XML it was compiled from
std::uint32_t instance_size = 0;
std::uint32_t relocation_size = 0;
std::uint32_t imports_size = 0;
bool tokenized = false;
std::uint32_t instance_offset = 4; ///< into the decompressed `.bin` (after its 4-byte checksum)
std::uint32_t relocation_offset = 4; ///< into the `.relo`
std::uint32_t imports_offset = 4; ///< into the `.imp`
[[nodiscard]] auto type_name() const -> std::string_view {
const auto colon = this->name.find(':');
return std::string_view{this->name}.substr(0, colon);
}
[[nodiscard]] auto instance_name() const -> std::string_view {
const auto colon = this->name.find(':');
return colon == std::string::npos ? std::string_view{this->name} : std::string_view{this->name}.substr(colon + 1U);
}
/**
* The `cdata` blob name for a custom-data asset (`AudioFile`,
* `OnDemandTexture`, ...):
* `data\<stream>\cdata\<typeId>.<typeHash>.<instanceId>.<instanceHash>.cdata`.
*/
[[nodiscard]] auto cdata_name(std::string_view stream) const -> std::string {
return std::format("data\\{}\\cdata\\{:08x}.{:08x}.{:08x}.{:08x}.cdata", stream, this->type_id, this->type_hash, this->instance_id,
this->instance_hash);
}
};
/** A callback that resolves a `cdata` blob name to its decompressed bytes. */
using cdata_source = std::function<std::optional<std::vector<std::byte>>(const std::string &name)>;
/**
* A parsed `.manifest` (+ optional `.bin`/`.relo`/`.imp`).
*
* Only the manifest is mandatory; `need_data = false` when constructing
* skips the (up to hundreds of megabytes) instance stream, which is all
* listing the assets requires.
*/
class binary_container {
public:
/** Parse a manifest, its instance stream and its optional fixup streams. */
[[nodiscard]] static auto from_bytes(std::vector<std::byte> manifest, std::vector<std::byte> data = {}, std::vector<std::byte> relocation = {},
std::vector<std::byte> imports = {}, std::string stream = "static") -> binary_container {
binary_container container;
container.stream_ = std::move(stream);
container.manifest_ = std::move(manifest);
container.data_ = std::move(data);
container.relocation_ = std::move(relocation);
container.imports_ = std::move(imports);
container.parse();
return container;
}
[[nodiscard]] auto stream() const -> const std::string & { return this->stream_; }
[[nodiscard]] auto version() const -> std::uint16_t { return this->version_; }
[[nodiscard]] auto is_linked() const -> bool { return this->is_linked_; }
[[nodiscard]] auto assets() const -> const std::vector<binary_asset> & { return this->assets_; }
[[nodiscard]] auto size() const -> std::size_t { return this->assets_.size(); }
/** Count of assets per `Type` (the part before `:`). */
[[nodiscard]] auto type_counts() const -> std::map<std::string, std::size_t> {
std::map<std::string, std::size_t> counts;
for (const auto &asset: this->assets_) ++counts[std::string{asset.type_name()}];
return counts;
}
/** Map `(typeId << 32 | instanceId)` to the owning asset. */
[[nodiscard]] auto asset_index() const -> const std::unordered_map<std::uint64_t, std::size_t> & { return this->asset_index_; }
/** Every asset, in manifest order, whose `Type` matches (case-insensitive). */
[[nodiscard]] auto of_type(std::string_view type) const -> std::vector<const binary_asset *> {
std::vector<const binary_asset *> matches;
for (const auto &asset: this->assets_) {
if (std::ranges::equal(asset.type_name(), type, [](char a, char b) { return std::tolower(static_cast<unsigned char>(a)) == std::tolower(static_cast<unsigned char>(b)); }))
matches.push_back(&asset);
}
return matches;
}
/** Resolve `"Type:Instance"` (case-sensitive, then insensitive) or `"#index"`. */
[[nodiscard]] auto find(std::string_view selector) const -> const binary_asset & {
if (selector.starts_with('#')) {
const auto index = static_cast<std::size_t>(std::stoul(std::string{selector.substr(1)}));
if (index >= this->assets_.size()) throw binary_error("asset index out of range: " + std::string{selector});
return this->assets_[index];
}
for (const auto &asset: this->assets_) {
if (asset.name == selector) return asset;
}
for (const auto &asset: this->assets_) {
if (std::ranges::equal(asset.name, selector, [](char a, char b) { return std::tolower(static_cast<unsigned char>(a)) == std::tolower(static_cast<unsigned char>(b)); }))
return asset;
}
throw binary_error("no such asset: " + std::string{selector});
}
auto set_cdata_source(cdata_source source) -> void { this->cdata_ = std::move(source); }
[[nodiscard]] auto read_instance(const binary_asset &asset) const -> std::span<const std::byte> {
if (asset.instance_size == 0U) return {};
if (static_cast<std::uint64_t>(asset.instance_offset) + asset.instance_size > this->data_.size()) throw binary_error("instance extends past stream: " + asset.name);
return std::span<const std::byte>{this->data_}.subspan(asset.instance_offset, asset.instance_size);
}
[[nodiscard]] auto read_relocation(const binary_asset &asset) const -> std::span<const std::byte> {
if (asset.relocation_size == 0U) return {};
if (static_cast<std::uint64_t>(asset.relocation_offset) + asset.relocation_size > this->relocation_.size()) throw binary_error("relocation extends past stream: " + asset.name);
return std::span<const std::byte>{this->relocation_}.subspan(asset.relocation_offset, asset.relocation_size);
}
[[nodiscard]] auto read_imports(const binary_asset &asset) const -> std::span<const std::byte> {
if (asset.imports_size == 0U) return {};
if (static_cast<std::uint64_t>(asset.imports_offset) + asset.imports_size > this->imports_.size()) throw binary_error("imports extend past stream: " + asset.name);
return std::span<const std::byte>{this->imports_}.subspan(asset.imports_offset, asset.imports_size);
}
/** The asset's `cdata` blob, or `std::nullopt` when it has none. */
[[nodiscard]] auto read_cdata(const binary_asset &asset) const -> std::optional<std::vector<std::byte>> {
if (!this->cdata_) return std::nullopt;
return this->cdata_(asset.cdata_name(this->stream_));
}
/** The asset's real payload: its `cdata` blob when present, else its instance. */
[[nodiscard]] auto read_payload(const binary_asset &asset) const -> std::vector<std::byte> {
if (auto cdata = this->read_cdata(asset)) return *cdata;
const auto instance = this->read_instance(asset);
return {instance.begin(), instance.end()};
}
private:
binary_container() = default;
auto parse() -> void {
byte_reader reader{this->manifest_};
const auto is_big_endian = reader.read_u8();
this->is_linked_ = reader.read_bool();
this->version_ = reader.read_u16();
if (is_big_endian != 0U) throw binary_error("big-endian manifests are not supported");
if (this->version_ != 5U && this->version_ != 6U) throw binary_error("unsupported manifest version " + std::to_string(this->version_));
(void) reader.read_u32(); // stream checksum
(void) reader.read_u32(); // all-types hash
const auto count = reader.read_u32();
(void) reader.read_u32(); // total instance data size
(void) reader.read_u32(); // max instance chunk size
(void) reader.read_u32(); // max relocation chunk size
(void) reader.read_u32(); // max imports chunk size
const auto reference_buffer_size = reader.read_u32();
const auto reference_name_buffer_size = reader.read_u32();
const auto name_buffer_size = reader.read_u32();
const auto source_buffer_size = reader.read_u32();
constexpr std::size_t header_size = 48U;
constexpr std::size_t entry_size = 48U;
if (this->manifest_.size() < header_size + static_cast<std::size_t>(count) * entry_size) throw binary_error("manifest entry table is truncated");
const auto entries_off = header_size;
const auto references_off = entries_off + static_cast<std::size_t>(count) * entry_size;
const auto reference_names_off = references_off + reference_buffer_size;
const auto names_off = reference_names_off + reference_name_buffer_size;
const auto sources_off = names_off + name_buffer_size;
if (sources_off + source_buffer_size > this->manifest_.size()) throw binary_error("manifest string buffers are truncated");
const auto cstr = [&](std::size_t base, std::int32_t offset) -> std::string {
if (offset < 0) return {};
const auto start = base + static_cast<std::size_t>(offset);
if (start >= this->manifest_.size()) return {};
std::size_t end = start;
while (end < this->manifest_.size() && std::to_integer<std::uint8_t>(this->manifest_[end]) != 0U) ++end;
return std::string{reinterpret_cast<const char *>(this->manifest_.data() + start), end - start};
};
this->assets_.reserve(count);
std::uint32_t instance_offset = 4;
std::uint32_t relocation_offset = 4;
std::uint32_t imports_offset = 4;
for (std::uint32_t i = 0; i < count; ++i) {
byte_reader entry{std::span<const std::byte>{this->manifest_}.subspan(entries_off + static_cast<std::size_t>(i) * entry_size, entry_size)};
binary_asset asset;
asset.index = i;
asset.type_id = entry.read_u32();
asset.instance_id = entry.read_u32();
asset.type_hash = entry.read_u32();
asset.instance_hash = entry.read_u32();
const auto reference_offset = entry.read_i32();
const auto reference_count = entry.read_i32();
const auto name_offset = entry.read_i32();
const auto source_offset = entry.read_i32();
asset.instance_size = entry.read_u32();
asset.relocation_size = entry.read_u32();
asset.imports_size = entry.read_u32();
asset.tokenized = entry.read_u32() != 0U;
asset.name = cstr(names_off, name_offset);
asset.source = cstr(sources_off, source_offset);
asset.instance_offset = instance_offset;
asset.relocation_offset = relocation_offset;
asset.imports_offset = imports_offset;
instance_offset += asset.instance_size;
relocation_offset += asset.relocation_size;
imports_offset += asset.imports_size;
for (std::int32_t j = 0; j < reference_count; ++j) {
byte_reader ref{std::span<const std::byte>{this->manifest_}.subspan(references_off + static_cast<std::size_t>(reference_offset) +
static_cast<std::size_t>(j) * 8U,
8U)};
asset.references.emplace_back(ref.read_u32(), ref.read_u32());
}
this->asset_index_.emplace((static_cast<std::uint64_t>(asset.type_id) << 32U) | asset.instance_id, this->assets_.size());
this->assets_.push_back(std::move(asset));
}
}
std::string stream_ = "static";
std::vector<std::byte> manifest_;
std::vector<std::byte> data_;
std::vector<std::byte> relocation_;
std::vector<std::byte> imports_;
std::uint16_t version_ = 0;
bool is_linked_ = false;
std::vector<binary_asset> assets_;
std::unordered_map<std::uint64_t, std::size_t> asset_index_;
cdata_source cdata_;
};
namespace detail {
[[nodiscard]] inline auto pick_manifest(std::string_view stream) -> std::string { return std::format("data\\{}.manifest", stream); }
}
/**
* Load one BinaryAsset stream out of a `BIG4` archive.
*
* `stream` selects the `.manifest` (default: the first of
* `binary_stream_order` found). When `need_data` is false only the manifest
* is read, which is enough to enumerate assets without decompressing the
* multi-hundred-megabyte instance stream.
*
* The returned container keeps a `cdata` reader that borrows `archive`; keep
* the archive alive for as long as the container is used.
*/
[[nodiscard]] inline auto binary_stream_from_big(const big_archive &archive, std::optional<binary_stream> stream = std::nullopt, bool need_data = true) -> binary_container {
std::optional<binary_stream> chosen = stream;
if (!chosen) {
for (const auto candidate: binary_stream_order) {
if (archive.contains(detail::pick_manifest(to_string(candidate)))) {
chosen = candidate;
break;
}
}
}
if (!chosen) throw binary_error("archive has no known .manifest entry");
const auto manifest_name = detail::pick_manifest(to_string(*chosen));
const auto base = manifest_name.substr(0, manifest_name.size() - std::string_view{".manifest"}.size());
const auto manifest = maybe_decompress(archive.read(manifest_name, false));
std::vector<std::byte> data;
if (need_data) data = maybe_decompress(archive.read(base + ".bin", false));
const auto relocation = maybe_decompress(archive.read(base + ".relo", false));
const auto imports = maybe_decompress(archive.read(base + ".imp", false));
auto container = binary_container::from_bytes(std::move(manifest), std::move(data), std::move(relocation), std::move(imports), std::string{to_string(*chosen)});
container.set_cdata_source([&archive](const std::string &name) -> std::optional<std::vector<std::byte>> {
const auto *entry = archive.find_exact(name);
if (entry == nullptr) return std::nullopt;
return maybe_decompress(archive.read(*entry, false));
});
return container;
}
} // namespace ra3::assets
third_party/libra3assets/src/bytes.cppm
Vendored
+220
View File
@@ -0,0 +1,220 @@
/**
* Bounds-checked little-/big-endian byte readers and writers.
*
* Red Alert 3's asset formats are little-endian, with one notable exception:
* the `BIG4` index stores its counts, offsets and sizes big-endian. Both byte
* orders are provided here. Every read is bounds-checked and throws
* `format_error` rather than reading past the end of the buffer.
*/
export module ra3.assets:bytes;
import std;
import :error;
export namespace ra3::assets {
/** A cursor over a read-only byte range with checked accessors. */
class byte_reader {
public:
explicit byte_reader(std::span<const std::byte> data)
: data_(data) {
}
[[nodiscard]] auto size() const -> std::size_t { return this->data_.size(); }
[[nodiscard]] auto position() const -> std::size_t { return this->pos_; }
[[nodiscard]] auto remaining() const -> std::size_t { return this->data_.size() - this->pos_; }
[[nodiscard]] auto empty() const -> bool { return this->pos_ >= this->data_.size(); }
auto seek(std::size_t at) -> void {
if (at > this->data_.size()) throw format_error("seek past end of buffer");
this->pos_ = at;
}
auto skip(std::size_t count) -> void {
this->require(count);
this->pos_ += count;
}
[[nodiscard]] auto read_u8() -> std::uint8_t {
this->require(1);
return std::to_integer<std::uint8_t>(this->data_[this->pos_++]);
}
[[nodiscard]] auto read_bool() -> bool { return this->read_u8() != 0U; }
[[nodiscard]] auto read_u16() -> std::uint16_t {
return static_cast<std::uint16_t>(this->read_le(2));
}
[[nodiscard]] auto read_u24() -> std::uint32_t { return this->read_le(3); }
[[nodiscard]] auto read_u32() -> std::uint32_t { return this->read_le(4); }
[[nodiscard]] auto read_u64() -> std::uint64_t { return this->read_le(8); }
[[nodiscard]] auto read_i16() -> std::int16_t { return static_cast<std::int16_t>(this->read_u16()); }
[[nodiscard]] auto read_i32() -> std::int32_t { return static_cast<std::int32_t>(this->read_u32()); }
[[nodiscard]] auto read_f32() -> float { return std::bit_cast<float>(this->read_u32()); }
[[nodiscard]] auto read_be_u16() -> std::uint16_t { return static_cast<std::uint16_t>(this->read_be(2)); }
[[nodiscard]] auto read_be_u32() -> std::uint32_t { return this->read_be(4); }
/** A view of the next `count` bytes; the cursor advances past them. */
[[nodiscard]] auto read_bytes(std::size_t count) -> std::span<const std::byte> {
this->require(count);
const auto view = this->data_.subspan(this->pos_, count);
this->pos_ += count;
return view;
}
/** `count` bytes decoded as Latin-1 (one byte per character). */
[[nodiscard]] auto read_ascii(std::size_t count) -> std::string {
const auto view = this->read_bytes(count);
return std::string{reinterpret_cast<const char *>(view.data()), view.size()};
}
/** A `u16`-length-prefixed ASCII string. */
[[nodiscard]] auto read_u16_prefixed_ascii() -> std::string { return this->read_ascii(this->read_u16()); }
/** A `u16`-length-prefixed string, decoded as UTF-8 (ASCII-compatible). */
[[nodiscard]] auto read_u16_prefixed_ascii_as_utf8() -> std::string { return this->read_u16_prefixed_ascii(); }
/** A `u16`-length-prefixed UTF-16LE string (length counted in code units). */
[[nodiscard]] auto read_u16_prefixed_utf16() -> std::u16string {
const auto count = this->read_u16();
this->require(static_cast<std::size_t>(count) * 2U);
std::u16string text(count, u'\0');
for (std::uint16_t i = 0; i < count; ++i) text[i] = static_cast<char16_t>(this->read_u16());
return text;
}
/** A NUL-terminated ASCII string; the cursor stops just past the NUL. */
[[nodiscard]] auto read_cstring() -> std::string {
std::string text;
while (this->pos_ < this->data_.size()) {
const auto ch = static_cast<char>(this->read_u8());
if (ch == '\0') break;
text.push_back(ch);
}
return text;
}
private:
auto require(std::size_t count) const -> void {
if (this->pos_ + count > this->data_.size()) throw format_error("unexpected end of buffer");
}
[[nodiscard]] auto read_le(std::size_t width) -> std::uint64_t {
this->require(width);
std::uint64_t value = 0;
for (std::size_t i = 0; i < width; ++i) value |= static_cast<std::uint64_t>(std::to_integer<std::uint8_t>(this->data_[this->pos_ + i])) << (8U * i);
this->pos_ += width;
return value;
}
[[nodiscard]] auto read_be(std::size_t width) -> std::uint64_t {
this->require(width);
std::uint64_t value = 0;
for (std::size_t i = 0; i < width; ++i) value = (value << 8U) | static_cast<std::uint64_t>(std::to_integer<std::uint8_t>(this->data_[this->pos_ + i]));
this->pos_ += width;
return value;
}
std::span<const std::byte> data_;
std::size_t pos_ = 0;
};
/** An append-only little-endian byte buffer. */
class byte_writer {
public:
[[nodiscard]] auto size() const -> std::size_t { return this->buffer_.size(); }
[[nodiscard]] auto data() const -> std::span<const std::byte> { return this->buffer_; }
[[nodiscard]] auto take() -> std::vector<std::byte> { return std::move(this->buffer_); }
auto write_u8(std::uint8_t value) -> void { this->buffer_.push_back(static_cast<std::byte>(value)); }
auto write_bool(bool value) -> void { this->write_u8(value ? 1U : 0U); }
auto write_u16(std::uint16_t value) -> void { this->write_le(value, 2); }
auto write_u24(std::uint32_t value) -> void { this->write_le(value, 3); }
auto write_u32(std::uint32_t value) -> void { this->write_le(value, 4); }
auto write_u64(std::uint64_t value) -> void { this->write_le(value, 8); }
auto write_i16(std::int16_t value) -> void { this->write_u16(static_cast<std::uint16_t>(value)); }
auto write_i32(std::int32_t value) -> void { this->write_u32(static_cast<std::uint32_t>(value)); }
auto write_f32(float value) -> void { this->write_u32(std::bit_cast<std::uint32_t>(value)); }
auto write_bytes(std::span<const std::byte> bytes) -> void { this->buffer_.insert(this->buffer_.end(), bytes.begin(), bytes.end()); }
auto write_ascii(std::string_view text) -> void {
for (const auto ch: text) this->buffer_.push_back(static_cast<std::byte>(static_cast<unsigned char>(ch)));
}
auto write_u16_prefixed_ascii(std::string_view text) -> void {
this->write_u16(static_cast<std::uint16_t>(text.size()));
this->write_ascii(text);
}
auto write_u16_prefixed_utf16(std::u16string_view text) -> void {
this->write_u16(static_cast<std::uint16_t>(text.size()));
for (const auto unit: text) {
this->write_u16(static_cast<std::uint16_t>(unit));
}
}
/** Overwrite a previously written `u32` (used to backpatch sizes). */
auto patch_u32(std::size_t offset, std::uint32_t value) -> void {
if (offset + 4U > this->buffer_.size()) throw format_error("patch offset past end of buffer");
for (std::size_t i = 0; i < 4U; ++i) this->buffer_[offset + i] = static_cast<std::byte>((value >> (8U * i)) & 0xFFU);
}
private:
auto write_le(std::uint64_t value, std::size_t width) -> void {
for (std::size_t i = 0; i < width; ++i) this->buffer_.push_back(static_cast<std::byte>((value >> (8U * i)) & 0xFFU));
}
std::vector<std::byte> buffer_;
};
/** Convenience: encode a UTF-8 string (ASCII subset) as UTF-16LE code units. */
[[nodiscard]] inline auto to_utf16(std::string_view text) -> std::u16string {
std::u16string out;
out.reserve(text.size());
for (const auto ch: text) out.push_back(static_cast<char16_t>(static_cast<unsigned char>(ch)));
return out;
}
/** Convenience: narrow a UTF-16 string that is known to be within Latin-1. */
[[nodiscard]] inline auto to_ascii(std::u16string_view text) -> std::string {
std::string out;
out.reserve(text.size());
for (const auto unit: text) out.push_back(static_cast<char>(unit & 0xFFU));
return out;
}
/** Read a whole file as bytes. */
[[nodiscard]] inline auto read_file(const std::filesystem::path &path) -> std::vector<std::byte> {
std::ifstream in(path, std::ios::binary);
if (!in) throw format_error("cannot open file: " + path.string());
in.seekg(0, std::ios::end);
const auto end = in.tellg();
if (end < 0) throw format_error("cannot size file: " + path.string());
std::vector<std::byte> bytes(static_cast<std::size_t>(end));
in.seekg(0, std::ios::beg);
if (!bytes.empty()) in.read(reinterpret_cast<char *>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
return bytes;
}
/** Write bytes to a file, creating parent directories as needed. */
inline auto write_file(const std::filesystem::path &path, std::span<const std::byte> bytes) -> void {
std::error_code ec;
if (!path.parent_path().empty()) std::filesystem::create_directories(path.parent_path(), ec);
std::ofstream out(path, std::ios::binary);
if (!out) throw format_error("cannot write file: " + path.string());
out.write(reinterpret_cast<const char *>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
if (!out) throw format_error("cannot write file: " + path.string());
}
} // namespace ra3::assets
third_party/libra3assets/src/csf.cppm
Vendored
+180
View File
@@ -0,0 +1,180 @@
/**
* SAGE `.csf` string tables (`data\gamestrings.csf`).
*
* The compressed unit/label localization table Red Alert 3 ships per language
* in `Lang-*.big` / `English.big`. Each value is a UTF-16 string whose every
* code unit is bit-inverted (`code ^ 0xFFFF`); labels are plain ASCII and are
* matched case-insensitively.
*/
export module ra3.assets:csf;
import std;
import :big;
import :bytes;
import :error;
import :refpack;
export namespace ra3::assets {
/** Magic of the label block (`" LBL"` little-endian). */
inline constexpr std::uint32_t csf_label_flag = 0x4C424C20U;
/** Magic of a value block (`" RTS"` little-endian). */
inline constexpr std::uint32_t csf_value_flag = 0x53545220U;
/** One label and its (usually single) string value. */
struct csf_entry {
std::string label;
std::vector<std::u16string> values;
/** The concatenation of every value block, the way the game renders it. */
[[nodiscard]] auto value() const -> std::u16string {
std::u16string joined;
for (const auto &chunk: this->values) joined += chunk;
return joined;
}
};
/** Decode a UTF-16 string (with surrogate pairs) to UTF-8. */
[[nodiscard]] inline auto utf16_to_utf8(std::u16string_view text) -> std::string {
std::string out;
for (std::size_t i = 0; i < text.size(); ++i) {
auto code = static_cast<std::uint32_t>(text[i]);
if (code >= 0xD800U && code <= 0xDBFFU && i + 1U < text.size()) {
const auto low = static_cast<std::uint32_t>(text[i + 1U]);
if (low >= 0xDC00U && low <= 0xDFFFU) {
code = 0x10000U + ((code - 0xD800U) << 10U) + (low - 0xDC00U);
++i;
}
}
if (code < 0x80U) {
out.push_back(static_cast<char>(code));
} else if (code < 0x800U) {
out.push_back(static_cast<char>(0xC0U | (code >> 6U)));
out.push_back(static_cast<char>(0x80U | (code & 0x3FU)));
} else if (code < 0x10000U) {
out.push_back(static_cast<char>(0xE0U | (code >> 12U)));
out.push_back(static_cast<char>(0x80U | ((code >> 6U) & 0x3FU)));
out.push_back(static_cast<char>(0x80U | (code & 0x3FU)));
} else {
out.push_back(static_cast<char>(0xF0U | (code >> 18U)));
out.push_back(static_cast<char>(0x80U | ((code >> 12U) & 0x3FU)));
out.push_back(static_cast<char>(0x80U | ((code >> 6U) & 0x3FU)));
out.push_back(static_cast<char>(0x80U | (code & 0x3FU)));
}
}
return out;
}
/**
* A parsed `.csf` table.
*
* Labels are stored verbatim; lookup upper-cases the key because the engine
* treats them case-insensitively.
*/
class csf_table {
public:
/** Parse a `.csf` image. */
[[nodiscard]] static auto parse(std::span<const std::byte> bytes) -> csf_table {
byte_reader reader{bytes};
const auto magic = reader.read_ascii(4);
if (magic != "CSF " && magic != " FSC") throw csf_error("not a CSF file");
csf_table table;
table.version_ = reader.read_u32();
const auto num_labels = reader.read_u32();
(void) reader.read_u32(); // number of value blocks
(void) reader.read_u32(); // reserved
(void) reader.read_u32(); // reserved
table.entries_.reserve(num_labels);
for (std::uint32_t i = 0; i < num_labels; ++i) {
(void) reader.read_u32(); // label flag (" LBL")
const auto value_count = reader.read_u32();
const auto label_length = reader.read_u32();
csf_entry entry;
entry.label = reader.read_ascii(label_length);
entry.values.reserve(value_count);
for (std::uint32_t j = 0; j < value_count; ++j) {
(void) reader.read_u32(); // value flag (" RTS")
const auto char_count = reader.read_u32();
std::u16string value(char_count, u'\0');
for (std::uint32_t k = 0; k < char_count; ++k) value[k] = static_cast<char16_t>(reader.read_u16() ^ 0xFFFFU);
entry.values.push_back(std::move(value));
}
table.entries_.push_back(std::move(entry));
}
table.reindex();
return table;
}
/** Read and parse a `.csf` file from disk. */
[[nodiscard]] static auto open(const std::filesystem::path &path) -> csf_table {
return csf_table::parse(read_file(path));
}
/** Load `data\gamestrings.csf` out of an archive. */
[[nodiscard]] static auto from_big(const big_archive &archive, std::string_view entry = "data\\gamestrings.csf") -> csf_table {
return csf_table::parse(maybe_decompress(archive.read(entry, false)));
}
/** Serialise the table back to a `.csf` image. */
[[nodiscard]] auto write() const -> std::vector<std::byte> {
byte_writer writer;
writer.write_ascii(" FSC");
writer.write_u32(this->version_);
writer.write_u32(static_cast<std::uint32_t>(this->entries_.size()));
std::uint32_t value_count = 0;
for (const auto &entry: this->entries_) value_count += static_cast<std::uint32_t>(entry.values.size());
writer.write_u32(value_count);
writer.write_u32(0);
writer.write_u32(0);
for (const auto &entry: this->entries_) {
writer.write_u32(csf_label_flag);
writer.write_u32(static_cast<std::uint32_t>(entry.values.size()));
writer.write_u32(static_cast<std::uint32_t>(entry.label.size()));
writer.write_ascii(entry.label);
for (const auto &value: entry.values) {
writer.write_u32(csf_value_flag);
writer.write_u32(static_cast<std::uint32_t>(value.size()));
for (const auto unit: value) writer.write_u16(static_cast<std::uint16_t>(unit) ^ 0xFFFFU);
}
}
return writer.take();
}
[[nodiscard]] auto version() const -> std::uint32_t { return this->version_; }
[[nodiscard]] auto entries() const -> const std::vector<csf_entry> & { return this->entries_; }
[[nodiscard]] auto size() const -> std::size_t { return this->entries_.size(); }
/** The entry for `label`, or `nullptr`; the match is case-insensitive. */
[[nodiscard]] auto find(std::string_view label) const -> const csf_entry * {
auto key = std::string{label};
std::ranges::transform(key, key.begin(), [](unsigned char ch) { return static_cast<char>(std::toupper(ch)); });
const auto it = this->index_.find(key);
return it == this->index_.end() ? nullptr : &this->entries_[it->second];
}
/** The UTF-8 value for `label`, or `fallback` when absent. */
[[nodiscard]] auto lookup(std::string_view label, std::string fallback = {}) const -> std::string {
const auto *entry = this->find(label);
if (entry == nullptr) return fallback;
auto text = utf16_to_utf8(entry->value());
return text.empty() ? std::move(fallback) : text;
}
private:
auto reindex() -> void {
this->index_.clear();
for (std::size_t i = 0; i < this->entries_.size(); ++i) {
auto key = this->entries_[i].label;
std::ranges::transform(key, key.begin(), [](unsigned char ch) { return static_cast<char>(std::toupper(ch)); });
this->index_.try_emplace(std::move(key), i);
}
}
std::uint32_t version_ = 3;
std::vector<csf_entry> entries_;
std::unordered_map<std::string, std::size_t> index_;
};
} // namespace ra3::assets
third_party/libra3assets/src/error.cppm
Vendored
+56
View File
@@ -0,0 +1,56 @@
/**
* Exception hierarchy of libra3assets.
*
* Every failure the library reports derives from `asset_error` (itself a
* `std::runtime_error`), so a caller can catch the whole family with one
* handler and still discriminate by the concrete type when it matters.
*/
export module ra3.assets:error;
import std;
export namespace ra3::assets {
/** Base class of every failure libra3assets reports. */
class asset_error: public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
/** A byte range is not a well-formed Red Alert 3 asset. */
class format_error: public asset_error {
public:
using asset_error::asset_error;
};
/** An EA RefPack stream is malformed or its length disagrees. */
class refpack_error: public asset_error {
public:
using asset_error::asset_error;
};
/** A `BIG4` archive is malformed, truncated or missing an entry. */
class big_error: public asset_error {
public:
using asset_error::asset_error;
};
/** A compiled `BinaryAsset` manifest/stream is malformed. */
class binary_error: public asset_error {
public:
using asset_error::asset_error;
};
/** A SAGE `.csf` string table is malformed. */
class csf_error: public asset_error {
public:
using asset_error::asset_error;
};
/** A SAGE `.map` container or one of its chunks is malformed. */
class map_error: public asset_error {
public:
using asset_error::asset_error;
};
} // namespace ra3::assets
third_party/libra3assets/src/map.cppm
Vendored
+778
View File
@@ -0,0 +1,778 @@
/**
* SAGE `.map` containers - the core of the map editor.
*
* A `.map` is a `CkMp` chunk tree. On disk it may be wrapped twice: inside a
* `BIG4` the payload is one RefPack stream; the `.map` file it yields is either
* a bare `CkMp` tree or an `EAR\0`-wrapped RefPack stream around one. This
* partition unwraps all three shapes and models the tree as an ordered list of
* named, versioned chunks.
*
* The container round-trips losslessly: chunks this library does not model are
* preserved byte-for-byte, and the asset-name table keeps its original indices
* so unparsed chunks (which embed name indices) stay valid. On top of the
* container sit typed accessors for the chunks a map editor needs:
* `HeightMapData` (the terrain grid), `ObjectsList` (every placed prop, building
* and `*Waypoints/Waypoint`), `MPPositionList`, `WorldInfo` and `WaypointsList`.
*/
export module ra3.assets:map;
import std;
import :big;
import :bytes;
import :error;
import :refpack;
export namespace ra3::assets {
/** A world-space coordinate. */
struct coord3d {
float x = 0.0F;
float y = 0.0F;
float z = 0.0F;
};
/**
* A map object's `RoadType` bit flags.
*
* Zero for most objects; set on objects the terrain road system connects
* (`Start`/`End`/`BridgeStart`/`BridgeEnd`/`Angled`/`TightCurve`/`EndCap`).
*/
enum class road_type : std::uint32_t {
none = 0,
start = 2,
end = 4,
angled = 8,
bridge_start = 16,
bridge_end = 32,
tight_curve = 64,
end_cap = 128,
unknown3 = 256,
unknown4 = 512,
};
[[nodiscard]] constexpr auto operator|(road_type a, road_type b) -> road_type {
return static_cast<road_type>(static_cast<std::uint32_t>(a) | static_cast<std::uint32_t>(b));
}
[[nodiscard]] constexpr auto operator&(road_type a, road_type b) -> road_type {
return static_cast<road_type>(static_cast<std::uint32_t>(a) & static_cast<std::uint32_t>(b));
}
/** True when `value` has every bit of `flag` set. */
[[nodiscard]] constexpr auto has_flag(road_type value, road_type flag) -> bool {
return (static_cast<std::uint32_t>(value) & static_cast<std::uint32_t>(flag)) != 0U;
}
/** The `CkMp` chunk types SAGE defines, plus `unknown` for anything else. */
enum class chunk_kind {
asset_list,
global_version,
height_map_data,
blend_tile_data,
world_info,
mp_position_list,
sides_list,
library_map_lists,
teams,
player_scripts_list,
build_lists,
objects_list,
polygon_triggers,
trigger_areas,
global_water_settings,
fog_settings,
mission_hot_spots,
mission_objectives,
standing_water_areas,
river_areas,
standing_wave_areas,
global_lighting,
post_effects_chunk,
environment_data,
named_cameras,
camera_animation_list,
castle_templates,
waypoints_list,
skybox_settings,
unknown,
};
/** The canonical chunk name (the `unknown` kind has no name). */
[[nodiscard]] inline auto to_string(chunk_kind kind) -> std::string_view {
switch (kind) {
case chunk_kind::asset_list: return "AssetList";
case chunk_kind::global_version: return "GlobalVersion";
case chunk_kind::height_map_data: return "HeightMapData";
case chunk_kind::blend_tile_data: return "BlendTileData";
case chunk_kind::world_info: return "WorldInfo";
case chunk_kind::mp_position_list: return "MPPositionList";
case chunk_kind::sides_list: return "SidesList";
case chunk_kind::library_map_lists: return "LibraryMapLists";
case chunk_kind::teams: return "Teams";
case chunk_kind::player_scripts_list: return "PlayerScriptsList";
case chunk_kind::build_lists: return "BuildLists";
case chunk_kind::objects_list: return "ObjectsList";
case chunk_kind::polygon_triggers: return "PolygonTriggers";
case chunk_kind::trigger_areas: return "TriggerAreas";
case chunk_kind::global_water_settings: return "GlobalWaterSettings";
case chunk_kind::fog_settings: return "FogSettings";
case chunk_kind::mission_hot_spots: return "MissionHotSpots";
case chunk_kind::mission_objectives: return "MissionObjectives";
case chunk_kind::standing_water_areas: return "StandingWaterAreas";
case chunk_kind::river_areas: return "RiverAreas";
case chunk_kind::standing_wave_areas: return "StandingWaveAreas";
case chunk_kind::global_lighting: return "GlobalLighting";
case chunk_kind::post_effects_chunk: return "PostEffectsChunk";
case chunk_kind::environment_data: return "EnvironmentData";
case chunk_kind::named_cameras: return "NamedCameras";
case chunk_kind::camera_animation_list: return "CameraAnimationList";
case chunk_kind::castle_templates: return "CastleTemplates";
case chunk_kind::waypoints_list: return "WaypointsList";
case chunk_kind::skybox_settings: return "SkyboxSettings";
case chunk_kind::unknown: break;
}
return {};
}
/** Classify a chunk name. */
[[nodiscard]] inline auto chunk_kind_of(std::string_view name) -> chunk_kind {
for (int i = 0; i < static_cast<int>(chunk_kind::unknown); ++i) {
const auto kind = static_cast<chunk_kind>(i);
if (to_string(kind) == name) return kind;
}
return chunk_kind::unknown;
}
/** The map's asset-name table: `index -> name`, indices preserved on write. */
class name_table {
public:
[[nodiscard]] auto name(std::uint32_t index) const -> std::string_view {
if (index >= this->names_.size()) throw map_error("asset name index out of range: " + std::to_string(index));
return this->names_[index];
}
/** The existing index of `name`, or a freshly appended one. */
[[nodiscard]] auto get_or_create(std::string_view name) -> std::uint32_t {
for (std::uint32_t i = 1; i < this->names_.size(); ++i) {
if (this->names_[i] == name) return i;
}
this->names_.emplace_back(name);
return static_cast<std::uint32_t>(this->names_.size() - 1U);
}
[[nodiscard]] auto entries() const -> const std::vector<std::string> & { return this->names_; }
[[nodiscard]] auto size() const -> std::size_t { return this->names_.empty() ? 0U : this->names_.size() - 1U; }
/** Resize the table (used while parsing); index 0 stays the unused slot. */
auto resize(std::size_t count) -> void { this->names_.resize(count); }
/** Assign the name at `index` (used while parsing). */
auto set_name(std::uint32_t index, std::string name) -> void { this->names_.at(index) = std::move(name); }
private:
std::vector<std::string> names_{std::string{}}; // index 0 is unused
};
/** One `CkMp` chunk: a named, versioned, opaque payload. */
struct map_chunk {
std::string name;
chunk_kind kind = chunk_kind::unknown;
std::uint32_t asset_index = 0;
std::uint16_t version = 0;
std::vector<std::byte> payload;
};
/** A `HeightMapData` border rectangle. */
struct height_map_border {
std::uint32_t corner1_x = 0;
std::uint32_t corner1_y = 0;
std::uint32_t x = 0;
std::uint32_t y = 0;
};
/** The parsed `HeightMapData` chunk: the terrain elevation grid. */
struct height_map_data {
std::uint32_t width = 0;
std::uint32_t height = 0;
std::uint32_t border_width = 0;
std::vector<height_map_border> borders;
std::vector<std::uint16_t> elevations; ///< `width * height`, row-major (y outer, x inner)
std::uint16_t version = 6;
/** Metres per elevation unit (uint16 grids scale by 0.0390625). */
[[nodiscard]] auto vertical_scale() const -> float { return this->version >= 5U ? 0.0390625F : 0.625F; }
[[nodiscard]] auto at(std::uint32_t x, std::uint32_t y) const -> std::uint16_t { return this->elevations.at(static_cast<std::size_t>(y) * this->width + x); }
auto set(std::uint32_t x, std::uint32_t y, std::uint16_t value) -> void { this->elevations.at(static_cast<std::size_t>(y) * this->width + x) = value; }
/** Parse a `HeightMapData` chunk payload. */
[[nodiscard]] static auto parse(std::uint16_t version, std::span<const std::byte> payload) -> height_map_data {
byte_reader reader{payload};
height_map_data data;
data.version = version;
data.width = reader.read_u32();
data.height = reader.read_u32();
data.border_width = reader.read_u32();
const auto border_count = reader.read_u32();
data.borders.reserve(border_count);
for (std::uint32_t i = 0; i < border_count; ++i) {
height_map_border border;
if (version >= 6U) {
border.corner1_x = reader.read_u32();
border.corner1_y = reader.read_u32();
}
border.x = reader.read_u32();
border.y = reader.read_u32();
data.borders.push_back(border);
}
const auto area = reader.read_u32();
const auto expected = static_cast<std::uint64_t>(data.width) * data.height;
if (area != expected) throw map_error("HeightMapData area does not match width * height");
data.elevations.resize(static_cast<std::size_t>(expected));
for (std::uint32_t y = 0; y < data.height; ++y) {
for (std::uint32_t x = 0; x < data.width; ++x) {
data.elevations[static_cast<std::size_t>(y) * data.width + x] = version >= 5U ? reader.read_u16() : reader.read_u8();
}
}
return data;
}
/** Serialise the chunk payload (without the 10-byte chunk header). */
[[nodiscard]] auto serialize() const -> std::vector<std::byte> {
byte_writer writer;
writer.write_u32(this->width);
writer.write_u32(this->height);
writer.write_u32(this->border_width);
writer.write_u32(static_cast<std::uint32_t>(this->borders.size()));
for (const auto &border: this->borders) {
if (this->version >= 6U) {
writer.write_u32(border.corner1_x);
writer.write_u32(border.corner1_y);
}
writer.write_u32(border.x);
writer.write_u32(border.y);
}
writer.write_u32(this->width * this->height);
for (const auto elevation: this->elevations) {
if (this->version >= 5U) {
writer.write_u16(elevation);
} else {
writer.write_u8(static_cast<std::uint8_t>(elevation));
}
}
return writer.take();
}
};
/** The value type of an asset property. */
enum class property_type : std::uint8_t {
boolean = 0,
integer = 1,
real_number = 2,
ascii_string = 3,
unicode_string = 4,
unknown = 5,
};
/** The payload of an `asset_property`; the active alternative tracks `property_type`. */
using property_value = std::variant<bool, std::int32_t, float, std::string, std::u16string>;
/** One key/value pair carried by a map object or the world info. */
struct asset_property {
std::string name;
property_type type = property_type::integer;
property_value value{}; ///< bool, int32, float, ASCII/UTF-8 string, or UTF-16 string
[[nodiscard]] static auto boolean(std::string name, bool value) -> asset_property {
return {std::move(name), property_type::boolean, value};
}
[[nodiscard]] static auto integer(std::string name, std::int32_t value) -> asset_property {
return {std::move(name), property_type::integer, value};
}
[[nodiscard]] static auto real(std::string name, float value) -> asset_property {
return {std::move(name), property_type::real_number, value};
}
[[nodiscard]] static auto text(std::string name, std::string value) -> asset_property {
return {std::move(name), property_type::ascii_string, std::move(value)};
}
[[nodiscard]] static auto wide_text(std::string name, std::u16string value) -> asset_property {
return {std::move(name), property_type::unicode_string, std::move(value)};
}
[[nodiscard]] auto as_bool() const -> bool {
const auto *held = std::get_if<bool>(&this->value);
return held != nullptr && *held;
}
[[nodiscard]] auto as_int() const -> std::int32_t {
const auto *held = std::get_if<std::int32_t>(&this->value);
return held != nullptr ? *held : 0;
}
[[nodiscard]] auto as_real() const -> float {
const auto *held = std::get_if<float>(&this->value);
return held != nullptr ? *held : 0.0F;
}
[[nodiscard]] auto as_ascii() const -> std::string_view {
const auto *held = std::get_if<std::string>(&this->value);
return held != nullptr ? std::string_view{*held} : std::string_view{};
}
[[nodiscard]] auto as_unicode() const -> const std::u16string * { return std::get_if<std::u16string>(&this->value); }
};
/** One object placed on the map (building, prop, waypoint, ...). */
struct map_object {
std::string type_name; ///< the SAGE `ThingTemplate` name, e.g. `*Waypoints/Waypoint`
coord3d position;
float angle = 0.0F;
road_type road = road_type::none; ///< road connectivity flags (0 for most objects)
std::uint16_t version = 1;
std::vector<asset_property> properties;
[[nodiscard]] auto property(std::string_view key) const -> const asset_property * {
for (const auto &entry: this->properties) {
if (entry.name == key) return &entry;
}
return nullptr;
}
};
/** A player start position recovered from a `Player_N_Start` waypoint. */
struct player_start {
int index = 0; ///< the `N` in `Player_N_Start` (1-based)
coord3d position;
};
/** One `MPPositionInfo` entry. */
struct mp_position {
bool is_human = false;
bool is_computer = false;
bool load_ai_script = false;
std::uint32_t team = 0;
std::vector<std::string> side_restrictions;
std::uint16_t version = 1;
};
/** One `WaypointPath` (a link between two waypoints by id). */
struct waypoint_path {
std::int32_t start_id = 0;
std::int32_t end_id = 0;
};
namespace detail {
/** True when `data` begins with the ASCII tag (avoids `memcmp` with a literal). */
[[nodiscard]] inline auto starts_with(std::span<const std::byte> data, std::string_view tag) -> bool {
if (data.size() < tag.size()) return false;
for (std::size_t i = 0; i < tag.size(); ++i) {
if (std::to_integer<std::uint8_t>(data[i]) != static_cast<std::uint8_t>(tag[i])) return false;
}
return true;
}
[[nodiscard]] inline auto read_property(byte_reader &reader, const name_table &names) -> asset_property {
asset_property property;
property.type = static_cast<property_type>(reader.read_u8());
property.name = names.name(reader.read_u24());
switch (property.type) {
case property_type::boolean: property.value = reader.read_bool(); break;
case property_type::integer: property.value = reader.read_i32(); break;
case property_type::real_number: property.value = reader.read_f32(); break;
case property_type::ascii_string:
case property_type::unknown: property.value = reader.read_u16_prefixed_ascii(); break;
case property_type::unicode_string: property.value = reader.read_u16_prefixed_utf16(); break;
}
return property;
}
inline auto write_property(byte_writer &writer, const asset_property &property, name_table &names) -> void {
writer.write_u8(static_cast<std::uint8_t>(property.type));
writer.write_u24(names.get_or_create(property.name));
switch (property.type) {
case property_type::boolean: writer.write_bool(property.as_bool()); break;
case property_type::integer: writer.write_i32(property.as_int()); break;
case property_type::real_number: writer.write_f32(property.as_real()); break;
case property_type::ascii_string:
case property_type::unknown: writer.write_u16_prefixed_ascii(property.as_ascii()); break;
case property_type::unicode_string: {
const auto *text = property.as_unicode();
writer.write_u16_prefixed_utf16(text != nullptr ? *text : std::u16string{});
break;
}
}
}
[[nodiscard]] inline auto parse_property_list(byte_reader &reader, const name_table &names) -> std::vector<asset_property> {
const auto count = reader.read_u16();
std::vector<asset_property> properties;
properties.reserve(count);
for (std::uint16_t i = 0; i < count; ++i) properties.push_back(read_property(reader, names));
return properties;
}
inline auto write_property_list(byte_writer &writer, const std::vector<asset_property> &properties, name_table &names) -> void {
writer.write_u16(static_cast<std::uint16_t>(properties.size()));
for (const auto &property: properties) write_property(writer, property, names);
}
/** Split `Player_<n>_Start` into `n`, or 0 when the name does not match. */
[[nodiscard]] inline auto parse_player_start_name(std::string_view name) -> int {
constexpr std::string_view prefix = "Player_";
constexpr std::string_view suffix = "_Start";
if (!name.starts_with(prefix) || !name.ends_with(suffix)) return 0;
const auto digits = name.substr(prefix.size(), name.size() - prefix.size() - suffix.size());
if (digits.empty() || !std::ranges::all_of(digits, [](char ch) { return ch >= '0' && ch <= '9'; })) return 0;
return std::stoi(std::string{digits});
}
}
/**
* A parsed `.map`: the asset-name table plus the ordered chunk list.
*/
class map_document {
public:
/** Unwrap (BIG payload / `EAR\0` / bare RefPack / bare `CkMp`) and parse. */
[[nodiscard]] static auto parse(std::span<const std::byte> file) -> map_document {
auto data = std::vector<std::byte>{file.begin(), file.end()};
if (is_refpack(data)) data = refpack_decompress(data); // a BIG payload
if (detail::starts_with(data, std::string_view{"EAR\0", 4})) {
const auto *body = reinterpret_cast<const std::byte *>(data.data()) + 8;
std::span<const std::byte> payload{body, data.size() - 8U};
data = maybe_decompress(payload);
} else if (is_refpack(data)) {
data = refpack_decompress(data);
}
return map_document::from_ckmp(data);
}
[[nodiscard]] static auto open(const std::filesystem::path &path) -> map_document {
return map_document::parse(read_file(path));
}
[[nodiscard]] auto names() const -> const name_table & { return this->names_; }
[[nodiscard]] auto names() -> name_table & { return this->names_; }
[[nodiscard]] auto chunks() const -> const std::vector<map_chunk> & { return this->chunks_; }
[[nodiscard]] auto chunks() -> std::vector<map_chunk> & { return this->chunks_; }
[[nodiscard]] auto find_chunk(std::string_view name) const -> const map_chunk * {
for (const auto &chunk: this->chunks_) {
if (chunk.name == name) return &chunk;
}
return nullptr;
}
[[nodiscard]] auto find_chunk(std::string_view name) -> map_chunk * {
for (auto &chunk: this->chunks_) {
if (chunk.name == name) return &chunk;
}
return nullptr;
}
[[nodiscard]] auto has_chunk(std::string_view name) const -> bool { return this->find_chunk(name) != nullptr; }
/** The chunk of a known kind, or `nullptr`. */
[[nodiscard]] auto find_chunk(chunk_kind kind) const -> const map_chunk * {
for (const auto &chunk: this->chunks_) {
if (chunk.kind == kind) return &chunk;
}
return nullptr;
}
[[nodiscard]] auto find_chunk(chunk_kind kind) -> map_chunk * {
for (auto &chunk: this->chunks_) {
if (chunk.kind == kind) return &chunk;
}
return nullptr;
}
[[nodiscard]] auto has_chunk(chunk_kind kind) const -> bool { return this->find_chunk(kind) != nullptr; }
// -- typed chunks ---------------------------------------------------
/** The terrain grid, or `std::nullopt` when the map has no `HeightMapData`. */
[[nodiscard]] auto height_map() const -> std::optional<height_map_data> {
const auto *chunk = this->find_chunk("HeightMapData");
if (chunk == nullptr) return std::nullopt;
return height_map_data::parse(chunk->version, chunk->payload);
}
/** Replace (or create) the `HeightMapData` chunk from a grid. */
auto set_height_map(height_map_data data) -> void {
auto *chunk = this->find_chunk("HeightMapData");
if (chunk == nullptr) {
chunk = &this->chunks_.emplace_back();
chunk->name = "HeightMapData";
chunk->kind = chunk_kind::height_map_data;
chunk->asset_index = this->names_.get_or_create("HeightMapData");
chunk->version = data.version;
} else {
data.version = chunk->version; // keep the file's on-disk version
}
chunk->payload = data.serialize();
}
/** Every object the map places. */
[[nodiscard]] auto objects() const -> std::vector<map_object> {
const auto *chunk = this->find_chunk("ObjectsList");
if (chunk == nullptr) return {};
return map_document::decode_objects(this->names_, chunk->payload);
}
/** Replace (or create) the `ObjectsList` chunk. */
auto set_objects(const std::vector<map_object> &objects) -> void {
auto *chunk = this->find_chunk("ObjectsList");
if (chunk == nullptr) {
chunk = &this->chunks_.emplace_back();
chunk->name = "ObjectsList";
chunk->kind = chunk_kind::objects_list;
chunk->asset_index = this->names_.get_or_create("ObjectsList");
chunk->version = 1;
}
chunk->payload = map_document::encode_objects(this->names_, objects);
}
/** The `Player_N_Start` waypoints, ordered by index. */
[[nodiscard]] auto player_starts() const -> std::vector<player_start> {
std::vector<player_start> starts;
for (const auto &object: this->objects()) {
if (object.type_name != "*Waypoints/Waypoint") continue;
const auto *name_property = object.property("waypointName");
if (name_property == nullptr) continue;
const auto index = detail::parse_player_start_name(name_property->as_ascii());
if (index == 0) continue;
starts.push_back({index, object.position});
}
std::ranges::sort(starts, {}, &player_start::index);
return starts;
}
[[nodiscard]] auto mp_positions() const -> std::vector<mp_position> {
const auto *chunk = this->find_chunk("MPPositionList");
if (chunk == nullptr) return {};
return map_document::decode_mp_positions(chunk->payload);
}
auto set_mp_positions(const std::vector<mp_position> &positions) -> void {
auto *chunk = this->find_chunk("MPPositionList");
if (chunk == nullptr) {
chunk = &this->chunks_.emplace_back();
chunk->name = "MPPositionList";
chunk->kind = chunk_kind::mp_position_list;
chunk->asset_index = this->names_.get_or_create("MPPositionList");
chunk->version = 1;
}
chunk->payload = map_document::encode_mp_positions(this->names_, positions);
}
/** The `WorldInfo` property list. */
[[nodiscard]] auto world_info() const -> std::vector<asset_property> {
const auto *chunk = this->find_chunk("WorldInfo");
if (chunk == nullptr) return {};
byte_reader reader{chunk->payload};
return detail::parse_property_list(reader, this->names_);
}
auto set_world_info(const std::vector<asset_property> &properties) -> void {
auto *chunk = this->find_chunk("WorldInfo");
if (chunk == nullptr) {
chunk = &this->chunks_.emplace_back();
chunk->name = "WorldInfo";
chunk->kind = chunk_kind::world_info;
chunk->asset_index = this->names_.get_or_create("WorldInfo");
chunk->version = 1;
}
byte_writer writer;
detail::write_property_list(writer, properties, this->names_);
chunk->payload = writer.take();
}
/** The waypoint paths (`WaypointsList`). */
[[nodiscard]] auto waypoint_paths() const -> std::vector<waypoint_path> {
const auto *chunk = this->find_chunk("WaypointsList");
if (chunk == nullptr) return {};
byte_reader reader{chunk->payload};
const auto count = reader.read_u32();
std::vector<waypoint_path> paths;
paths.reserve(count);
for (std::uint32_t i = 0; i < count; ++i) paths.push_back({reader.read_i32(), reader.read_i32()});
return paths;
}
// -- serialisation --------------------------------------------------
/** The bare `CkMp` byte image. */
[[nodiscard]] auto to_ckmp() const -> std::vector<std::byte> {
byte_writer writer;
writer.write_ascii("CkMp");
const auto count = static_cast<std::uint32_t>(this->names_.size());
writer.write_u32(count);
for (std::uint32_t i = count; i >= 1U; --i) {
const auto &name = this->names_.entries()[i];
writer.write_u8(static_cast<std::uint8_t>(name.size()));
writer.write_ascii(name);
writer.write_u32(i);
}
for (const auto &chunk: this->chunks_) {
writer.write_u32(chunk.asset_index);
writer.write_u16(chunk.version);
writer.write_u32(static_cast<std::uint32_t>(chunk.payload.size()));
writer.write_bytes(chunk.payload);
}
return writer.take();
}
/**
* A standalone `.map` image.
*
* When `compress` is true the result is `EAR\0` + `u32` decompressed size
* + a RefPack stream (the retail compression). When false it is a bare
* `CkMp` tree. To store a map inside a `BIG4`, pass the *uncompressed*
* image and let `big_writer::add(..., compress = true)` wrap it once.
*/
[[nodiscard]] auto serialize(bool compress = false) const -> std::vector<std::byte> {
auto ckmp = this->to_ckmp();
if (!compress) return ckmp;
byte_writer writer;
writer.write_ascii(std::string_view{"EAR\0", 4});
writer.write_u32(static_cast<std::uint32_t>(ckmp.size()));
const auto packed = refpack_compress(ckmp);
writer.write_bytes(packed);
return writer.take();
}
/** True when the name table carries `name` (top-level chunk, property, ...). */
[[nodiscard]] auto has_name(std::string_view name) const -> bool {
for (std::uint32_t i = 1; i < this->names_.entries().size(); ++i) {
if (this->names_.entries()[i] == name) return true;
}
return false;
}
private:
map_document() = default;
[[nodiscard]] static auto from_ckmp(std::span<const std::byte> data) -> map_document {
if (data.size() < 8U || !detail::starts_with(data, "CkMp")) throw map_error("not a CkMp map");
byte_reader reader{data};
(void) reader.read_ascii(4);
const auto count = reader.read_u32();
map_document document;
document.names_.resize(count + 1U);
for (std::uint32_t i = count; i >= 1U; --i) {
const auto length = reader.read_u8();
auto name = reader.read_ascii(length);
const auto index = reader.read_u32();
if (index != i) throw map_error("asset name table index mismatch");
document.names_.set_name(i, std::move(name));
}
while (reader.remaining() >= 10U) {
const auto index = reader.read_u32();
const auto version = reader.read_u16();
const auto size = reader.read_u32();
if (index >= document.names_.entries().size() || reader.remaining() < size) throw map_error("truncated CkMp chunk");
map_chunk chunk;
chunk.asset_index = index;
chunk.name = std::string{document.names_.entries()[index]};
chunk.kind = chunk_kind_of(chunk.name);
chunk.version = version;
const auto payload = reader.read_bytes(size);
chunk.payload.assign(payload.begin(), payload.end());
document.chunks_.push_back(std::move(chunk));
}
return document;
}
[[nodiscard]] static auto decode_objects(const name_table &names, std::span<const std::byte> payload) -> std::vector<map_object> {
byte_reader reader{payload};
std::vector<map_object> objects;
while (reader.remaining() >= 10U) {
(void) reader.read_u32(); // nested asset index (always "Object")
const auto version = reader.read_u16();
const auto size = reader.read_u32();
if (reader.remaining() < size) throw map_error("truncated Object asset");
const auto body = reader.read_bytes(size);
byte_reader body_reader{body};
map_object object;
object.version = version;
object.position.x = body_reader.read_f32();
object.position.y = body_reader.read_f32();
object.position.z = body_reader.read_f32();
object.angle = body_reader.read_f32();
object.road = static_cast<road_type>(body_reader.read_u32());
object.type_name = body_reader.read_ascii(body_reader.read_u16());
object.properties = detail::parse_property_list(body_reader, names);
objects.push_back(std::move(object));
}
return objects;
}
[[nodiscard]] static auto encode_objects(name_table &names, const std::vector<map_object> &objects) -> std::vector<std::byte> {
const auto object_index = names.get_or_create("Object");
byte_writer writer;
for (const auto &object: objects) {
byte_writer body;
body.write_f32(object.position.x);
body.write_f32(object.position.y);
body.write_f32(object.position.z);
body.write_f32(object.angle);
body.write_u32(static_cast<std::uint32_t>(object.road));
body.write_u16(static_cast<std::uint16_t>(object.type_name.size()));
body.write_ascii(object.type_name);
detail::write_property_list(body, object.properties, names);
writer.write_u32(object_index);
writer.write_u16(object.version);
writer.write_u32(static_cast<std::uint32_t>(body.size()));
writer.write_bytes(body.data());
}
return writer.take();
}
[[nodiscard]] static auto decode_mp_positions(std::span<const std::byte> payload) -> std::vector<mp_position> {
byte_reader reader{payload};
std::vector<mp_position> positions;
while (reader.remaining() >= 10U) {
(void) reader.read_u32(); // nested asset index (always "MPPositionInfo")
const auto version = reader.read_u16();
const auto size = reader.read_u32();
if (reader.remaining() < size) throw map_error("truncated MPPositionInfo asset");
const auto body = reader.read_bytes(size);
byte_reader body_reader{body};
mp_position position;
position.version = version;
position.is_human = body_reader.read_bool();
position.is_computer = body_reader.read_bool();
if (version > 0U) position.load_ai_script = body_reader.read_bool();
position.team = body_reader.read_u32();
if (version > 0U) {
const auto count = body_reader.read_u32();
position.side_restrictions.reserve(count);
for (std::uint32_t i = 0; i < count; ++i) position.side_restrictions.push_back(body_reader.read_u16_prefixed_ascii());
}
positions.push_back(std::move(position));
}
return positions;
}
[[nodiscard]] static auto encode_mp_positions(name_table &names, const std::vector<mp_position> &positions) -> std::vector<std::byte> {
const auto info_index = names.get_or_create("MPPositionInfo");
byte_writer writer;
for (const auto &position: positions) {
byte_writer body;
body.write_bool(position.is_human);
body.write_bool(position.is_computer);
if (position.version > 0U) body.write_bool(position.load_ai_script);
body.write_u32(position.team);
if (position.version > 0U) {
body.write_u32(static_cast<std::uint32_t>(position.side_restrictions.size()));
for (const auto &side: position.side_restrictions) body.write_u16_prefixed_ascii(side);
}
writer.write_u32(info_index);
writer.write_u16(position.version);
writer.write_u32(static_cast<std::uint32_t>(body.size()));
writer.write_bytes(body.data());
}
return writer.take();
}
name_table names_;
std::vector<map_chunk> chunks_;
};
} // namespace ra3::assets
third_party/libra3assets/src/refpack.cppm
Vendored
+284
View File
@@ -0,0 +1,284 @@
/**
* EA's RefPack compression codec.
*
* Red Alert 3 compresses individual `BIG4` payloads and every SAGE `.map` with
* RefPack (the `10 FB` stream shared across EA titles). This partition decodes
* and encodes that stream. The decoder is a port of the reference used across
* the project (`ra3tools/ra3_big.py`, `OpenRA3`'s `ra3.fs`); the encoder is a
* greedy LZ77 matcher that emits only canonical tokens, so anything it produces
* is readable by the same decoder (and by the game).
*/
export module ra3.assets:refpack;
import std;
import :bytes;
import :error;
export namespace ra3::assets {
/** Max back-reference distance (the 17-bit RefPack window). */
inline constexpr std::size_t refpack_window = 1U << 17U;
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const std::byte> data) -> bool {
return data.size() >= 2U && (std::to_integer<std::uint8_t>(data[0]) & 0x3EU) == 0x10U && std::to_integer<std::uint8_t>(data[1]) == 0xFBU;
}
/** Read the declared output size from a RefPack header without decompressing. */
[[nodiscard]] inline auto refpack_output_size(std::span<const std::byte> data) -> std::uint32_t {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
std::size_t pos = 0;
const auto header = std::to_integer<std::uint8_t>(data[pos++]);
const bool large_files = (header & 0x80U) != 0U;
const bool compressed_size_present = (header & 0x01U) != 0U;
pos++; // 0xFB
const std::size_t size_bytes = large_files ? 4U : 3U;
const auto read_size = [&]() -> std::uint32_t {
std::uint32_t value = 0;
for (std::size_t i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8U) | std::to_integer<std::uint8_t>(data[pos++]);
}
return value;
};
if (compressed_size_present) (void) read_size();
return read_size();
}
/**
* Decompress an EA RefPack stream.
*
* @throws refpack_error if the stream is malformed or the length disagrees.
*/
[[nodiscard]] inline auto refpack_decompress(std::span<const std::byte> data) -> std::vector<std::byte> {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
std::size_t pos = 0;
const auto header = std::to_integer<std::uint8_t>(data[pos++]);
const bool large_files = (header & 0x80U) != 0U;
const bool compressed_size_present = (header & 0x01U) != 0U;
pos++; // 0xFB
const std::size_t size_bytes = large_files ? 4U : 3U;
const auto read_size = [&]() -> std::uint32_t {
std::uint32_t value = 0;
for (std::size_t i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8U) | std::to_integer<std::uint8_t>(data[pos++]);
}
return value;
};
if (compressed_size_present) (void) read_size();
const auto out_len = read_size();
std::vector<std::byte> out;
out.reserve(out_len); // guarantees no reallocation, so overlapping reads stay valid
const auto copy_literals = [&](std::size_t count) {
if (pos + count > data.size()) throw refpack_error("truncated RefPack literals");
out.insert(out.end(), data.begin() + static_cast<std::ptrdiff_t>(pos), data.begin() + static_cast<std::ptrdiff_t>(pos + count));
pos += count;
};
// A back-reference may overlap its own output (an RLE run): out[start + i]
// is read one byte at a time, so the pattern repeats correctly.
const auto copy_reference = [&](std::size_t length, std::size_t distance) {
if (distance == 0U || distance > out.size()) throw refpack_error("RefPack back-reference out of range");
const auto start = out.size() - distance;
if (out.size() + length > out_len) throw refpack_error("RefPack output overrun");
for (std::size_t i = 0; i < length; ++i) out.push_back(out[start + i]);
};
while (pos < data.size()) {
const auto cmd = std::to_integer<std::uint8_t>(data[pos++]);
if ((cmd & 0x80U) == 0U) { // 2-byte command
if (pos >= data.size()) throw refpack_error("truncated 2-byte command");
const auto b2 = std::to_integer<std::uint8_t>(data[pos++]);
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x1CU) >> 2U) + 3U, ((cmd & 0x60U) << 3U) + b2 + 1U);
} else if ((cmd & 0x40U) == 0U) { // 3-byte command
if (pos + 1U >= data.size()) throw refpack_error("truncated 3-byte command");
const auto b2 = std::to_integer<std::uint8_t>(data[pos]);
const auto b3 = std::to_integer<std::uint8_t>(data[pos + 1U]);
pos += 2U;
copy_literals((b2 & 0xC0U) >> 6U);
copy_reference((cmd & 0x3FU) + 4U, ((b2 & 0x3FU) << 8U) + b3 + 1U);
} else if ((cmd & 0x20U) == 0U) { // 4-byte command
if (pos + 2U >= data.size()) throw refpack_error("truncated 4-byte command");
const auto b2 = std::to_integer<std::uint8_t>(data[pos]);
const auto b3 = std::to_integer<std::uint8_t>(data[pos + 1U]);
const auto b4 = std::to_integer<std::uint8_t>(data[pos + 2U]);
pos += 3U;
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x0CU) << 6U) + b4 + 5U, ((cmd & 0x10U) << 12U) + (static_cast<std::size_t>(b2) << 8U) + b3 + 1U);
} else if (cmd < 0xFCU) { // long literal run
copy_literals((static_cast<std::size_t>(cmd & 0x1FU) + 1U) << 2U);
} else { // stop
copy_literals(cmd & 0x03U);
break;
}
}
if (out.size() != out_len) throw refpack_error("RefPack length mismatch");
return out;
}
/** Decompress `data` when it is RefPack, otherwise copy it unchanged. */
[[nodiscard]] inline auto maybe_decompress(std::span<const std::byte> data) -> std::vector<std::byte> {
if (is_refpack(data)) return refpack_decompress(data);
return {data.begin(), data.end()};
}
namespace detail {
/** True when `(length, distance)` maps to one of the three canonical tokens. */
[[nodiscard]] inline constexpr auto refpack_match_encodable(std::size_t length, std::size_t distance) -> bool {
if (length >= 3U && length <= 10U && distance >= 1U && distance <= 1024U) return true;
if (length >= 4U && length <= 67U && distance >= 1U && distance <= 16384U) return true;
if (length >= 5U && length <= 1028U && distance >= 1U && distance <= 131072U) return true;
return false;
}
/** Greedy RefPack encoder state. */
class refpack_encoder {
public:
explicit refpack_encoder(std::span<const std::byte> input)
: input_(input), chain_(input.size(), -1) {
}
[[nodiscard]] auto run() -> std::vector<std::byte> {
const auto n = this->input_.size();
const bool large = n >= (1U << 24U);
this->out_.write_u8(static_cast<std::uint8_t>(0x10U | (large ? 0x80U : 0x00U)));
this->out_.write_u8(0xFBU);
const std::size_t size_bytes = large ? 4U : 3U;
for (std::size_t i = size_bytes; i-- > 0U;) this->out_.write_u8(static_cast<std::uint8_t>((n >> (8U * i)) & 0xFFU));
std::size_t pos = 0;
std::size_t literals_start = 0;
while (pos < n) {
std::size_t best_len = 0;
std::size_t best_dist = 0;
if (pos + 2U < n) this->find_match(pos, best_len, best_dist);
if (best_len >= 3U) {
const auto pending = pos - literals_start;
const auto carry = pending % 4U; // 0..3 literals ride with the token
const auto run = pending - carry; // always a multiple of 4
this->emit_literal_run(literals_start, run);
this->emit_match(literals_start + run, carry, best_len, best_dist);
for (std::size_t i = pos; i < pos + best_len; ++i) this->insert(i);
pos += best_len;
literals_start = pos;
} else {
this->insert(pos);
++pos;
}
}
const auto pending = n - literals_start;
const auto carry = pending % 4U;
this->emit_literal_run(literals_start, pending - carry);
this->out_.write_u8(static_cast<std::uint8_t>(0xFCU | carry));
this->write_literals(literals_start + (pending - carry), carry);
return this->out_.take();
}
private:
void insert(std::size_t pos) {
if (pos + 2U >= this->input_.size()) return;
const auto bucket = this->hash3(pos);
this->chain_[pos] = this->head_[bucket];
this->head_[bucket] = static_cast<std::int32_t>(pos);
}
void find_match(std::size_t pos, std::size_t &best_len, std::size_t &best_dist) const {
const auto n = this->input_.size();
const auto max_len = std::min<std::size_t>(1028U, n - pos);
auto candidate = this->head_[this->hash3(pos)];
int depth = 0;
while (candidate >= 0 && depth < 64) {
const auto c = static_cast<std::size_t>(candidate);
const auto distance = pos - c;
if (distance > refpack_window) break; // the chain only walks backwards
std::size_t length = 0;
while (length < max_len && this->input_[c + length] == this->input_[pos + length]) ++length;
if (length > best_len && refpack_match_encodable(length, distance)) {
best_len = length;
best_dist = distance;
if (length == max_len) break;
}
candidate = this->chain_[c];
++depth;
}
}
void emit_literal_run(std::size_t offset, std::size_t count) {
std::size_t remaining = count;
std::size_t at = offset;
while (remaining >= 4U) {
std::size_t step = std::min<std::size_t>(112U, remaining);
step -= step % 4U;
if (step < 4U) step = 4U;
this->out_.write_u8(static_cast<std::uint8_t>(0xE0U | ((step / 4U) - 1U)));
this->write_literals(at, step);
at += step;
remaining -= step;
}
}
void emit_match(std::size_t literal_offset, std::size_t literal_count, std::size_t length, std::size_t distance) {
const auto d = static_cast<std::uint32_t>(distance - 1U);
if (length >= 3U && length <= 10U && distance <= 1024U) { // 2-byte token
const auto cmd = static_cast<std::uint8_t>((((d >> 8U) & 0x03U) << 5U) | (static_cast<std::uint32_t>(length - 3U) << 2U) |
static_cast<std::uint32_t>(literal_count));
this->out_.write_u8(cmd);
this->out_.write_u8(static_cast<std::uint8_t>(d & 0xFFU));
} else if (length >= 4U && length <= 67U && distance <= 16384U) { // 3-byte token
this->out_.write_u8(static_cast<std::uint8_t>(0x80U | (length - 4U)));
this->out_.write_u8(static_cast<std::uint8_t>((literal_count << 6U) | ((d >> 8U) & 0x3FU)));
this->out_.write_u8(static_cast<std::uint8_t>(d & 0xFFU));
} else if (length >= 5U && length <= 1028U && distance <= 131072U) { // 4-byte token
const auto l = static_cast<std::uint32_t>(length - 5U);
const auto cmd = static_cast<std::uint8_t>(0xC0U | (((l >> 8U) & 0x03U) << 2U) | ((d >> 12U) & 0x10U) |
static_cast<std::uint32_t>(literal_count));
this->out_.write_u8(cmd);
this->out_.write_u8(static_cast<std::uint8_t>((d >> 8U) & 0xFFU));
this->out_.write_u8(static_cast<std::uint8_t>(d & 0xFFU));
this->out_.write_u8(static_cast<std::uint8_t>(l & 0xFFU));
} else {
throw refpack_error("internal error: unencodable RefPack match");
}
this->write_literals(literal_offset, literal_count);
}
void write_literals(std::size_t offset, std::size_t count) {
for (std::size_t i = 0; i < count; ++i) this->out_.write_u8(std::to_integer<std::uint8_t>(this->input_[offset + i]));
}
[[nodiscard]] auto hash3(std::size_t pos) const -> std::size_t {
const auto a = static_cast<std::uint32_t>(std::to_integer<std::uint8_t>(this->input_[pos]));
const auto b = static_cast<std::uint32_t>(std::to_integer<std::uint8_t>(this->input_[pos + 1U]));
const auto c = static_cast<std::uint32_t>(std::to_integer<std::uint8_t>(this->input_[pos + 2U]));
return ((a | (b << 8U) | (c << 16U)) * 2654435761U) >> (32U - 16U);
}
std::span<const std::byte> input_;
std::vector<std::int32_t> chain_;
std::vector<std::int32_t> head_ = std::vector<std::int32_t>(1U << 16U, -1);
byte_writer out_;
};
}
/**
* Compress a byte range into a canonical RefPack stream.
*
* The encoder is a plain greedy LZ77: it never emits a token form the
* decoder above cannot read, so `refpack_decompress(refpack_compress(x))`
* is lossless for every input.
*/
[[nodiscard]] inline auto refpack_compress(std::span<const std::byte> input) -> std::vector<std::byte> {
return detail::refpack_encoder{input}.run();
}
} // namespace ra3::assets
third_party/libra3assets/tests/test_assets.cpp
Vendored
+310
View File
@@ -0,0 +1,310 @@
#include "test_main.hpp"
import std;
import ra3.assets;
using namespace ra3::assets;
namespace {
auto bytes_of(std::string_view text) -> std::vector<std::byte> {
std::vector<std::byte> out;
out.reserve(text.size());
for (const auto ch: text) out.push_back(static_cast<std::byte>(static_cast<unsigned char>(ch)));
return out;
}
auto text_of(std::span<const std::byte> bytes) -> std::string {
return std::string{reinterpret_cast<const char *>(bytes.data()), bytes.size()};
}
/** A deterministic pseudo-random buffer (no external RNG needed). */
auto pseudo_random(std::size_t size, std::uint32_t seed) -> std::vector<std::byte> {
std::vector<std::byte> out;
out.reserve(size);
std::uint32_t state = seed;
for (std::size_t i = 0; i < size; ++i) {
state = state * 1664525U + 1013904223U;
out.push_back(static_cast<std::byte>((state >> 16U) & 0xFFU));
}
return out;
}
auto make_empty_ckmp(const std::vector<std::string> &names) -> std::vector<std::byte> {
byte_writer writer;
writer.write_ascii("CkMp");
writer.write_u32(static_cast<std::uint32_t>(names.size()));
for (std::uint32_t i = static_cast<std::uint32_t>(names.size()); i >= 1U; --i) {
writer.write_u8(static_cast<std::uint8_t>(names[i - 1].size()));
writer.write_ascii(names[i - 1]);
writer.write_u32(i);
}
return writer.take();
}
auto make_csf() -> std::vector<std::byte> {
byte_writer writer;
writer.write_ascii(" FSC");
writer.write_u32(3);
writer.write_u32(2); // labels
writer.write_u32(2); // value blocks
writer.write_u32(0);
writer.write_u32(0);
writer.write_u32(csf_label_flag);
writer.write_u32(1);
writer.write_u32(3);
writer.write_ascii("ABC");
writer.write_u32(csf_value_flag);
writer.write_u32(5);
for (const auto ch: std::string_view{"Hello"}) writer.write_u16(static_cast<std::uint16_t>(ch) ^ 0xFFFFU);
writer.write_u32(csf_label_flag);
writer.write_u32(1);
writer.write_u32(3);
writer.write_ascii("UNI");
writer.write_u32(csf_value_flag);
writer.write_u32(1);
writer.write_u16(0x4E2DU ^ 0xFFFFU); // U+4E2D (CJK)
return writer.take();
}
} // namespace
TEST(refpack_roundtrip_small) {
const std::vector<std::string> samples{"", "A", "AB", "ABC", "ABCD", "abcabcabc", std::string(300, 'z')};
for (const auto &sample: samples) {
const auto input = bytes_of(sample);
const auto packed = refpack_compress(input);
const auto restored = refpack_decompress(packed);
CHECK_EQ(restored.size(), input.size());
CHECK(std::ranges::equal(restored, input));
}
}
TEST(refpack_roundtrip_large_and_repetitive) {
std::vector<std::byte> input;
for (int i = 0; i < 2000; ++i) {
for (const auto ch: std::string_view{"lorem ipsum dolor sit amet "}) input.push_back(static_cast<std::byte>(ch));
}
const auto random = pseudo_random(50000, 0x1234ABCDU);
input.insert(input.end(), random.begin(), random.end());
const auto packed = refpack_compress(input);
CHECK(packed.size() < input.size()); // the repeated prefix must compress
CHECK(std::ranges::equal(refpack_decompress(packed), input));
}
TEST(refpack_decode_known_stream) {
// "abcd" literals, then a back-reference (distance 4, length 3) -> "abcdabc".
const std::vector<std::byte> stream{
std::byte{0x10}, std::byte{0xFB}, std::byte{0x00}, std::byte{0x00}, std::byte{0x07}, std::byte{0xE0},
std::byte{'a'}, std::byte{'b'}, std::byte{'c'}, std::byte{'d'}, std::byte{0x00}, std::byte{0x03},
std::byte{0xFC},
};
CHECK(is_refpack(stream));
CHECK_EQ(text_of(refpack_decompress(stream)), "abcdabc");
CHECK_EQ(refpack_output_size(stream), 7U);
}
TEST(big_roundtrip) {
big_writer writer;
writer.add("data\\a.txt", bytes_of("hello world"));
writer.add("data\\b.bin", bytes_of(std::string(1000, 'x')), true);
const auto image = writer.write();
CHECK(image.size() >= 16U);
const std::string magic{reinterpret_cast<const char *>(image.data()), 4};
CHECK_EQ(magic, "BIG4");
const auto archive = big_archive::from_bytes(image);
CHECK_EQ(archive.size(), 2U);
CHECK(archive.contains("data\\a.txt"));
CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello world");
CHECK_EQ(archive.read("data\\b.bin").size(), 1000U);
CHECK_EQ(archive.find("data\\").size(), 2U);
// The first payload starts 64-byte aligned, as the retail archives do.
CHECK_EQ(archive.entries()[0].offset % 64U, 0U);
}
TEST(big_open_from_disk_is_lazy) {
big_writer writer;
writer.add("data\\a.txt", bytes_of("hello disk"));
writer.add("data\\b.bin", bytes_of(std::string(4096, 'q')), true);
const auto image = writer.write();
const auto path = std::filesystem::temp_directory_path() / "libra3assets_big_lazy.big";
write_file(path, image);
const auto archive = big_archive::open(path);
CHECK_EQ(archive.size(), 2U);
CHECK(archive.path() == path);
CHECK(archive.contains("data\\a.txt"));
CHECK_EQ(archive.entries()[0].offset % 64U, 0U);
CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello disk");
CHECK_EQ(archive.read("data\\b.bin").size(), 4096U);
CHECK_EQ(text_of(archive.read_prefix("data\\a.txt", 5)), "hello");
CHECK(archive.read_prefix("data\\b.bin", 8U).size() == 8U);
std::error_code ec;
std::filesystem::remove(path, ec);
}
TEST(binary_manifest_and_hash) {
CHECK_EQ(hash_string("W3DMesh"), 0xC2B1A262U);
CHECK_EQ(hash_string("ABAIRFIELD", false), 0x2B479BD3U);
CHECK(hash_string("Texture") != hash_string("Texture", false));
const std::string name = "W3DMesh:TEST";
const std::string source = "art:test.w3x";
byte_writer manifest;
manifest.write_u8(0); // not big-endian
manifest.write_u8(0); // not linked
manifest.write_u16(5); // version
manifest.write_u32(0); // checksum
manifest.write_u32(0); // all-types hash
manifest.write_u32(1); // count
manifest.write_u32(12);
manifest.write_u32(0);
manifest.write_u32(0);
manifest.write_u32(0);
manifest.write_u32(0); // reference buffer
manifest.write_u32(0); // reference-name buffer
manifest.write_u32(static_cast<std::uint32_t>(name.size() + 1));
manifest.write_u32(static_cast<std::uint32_t>(source.size() + 1));
manifest.write_u32(hash_string("W3DMesh"));
manifest.write_u32(hash_string("TEST", false));
manifest.write_u32(hash_string("W3DMesh"));
manifest.write_u32(hash_string("TEST", false));
manifest.write_i32(0); // reference offset
manifest.write_i32(0); // reference count
manifest.write_i32(0); // name offset
manifest.write_i32(0); // source offset
manifest.write_u32(8); // instance size
manifest.write_u32(0); // relocation size
manifest.write_u32(0); // imports size
manifest.write_u32(0); // tokenized
manifest.write_ascii(name);
manifest.write_u8(0);
manifest.write_ascii(source);
manifest.write_u8(0);
byte_writer data;
data.write_u32(0); // stream checksum
data.write_ascii("PAYLOAD!");
auto container = binary_container::from_bytes(manifest.take(), data.take());
CHECK_EQ(container.size(), 1U);
const auto &asset = container.assets()[0];
CHECK_EQ(asset.name, "W3DMesh:TEST");
CHECK_EQ(asset.type_name(), "W3DMesh");
CHECK_EQ(asset.instance_name(), "TEST");
CHECK_EQ(asset.source, "art:test.w3x");
CHECK_EQ(text_of(container.read_instance(asset)), "PAYLOAD!");
CHECK_EQ(container.find("#0").name, "W3DMesh:TEST");
CHECK_EQ(container.of_type("w3dmesh").size(), 1U);
CHECK_EQ(container.type_counts().at("W3DMesh"), 1U);
CHECK(container.read_relocation(asset).empty());
container.set_cdata_source([&asset](const std::string &cdata_name) -> std::optional<std::vector<std::byte>> {
if (cdata_name == asset.cdata_name("static")) return bytes_of("CDATA");
return std::nullopt;
});
CHECK(container.read_cdata(asset).has_value());
CHECK_EQ(text_of(container.read_payload(asset)), "CDATA");
}
TEST(csf_roundtrip) {
auto table = csf_table::parse(make_csf());
CHECK_EQ(table.size(), 2U);
CHECK_EQ(table.version(), 3U);
CHECK_EQ(table.lookup("abc"), "Hello"); // lookup is case-insensitive
CHECK_EQ(table.lookup("ABC"), "Hello");
CHECK_EQ(table.lookup("uni"), "\xE4\xB8\xAD");
CHECK(table.find("missing") == nullptr);
const auto rewritten = table.write();
const auto reparsed = csf_table::parse(rewritten);
CHECK_EQ(reparsed.size(), 2U);
CHECK_EQ(reparsed.lookup("abc"), "Hello");
CHECK_EQ(reparsed.lookup("uni"), "\xE4\xB8\xAD");
}
TEST(csf_decode_utf16_surrogate) {
const std::u16string emoji = u"\U0001F600";
CHECK_EQ(utf16_to_utf8(emoji), "\xF0\x9F\x98\x80");
}
TEST(map_roundtrip) {
const std::vector<std::string> names{"HeightMapData", "ObjectsList", "Object", "waypointName", "MPPositionList", "MPPositionInfo", "WorldInfo", "waypointID"};
auto document = map_document::parse(make_empty_ckmp(names));
CHECK_EQ(document.names().size(), names.size());
height_map_data height;
height.width = 2;
height.height = 2;
height.version = 6;
height.elevations = {1, 2, 3, 4};
document.set_height_map(height);
map_object waypoint;
waypoint.type_name = "*Waypoints/Waypoint";
waypoint.position = {100.0F, 200.0F, 0.0F};
waypoint.properties.push_back(asset_property::text("waypointName", "Player_1_Start"));
waypoint.properties.push_back(asset_property::integer("waypointID", 0));
document.set_objects({waypoint});
mp_position position;
position.is_human = true;
position.team = 1;
document.set_mp_positions({position});
const auto ckmp = document.to_ckmp();
const auto reparsed = map_document::parse(ckmp);
CHECK(reparsed.has_chunk(chunk_kind::height_map_data));
CHECK(reparsed.has_chunk(chunk_kind::objects_list));
CHECK(reparsed.has_chunk(chunk_kind::mp_position_list));
CHECK_EQ(reparsed.find_chunk("HeightMapData")->kind, chunk_kind::height_map_data);
CHECK_EQ(chunk_kind_of("BlendTileData"), chunk_kind::blend_tile_data);
CHECK_EQ(chunk_kind_of("NotAChunk"), chunk_kind::unknown);
const auto restored = reparsed.height_map();
CHECK(restored.has_value());
CHECK_EQ(restored->width, 2U);
CHECK_EQ(restored->at(1, 1), 4U);
CHECK_EQ(restored->vertical_scale(), 0.0390625F);
const auto objects = reparsed.objects();
CHECK_EQ(objects.size(), 1U);
CHECK_EQ(objects[0].type_name, "*Waypoints/Waypoint");
CHECK_EQ(objects[0].property("waypointID")->as_int(), 0);
const auto starts = reparsed.player_starts();
CHECK_EQ(starts.size(), 1U);
CHECK_EQ(starts[0].index, 1);
CHECK_EQ(starts[0].position.x, 100.0F);
const auto positions = reparsed.mp_positions();
CHECK_EQ(positions.size(), 1U);
CHECK(positions[0].is_human);
CHECK_EQ(positions[0].team, 1U);
}
TEST(map_compressed_and_big_payload) {
const std::vector<std::string> names{"HeightMapData"};
auto document = map_document::parse(make_empty_ckmp(names));
height_map_data height;
height.width = 1;
height.height = 1;
height.version = 6;
height.elevations = {7};
document.set_height_map(height);
// EAR\0 + RefPack round-trip.
const auto compressed = document.serialize(true);
const auto from_compressed = map_document::parse(compressed);
CHECK(from_compressed.height_map().has_value());
CHECK_EQ(from_compressed.height_map()->at(0, 0), 7U);
// A BIG payload is one more RefPack layer over the map file.
const auto big_payload = refpack_compress(document.to_ckmp());
const auto from_big = map_document::parse(big_payload);
CHECK_EQ(from_big.height_map()->at(0, 0), 7U);
}
third_party/libra3assets/tests/test_main.cpp
Vendored
+5
View File
@@ -0,0 +1,5 @@
#include "test_main.hpp"
int main() {
return ra3test::run_all();
}
third_party/libra3assets/tests/test_main.hpp
Vendored
+83
View File
@@ -0,0 +1,83 @@
#ifndef RA3TEST_HPP
#define RA3TEST_HPP
// A tiny dependency-free test harness (registry + CHECK macros). Each unit-test
// executable compiles test_main.cpp and links the sources under test.
#include <functional>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
namespace ra3test {
struct test_case {
std::string name;
std::function<void()> fn;
};
inline auto registry() -> std::vector<test_case> & {
static std::vector<test_case> tests;
return tests;
}
inline auto failure_count() -> int & {
static int count = 0;
return count;
}
struct registrar {
registrar(std::string name, std::function<void()> fn) { registry().push_back({std::move(name), std::move(fn)}); }
};
inline auto report_failure(const std::string &expr, const std::string &file, int line) -> void {
++failure_count();
std::cerr << " FAIL " << file << ':' << line << " " << expr << '\n';
}
inline auto check(bool condition, const std::string &expr, const std::string &file, int line) -> void {
if (!condition) report_failure(expr, file, line);
}
inline auto run_all() -> int {
int passed = 0;
for (auto &test: registry()) {
const int before = failure_count();
std::cout << "[ RUN ] " << test.name << '\n';
try {
test.fn();
} catch (const std::exception &exc) {
report_failure(std::string("uncaught exception: ") + exc.what(), __FILE__, __LINE__);
}
if (failure_count() == before) {
++passed;
std::cout << "[ OK ] " << test.name << '\n';
}
}
std::cout << "\n" << passed << '/' << registry().size() << " tests passed, " << failure_count() << " failure(s)\n";
return failure_count() == 0 ? 0 : 1;
}
} // namespace ra3test
#define TEST(name) \
static void name(); \
static ::ra3test::registrar ra3test_reg_##name(#name, name); \
static void name()
#define CHECK(cond) ::ra3test::check((cond), #cond, __FILE__, __LINE__)
#define CHECK_EQ(a, b) ::ra3test::check_eq((a), (b), #a, #b, __FILE__, __LINE__)
namespace ra3test {
template<typename A, typename B>
auto check_eq(const A &a, const B &b, const std::string &ea, const std::string &eb, const std::string &file, int line) -> void {
if (!(a == b)) {
std::ostringstream os;
os << ea << " == " << eb;
report_failure(os.str(), file, line);
}
}
}
#endif // RA3TEST_HPP