EnderTheCoder/OpenRA3
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(assets): read retail assets through the vendored libra3assets library

Browse Source 
Vendor libra3assets (C++26 modules: BIG4, RefPack, BinaryAsset, CSF, CkMp map)
under third_party/ and add the ra3_assets target. ra3.fs, ra3.map and ra3.terrain
become thin adapters over it:

- ra3.fs delegates RefPack and the BIG4 index/payload reads (index-only, payloads
  read on demand).
- ra3.map decodes ObjectsList and CSF via map_document/csf_table; starts come from
  player_starts(), replacing the off-by-one whole-buffer scan.
- ra3.terrain takes the CkMp container and HeightMapData from map_document, with
  BlendTileData (not modelled by the library) parsed from the chunk payload.

Adds an opt-in real-asset check (OPENRA3_TEST_ASSETS) plus the library's own unit
suite as ra3assets_unit.
This commit is contained in:
EnderTheCoder
2026-09-29 23:58:13 +08:00
parent 8047e80ed6
commit 23f8be394c
17 changed files with 3078 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fs/ra3.fs.cppm
+58 -190
View File
@@ -3,13 +3,16 @@ export module ra3.fs;
import std;
export import ra3.core;
import ra3.assets;
/**
* Reading of the retail game's on-disk assets.
*
* Red Alert 3 ships its data in `BIG4` archives under `<install>\Data`, with
* individual payloads compressed by EA's RefPack codec. This module implements
* the container and codec so OpenRA3 can read a user's own installation.
* individual payloads compressed by EA's RefPack codec. The container and codec
* are implemented by the vendored `libra3assets` (`ra3.assets`, a sibling of
* `libenderlog`); this module is the thin adapter OpenRA3's loader talks to, so
* the format knowledge lives in one place.
*
* No game data is ever written into the repository; callers point the loader at
* their local install (`--game-dir` / `RA3_GAME_DIR`, default `C:\Red Alert 3`).
@@ -42,11 +45,21 @@ export namespace ra3::fs {
uint32 size = 0;
};
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const uint8> data) -> bool {
return data.size() >= 2 && (data[0] & refpack_mask) == 0x10U && data[1] == refpack_magic2;
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
/** Copy a `libra3assets` byte buffer into OpenRA3's `uint8` vector. */
[[nodiscard]] inline auto to_u8(std::vector<std::byte> bytes) -> std::vector<uint8> {
std::vector<uint8> out(bytes.size());
if (!bytes.empty()) std::memcpy(out.data(), bytes.data(), bytes.size());
return out;
}
}
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const uint8> data) -> bool { return ra3::assets::is_refpack(detail::as_bytes(data)); }
/**
* Decompress an EA RefPack stream.
*
@@ -55,80 +68,17 @@ export namespace ra3::fs {
* @throws refpack_error if the stream is malformed or the length disagrees.
*/
[[nodiscard]] inline auto refpack_decompress(std::span<const uint8> data) -> std::vector<uint8> {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
usize pos = 0;
const auto header = data[pos++];
const bool large_files = (header & 0x80U) != 0;
const bool compressed_size_present = (header & 0x01U) != 0;
pos++; // 0xFB
const usize size_bytes = large_files ? 4U : 3U;
auto read_size = [&]() -> uint32 {
uint32 value = 0;
for (usize i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8) | data[pos++];
}
return value;
};
if (compressed_size_present) (void)read_size();
const auto out_len = read_size();
std::vector<uint8> out;
out.reserve(out_len);
auto copy_literals = [&](usize count) {
if (pos + count > data.size()) throw refpack_error("truncated RefPack literals");
out.insert(out.end(), data.begin() + static_cast<std::ptrdiff_t>(pos), data.begin() + static_cast<std::ptrdiff_t>(pos + count));
pos += count;
};
auto copy_reference = [&](usize length, usize distance) {
if (distance == 0 || distance > out.size()) throw refpack_error("RefPack back-reference out of range");
usize start = out.size() - distance;
for (usize i = 0; i < length; ++i) out.push_back(out[start + i]);
};
while (pos < data.size()) {
const auto cmd = data[pos++];
if ((cmd & 0x80U) == 0) { // 2-byte command
if (pos >= data.size()) throw refpack_error("truncated 2-byte command");
const auto b2 = data[pos++];
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x1CU) >> 2) + 3, ((cmd & 0x60U) << 3) + b2 + 1);
} else if ((cmd & 0x40U) == 0) { // 3-byte command
if (pos + 1 >= data.size()) throw refpack_error("truncated 3-byte command");
const auto b2 = data[pos];
const auto b3 = data[pos + 1];
pos += 2;
copy_literals((b2 & 0xC0U) >> 6);
copy_reference((cmd & 0x3FU) + 4, ((b2 & 0x3FU) << 8) + b3 + 1);
} else if ((cmd & 0x20U) == 0) { // 4-byte command
if (pos + 2 >= data.size()) throw refpack_error("truncated 4-byte command");
const auto b2 = data[pos];
const auto b3 = data[pos + 1];
const auto b4 = data[pos + 2];
pos += 3;
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x0CU) << 6) + b4 + 5, ((cmd & 0x10U) << 12) + (b2 << 8) + b3 + 1);
} else if (cmd < 0xFCU) { // long literal run
copy_literals(((cmd & 0x1FU) + 1) << 2);
} else { // stop
copy_literals(cmd & 0x03U);
break;
}
try {
return detail::to_u8(ra3::assets::refpack_decompress(detail::as_bytes(data)));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
}
if (out.size() != out_len) throw refpack_error("RefPack length mismatch");
return out;
}
/** Decompress `data` when it is RefPack, otherwise copy it unchanged. */
[[nodiscard]] inline auto maybe_decompress(std::span<const uint8> data) -> std::vector<uint8> {
if (is_refpack(data)) return refpack_decompress(data);
return {data.begin(), data.end()};
if (!is_refpack(data)) return {data.begin(), data.end()};
return refpack_decompress(data);
}
/**
@@ -137,23 +87,11 @@ export namespace ra3::fs {
* @throws refpack_error if the stream is malformed.
*/
[[nodiscard]] inline auto refpack_output_size(std::span<const uint8> data) -> uint32 {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
usize pos = 0;
const auto header = data[pos++];
const bool large_files = (header & 0x80U) != 0;
const bool compressed_size_present = (header & 0x01U) != 0;
pos++; // 0xFB
const usize size_bytes = large_files ? 4U : 3U;
auto read_size = [&]() -> uint32 {
uint32 value = 0;
for (usize i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8) | data[pos++];
}
return value;
};
if (compressed_size_present) (void)read_size();
return read_size();
try {
return ra3::assets::refpack_output_size(detail::as_bytes(data));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
}
}
[[nodiscard]] constexpr auto read_be32(const uint8 *p) -> uint32 {
@@ -165,81 +103,32 @@ export namespace ra3::fs {
}
/**
* A parsed `BIG4` archive. Only the index is held in memory; payloads are
* read from disk on demand so multi-hundred-megabyte archives stay cheap.
* A parsed `BIG4` archive.
*
* The index and payloads are held by a `ra3::assets::big_archive`; this
* adapter exposes the OpenRA3-facing surface (`big_entry`, `uint8` buffers)
* over it. Payloads are RefPack-decompressed on request.
*/
class big_archive {
public:
/**
* Parse the index of a `BIG4` archive.
* Parse a `BIG4` archive.
*
* @param path Archive path.
* @throws archive_error if the file is missing, not `BIG4`, or truncated.
*/
[[nodiscard]] static auto open(const std::filesystem::path &path) -> big_archive {
big_archive archive;
archive.path_ = path;
std::error_code ec;
archive.file_size_ = static_cast<usize>(std::filesystem::file_size(path, ec));
if (ec) throw archive_error("cannot stat archive: " + path.string());
std::ifstream in(path, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path.string());
std::array<uint8, 16> header{};
in.read(reinterpret_cast<char *>(header.data()), static_cast<std::streamsize>(header.size()));
if (!in || std::memcmp(header.data(), big_magic.data(), big_magic.size()) != 0) throw archive_error("not a BIG4 archive: " + path.string());
const auto count = read_be32(header.data() + 8);
// Read the variable-length index, growing the window until parsed.
usize window = std::min(archive.file_size_, std::max<usize>(1U << 20U, static_cast<usize>(count) * 256U));
std::vector<uint8> index;
for (;;) {
index.resize(window);
in.clear();
in.seekg(0);
in.read(reinterpret_cast<char *>(index.data()), static_cast<std::streamsize>(window));
const auto got = static_cast<usize>(in.gcount());
index.resize(got);
archive.entries_.clear();
archive.entries_.reserve(count);
usize pos = 16;
bool complete = true;
for (uint32 i = 0; i < count; ++i) {
if (pos + 8 > index.size()) {
complete = false;
break;
}
big_entry entry;
entry.offset = read_be32(index.data() + pos);
entry.size = read_be32(index.data() + pos + 4);
pos += 8;
const auto *begin = reinterpret_cast<const char *>(index.data() + pos);
const auto *end = reinterpret_cast<const char *>(std::memchr(begin, '\0', index.size() - pos));
if (end == nullptr) {
complete = false;
break;
}
entry.name.assign(begin, end);
pos += static_cast<usize>(end - begin) + 1U;
archive.entries_.push_back(std::move(entry));
}
if (complete) break;
if (window >= archive.file_size_) throw archive_error("truncated BIG4 index: " + path.string());
window = std::min(archive.file_size_, window * 2U);
try {
return big_archive{ra3::assets::big_archive::open(path), path};
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
archive.index_.reserve(archive.entries_.size());
for (usize i = 0; i < archive.entries_.size(); ++i) archive.index_.emplace(archive.entries_[i].name, i);
return archive;
}
[[nodiscard]] auto path() const -> const std::filesystem::path & { return path_; }
[[nodiscard]] auto entries() const -> const std::vector<big_entry> & { return entries_; }
[[nodiscard]] auto size() const -> usize { return entries_.size(); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return index_.contains(std::string{name}); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return archive_.contains(name); }
/** Entry names whose path contains `needle`, in index order. */
[[nodiscard]] auto find(std::string_view needle) const -> std::vector<const big_entry *> {
@@ -258,42 +147,33 @@ export namespace ra3::fs {
* @throws archive_error if the entry is missing or unreadable.
*/
[[nodiscard]] auto read(std::string_view name, bool decompress = true) const -> std::vector<uint8> {
const auto it = index_.find(std::string{name});
if (it == index_.end()) throw archive_error("no such entry: " + std::string{name});
const auto &entry = entries_[it->second];
std::ifstream in(path_, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path_.string());
in.seekg(static_cast<std::streamoff>(entry.offset));
std::vector<uint8> raw(entry.size);
in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(raw.size()));
if (!in) throw archive_error("short read for entry: " + entry.name);
return decompress ? maybe_decompress(raw) : raw;
try {
return detail::to_u8(archive_.read(name, decompress));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
}
/** Read the first `count` stored bytes of an entry (no decompression). */
[[nodiscard]] auto read_prefix(std::string_view name, usize count) const -> std::vector<uint8> {
const auto it = index_.find(std::string{name});
if (it == index_.end()) throw archive_error("no such entry: " + std::string{name});
const auto &entry = entries_[it->second];
std::ifstream in(path_, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path_.string());
in.seekg(static_cast<std::streamoff>(entry.offset));
const auto want = std::min(count, static_cast<usize>(entry.size));
std::vector<uint8> raw(want);
in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(raw.size()));
raw.resize(static_cast<usize>(in.gcount()));
return raw;
try {
return detail::to_u8(archive_.read_prefix(name, count));
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
}
private:
big_archive() = default;
big_archive(ra3::assets::big_archive archive, std::filesystem::path path) : archive_(std::move(archive)), path_(std::move(path)) {
entries_.reserve(archive_.size());
for (const auto &entry: archive_.entries()) entries_.push_back({entry.name, entry.offset, entry.size});
}
ra3::assets::big_archive archive_;
std::filesystem::path path_;
usize file_size_ = 0;
std::vector<big_entry> entries_;
std::unordered_map<std::string, usize> index_;
};
/**
@@ -306,18 +186,6 @@ export namespace ra3::fs {
* @return The install root, or `std::nullopt` when none is found.
*/
[[nodiscard]] inline auto find_game_dir(const std::optional<std::filesystem::path> &explicit_dir = std::nullopt) -> std::optional<std::filesystem::path> {
auto qualifies = [](const std::filesystem::path &candidate) {
std::error_code ec;
return !candidate.empty() && std::filesystem::is_directory(candidate / "Data", ec);
};
if (explicit_dir && qualifies(*explicit_dir)) return explicit_dir;
if (const auto *env = std::getenv("RA3_GAME_DIR"); env != nullptr && *env != '\0') {
const std::filesystem::path candidate{env};
if (qualifies(candidate)) return candidate;
}
const std::filesystem::path default_dir{"C:/Red Alert 3"};
if (qualifies(default_dir)) return default_dir;
return std::nullopt;
return ra3::assets::find_game_dir(explicit_dir);
}
}