EnderTheCoder/OpenRA3
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(assets): read retail assets through the vendored libra3assets library

Browse Source 
Vendor libra3assets (C++26 modules: BIG4, RefPack, BinaryAsset, CSF, CkMp map)
under third_party/ and add the ra3_assets target. ra3.fs, ra3.map and ra3.terrain
become thin adapters over it:

- ra3.fs delegates RefPack and the BIG4 index/payload reads (index-only, payloads
  read on demand).
- ra3.map decodes ObjectsList and CSF via map_document/csf_table; starts come from
  player_starts(), replacing the off-by-one whole-buffer scan.
- ra3.terrain takes the CkMp container and HeightMapData from map_document, with
  BlendTileData (not modelled by the library) parsed from the chunk payload.

Adds an opt-in real-asset check (OPENRA3_TEST_ASSETS) plus the library's own unit
suite as ra3assets_unit.
This commit is contained in:
EnderTheCoder
2026-09-29 23:58:13 +08:00
parent 8047e80ed6
commit 23f8be394c
17 changed files with 3078 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fs/ra3.fs.cppm
+58 -190
View File
@@ -3,13 +3,16 @@ export module ra3.fs;
import std;
export import ra3.core;
import ra3.assets;
/**
* Reading of the retail game's on-disk assets.
*
* Red Alert 3 ships its data in `BIG4` archives under `<install>\Data`, with
* individual payloads compressed by EA's RefPack codec. This module implements
* the container and codec so OpenRA3 can read a user's own installation.
* individual payloads compressed by EA's RefPack codec. The container and codec
* are implemented by the vendored `libra3assets` (`ra3.assets`, a sibling of
* `libenderlog`); this module is the thin adapter OpenRA3's loader talks to, so
* the format knowledge lives in one place.
*
* No game data is ever written into the repository; callers point the loader at
* their local install (`--game-dir` / `RA3_GAME_DIR`, default `C:\Red Alert 3`).
@@ -42,11 +45,21 @@ export namespace ra3::fs {
uint32 size = 0;
};
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const uint8> data) -> bool {
return data.size() >= 2 && (data[0] & refpack_mask) == 0x10U && data[1] == refpack_magic2;
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
/** Copy a `libra3assets` byte buffer into OpenRA3's `uint8` vector. */
[[nodiscard]] inline auto to_u8(std::vector<std::byte> bytes) -> std::vector<uint8> {
std::vector<uint8> out(bytes.size());
if (!bytes.empty()) std::memcpy(out.data(), bytes.data(), bytes.size());
return out;
}
}
/** True when `data` starts with a RefPack header (`0b??010000`, `0xFB`). */
[[nodiscard]] inline auto is_refpack(std::span<const uint8> data) -> bool { return ra3::assets::is_refpack(detail::as_bytes(data)); }
/**
* Decompress an EA RefPack stream.
*
@@ -55,80 +68,17 @@ export namespace ra3::fs {
* @throws refpack_error if the stream is malformed or the length disagrees.
*/
[[nodiscard]] inline auto refpack_decompress(std::span<const uint8> data) -> std::vector<uint8> {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
usize pos = 0;
const auto header = data[pos++];
const bool large_files = (header & 0x80U) != 0;
const bool compressed_size_present = (header & 0x01U) != 0;
pos++; // 0xFB
const usize size_bytes = large_files ? 4U : 3U;
auto read_size = [&]() -> uint32 {
uint32 value = 0;
for (usize i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8) | data[pos++];
}
return value;
};
if (compressed_size_present) (void)read_size();
const auto out_len = read_size();
std::vector<uint8> out;
out.reserve(out_len);
auto copy_literals = [&](usize count) {
if (pos + count > data.size()) throw refpack_error("truncated RefPack literals");
out.insert(out.end(), data.begin() + static_cast<std::ptrdiff_t>(pos), data.begin() + static_cast<std::ptrdiff_t>(pos + count));
pos += count;
};
auto copy_reference = [&](usize length, usize distance) {
if (distance == 0 || distance > out.size()) throw refpack_error("RefPack back-reference out of range");
usize start = out.size() - distance;
for (usize i = 0; i < length; ++i) out.push_back(out[start + i]);
};
while (pos < data.size()) {
const auto cmd = data[pos++];
if ((cmd & 0x80U) == 0) { // 2-byte command
if (pos >= data.size()) throw refpack_error("truncated 2-byte command");
const auto b2 = data[pos++];
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x1CU) >> 2) + 3, ((cmd & 0x60U) << 3) + b2 + 1);
} else if ((cmd & 0x40U) == 0) { // 3-byte command
if (pos + 1 >= data.size()) throw refpack_error("truncated 3-byte command");
const auto b2 = data[pos];
const auto b3 = data[pos + 1];
pos += 2;
copy_literals((b2 & 0xC0U) >> 6);
copy_reference((cmd & 0x3FU) + 4, ((b2 & 0x3FU) << 8) + b3 + 1);
} else if ((cmd & 0x20U) == 0) { // 4-byte command
if (pos + 2 >= data.size()) throw refpack_error("truncated 4-byte command");
const auto b2 = data[pos];
const auto b3 = data[pos + 1];
const auto b4 = data[pos + 2];
pos += 3;
copy_literals(cmd & 0x03U);
copy_reference(((cmd & 0x0CU) << 6) + b4 + 5, ((cmd & 0x10U) << 12) + (b2 << 8) + b3 + 1);
} else if (cmd < 0xFCU) { // long literal run
copy_literals(((cmd & 0x1FU) + 1) << 2);
} else { // stop
copy_literals(cmd & 0x03U);
break;
}
try {
return detail::to_u8(ra3::assets::refpack_decompress(detail::as_bytes(data)));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
}
if (out.size() != out_len) throw refpack_error("RefPack length mismatch");
return out;
}
/** Decompress `data` when it is RefPack, otherwise copy it unchanged. */
[[nodiscard]] inline auto maybe_decompress(std::span<const uint8> data) -> std::vector<uint8> {
if (is_refpack(data)) return refpack_decompress(data);
return {data.begin(), data.end()};
if (!is_refpack(data)) return {data.begin(), data.end()};
return refpack_decompress(data);
}
/**
@@ -137,23 +87,11 @@ export namespace ra3::fs {
* @throws refpack_error if the stream is malformed.
*/
[[nodiscard]] inline auto refpack_output_size(std::span<const uint8> data) -> uint32 {
if (!is_refpack(data)) throw refpack_error("not a RefPack stream");
usize pos = 0;
const auto header = data[pos++];
const bool large_files = (header & 0x80U) != 0;
const bool compressed_size_present = (header & 0x01U) != 0;
pos++; // 0xFB
const usize size_bytes = large_files ? 4U : 3U;
auto read_size = [&]() -> uint32 {
uint32 value = 0;
for (usize i = 0; i < size_bytes; ++i) {
if (pos >= data.size()) throw refpack_error("truncated RefPack size field");
value = (value << 8) | data[pos++];
}
return value;
};
if (compressed_size_present) (void)read_size();
return read_size();
try {
return ra3::assets::refpack_output_size(detail::as_bytes(data));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
}
}
[[nodiscard]] constexpr auto read_be32(const uint8 *p) -> uint32 {
@@ -165,81 +103,32 @@ export namespace ra3::fs {
}
/**
* A parsed `BIG4` archive. Only the index is held in memory; payloads are
* read from disk on demand so multi-hundred-megabyte archives stay cheap.
* A parsed `BIG4` archive.
*
* The index and payloads are held by a `ra3::assets::big_archive`; this
* adapter exposes the OpenRA3-facing surface (`big_entry`, `uint8` buffers)
* over it. Payloads are RefPack-decompressed on request.
*/
class big_archive {
public:
/**
* Parse the index of a `BIG4` archive.
* Parse a `BIG4` archive.
*
* @param path Archive path.
* @throws archive_error if the file is missing, not `BIG4`, or truncated.
*/
[[nodiscard]] static auto open(const std::filesystem::path &path) -> big_archive {
big_archive archive;
archive.path_ = path;
std::error_code ec;
archive.file_size_ = static_cast<usize>(std::filesystem::file_size(path, ec));
if (ec) throw archive_error("cannot stat archive: " + path.string());
std::ifstream in(path, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path.string());
std::array<uint8, 16> header{};
in.read(reinterpret_cast<char *>(header.data()), static_cast<std::streamsize>(header.size()));
if (!in || std::memcmp(header.data(), big_magic.data(), big_magic.size()) != 0) throw archive_error("not a BIG4 archive: " + path.string());
const auto count = read_be32(header.data() + 8);
// Read the variable-length index, growing the window until parsed.
usize window = std::min(archive.file_size_, std::max<usize>(1U << 20U, static_cast<usize>(count) * 256U));
std::vector<uint8> index;
for (;;) {
index.resize(window);
in.clear();
in.seekg(0);
in.read(reinterpret_cast<char *>(index.data()), static_cast<std::streamsize>(window));
const auto got = static_cast<usize>(in.gcount());
index.resize(got);
archive.entries_.clear();
archive.entries_.reserve(count);
usize pos = 16;
bool complete = true;
for (uint32 i = 0; i < count; ++i) {
if (pos + 8 > index.size()) {
complete = false;
break;
}
big_entry entry;
entry.offset = read_be32(index.data() + pos);
entry.size = read_be32(index.data() + pos + 4);
pos += 8;
const auto *begin = reinterpret_cast<const char *>(index.data() + pos);
const auto *end = reinterpret_cast<const char *>(std::memchr(begin, '\0', index.size() - pos));
if (end == nullptr) {
complete = false;
break;
}
entry.name.assign(begin, end);
pos += static_cast<usize>(end - begin) + 1U;
archive.entries_.push_back(std::move(entry));
}
if (complete) break;
if (window >= archive.file_size_) throw archive_error("truncated BIG4 index: " + path.string());
window = std::min(archive.file_size_, window * 2U);
try {
return big_archive{ra3::assets::big_archive::open(path), path};
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
archive.index_.reserve(archive.entries_.size());
for (usize i = 0; i < archive.entries_.size(); ++i) archive.index_.emplace(archive.entries_[i].name, i);
return archive;
}
[[nodiscard]] auto path() const -> const std::filesystem::path & { return path_; }
[[nodiscard]] auto entries() const -> const std::vector<big_entry> & { return entries_; }
[[nodiscard]] auto size() const -> usize { return entries_.size(); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return index_.contains(std::string{name}); }
[[nodiscard]] auto contains(std::string_view name) const -> bool { return archive_.contains(name); }
/** Entry names whose path contains `needle`, in index order. */
[[nodiscard]] auto find(std::string_view needle) const -> std::vector<const big_entry *> {
@@ -258,42 +147,33 @@ export namespace ra3::fs {
* @throws archive_error if the entry is missing or unreadable.
*/
[[nodiscard]] auto read(std::string_view name, bool decompress = true) const -> std::vector<uint8> {
const auto it = index_.find(std::string{name});
if (it == index_.end()) throw archive_error("no such entry: " + std::string{name});
const auto &entry = entries_[it->second];
std::ifstream in(path_, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path_.string());
in.seekg(static_cast<std::streamoff>(entry.offset));
std::vector<uint8> raw(entry.size);
in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(raw.size()));
if (!in) throw archive_error("short read for entry: " + entry.name);
return decompress ? maybe_decompress(raw) : raw;
try {
return detail::to_u8(archive_.read(name, decompress));
} catch (const ra3::assets::refpack_error &error) {
throw refpack_error(error.what());
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
}
/** Read the first `count` stored bytes of an entry (no decompression). */
[[nodiscard]] auto read_prefix(std::string_view name, usize count) const -> std::vector<uint8> {
const auto it = index_.find(std::string{name});
if (it == index_.end()) throw archive_error("no such entry: " + std::string{name});
const auto &entry = entries_[it->second];
std::ifstream in(path_, std::ios::binary);
if (!in) throw archive_error("cannot open archive: " + path_.string());
in.seekg(static_cast<std::streamoff>(entry.offset));
const auto want = std::min(count, static_cast<usize>(entry.size));
std::vector<uint8> raw(want);
in.read(reinterpret_cast<char *>(raw.data()), static_cast<std::streamsize>(raw.size()));
raw.resize(static_cast<usize>(in.gcount()));
return raw;
try {
return detail::to_u8(archive_.read_prefix(name, count));
} catch (const ra3::assets::asset_error &error) {
throw archive_error(error.what());
}
}
private:
big_archive() = default;
big_archive(ra3::assets::big_archive archive, std::filesystem::path path) : archive_(std::move(archive)), path_(std::move(path)) {
entries_.reserve(archive_.size());
for (const auto &entry: archive_.entries()) entries_.push_back({entry.name, entry.offset, entry.size});
}
ra3::assets::big_archive archive_;
std::filesystem::path path_;
usize file_size_ = 0;
std::vector<big_entry> entries_;
std::unordered_map<std::string, usize> index_;
};
/**
@@ -306,18 +186,6 @@ export namespace ra3::fs {
* @return The install root, or `std::nullopt` when none is found.
*/
[[nodiscard]] inline auto find_game_dir(const std::optional<std::filesystem::path> &explicit_dir = std::nullopt) -> std::optional<std::filesystem::path> {
auto qualifies = [](const std::filesystem::path &candidate) {
std::error_code ec;
return !candidate.empty() && std::filesystem::is_directory(candidate / "Data", ec);
};
if (explicit_dir && qualifies(*explicit_dir)) return explicit_dir;
if (const auto *env = std::getenv("RA3_GAME_DIR"); env != nullptr && *env != '\0') {
const std::filesystem::path candidate{env};
if (qualifies(candidate)) return candidate;
}
const std::filesystem::path default_dir{"C:/Red Alert 3"};
if (qualifies(default_dir)) return default_dir;
return std::nullopt;
return ra3::assets::find_game_dir(explicit_dir);
}
}
src/map/ra3.map.cppm
+55 -233
View File
@@ -4,6 +4,7 @@ import std;
export import ra3.core;
export import ra3.fs;
import ra3.assets;
/**
* Red Alert 3 map discovery and loading.
@@ -50,6 +51,9 @@ export namespace ra3::map {
};
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
[[nodiscard]] inline auto split_path(std::string_view path) -> std::vector<std::string> {
std::vector<std::string> parts;
usize start = 0;
@@ -82,54 +86,6 @@ export namespace ra3::map {
return file;
}
[[nodiscard]] inline auto find_bytes(std::span<const uint8> haystack, std::string_view needle) -> usize {
if (needle.empty()) return 0;
const auto *needle_begin = reinterpret_cast<const uint8 *>(needle.data());
const auto it = std::search(haystack.begin(), haystack.end(), needle_begin, needle_begin + needle.size());
return it == haystack.end() ? static_cast<usize>(-1) : static_cast<usize>(it - haystack.begin());
}
[[nodiscard]] inline auto read_le_f32(const uint8 *p) -> real {
const auto bits = fs::read_le32(p);
real value = 0.0F;
std::memcpy(&value, &bits, sizeof(value));
return value;
}
/** First plausible `(x, y, 0)` float triple at or after `from`. */
[[nodiscard]] inline auto scan_triple(std::span<const uint8> data, usize from, usize window) -> std::optional<start_position> {
const auto end = std::min(data.size(), from + window);
for (usize p = from; p + 12U <= end; ++p) {
const auto x = read_le_f32(data.data() + p);
const auto y = read_le_f32(data.data() + p + 4U);
const auto z = read_le_f32(data.data() + p + 8U);
if (z == 0.0F && std::isfinite(x) && std::isfinite(y) && x > 0.0F && x < 20000.0F && y > 0.0F && y < 20000.0F) {
return start_position{x, y, z};
}
}
return std::nullopt;
}
/** Extract `Player_1_Start` .. `Player_N_Start` waypoint coordinates. */
[[nodiscard]] inline auto extract_start_positions(std::span<const uint8> ckmp, int max_players = 8) -> std::vector<start_position> {
std::vector<start_position> result;
for (int n = 1; n <= max_players; ++n) {
const auto needle = std::string{"Player_"} + std::to_string(n) + "_Start";
usize cursor = 0;
while (cursor < ckmp.size()) {
const auto at = find_bytes(ckmp.subspan(cursor), needle);
if (at == static_cast<usize>(-1)) break;
const auto abs = cursor + at;
if (const auto triple = scan_triple(ckmp, abs + needle.size(), 128); triple) {
result.push_back(*triple);
break;
}
cursor = abs + 1;
}
}
return result;
}
/** Number of distinct integer positions (used to reject degenerate sets). */
[[nodiscard]] inline auto distinct_positions(const std::vector<start_position> &starts) -> usize {
std::vector<std::pair<int, int>> seen;
@@ -196,9 +152,24 @@ export namespace ra3::map {
return fs::maybe_decompress(payload);
}
/** Recover `Player_N_Start` waypoints from raw, uncompressed `CkMp` bytes. */
/**
* Recover `Player_N_Start` waypoints from raw, uncompressed `CkMp` bytes.
*
* `libra3assets` decodes the `ObjectsList` `*Waypoints/Waypoint` objects and
* returns the position of each `waypointName == "Player_<n>_Start"`. This
* replaces the previous whole-buffer heuristic, which scanned for the string
* and then took the first plausible float triple *after* it — off by one
* (the position precedes the name in each object) and prone to matching
* arbitrary bytes.
*/
[[nodiscard]] inline auto starts_from_ckmp(std::span<const uint8> ckmp) -> std::vector<start_position> {
auto starts = detail::extract_start_positions(ckmp);
std::vector<start_position> starts;
try {
const auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp));
for (const auto &start: document.player_starts()) starts.push_back({start.position.x, start.position.y, start.position.z});
} catch (const std::exception &) {
return {};
}
if (detail::distinct_positions(starts) < 2U) starts.clear();
return starts;
}
@@ -221,117 +192,34 @@ export namespace ra3::map {
uint32 road_type = 0U; ///< SAGE `RoadType` flags; 0 for a non-road object.
};
namespace detail {
struct ckmp_chunk {
std::string name;
uint16 version = 0;
usize offset = 0;
usize size = 0;
};
/** Parse the `CkMp` chunk tree and its `index -> name` table. */
[[nodiscard]] inline auto parse_ckmp_chunks(std::span<const uint8> data) -> std::pair<std::vector<std::string>, std::vector<ckmp_chunk>> {
if (data.size() < 8U || std::memcmp(data.data(), "CkMp", 4) != 0) return {};
usize pos = 4;
const auto read_u32 = [&](usize at) {
return static_cast<uint32>(data[at]) | (static_cast<uint32>(data[at + 1U]) << 8U) | (static_cast<uint32>(data[at + 2U]) << 16U) |
(static_cast<uint32>(data[at + 3U]) << 24U);
};
const auto read_u16 = [&](usize at) { return static_cast<uint16>(static_cast<uint32>(data[at]) | (static_cast<uint32>(data[at + 1U]) << 8U)); };
const auto count = read_u32(pos);
pos += 4;
std::vector<std::string> names(count + 1U);
for (uint32 i = count; i >= 1U && pos < data.size(); --i) {
const auto len = data[pos++];
if (pos + len + 4U > data.size()) break;
names[i] = std::string{reinterpret_cast<const char *>(data.data() + pos), len};
pos += len + 4U;
}
std::vector<ckmp_chunk> chunks;
while (pos + 10U <= data.size()) {
const auto index = read_u32(pos);
const auto version = read_u16(pos + 4U);
const auto size = read_u32(pos + 6U);
pos += 10U;
if (index >= names.size() || pos + size > data.size()) break;
chunks.push_back({names[index], version, pos, size});
pos += size;
}
return {std::move(names), std::move(chunks)};
}
}
/**
* Every object the map places (the `ObjectsList` chunk).
*
* Layout (OpenSAGE `Data/Map/{ObjectsList,MapObject,AssetProperty}.cs`):
* the chunk is a list of nested `Object` assets, each a `Coord3D`, a Z
* `angle`, a `RoadType`, a `u16`-prefixed type-name and a property list
* whose keys index the shared name table.
* Decoded by `libra3assets` (`ra3.assets`): the map is parsed as a
* `map_document`, whose `ObjectsList` accessor walks the nested `Object`
* assets (a `Coord3D`, a Z `angle`, a `RoadType`, a `u16`-prefixed
* type-name and a property list keyed into the shared name table).
*
* @return The objects in chunk order; empty when the map has no object list.
* @return The objects in chunk order; empty when the map is malformed or has
* no object list.
*/
[[nodiscard]] inline auto parse_objects(std::span<const uint8> ckmp) -> std::vector<map_object> {
const auto [names, chunks] = detail::parse_ckmp_chunks(ckmp);
(void) names;
std::vector<map_object> objects;
const auto read_u16 = [&](usize at) { return static_cast<uint16>(static_cast<uint32>(ckmp[at]) | (static_cast<uint32>(ckmp[at + 1U]) << 8U)); };
const auto read_u32 = [&](usize at) {
return static_cast<uint32>(ckmp[at]) | (static_cast<uint32>(ckmp[at + 1U]) << 8U) | (static_cast<uint32>(ckmp[at + 2U]) << 16U) |
(static_cast<uint32>(ckmp[at + 3U]) << 24U);
};
const auto read_f32 = [&](usize at) {
const auto bits = read_u32(at);
real value = 0.0F;
std::memcpy(&value, &bits, sizeof(value));
return value;
};
for (const auto &chunk: chunks) {
if (chunk.name != "ObjectsList") continue;
usize p = chunk.offset;
const auto end = chunk.offset + chunk.size;
while (p + 6U <= end) {
p += 4U; // asset index (always `Object`)
p += 2U; // asset version
const auto asset_size = read_u32(p);
p += 4U;
if (asset_size < 24U || p + asset_size > end) break;
const auto asset_end = p + asset_size;
map_object object;
object.x = read_f32(p);
object.y = read_f32(p + 4U);
object.z = read_f32(p + 8U);
object.angle = read_f32(p + 12U);
object.road_type = read_u32(p + 16U);
p += 20U; // Coord3D + angle + road type
const auto name_len = read_u16(p);
p += 2U;
if (p + name_len > asset_end) break;
object.type = std::string{reinterpret_cast<const char *>(ckmp.data() + p), name_len};
p += name_len;
const auto property_count = static_cast<uint32>(read_u16(p));
p += 2U;
for (uint32 i = 0; i < property_count && p + 4U <= asset_end; ++i) {
const auto type = ckmp[p++];
p += 3U; // property name index (u24) into the shared name table
usize value_size = 0;
if (type == 0U) {
value_size = 1U; // boolean
} else if (type == 1U || type == 2U) {
value_size = 4U; // integer / real
} else if (p + 2U <= asset_end) {
const auto len = read_u16(p);
value_size = 2U + static_cast<usize>(len) * ((type == 4U) ? 2U : 1U); // ascii/unicode/unknown
}
p += value_size;
if (p > asset_end) break;
}
objects.push_back(std::move(object));
p = asset_end;
try {
const auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp));
for (const auto &object: document.objects()) {
map_object out;
out.type = object.type_name;
out.x = object.position.x;
out.y = object.position.y;
out.z = object.position.z;
out.angle = object.angle;
out.scale = 1.0F;
out.road_type = static_cast<uint32>(object.road);
objects.push_back(std::move(out));
}
} catch (const std::exception &) {
return {};
}
return objects;
}
@@ -351,96 +239,30 @@ export namespace ra3::map {
}
};
namespace detail {
/**
* Decode one SAGE `.csf` string value.
*
* RA3's `gamestrings.csf` stores each UTF-16 code unit with the low byte
* XORed by `0xFF` (the high byte is the padding `0xFF`); undoing that
* yields the plain ASCII/UTF-8 text.
*/
[[nodiscard]] inline auto decode_csf_string(std::span<const uint8> raw) -> std::string {
std::string value;
for (usize i = 0; i + 1U < raw.size(); i += 2U) {
const auto ch = static_cast<char>(static_cast<uint8>(raw[i] ^ 0xFFU));
if (ch == '\0') break;
value.push_back(ch);
}
return value;
}
}
/**
* Parse `MAP:<id>` display names out of a SAGE `.csf` string table.
*
* The map list UI reads its labels from the install's `data\gamestrings.csf`
* under the key `MAP:<UPPERCASE_ID>` (e.g. `MAP:MAP_MP_2_FEASEL4` is
* "Battlebase Beta"). Values are byte-XORed with `0xFF`.
* "Battlebase Beta"). Decoding is `libra3assets`' `csf_table` (each UTF-16
* code unit is bit-inverted; the table lookup is case-insensitive).
*/
[[nodiscard]] inline auto parse_map_names(std::span<const uint8> csf) -> map_name_table {
map_name_table table;
if (csf.size() < 24U || std::memcmp(csf.data(), " FSC", 4) != 0) return table;
const auto read = [&](usize p) -> uint32 {
return static_cast<uint32>(csf[p]) | (static_cast<uint32>(csf[p + 1U]) << 8U) | (static_cast<uint32>(csf[p + 2U]) << 16U) |
(static_cast<uint32>(csf[p + 3U]) << 24U);
};
usize pos = 24; // header: magic + version + label/string counts + 8 reserved bytes
while (pos + 4U <= csf.size()) {
if (std::memcmp(csf.data() + pos, " LBL", 4) != 0) {
++pos;
continue;
}
pos += 4;
if (pos + 4U > csf.size()) break;
const auto count = read(pos);
pos += 4;
std::vector<std::string> labels;
labels.reserve(count);
bool ok = true;
for (uint32 i = 0; i < count; ++i) {
if (pos + 4U > csf.size()) {
ok = false;
break;
}
const auto len = read(pos);
pos += 4;
if (pos + len > csf.size()) {
ok = false;
break;
}
labels.emplace_back(reinterpret_cast<const char *>(csf.data() + pos), len);
pos += len;
}
if (!ok) break;
for (const auto &label: labels) {
if (pos + 4U > csf.size() || std::memcmp(csf.data() + pos, " RTS", 4) != 0) {
ok = false;
break;
}
pos += 4;
if (pos + 4U > csf.size()) {
ok = false;
break;
}
const auto chars = read(pos);
pos += 4;
const auto bytes = static_cast<usize>(chars) * 2U;
if (pos + bytes > csf.size()) {
ok = false;
break;
}
auto value = detail::decode_csf_string(csf.subspan(pos, bytes));
pos += bytes;
try {
const auto strings = ra3::assets::csf_table::parse(detail::as_bytes(csf));
for (const auto &entry: strings.entries()) {
constexpr std::string_view prefix = "MAP:";
if (label.size() > prefix.size() && label.compare(0, prefix.size(), prefix) == 0) {
auto id = label.substr(prefix.size());
std::transform(id.begin(), id.end(), id.begin(), [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
if (!value.empty()) table.names.try_emplace(std::move(id), std::move(value));
}
if (entry.label.size() <= prefix.size() || entry.label.compare(0, prefix.size(), prefix) != 0) continue;
auto id = entry.label.substr(prefix.size());
std::transform(id.begin(), id.end(), id.begin(), [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
auto value = ra3::assets::utf16_to_utf8(entry.value());
// CSF strings are NUL-terminated; the terminator is not part of the text.
if (const auto nul = value.find('\0'); nul != std::string::npos) value.resize(nul);
if (!value.empty()) table.names.try_emplace(std::move(id), std::move(value));
}
if (!ok) break;
} catch (const std::exception &) {
return {};
}
return table;
}
src/terrain/ra3.terrain.cppm
+50 -92
View File
@@ -6,6 +6,7 @@ export import ra3.core;
export import ra3.fs;
export import ra3.render;
export import ra3.models;
import ra3.assets;
/**
* The map's real terrain, read from the compiled `CkMp` chunk tree.
@@ -91,82 +92,24 @@ export namespace ra3::terrain {
};
namespace detail {
/** View a `uint8` range as bytes, the currency of `libra3assets`. */
[[nodiscard]] inline auto as_bytes(std::span<const uint8> data) -> std::span<const std::byte> { return std::as_bytes(data); }
/** View a `libra3assets` byte range as OpenRA3's `uint8`. */
[[nodiscard]] inline auto as_u8(std::span<const std::byte> data) -> std::span<const uint8> {
return {reinterpret_cast<const uint8 *>(data.data()), data.size()};
}
[[nodiscard]] inline auto read_u16(const uint8 *p) -> uint16 { return static_cast<uint16>(p[0]) | (static_cast<uint16>(p[1]) << 8U); }
[[nodiscard]] inline auto read_u32(const uint8 *p) -> uint32 {
return static_cast<uint32>(p[0]) | (static_cast<uint32>(p[1]) << 8U) | (static_cast<uint32>(p[2]) << 16U) |
(static_cast<uint32>(p[3]) << 24U);
}
struct chunk {
std::string name;
uint16 version = 0;
usize offset = 0;
usize size = 0;
};
/**
* Parse the `CkMp` chunk tree into a flat chunk list.
*
* Layout: `"CkMp"`, `u32 assetCount`, then `assetCount` entries of
* `{ u8 nameLen, name, u32 index }` (index descending from the count),
* then `{ u32 index, u16 version, u32 dataSize, data[dataSize] }` per
* chunk until the end.
*/
[[nodiscard]] inline auto parse_chunks(std::span<const uint8> data) -> std::vector<chunk> {
if (data.size() < 8U || std::memcmp(data.data(), "CkMp", 4) != 0) throw terrain_error("not a CkMp map");
usize pos = 4;
const auto count = read_u32(data.data() + pos);
pos += 4;
std::vector<std::string> names(count + 1U);
for (uint32 i = count; i >= 1U; --i) {
if (pos >= data.size()) throw terrain_error("truncated asset-name table");
const auto len = data[pos++];
if (pos + len + 4U > data.size()) throw terrain_error("truncated asset-name");
names[i] = std::string{reinterpret_cast<const char *>(data.data() + pos), len};
pos += len;
pos += 4; // asset index (== i)
}
std::vector<chunk> chunks;
while (pos + 10U <= data.size()) {
const auto index = read_u32(data.data() + pos);
const auto version = read_u16(data.data() + pos + 4U);
const auto size = read_u32(data.data() + pos + 6U);
pos += 10;
if (index >= names.size() || pos + size > data.size()) throw terrain_error("bad chunk header");
chunks.push_back({names[index], version, pos, size});
pos += size;
}
return chunks;
}
[[nodiscard]] inline auto find(const std::vector<chunk> &chunks, std::string_view name) -> const chunk * {
for (const auto &c: chunks) {
if (c.name == name) return &c;
}
return nullptr;
}
inline auto parse_heightmap(std::span<const uint8> data, const chunk &c, map_data &out) -> void {
usize p = c.offset;
out.width = read_u32(data.data() + p);
out.height = read_u32(data.data() + p + 4U);
out.border_width = read_u32(data.data() + p + 8U);
const auto border_count = read_u32(data.data() + p + 12U);
p += 16;
p += static_cast<usize>(border_count) * (c.version >= 6U ? 16U : 8U);
p += 4U; // area
if (out.width == 0U || out.height == 0U) throw terrain_error("empty heightmap");
const auto area = static_cast<usize>(out.width) * out.height;
out.elevations.resize(area);
for (usize i = 0; i < area; ++i) {
if (p >= data.size()) throw terrain_error("truncated heightmap");
out.elevations[i] = c.version >= 5U ? read_u16(data.data() + p) : data[p];
p += c.version >= 5U ? 2U : 1U;
}
}
// The `CkMp` container and `HeightMapData` chunk are modelled by
// `libra3assets` (`ra3.assets`); `parse_map` reads them through a
// `map_document`. Only `BlendTileData` (which the library does not type
// yet) is parsed here, over the chunk payload the document exposes.
/** Where the texture table ends and how many `BlendDescription`s follow. */
struct texture_table_info {
@@ -246,20 +189,20 @@ export namespace ra3::terrain {
}
}
inline auto parse_blend(std::span<const uint8> data, const chunk &c, map_data &out) -> void {
inline auto parse_blend(std::span<const uint8> payload, uint16 version, map_data &out) -> void {
const auto area = static_cast<usize>(out.width) * out.height;
usize p = c.offset;
const auto num_tiles = read_u32(data.data() + p);
usize p = 0;
const auto num_tiles = read_u32(payload.data() + p);
p += 4;
if (num_tiles != area) throw terrain_error("BlendTileData tile count mismatch");
out.tiles.resize(area);
for (usize i = 0; i < area; ++i) out.tiles[i] = read_u16(data.data() + p + i * 2U);
for (usize i = 0; i < area; ++i) out.tiles[i] = read_u16(payload.data() + p + i * 2U);
p += area * 2U;
const auto bits = (c.version >= 14U && c.version < 24U) ? 32U : 16U;
const auto bits = (version >= 14U && version < 24U) ? 32U : 16U;
const auto word = bits / 8U;
const auto read_index = [&](usize off) -> uint16 {
return word == 4U ? static_cast<uint16>(read_u32(data.data() + off)) : read_u16(data.data() + off);
return word == 4U ? static_cast<uint16>(read_u32(payload.data() + off)) : read_u16(payload.data() + off);
};
out.blends.resize(area);
for (usize i = 0; i < area; ++i) out.blends[i] = read_index(p + i * word);
@@ -269,33 +212,48 @@ export namespace ra3::terrain {
p += area * word;
p += area * word; // CliffTextures (not rendered yet)
const auto chunk_end = c.offset + c.size;
const auto table = parse_textures(data, p, chunk_end, out);
parse_blend_descriptions(data, table, chunk_end, out);
const auto chunk_end = payload.size();
const auto table = parse_textures(payload, p, chunk_end, out);
parse_blend_descriptions(payload, table, chunk_end, out);
}
}
/**
* Parse the terrain chunks out of a `CkMp` (uncompressed) map payload.
*
* The container and `HeightMapData` come from `libra3assets`
* (`map_document`); `BlendTileData` (which the library does not model yet)
* is decoded here from the chunk payload the document exposes.
*
* @throws terrain_error if the chunk tree or terrain chunks are malformed.
*/
[[nodiscard]] inline auto parse_map(std::span<const uint8> ckmp) -> map_data {
const auto chunks = detail::parse_chunks(ckmp);
const auto *heightmap = detail::find(chunks, "HeightMapData");
const auto *blend = detail::find(chunks, "BlendTileData");
if (heightmap == nullptr || blend == nullptr) throw terrain_error("map has no terrain chunks");
try {
auto document = ra3::assets::map_document::parse(detail::as_bytes(ckmp));
map_data out;
detail::parse_heightmap(ckmp, *heightmap, out);
detail::parse_blend(ckmp, *blend, out);
if (const auto *water = detail::find(chunks, "GlobalWaterSettings"); water != nullptr && water->size >= 8U) {
out.has_water = detail::read_u32(ckmp.data() + water->offset) != 0U;
const auto bits = detail::read_u32(ckmp.data() + water->offset + 4U);
std::memcpy(&out.water_plane_z, &bits, sizeof(out.water_plane_z));
map_data out;
const auto height = document.height_map();
if (!height) throw terrain_error("map has no HeightMapData chunk");
out.width = height->width;
out.height = height->height;
out.border_width = height->border_width;
out.elevations.assign(height->elevations.begin(), height->elevations.end());
const auto *blend = document.find_chunk("BlendTileData");
if (blend == nullptr) throw terrain_error("map has no BlendTileData chunk");
detail::parse_blend(detail::as_u8(blend->payload), blend->version, out);
if (const auto *water = document.find_chunk("GlobalWaterSettings"); water != nullptr && water->payload.size() >= 8U) {
const auto *bytes = reinterpret_cast<const uint8 *>(water->payload.data());
out.has_water = detail::read_u32(bytes) != 0U;
const auto bits = detail::read_u32(bytes + 4U);
std::memcpy(&out.water_plane_z, &bits, sizeof(out.water_plane_z));
}
out.valid = true;
return out;
} catch (const ra3::assets::asset_error &error) {
throw terrain_error(error.what());
}
out.valid = true;
return out;
}
/** The decoded terrain textures, parallel to `map_data::textures`. */