feat(assets): read retail assets through the vendored libra3assets library
Vendor libra3assets (C++26 modules: BIG4, RefPack, BinaryAsset, CSF, CkMp map) under third_party/ and add the ra3_assets target. ra3.fs, ra3.map and ra3.terrain become thin adapters over it: - ra3.fs delegates RefPack and the BIG4 index/payload reads (index-only, payloads read on demand). - ra3.map decodes ObjectsList and CSF via map_document/csf_table; starts come from player_starts(), replacing the off-by-one whole-buffer scan. - ra3.terrain takes the CkMp container and HeightMapData from map_document, with BlendTileData (not modelled by the library) parsed from the chunk payload. Adds an opt-in real-asset check (OPENRA3_TEST_ASSETS) plus the library's own unit suite as ra3assets_unit.
This commit is contained in:
EnderTheCoder
+310
@@ -0,0 +1,310 @@
|
||||
#include "test_main.hpp"
|
||||
|
||||
import std;
|
||||
import ra3.assets;
|
||||
|
||||
using namespace ra3::assets;
|
||||
|
||||
namespace {
|
||||
|
||||
auto bytes_of(std::string_view text) -> std::vector<std::byte> {
|
||||
std::vector<std::byte> out;
|
||||
out.reserve(text.size());
|
||||
for (const auto ch: text) out.push_back(static_cast<std::byte>(static_cast<unsigned char>(ch)));
|
||||
return out;
|
||||
}
|
||||
|
||||
auto text_of(std::span<const std::byte> bytes) -> std::string {
|
||||
return std::string{reinterpret_cast<const char *>(bytes.data()), bytes.size()};
|
||||
}
|
||||
|
||||
/** A deterministic pseudo-random buffer (no external RNG needed). */
|
||||
auto pseudo_random(std::size_t size, std::uint32_t seed) -> std::vector<std::byte> {
|
||||
std::vector<std::byte> out;
|
||||
out.reserve(size);
|
||||
std::uint32_t state = seed;
|
||||
for (std::size_t i = 0; i < size; ++i) {
|
||||
state = state * 1664525U + 1013904223U;
|
||||
out.push_back(static_cast<std::byte>((state >> 16U) & 0xFFU));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
auto make_empty_ckmp(const std::vector<std::string> &names) -> std::vector<std::byte> {
|
||||
byte_writer writer;
|
||||
writer.write_ascii("CkMp");
|
||||
writer.write_u32(static_cast<std::uint32_t>(names.size()));
|
||||
for (std::uint32_t i = static_cast<std::uint32_t>(names.size()); i >= 1U; --i) {
|
||||
writer.write_u8(static_cast<std::uint8_t>(names[i - 1].size()));
|
||||
writer.write_ascii(names[i - 1]);
|
||||
writer.write_u32(i);
|
||||
}
|
||||
return writer.take();
|
||||
}
|
||||
|
||||
auto make_csf() -> std::vector<std::byte> {
|
||||
byte_writer writer;
|
||||
writer.write_ascii(" FSC");
|
||||
writer.write_u32(3);
|
||||
writer.write_u32(2); // labels
|
||||
writer.write_u32(2); // value blocks
|
||||
writer.write_u32(0);
|
||||
writer.write_u32(0);
|
||||
writer.write_u32(csf_label_flag);
|
||||
writer.write_u32(1);
|
||||
writer.write_u32(3);
|
||||
writer.write_ascii("ABC");
|
||||
writer.write_u32(csf_value_flag);
|
||||
writer.write_u32(5);
|
||||
for (const auto ch: std::string_view{"Hello"}) writer.write_u16(static_cast<std::uint16_t>(ch) ^ 0xFFFFU);
|
||||
writer.write_u32(csf_label_flag);
|
||||
writer.write_u32(1);
|
||||
writer.write_u32(3);
|
||||
writer.write_ascii("UNI");
|
||||
writer.write_u32(csf_value_flag);
|
||||
writer.write_u32(1);
|
||||
writer.write_u16(0x4E2DU ^ 0xFFFFU); // U+4E2D (CJK)
|
||||
return writer.take();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(refpack_roundtrip_small) {
|
||||
const std::vector<std::string> samples{"", "A", "AB", "ABC", "ABCD", "abcabcabc", std::string(300, 'z')};
|
||||
for (const auto &sample: samples) {
|
||||
const auto input = bytes_of(sample);
|
||||
const auto packed = refpack_compress(input);
|
||||
const auto restored = refpack_decompress(packed);
|
||||
CHECK_EQ(restored.size(), input.size());
|
||||
CHECK(std::ranges::equal(restored, input));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(refpack_roundtrip_large_and_repetitive) {
|
||||
std::vector<std::byte> input;
|
||||
for (int i = 0; i < 2000; ++i) {
|
||||
for (const auto ch: std::string_view{"lorem ipsum dolor sit amet "}) input.push_back(static_cast<std::byte>(ch));
|
||||
}
|
||||
const auto random = pseudo_random(50000, 0x1234ABCDU);
|
||||
input.insert(input.end(), random.begin(), random.end());
|
||||
const auto packed = refpack_compress(input);
|
||||
CHECK(packed.size() < input.size()); // the repeated prefix must compress
|
||||
CHECK(std::ranges::equal(refpack_decompress(packed), input));
|
||||
}
|
||||
|
||||
TEST(refpack_decode_known_stream) {
|
||||
// "abcd" literals, then a back-reference (distance 4, length 3) -> "abcdabc".
|
||||
const std::vector<std::byte> stream{
|
||||
std::byte{0x10}, std::byte{0xFB}, std::byte{0x00}, std::byte{0x00}, std::byte{0x07}, std::byte{0xE0},
|
||||
std::byte{'a'}, std::byte{'b'}, std::byte{'c'}, std::byte{'d'}, std::byte{0x00}, std::byte{0x03},
|
||||
std::byte{0xFC},
|
||||
};
|
||||
CHECK(is_refpack(stream));
|
||||
CHECK_EQ(text_of(refpack_decompress(stream)), "abcdabc");
|
||||
CHECK_EQ(refpack_output_size(stream), 7U);
|
||||
}
|
||||
|
||||
TEST(big_roundtrip) {
|
||||
big_writer writer;
|
||||
writer.add("data\\a.txt", bytes_of("hello world"));
|
||||
writer.add("data\\b.bin", bytes_of(std::string(1000, 'x')), true);
|
||||
const auto image = writer.write();
|
||||
|
||||
CHECK(image.size() >= 16U);
|
||||
const std::string magic{reinterpret_cast<const char *>(image.data()), 4};
|
||||
CHECK_EQ(magic, "BIG4");
|
||||
|
||||
const auto archive = big_archive::from_bytes(image);
|
||||
CHECK_EQ(archive.size(), 2U);
|
||||
CHECK(archive.contains("data\\a.txt"));
|
||||
CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello world");
|
||||
CHECK_EQ(archive.read("data\\b.bin").size(), 1000U);
|
||||
CHECK_EQ(archive.find("data\\").size(), 2U);
|
||||
// The first payload starts 64-byte aligned, as the retail archives do.
|
||||
CHECK_EQ(archive.entries()[0].offset % 64U, 0U);
|
||||
}
|
||||
|
||||
TEST(big_open_from_disk_is_lazy) {
|
||||
big_writer writer;
|
||||
writer.add("data\\a.txt", bytes_of("hello disk"));
|
||||
writer.add("data\\b.bin", bytes_of(std::string(4096, 'q')), true);
|
||||
const auto image = writer.write();
|
||||
|
||||
const auto path = std::filesystem::temp_directory_path() / "libra3assets_big_lazy.big";
|
||||
write_file(path, image);
|
||||
|
||||
const auto archive = big_archive::open(path);
|
||||
CHECK_EQ(archive.size(), 2U);
|
||||
CHECK(archive.path() == path);
|
||||
CHECK(archive.contains("data\\a.txt"));
|
||||
CHECK_EQ(archive.entries()[0].offset % 64U, 0U);
|
||||
CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello disk");
|
||||
CHECK_EQ(archive.read("data\\b.bin").size(), 4096U);
|
||||
CHECK_EQ(text_of(archive.read_prefix("data\\a.txt", 5)), "hello");
|
||||
CHECK(archive.read_prefix("data\\b.bin", 8U).size() == 8U);
|
||||
std::error_code ec;
|
||||
std::filesystem::remove(path, ec);
|
||||
}
|
||||
|
||||
TEST(binary_manifest_and_hash) {
|
||||
CHECK_EQ(hash_string("W3DMesh"), 0xC2B1A262U);
|
||||
CHECK_EQ(hash_string("ABAIRFIELD", false), 0x2B479BD3U);
|
||||
CHECK(hash_string("Texture") != hash_string("Texture", false));
|
||||
|
||||
const std::string name = "W3DMesh:TEST";
|
||||
const std::string source = "art:test.w3x";
|
||||
byte_writer manifest;
|
||||
manifest.write_u8(0); // not big-endian
|
||||
manifest.write_u8(0); // not linked
|
||||
manifest.write_u16(5); // version
|
||||
manifest.write_u32(0); // checksum
|
||||
manifest.write_u32(0); // all-types hash
|
||||
manifest.write_u32(1); // count
|
||||
manifest.write_u32(12);
|
||||
manifest.write_u32(0);
|
||||
manifest.write_u32(0);
|
||||
manifest.write_u32(0);
|
||||
manifest.write_u32(0); // reference buffer
|
||||
manifest.write_u32(0); // reference-name buffer
|
||||
manifest.write_u32(static_cast<std::uint32_t>(name.size() + 1));
|
||||
manifest.write_u32(static_cast<std::uint32_t>(source.size() + 1));
|
||||
manifest.write_u32(hash_string("W3DMesh"));
|
||||
manifest.write_u32(hash_string("TEST", false));
|
||||
manifest.write_u32(hash_string("W3DMesh"));
|
||||
manifest.write_u32(hash_string("TEST", false));
|
||||
manifest.write_i32(0); // reference offset
|
||||
manifest.write_i32(0); // reference count
|
||||
manifest.write_i32(0); // name offset
|
||||
manifest.write_i32(0); // source offset
|
||||
manifest.write_u32(8); // instance size
|
||||
manifest.write_u32(0); // relocation size
|
||||
manifest.write_u32(0); // imports size
|
||||
manifest.write_u32(0); // tokenized
|
||||
manifest.write_ascii(name);
|
||||
manifest.write_u8(0);
|
||||
manifest.write_ascii(source);
|
||||
manifest.write_u8(0);
|
||||
|
||||
byte_writer data;
|
||||
data.write_u32(0); // stream checksum
|
||||
data.write_ascii("PAYLOAD!");
|
||||
|
||||
auto container = binary_container::from_bytes(manifest.take(), data.take());
|
||||
CHECK_EQ(container.size(), 1U);
|
||||
const auto &asset = container.assets()[0];
|
||||
CHECK_EQ(asset.name, "W3DMesh:TEST");
|
||||
CHECK_EQ(asset.type_name(), "W3DMesh");
|
||||
CHECK_EQ(asset.instance_name(), "TEST");
|
||||
CHECK_EQ(asset.source, "art:test.w3x");
|
||||
CHECK_EQ(text_of(container.read_instance(asset)), "PAYLOAD!");
|
||||
CHECK_EQ(container.find("#0").name, "W3DMesh:TEST");
|
||||
CHECK_EQ(container.of_type("w3dmesh").size(), 1U);
|
||||
CHECK_EQ(container.type_counts().at("W3DMesh"), 1U);
|
||||
CHECK(container.read_relocation(asset).empty());
|
||||
|
||||
container.set_cdata_source([&asset](const std::string &cdata_name) -> std::optional<std::vector<std::byte>> {
|
||||
if (cdata_name == asset.cdata_name("static")) return bytes_of("CDATA");
|
||||
return std::nullopt;
|
||||
});
|
||||
CHECK(container.read_cdata(asset).has_value());
|
||||
CHECK_EQ(text_of(container.read_payload(asset)), "CDATA");
|
||||
}
|
||||
|
||||
TEST(csf_roundtrip) {
|
||||
auto table = csf_table::parse(make_csf());
|
||||
CHECK_EQ(table.size(), 2U);
|
||||
CHECK_EQ(table.version(), 3U);
|
||||
CHECK_EQ(table.lookup("abc"), "Hello"); // lookup is case-insensitive
|
||||
CHECK_EQ(table.lookup("ABC"), "Hello");
|
||||
CHECK_EQ(table.lookup("uni"), "\xE4\xB8\xAD");
|
||||
CHECK(table.find("missing") == nullptr);
|
||||
|
||||
const auto rewritten = table.write();
|
||||
const auto reparsed = csf_table::parse(rewritten);
|
||||
CHECK_EQ(reparsed.size(), 2U);
|
||||
CHECK_EQ(reparsed.lookup("abc"), "Hello");
|
||||
CHECK_EQ(reparsed.lookup("uni"), "\xE4\xB8\xAD");
|
||||
}
|
||||
|
||||
TEST(csf_decode_utf16_surrogate) {
|
||||
const std::u16string emoji = u"\U0001F600";
|
||||
CHECK_EQ(utf16_to_utf8(emoji), "\xF0\x9F\x98\x80");
|
||||
}
|
||||
|
||||
TEST(map_roundtrip) {
|
||||
const std::vector<std::string> names{"HeightMapData", "ObjectsList", "Object", "waypointName", "MPPositionList", "MPPositionInfo", "WorldInfo", "waypointID"};
|
||||
auto document = map_document::parse(make_empty_ckmp(names));
|
||||
CHECK_EQ(document.names().size(), names.size());
|
||||
|
||||
height_map_data height;
|
||||
height.width = 2;
|
||||
height.height = 2;
|
||||
height.version = 6;
|
||||
height.elevations = {1, 2, 3, 4};
|
||||
document.set_height_map(height);
|
||||
|
||||
map_object waypoint;
|
||||
waypoint.type_name = "*Waypoints/Waypoint";
|
||||
waypoint.position = {100.0F, 200.0F, 0.0F};
|
||||
waypoint.properties.push_back(asset_property::text("waypointName", "Player_1_Start"));
|
||||
waypoint.properties.push_back(asset_property::integer("waypointID", 0));
|
||||
document.set_objects({waypoint});
|
||||
|
||||
mp_position position;
|
||||
position.is_human = true;
|
||||
position.team = 1;
|
||||
document.set_mp_positions({position});
|
||||
|
||||
const auto ckmp = document.to_ckmp();
|
||||
const auto reparsed = map_document::parse(ckmp);
|
||||
|
||||
CHECK(reparsed.has_chunk(chunk_kind::height_map_data));
|
||||
CHECK(reparsed.has_chunk(chunk_kind::objects_list));
|
||||
CHECK(reparsed.has_chunk(chunk_kind::mp_position_list));
|
||||
CHECK_EQ(reparsed.find_chunk("HeightMapData")->kind, chunk_kind::height_map_data);
|
||||
CHECK_EQ(chunk_kind_of("BlendTileData"), chunk_kind::blend_tile_data);
|
||||
CHECK_EQ(chunk_kind_of("NotAChunk"), chunk_kind::unknown);
|
||||
|
||||
const auto restored = reparsed.height_map();
|
||||
CHECK(restored.has_value());
|
||||
CHECK_EQ(restored->width, 2U);
|
||||
CHECK_EQ(restored->at(1, 1), 4U);
|
||||
CHECK_EQ(restored->vertical_scale(), 0.0390625F);
|
||||
|
||||
const auto objects = reparsed.objects();
|
||||
CHECK_EQ(objects.size(), 1U);
|
||||
CHECK_EQ(objects[0].type_name, "*Waypoints/Waypoint");
|
||||
CHECK_EQ(objects[0].property("waypointID")->as_int(), 0);
|
||||
|
||||
const auto starts = reparsed.player_starts();
|
||||
CHECK_EQ(starts.size(), 1U);
|
||||
CHECK_EQ(starts[0].index, 1);
|
||||
CHECK_EQ(starts[0].position.x, 100.0F);
|
||||
|
||||
const auto positions = reparsed.mp_positions();
|
||||
CHECK_EQ(positions.size(), 1U);
|
||||
CHECK(positions[0].is_human);
|
||||
CHECK_EQ(positions[0].team, 1U);
|
||||
}
|
||||
|
||||
TEST(map_compressed_and_big_payload) {
|
||||
const std::vector<std::string> names{"HeightMapData"};
|
||||
auto document = map_document::parse(make_empty_ckmp(names));
|
||||
height_map_data height;
|
||||
height.width = 1;
|
||||
height.height = 1;
|
||||
height.version = 6;
|
||||
height.elevations = {7};
|
||||
document.set_height_map(height);
|
||||
|
||||
// EAR\0 + RefPack round-trip.
|
||||
const auto compressed = document.serialize(true);
|
||||
const auto from_compressed = map_document::parse(compressed);
|
||||
CHECK(from_compressed.height_map().has_value());
|
||||
CHECK_EQ(from_compressed.height_map()->at(0, 0), 7U);
|
||||
|
||||
// A BIG payload is one more RefPack layer over the map file.
|
||||
const auto big_payload = refpack_compress(document.to_ckmp());
|
||||
const auto from_big = map_document::parse(big_payload);
|
||||
CHECK_EQ(from_big.height_map()->at(0, 0), 7U);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
#include "test_main.hpp"
|
||||
|
||||
int main() {
|
||||
return ra3test::run_all();
|
||||
}
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
#ifndef RA3TEST_HPP
|
||||
#define RA3TEST_HPP
|
||||
|
||||
// A tiny dependency-free test harness (registry + CHECK macros). Each unit-test
|
||||
// executable compiles test_main.cpp and links the sources under test.
|
||||
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace ra3test {
|
||||
|
||||
struct test_case {
|
||||
std::string name;
|
||||
std::function<void()> fn;
|
||||
};
|
||||
|
||||
inline auto registry() -> std::vector<test_case> & {
|
||||
static std::vector<test_case> tests;
|
||||
return tests;
|
||||
}
|
||||
|
||||
inline auto failure_count() -> int & {
|
||||
static int count = 0;
|
||||
return count;
|
||||
}
|
||||
|
||||
struct registrar {
|
||||
registrar(std::string name, std::function<void()> fn) { registry().push_back({std::move(name), std::move(fn)}); }
|
||||
};
|
||||
|
||||
inline auto report_failure(const std::string &expr, const std::string &file, int line) -> void {
|
||||
++failure_count();
|
||||
std::cerr << " FAIL " << file << ':' << line << " " << expr << '\n';
|
||||
}
|
||||
|
||||
inline auto check(bool condition, const std::string &expr, const std::string &file, int line) -> void {
|
||||
if (!condition) report_failure(expr, file, line);
|
||||
}
|
||||
|
||||
inline auto run_all() -> int {
|
||||
int passed = 0;
|
||||
for (auto &test: registry()) {
|
||||
const int before = failure_count();
|
||||
std::cout << "[ RUN ] " << test.name << '\n';
|
||||
try {
|
||||
test.fn();
|
||||
} catch (const std::exception &exc) {
|
||||
report_failure(std::string("uncaught exception: ") + exc.what(), __FILE__, __LINE__);
|
||||
}
|
||||
if (failure_count() == before) {
|
||||
++passed;
|
||||
std::cout << "[ OK ] " << test.name << '\n';
|
||||
}
|
||||
}
|
||||
std::cout << "\n" << passed << '/' << registry().size() << " tests passed, " << failure_count() << " failure(s)\n";
|
||||
return failure_count() == 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
} // namespace ra3test
|
||||
|
||||
#define TEST(name) \
|
||||
static void name(); \
|
||||
static ::ra3test::registrar ra3test_reg_##name(#name, name); \
|
||||
static void name()
|
||||
|
||||
#define CHECK(cond) ::ra3test::check((cond), #cond, __FILE__, __LINE__)
|
||||
#define CHECK_EQ(a, b) ::ra3test::check_eq((a), (b), #a, #b, __FILE__, __LINE__)
|
||||
|
||||
namespace ra3test {
|
||||
template<typename A, typename B>
|
||||
auto check_eq(const A &a, const B &b, const std::string &ea, const std::string &eb, const std::string &file, int line) -> void {
|
||||
if (!(a == b)) {
|
||||
std::ostringstream os;
|
||||
os << ea << " == " << eb;
|
||||
report_failure(os.str(), file, line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif // RA3TEST_HPP
|
||||
Reference in New Issue
Block a user