OpenRA3/third_party/libra3assets/tests/test_assets.cpp

#include "test_main.hpp"

import std;
import ra3.assets;

using namespace ra3::assets;

namespace {

    auto bytes_of(std::string_view text) -> std::vector<std::byte> {
        std::vector<std::byte> out;
        out.reserve(text.size());
        for (const auto ch: text) out.push_back(static_cast<std::byte>(static_cast<unsigned char>(ch)));
        return out;
    }

    auto text_of(std::span<const std::byte> bytes) -> std::string {
        return std::string{reinterpret_cast<const char *>(bytes.data()), bytes.size()};
    }

    /** A deterministic pseudo-random buffer (no external RNG needed). */
    auto pseudo_random(std::size_t size, std::uint32_t seed) -> std::vector<std::byte> {
        std::vector<std::byte> out;
        out.reserve(size);
        std::uint32_t state = seed;
        for (std::size_t i = 0; i < size; ++i) {
            state = state * 1664525U + 1013904223U;
            out.push_back(static_cast<std::byte>((state >> 16U) & 0xFFU));
        }
        return out;
    }

    auto make_empty_ckmp(const std::vector<std::string> &names) -> std::vector<std::byte> {
        byte_writer writer;
        writer.write_ascii("CkMp");
        writer.write_u32(static_cast<std::uint32_t>(names.size()));
        for (std::uint32_t i = static_cast<std::uint32_t>(names.size()); i >= 1U; --i) {
            writer.write_u8(static_cast<std::uint8_t>(names[i - 1].size()));
            writer.write_ascii(names[i - 1]);
            writer.write_u32(i);
        }
        return writer.take();
    }

    auto make_csf() -> std::vector<std::byte> {
        byte_writer writer;
        writer.write_ascii(" FSC");
        writer.write_u32(3);
        writer.write_u32(2); // labels
        writer.write_u32(2); // value blocks
        writer.write_u32(0);
        writer.write_u32(0);
        writer.write_u32(csf_label_flag);
        writer.write_u32(1);
        writer.write_u32(3);
        writer.write_ascii("ABC");
        writer.write_u32(csf_value_flag);
        writer.write_u32(5);
        for (const auto ch: std::string_view{"Hello"}) writer.write_u16(static_cast<std::uint16_t>(ch) ^ 0xFFFFU);
        writer.write_u32(csf_label_flag);
        writer.write_u32(1);
        writer.write_u32(3);
        writer.write_ascii("UNI");
        writer.write_u32(csf_value_flag);
        writer.write_u32(1);
        writer.write_u16(0x4E2DU ^ 0xFFFFU); // U+4E2D (CJK)
        return writer.take();
    }

} // namespace

TEST(refpack_roundtrip_small) {
    const std::vector<std::string> samples{"", "A", "AB", "ABC", "ABCD", "abcabcabc", std::string(300, 'z')};
    for (const auto &sample: samples) {
        const auto input = bytes_of(sample);
        const auto packed = refpack_compress(input);
        const auto restored = refpack_decompress(packed);
        CHECK_EQ(restored.size(), input.size());
        CHECK(std::ranges::equal(restored, input));
    }
}

TEST(refpack_roundtrip_large_and_repetitive) {
    std::vector<std::byte> input;
    for (int i = 0; i < 2000; ++i) {
        for (const auto ch: std::string_view{"lorem ipsum dolor sit amet "}) input.push_back(static_cast<std::byte>(ch));
    }
    const auto random = pseudo_random(50000, 0x1234ABCDU);
    input.insert(input.end(), random.begin(), random.end());
    const auto packed = refpack_compress(input);
    CHECK(packed.size() < input.size()); // the repeated prefix must compress
    CHECK(std::ranges::equal(refpack_decompress(packed), input));
}

TEST(refpack_decode_known_stream) {
    // "abcd" literals, then a back-reference (distance 4, length 3) -> "abcdabc".
    const std::vector<std::byte> stream{
        std::byte{0x10}, std::byte{0xFB}, std::byte{0x00}, std::byte{0x00}, std::byte{0x07}, std::byte{0xE0},
        std::byte{'a'},  std::byte{'b'},  std::byte{'c'},  std::byte{'d'},  std::byte{0x00}, std::byte{0x03},
        std::byte{0xFC},
    };
    CHECK(is_refpack(stream));
    CHECK_EQ(text_of(refpack_decompress(stream)), "abcdabc");
    CHECK_EQ(refpack_output_size(stream), 7U);
}

TEST(big_roundtrip) {
    big_writer writer;
    writer.add("data\\a.txt", bytes_of("hello world"));
    writer.add("data\\b.bin", bytes_of(std::string(1000, 'x')), true);
    const auto image = writer.write();

    CHECK(image.size() >= 16U);
    const std::string magic{reinterpret_cast<const char *>(image.data()), 4};
    CHECK_EQ(magic, "BIG4");

    const auto archive = big_archive::from_bytes(image);
    CHECK_EQ(archive.size(), 2U);
    CHECK(archive.contains("data\\a.txt"));
    CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello world");
    CHECK_EQ(archive.read("data\\b.bin").size(), 1000U);
    CHECK_EQ(archive.find("data\\").size(), 2U);
    // The first payload starts 64-byte aligned, as the retail archives do.
    CHECK_EQ(archive.entries()[0].offset % 64U, 0U);
}

TEST(big_open_from_disk_is_lazy) {
    big_writer writer;
    writer.add("data\\a.txt", bytes_of("hello disk"));
    writer.add("data\\b.bin", bytes_of(std::string(4096, 'q')), true);
    const auto image = writer.write();

    const auto path = std::filesystem::temp_directory_path() / "libra3assets_big_lazy.big";
    write_file(path, image);

    const auto archive = big_archive::open(path);
    CHECK_EQ(archive.size(), 2U);
    CHECK(archive.path() == path);
    CHECK(archive.contains("data\\a.txt"));
    CHECK_EQ(archive.entries()[0].offset % 64U, 0U);
    CHECK_EQ(text_of(archive.read("data\\a.txt")), "hello disk");
    CHECK_EQ(archive.read("data\\b.bin").size(), 4096U);
    CHECK_EQ(text_of(archive.read_prefix("data\\a.txt", 5)), "hello");
    CHECK(archive.read_prefix("data\\b.bin", 8U).size() == 8U);
    std::error_code ec;
    std::filesystem::remove(path, ec);
}

TEST(binary_manifest_and_hash) {
    CHECK_EQ(hash_string("W3DMesh"), 0xC2B1A262U);
    CHECK_EQ(hash_string("ABAIRFIELD", false), 0x2B479BD3U);
    CHECK(hash_string("Texture") != hash_string("Texture", false));

    const std::string name = "W3DMesh:TEST";
    const std::string source = "art:test.w3x";
    byte_writer manifest;
    manifest.write_u8(0);  // not big-endian
    manifest.write_u8(0);  // not linked
    manifest.write_u16(5); // version
    manifest.write_u32(0); // checksum
    manifest.write_u32(0); // all-types hash
    manifest.write_u32(1); // count
    manifest.write_u32(12);
    manifest.write_u32(0);
    manifest.write_u32(0);
    manifest.write_u32(0);
    manifest.write_u32(0); // reference buffer
    manifest.write_u32(0); // reference-name buffer
    manifest.write_u32(static_cast<std::uint32_t>(name.size() + 1));
    manifest.write_u32(static_cast<std::uint32_t>(source.size() + 1));
    manifest.write_u32(hash_string("W3DMesh"));
    manifest.write_u32(hash_string("TEST", false));
    manifest.write_u32(hash_string("W3DMesh"));
    manifest.write_u32(hash_string("TEST", false));
    manifest.write_i32(0); // reference offset
    manifest.write_i32(0); // reference count
    manifest.write_i32(0); // name offset
    manifest.write_i32(0); // source offset
    manifest.write_u32(8); // instance size
    manifest.write_u32(0); // relocation size
    manifest.write_u32(0); // imports size
    manifest.write_u32(0); // tokenized
    manifest.write_ascii(name);
    manifest.write_u8(0);
    manifest.write_ascii(source);
    manifest.write_u8(0);

    byte_writer data;
    data.write_u32(0); // stream checksum
    data.write_ascii("PAYLOAD!");

    auto container = binary_container::from_bytes(manifest.take(), data.take());
    CHECK_EQ(container.size(), 1U);
    const auto &asset = container.assets()[0];
    CHECK_EQ(asset.name, "W3DMesh:TEST");
    CHECK_EQ(asset.type_name(), "W3DMesh");
    CHECK_EQ(asset.instance_name(), "TEST");
    CHECK_EQ(asset.source, "art:test.w3x");
    CHECK_EQ(text_of(container.read_instance(asset)), "PAYLOAD!");
    CHECK_EQ(container.find("#0").name, "W3DMesh:TEST");
    CHECK_EQ(container.of_type("w3dmesh").size(), 1U);
    CHECK_EQ(container.type_counts().at("W3DMesh"), 1U);
    CHECK(container.read_relocation(asset).empty());

    container.set_cdata_source([&asset](const std::string &cdata_name) -> std::optional<std::vector<std::byte>> {
        if (cdata_name == asset.cdata_name("static")) return bytes_of("CDATA");
        return std::nullopt;
    });
    CHECK(container.read_cdata(asset).has_value());
    CHECK_EQ(text_of(container.read_payload(asset)), "CDATA");
}

TEST(csf_roundtrip) {
    auto table = csf_table::parse(make_csf());
    CHECK_EQ(table.size(), 2U);
    CHECK_EQ(table.version(), 3U);
    CHECK_EQ(table.lookup("abc"), "Hello");      // lookup is case-insensitive
    CHECK_EQ(table.lookup("ABC"), "Hello");
    CHECK_EQ(table.lookup("uni"), "\xE4\xB8\xAD");
    CHECK(table.find("missing") == nullptr);

    const auto rewritten = table.write();
    const auto reparsed = csf_table::parse(rewritten);
    CHECK_EQ(reparsed.size(), 2U);
    CHECK_EQ(reparsed.lookup("abc"), "Hello");
    CHECK_EQ(reparsed.lookup("uni"), "\xE4\xB8\xAD");
}

TEST(csf_decode_utf16_surrogate) {
    const std::u16string emoji = u"\U0001F600";
    CHECK_EQ(utf16_to_utf8(emoji), "\xF0\x9F\x98\x80");
}

TEST(map_roundtrip) {
    const std::vector<std::string> names{"HeightMapData", "ObjectsList", "Object", "waypointName", "MPPositionList", "MPPositionInfo", "WorldInfo", "waypointID"};
    auto document = map_document::parse(make_empty_ckmp(names));
    CHECK_EQ(document.names().size(), names.size());

    height_map_data height;
    height.width = 2;
    height.height = 2;
    height.version = 6;
    height.elevations = {1, 2, 3, 4};
    document.set_height_map(height);

    map_object waypoint;
    waypoint.type_name = "*Waypoints/Waypoint";
    waypoint.position = {100.0F, 200.0F, 0.0F};
    waypoint.properties.push_back(asset_property::text("waypointName", "Player_1_Start"));
    waypoint.properties.push_back(asset_property::integer("waypointID", 0));
    document.set_objects({waypoint});

    mp_position position;
    position.is_human = true;
    position.team = 1;
    document.set_mp_positions({position});

    const auto ckmp = document.to_ckmp();
    const auto reparsed = map_document::parse(ckmp);

    CHECK(reparsed.has_chunk(chunk_kind::height_map_data));
    CHECK(reparsed.has_chunk(chunk_kind::objects_list));
    CHECK(reparsed.has_chunk(chunk_kind::mp_position_list));
    CHECK_EQ(reparsed.find_chunk("HeightMapData")->kind, chunk_kind::height_map_data);
    CHECK_EQ(chunk_kind_of("BlendTileData"), chunk_kind::blend_tile_data);
    CHECK_EQ(chunk_kind_of("NotAChunk"), chunk_kind::unknown);

    const auto restored = reparsed.height_map();
    CHECK(restored.has_value());
    CHECK_EQ(restored->width, 2U);
    CHECK_EQ(restored->at(1, 1), 4U);
    CHECK_EQ(restored->vertical_scale(), 0.0390625F);

    const auto objects = reparsed.objects();
    CHECK_EQ(objects.size(), 1U);
    CHECK_EQ(objects[0].type_name, "*Waypoints/Waypoint");
    CHECK_EQ(objects[0].property("waypointID")->as_int(), 0);

    const auto starts = reparsed.player_starts();
    CHECK_EQ(starts.size(), 1U);
    CHECK_EQ(starts[0].index, 1);
    CHECK_EQ(starts[0].position.x, 100.0F);

    const auto positions = reparsed.mp_positions();
    CHECK_EQ(positions.size(), 1U);
    CHECK(positions[0].is_human);
    CHECK_EQ(positions[0].team, 1U);
}

TEST(map_compressed_and_big_payload) {
    const std::vector<std::string> names{"HeightMapData"};
    auto document = map_document::parse(make_empty_ckmp(names));
    height_map_data height;
    height.width = 1;
    height.height = 1;
    height.version = 6;
    height.elevations = {7};
    document.set_height_map(height);

    // EAR\0 + RefPack round-trip.
    const auto compressed = document.serialize(true);
    const auto from_compressed = map_document::parse(compressed);
    CHECK(from_compressed.height_map().has_value());
    CHECK_EQ(from_compressed.height_map()->at(0, 0), 7U);

    // A BIG payload is one more RefPack layer over the map file.
    const auto big_payload = refpack_compress(document.to_ckmp());
    const auto from_big = map_document::parse(big_payload);
    CHECK_EQ(from_big.height_map()->at(0, 0), 7U);
}