EnderTheCoder/OpenRA3
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

render map objects: decode compiled W3DMesh art, skin and draw over terrain

Browse Source 
Read the buildings and props a map places from its compiled art (the
uncompressed worldbuilder stream + embedded DDS textures):

- ra3.models: BAB asset-stream parser (lazy slices), W3DMesh + D3DHierarchy
  decode, DDS (DXT1/3/5 + uncompressed) decode, per-mesh bone remap and
  bind-pose single-joint skinning, flattened to one world-space triangle soup.
- ra3.map: parse the ObjectsList chunk into (type, x, y, z, angle).
- ra3.terrain: render3d rasterises the scene over the raymarched terrain with
  a z-buffer; gpu_terrain carries the scene for the GPU backends.
- ra3.vulkan: second pipeline + depth attachment, terrain.frag writes
  gl_FragDepth, and a small depth bias keeps ground decals from z-fighting.
- objects.fx shaders (compiled to SPIR-V), embedded like scene/terrain.

Only opaque parts are drawn: FX-light billboards (DefaultW3D.fx / BasicW3D.fx)
and damage-fill shells (BuildingsGenericDamageFill.fx) are skipped, since the
latter paint the wrecked interior (e.g. orange CBBuilding_Wood) over the shell.

Ground-decal meshes with no diffuse role and the Road templates themselves are
still not drawn.
This commit is contained in:
EnderTheCoder
2026-09-29 18:08:11 +08:00
parent 6e5df350c7
commit 70f35d8382
18 changed files with 2027 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/ra3.models.cppm
+939
View File
@@ -0,0 +1,939 @@
export module ra3.models;
import std;
export import ra3.core;
export import ra3.render;
export import ra3.fs;
/**
* The map's static art: the compiled W3D models that draw the buildings and
* props a map places on the ground.
*
* Retail RA3 does not ship `.w3x`/`.w3d` files. BinaryAssetBuilder bakes every
* model into a *binary asset stream* — a `.manifest` index plus a `.bin` of
* relocatable instance data — and each map carries its own stream in
* `Maps*.big` under `data\maps\official\<id>\map.{manifest,bin}`. Those streams
* hold the `W3DMesh` assets for the map's props (sidewalks, deck props, lights,
* walls, civilian buildings, ...) and the `Texture` assets they sample.
*
* This module parses that stream, decodes the compiled `W3DMesh` vertex/index
* buffers and the embedded DDS textures, and flattens the map's objects into a
* single world-space triangle soup the renderers upload directly.
*
* Format references: ra3-headless `ra3tools/ra3_binary.py` (BAB
* `ManifestHeader`/`AssetEntry`, `SageBinaryData/W3D.cs`) and OpenSAGE
* `Data/Map/MapObject.cs`. Instance pointers are stored as offsets from the
* start of the instance data, so no relocation pass is needed.
*/
export namespace ra3::models {
using ra3::core::uint8;
using ra3::core::uint16;
using ra3::core::uint32;
using ra3::core::int32;
using ra3::core::usize;
using ra3::render::argb;
using ra3::render::image;
/** Thrown when a compiled asset stream or a model payload is malformed. */
class model_error : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
namespace detail {
[[nodiscard]] inline auto u16(std::span<const uint8> b, usize off) -> uint16 {
return off + 2U <= b.size() ? static_cast<uint16>(b[off]) | (static_cast<uint16>(b[off + 1U]) << 8U) : 0U;
}
[[nodiscard]] inline auto u32(std::span<const uint8> b, usize off) -> uint32 {
return off + 4U <= b.size() ? static_cast<uint32>(b[off]) | (static_cast<uint32>(b[off + 1U]) << 8U) | (static_cast<uint32>(b[off + 2U]) << 16U) |
(static_cast<uint32>(b[off + 3U]) << 24U)
: 0U;
}
[[nodiscard]] inline auto i32(std::span<const uint8> b, usize off) -> ra3::core::int32 { return static_cast<ra3::core::int32>(u32(b, off)); }
[[nodiscard]] inline auto f32(std::span<const uint8> b, usize off) -> float {
const auto bits = u32(b, off);
float value = 0.0F;
std::memcpy(&value, &bits, sizeof(value));
return value;
}
[[nodiscard]] inline auto cstr(std::span<const uint8> b, usize off, ra3::core::int32 length) -> std::string {
if (off == 0U || length <= 0 || off + static_cast<usize>(length) > b.size()) return {};
return std::string{reinterpret_cast<const char *>(b.data() + off), static_cast<usize>(length)};
}
/** NUL-terminated string in the manifest name/source buffers. */
[[nodiscard]] inline auto nul_string(std::span<const uint8> b, usize off) -> std::string {
if (off >= b.size()) return {};
usize end = off;
while (end < b.size() && b[end] != 0U) ++end;
return std::string{reinterpret_cast<const char *>(b.data() + off), end - off};
}
}
/**
* A parsed `BinaryAsset` stream: the manifest index plus the concatenated
* instance data. Assets are addressed by `Type:Instance` or by their
* `(typeId, instanceId)` pair (used by cross-asset references).
*/
class asset_stream {
public:
struct asset {
std::string name; ///< `Type:Instance`
std::string source;
uint32 type_id = 0;
uint32 instance_id = 0;
usize instance_offset = 0;
usize instance_size = 0;
std::vector<std::pair<uint32, uint32>> references; ///< `(typeId, instanceId)` targets.
[[nodiscard]] auto type_name() const -> std::string_view { return std::string_view{name}.substr(0, name.find(':')); }
[[nodiscard]] auto instance_name() const -> std::string_view {
const auto at = name.find(':');
return at == std::string::npos ? std::string_view{name} : std::string_view{name}.substr(at + 1U);
}
};
static constexpr uint32 header_size = 48U;
static constexpr uint32 entry_size = 48U;
/** Parse an in-memory `.manifest` + `.bin` pair. */
[[nodiscard]] static auto load(std::span<const uint8> manifest_raw, std::span<const uint8> data_raw) -> asset_stream {
asset_stream stream;
stream.data_ = fs::maybe_decompress(data_raw);
stream.parse(fs::maybe_decompress(manifest_raw));
return stream;
}
/**
* Parse a `.manifest` + `.bin` pair from disk.
*
* Retail's worldbuilder stream is ~1.1 GB and uncompressed, so a small
* `.bin` is read into memory but a large one is read lazily per asset
* (its pointers are offsets, so a slice needs no random-access decode).
*/
[[nodiscard]] static auto load_files(const std::filesystem::path &manifest_path, const std::filesystem::path &bin_path,
usize inline_limit = 128U * 1024U * 1024U) -> asset_stream {
const auto read_all = [](const std::filesystem::path &path) {
std::ifstream in(path, std::ios::binary);
if (!in) throw model_error("cannot open " + path.string());
return std::vector<uint8>{std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};
};
std::error_code ec;
const auto bin_size = std::filesystem::file_size(bin_path, ec);
asset_stream stream;
if (ec || bin_size <= inline_limit) {
stream.data_ = fs::maybe_decompress(read_all(bin_path));
} else {
stream.bin_path_ = bin_path;
// The large retail streams (worldbuilder.bin) are stored raw; a
// RefPack stream would not be randomly addressable anyway.
const auto probe = [&] {
std::ifstream in(bin_path, std::ios::binary);
std::array<uint8, 16> head{};
in.read(reinterpret_cast<char *>(head.data()), static_cast<std::streamsize>(head.size()));
return fs::is_refpack(head);
}();
if (probe) throw model_error("compressed asset stream is too large to map");
}
stream.parse(read_all(manifest_path));
return stream;
}
[[nodiscard]] auto assets() const -> const std::vector<asset> & { return assets_; }
[[nodiscard]] auto find(uint32 type_id, uint32 instance_id) const -> const asset * {
const auto it = index_.find((static_cast<unsigned long long>(type_id) << 32U) | instance_id);
return it == index_.end() ? nullptr : &assets_[it->second];
}
/** First asset whose `Type:Instance` name equals `name` (case-insensitive). */
[[nodiscard]] auto find_name(std::string_view name) const -> const asset * {
const auto want = lower(name);
for (const auto &a: assets_) {
if (lower(a.name) == want) return &a;
}
return nullptr;
}
/** Every `W3DMesh` whose instance name equals `base` or starts with `base.`. */
[[nodiscard]] auto meshes_for(std::string_view base) const -> const std::vector<const asset *> & {
static const std::vector<const asset *> none;
const auto it = mesh_index_.find(lower(base));
return it == mesh_index_.end() ? none : it->second;
}
/** The asset's instance bytes (from memory, or lazily from disk). */
[[nodiscard]] auto read_instance(const asset &a) const -> std::vector<uint8> {
if (!data_.empty()) {
if (a.instance_offset + a.instance_size > data_.size()) return {};
return {data_.begin() + static_cast<std::ptrdiff_t>(a.instance_offset),
data_.begin() + static_cast<std::ptrdiff_t>(a.instance_offset + a.instance_size)};
}
if (bin_path_.empty() || a.instance_size == 0U) return {};
std::ifstream in(bin_path_, std::ios::binary);
if (!in) return {};
in.seekg(static_cast<std::streamoff>(a.instance_offset));
std::vector<uint8> bytes(a.instance_size);
in.read(reinterpret_cast<char *>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
return bytes;
}
[[nodiscard]] static auto lower(std::string_view text) -> std::string {
std::string out{text};
std::transform(out.begin(), out.end(), out.begin(), [](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
return out;
}
private:
/** Parse the manifest into `assets_` + the lookup indexes. */
auto parse(std::span<const uint8> manifest) -> void {
if (manifest.size() < header_size) throw model_error("asset manifest is too short");
if (manifest[0] != 0U) throw model_error("big-endian asset manifest is unsupported");
const auto count = detail::u32(manifest, 12U);
const auto ref_buf_size = detail::u32(manifest, 32U);
const auto ref_name_buf_size = detail::u32(manifest, 36U);
const auto name_buf_size = detail::u32(manifest, 40U);
const auto src_buf_size = detail::u32(manifest, 44U);
(void) src_buf_size;
if (header_size + static_cast<usize>(count) * entry_size > manifest.size()) throw model_error("asset manifest entry table is truncated");
const usize entries_off = header_size;
const usize ref_off = entries_off + static_cast<usize>(count) * entry_size;
const usize refname_off = ref_off + ref_buf_size;
const usize name_off = refname_off + ref_name_buf_size;
const usize src_off = name_off + name_buf_size;
assets_.reserve(count);
usize instance_offset = 4U; // the first four bytes of `.bin` are the stream checksum
for (uint32 i = 0; i < count; ++i) {
const usize e = entries_off + static_cast<usize>(i) * entry_size;
asset a;
a.type_id = detail::u32(manifest, e);
a.instance_id = detail::u32(manifest, e + 4U);
const auto aref_off = detail::i32(manifest, e + 16U);
const auto aref_count = detail::i32(manifest, e + 20U);
a.instance_size = detail::u32(manifest, e + 32U);
a.instance_offset = instance_offset;
a.name = detail::nul_string(manifest, name_off + static_cast<usize>(std::max(0, detail::i32(manifest, e + 24U))));
a.source = detail::nul_string(manifest, src_off + static_cast<usize>(std::max(0, detail::i32(manifest, e + 28U))));
if (aref_off >= 0 && aref_count > 0 && ref_off + static_cast<usize>(aref_off) + static_cast<usize>(aref_count) * 8U <= manifest.size()) {
a.references.reserve(static_cast<usize>(aref_count));
for (int r = 0; r < aref_count; ++r) {
const usize ro = ref_off + static_cast<usize>(aref_off) + static_cast<usize>(r) * 8U;
a.references.emplace_back(detail::u32(manifest, ro), detail::u32(manifest, ro + 4U));
}
}
index_.try_emplace((static_cast<unsigned long long>(a.type_id) << 32U) | a.instance_id, assets_.size());
assets_.push_back(std::move(a));
instance_offset += assets_.back().instance_size;
}
// Index meshes by the base of their instance name (`A.B` -> `a` and
// `a.b`) so a map object type resolves to its mesh parts in one look.
for (const auto &a: assets_) {
if (a.type_name() != "W3DMesh") continue;
const auto instance = lower(a.instance_name());
mesh_index_[instance].push_back(&a);
const auto dot = instance.find('.');
if (dot != std::string::npos) mesh_index_[instance.substr(0, dot)].push_back(&a);
}
}
std::vector<asset> assets_;
std::unordered_map<unsigned long long, usize> index_;
std::unordered_map<std::string, std::vector<const asset *>> mesh_index_;
std::vector<uint8> data_;
std::filesystem::path bin_path_;
};
/** A decoded compiled `W3DMesh`: a vertex buffer plus a triangle list, in model space. */
struct mesh {
std::vector<float> positions; ///< 3 floats per vertex.
std::vector<float> normals; ///< 3 floats per vertex, or empty.
std::vector<float> uvs; ///< 2 floats per vertex, or empty.
std::vector<uint32> indices;
std::string shader; ///< FX shader name (`BuildingsGeneric.fx`, ...).
std::vector<uint8> blend_indices; ///< 4 bone indices per vertex, or empty (static mesh).
std::vector<uint16> bone_remap; ///< Per-mesh blend index -> hierarchy bone map, or empty.
std::vector<float> blend_weights; ///< 4 weights per vertex, or empty.
[[nodiscard]] auto skinned() const -> bool { return !blend_indices.empty(); }
};
/** One bone of a `W3DHierarchy`: name hash, parent and the local (bind) transform. */
struct bone {
uint32 name_hash = 0;
int32 parent = -1; ///< -1 for the root.
float tx = 0.0F;
float ty = 0.0F;
float tz = 0.0F;
float qx = 0.0F;
float qy = 0.0F;
float qz = 0.0F;
float qw = 1.0F;
};
/** A decoded `W3DHierarchy` (the skeleton the meshes are bound to). */
struct hierarchy {
std::vector<bone> bones;
};
/**
* Decode a compiled `W3DHierarchy`.
*
* Layout (from the retail data): `u32 pad, u32 boneCount, u32 headerBytes`,
* then `boneCount` 100-byte records: `u32 nameHash, i32 parent, f32 t[3],
* f32 quaternion[4] (x,y,z,w), f32 matrix[12]`. Meshes are bound to bones in
* *bone space*, so a static render must rebuild each bone's world transform
* from this default pose (see `bone_world_matrices`).
*/
[[nodiscard]] inline auto decode_hierarchy(std::span<const uint8> data) -> hierarchy {
hierarchy out;
const auto bone_count = detail::u32(data, 4U);
const auto header = detail::u32(data, 8U);
if (bone_count == 0U || bone_count > 4096U || header + static_cast<usize>(bone_count) * 100U > data.size()) return out;
out.bones.reserve(bone_count);
for (uint32 i = 0; i < bone_count; ++i) {
const auto o = header + static_cast<usize>(i) * 100U;
bone b;
b.name_hash = detail::u32(data, o);
b.parent = detail::i32(data, o + 4U);
b.tx = detail::f32(data, o + 8U);
b.ty = detail::f32(data, o + 12U);
b.tz = detail::f32(data, o + 16U);
b.qx = detail::f32(data, o + 20U);
b.qy = detail::f32(data, o + 24U);
b.qz = detail::f32(data, o + 28U);
b.qw = detail::f32(data, o + 32U);
out.bones.push_back(b);
}
return out;
}
/** A bone's world transform: 3x3 rotation (row-major) followed by a translation. */
struct bone_matrix {
std::array<float, 9> rotation{1.0F, 0.0F, 0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 0.0F, 1.0F};
std::array<float, 3> translation{0.0F, 0.0F, 0.0F};
};
namespace detail {
/** Quaternion `(x, y, z, w)` to a row-major 3x3 rotation matrix. */
[[nodiscard]] inline auto quaternion_matrix(float x, float y, float z, float w) -> std::array<float, 9> {
const auto n = std::sqrt(x * x + y * y + z * z + w * w);
if (n <= 1.0e-8F) return {1.0F, 0.0F, 0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 0.0F, 1.0F};
x /= n;
y /= n;
z /= n;
w /= n;
return {1.0F - 2.0F * (y * y + z * z), 2.0F * (x * y - z * w), 2.0F * (x * z + y * w),
2.0F * (x * y + z * w), 1.0F - 2.0F * (x * x + z * z), 2.0F * (y * z - x * w),
2.0F * (x * z - y * w), 2.0F * (y * z + x * w), 1.0F - 2.0F * (x * x + y * y)};
}
[[nodiscard]] inline auto multiply(const std::array<float, 9> &a, const std::array<float, 9> &b) -> std::array<float, 9> {
std::array<float, 9> out{};
for (int i = 0; i < 3; ++i) {
for (int j = 0; j < 3; ++j) {
out[static_cast<usize>(i) * 3U + static_cast<usize>(j)] = a[static_cast<usize>(i) * 3U] * b[static_cast<usize>(j)] +
a[static_cast<usize>(i) * 3U + 1U] * b[3U + static_cast<usize>(j)] +
a[static_cast<usize>(i) * 3U + 2U] * b[6U + static_cast<usize>(j)];
}
}
return out;
}
[[nodiscard]] inline auto rotate(const std::array<float, 9> &m, const std::array<float, 3> &v) -> std::array<float, 3> {
return {m[0] * v[0] + m[1] * v[1] + m[2] * v[2], m[3] * v[0] + m[4] * v[1] + m[5] * v[2], m[6] * v[0] + m[7] * v[1] + m[8] * v[2]};
}
}
/** Each bone's world transform in the hierarchy's default (bind) pose. */
[[nodiscard]] inline auto bone_world_matrices(const hierarchy &h) -> std::vector<bone_matrix> {
std::vector<bone_matrix> world(h.bones.size());
for (usize i = 0; i < h.bones.size(); ++i) {
const auto &b = h.bones[i];
const auto local_rotation = detail::quaternion_matrix(b.qx, b.qy, b.qz, b.qw);
const std::array<float, 3> local_translation{b.tx, b.ty, b.tz};
if (b.parent >= 0 && static_cast<usize>(b.parent) < i) {
const auto &parent = world[static_cast<usize>(b.parent)];
world[i].rotation = detail::multiply(parent.rotation, local_rotation);
const auto rotated = detail::rotate(parent.rotation, local_translation);
world[i].translation = {parent.translation[0] + rotated[0], parent.translation[1] + rotated[1], parent.translation[2] + rotated[2]};
} else {
world[i].rotation = local_rotation;
world[i].translation = local_translation;
}
}
return world;
}
namespace detail {
/** One vertex element: which attribute, its byte offset and its type code. */
struct vertex_element {
char usage = 0;
uint32 index = 0;
uint32 offset = 0;
std::string type;
};
[[nodiscard]] inline auto usage_of(char letter) -> bool {
return letter == 'p' || letter == 'n' || letter == 't' || letter == 'c' || letter == 'g' || letter == 'b' || letter == 'i' || letter == 'w';
}
/** Parse the D3D9 text vertex declaration `"p0:00:3f32 n0:0C:3f32 t0:1C:2f32"`. */
[[nodiscard]] inline auto parse_vertex_format(std::string_view decl) -> std::vector<vertex_element> {
std::vector<vertex_element> elements;
for (usize start = 0; start < decl.size();) {
const auto end = decl.find(' ', start);
const auto token = decl.substr(start, end == std::string_view::npos ? std::string_view::npos : end - start);
start = end == std::string_view::npos ? decl.size() : end + 1U;
const auto c1 = token.find(':');
if (c1 == std::string_view::npos) continue;
const auto c2 = token.find(':', c1 + 1U);
if (c2 == std::string_view::npos) continue;
const auto name = token.substr(0, c1);
if (name.empty() || !usage_of(name[0])) continue;
vertex_element element;
element.usage = name[0];
element.index = name.size() > 1U ? static_cast<uint32>(std::strtoul(std::string{name.substr(1)}.c_str(), nullptr, 10)) : 0U;
element.offset = static_cast<uint32>(std::strtoul(std::string{token.substr(c1 + 1U, c2 - c1 - 1U)}.c_str(), nullptr, 16));
element.type = std::string{token.substr(c2 + 1U)};
elements.push_back(std::move(element));
}
return elements;
}
/** Decode one vertex element (a small type grammar: `<count><kind>`). */
[[nodiscard]] inline auto decode_vertex_element(std::span<const uint8> buf, usize off, std::string_view type, float *out, uint32 wanted) -> uint32 {
usize digits = 0;
while (digits < type.size() && std::isdigit(static_cast<unsigned char>(type[digits]))) ++digits;
if (digits == 0U) return 0U;
const auto count = static_cast<uint32>(std::strtoul(std::string{type.substr(0, digits)}.c_str(), nullptr, 10));
const auto kind = type.substr(digits);
const auto take = std::min(count, wanted);
const auto bytes_per_element = (kind == "f32") ? 4U : ((kind == "u8n" || kind == "u8") ? 1U : 2U);
const auto lerp = [&](uint32 i, float value) { out[i] = value; };
for (uint32 i = 0; i < take; ++i) {
const auto at = off + static_cast<usize>(i) * bytes_per_element;
if (kind == "f32") {
lerp(i, f32(buf, off + static_cast<usize>(i) * 4U));
} else if (kind == "u8n") {
lerp(i, at < buf.size() ? static_cast<float>(buf[at]) / 255.0F : 0.0F);
} else if (kind == "u8") {
lerp(i, at < buf.size() ? static_cast<float>(buf[at]) : 0.0F);
} else if (kind == "s16n") {
lerp(i, at + 2U <= buf.size() ? static_cast<float>(static_cast<std::int16_t>(u16(buf, at))) / 32767.0F : 0.0F);
} else if (kind == "u16n") {
lerp(i, at + 2U <= buf.size() ? static_cast<float>(u16(buf, at)) / 65535.0F : 0.0F);
} else if (kind == "s16") {
lerp(i, at + 2U <= buf.size() ? static_cast<float>(static_cast<std::int16_t>(u16(buf, at))) : 0.0F);
} else if (kind == "u16") {
lerp(i, at + 2U <= buf.size() ? static_cast<float>(u16(buf, at)) : 0.0F);
} else if (kind == "f16") {
const auto raw = at + 2U <= buf.size() ? u16(buf, at) : 0U;
const auto sign = (raw & 0x8000U) != 0U ? -1.0F : 1.0F;
const auto exp = (raw >> 10U) & 0x1FU;
const auto mant = raw & 0x3FFU;
float value = exp == 0U ? static_cast<float>(mant) / 1024.0F * std::pow(2.0F, -14.0F)
: (1.0F + static_cast<float>(mant) / 1024.0F) * std::pow(2.0F, static_cast<float>(exp) - 15.0F);
lerp(i, sign * value);
}
}
return take;
}
}
/**
* Decode a compiled `W3DMesh` instance.
*
* The compiled struct stores a GPU vertex buffer (declaration string +
* packed vertices) and a triangle list; every pointer is an offset from the
* instance start, so the payload is read in place.
*/
[[nodiscard]] inline auto decode_mesh(std::span<const uint8> data) -> mesh {
mesh out;
const auto vertex_off = detail::u32(data, 4U);
const auto triangle_count = detail::u32(data, 52U);
const auto triangle_items = detail::u32(data, 56U);
if (vertex_off == 0U || vertex_off + 20U > data.size()) return out;
const auto vertex_count = detail::u32(data, vertex_off);
const auto stride = detail::u32(data, vertex_off + 4U);
const auto element_items = detail::u32(data, vertex_off + 8U);
const auto decl_bytes = detail::u32(data, vertex_off + 12U);
const auto decl_items = detail::u32(data, vertex_off + 16U);
if (vertex_count == 0U || stride == 0U || decl_items + decl_bytes > data.size()) return out;
const std::string_view decl{reinterpret_cast<const char *>(data.data() + decl_items), decl_bytes};
const auto elements = detail::parse_vertex_format(decl);
const auto shader_len = detail::i32(data, 60U);
const auto shader_ptr = detail::u32(data, 64U);
if (shader_len > 0 && shader_ptr != 0U && shader_ptr + static_cast<usize>(shader_len) <= data.size()) {
out.shader.assign(reinterpret_cast<const char *>(data.data() + shader_ptr), static_cast<usize>(shader_len));
}
// Per-mesh bone remap: a vertex's blend index selects an entry here, and
// that entry is the hierarchy bone. Without it the wrong bones are used
// and skinned models tear apart.
const auto bone_count = detail::u32(data, vertex_off + 0x14U);
const auto bone_ptr = detail::u32(data, vertex_off + 0x18U);
if (bone_count > 0U && bone_count < 4096U && bone_ptr != 0U && bone_ptr + static_cast<usize>(bone_count) * 2U <= data.size()) {
out.bone_remap.resize(bone_count);
for (uint32 i = 0; i < bone_count; ++i) out.bone_remap[i] = detail::u16(data, bone_ptr + static_cast<usize>(i) * 2U);
}
out.positions.assign(static_cast<usize>(vertex_count) * 3U, 0.0F);
std::vector<float> normals(static_cast<usize>(vertex_count) * 3U, 0.0F);
std::vector<float> uvs(static_cast<usize>(vertex_count) * 2U, 0.0F);
std::vector<uint8> blend_indices(static_cast<usize>(vertex_count) * 4U, 0U);
std::vector<float> blend_weights(static_cast<usize>(vertex_count) * 4U, 0.0F);
bool have_normals = false;
bool have_uvs = false;
bool have_blend = false;
for (uint32 i = 0; i < vertex_count; ++i) {
const auto base = static_cast<usize>(element_items) + static_cast<usize>(i) * stride;
float normal[3] = {0.0F, 0.0F, 1.0F};
float uv[2] = {0.0F, 0.0F};
for (const auto &element: elements) {
float value[4] = {0.0F, 0.0F, 0.0F, 0.0F};
if (element.usage == 'p' && element.index == 0U) {
if (detail::decode_vertex_element(data, base + element.offset, element.type, value, 3U) >= 3U) {
out.positions[static_cast<usize>(i) * 3U + 0U] = value[0];
out.positions[static_cast<usize>(i) * 3U + 1U] = value[1];
out.positions[static_cast<usize>(i) * 3U + 2U] = value[2];
}
} else if (element.usage == 'n' && element.index == 0U) {
if (detail::decode_vertex_element(data, base + element.offset, element.type, value, 3U) >= 3U) {
normal[0] = value[0];
normal[1] = value[1];
normal[2] = value[2];
have_normals = true;
}
} else if (element.usage == 't' && element.index == 0U) {
if (detail::decode_vertex_element(data, base + element.offset, element.type, value, 2U) >= 2U) {
uv[0] = value[0];
uv[1] = value[1];
have_uvs = true;
}
} else if (element.usage == 'i' && element.index == 0U) {
if (detail::decode_vertex_element(data, base + element.offset, element.type, value, 4U) >= 4U) {
for (usize k = 0; k < 4U; ++k) blend_indices[static_cast<usize>(i) * 4U + k] = static_cast<uint8>(std::clamp(value[k], 0.0F, 255.0F));
have_blend = true;
}
} else if (element.usage == 'w' && element.index == 0U) {
if (detail::decode_vertex_element(data, base + element.offset, element.type, value, 4U) >= 4U) {
for (usize k = 0; k < 4U; ++k) blend_weights[static_cast<usize>(i) * 4U + k] = value[k];
}
}
}
normals[static_cast<usize>(i) * 3U + 0U] = normal[0];
normals[static_cast<usize>(i) * 3U + 1U] = normal[1];
normals[static_cast<usize>(i) * 3U + 2U] = normal[2];
uvs[static_cast<usize>(i) * 2U + 0U] = uv[0];
uvs[static_cast<usize>(i) * 2U + 1U] = uv[1];
}
for (uint32 i = 0; i < triangle_count; ++i) {
const auto base = static_cast<usize>(triangle_items) + static_cast<usize>(i) * 24U;
const auto count = detail::u32(data, base);
const auto ptr = detail::u32(data, base + 4U);
if (count == 0U || ptr == 0U || ptr + static_cast<usize>(count) * 4U > data.size()) continue;
std::vector<uint32> ring(count);
for (uint32 k = 0; k < count; ++k) ring[k] = detail::u32(data, ptr + static_cast<usize>(k) * 4U);
for (uint32 k = 1; k + 1U < count; ++k) { // fan any n-gon
out.indices.push_back(ring[0]);
out.indices.push_back(ring[k]);
out.indices.push_back(ring[k + 1U]);
}
}
if (have_normals) out.normals = std::move(normals);
if (have_uvs) out.uvs = std::move(uvs);
if (have_blend) {
out.blend_indices = std::move(blend_indices);
out.blend_weights = std::move(blend_weights);
}
return out;
}
// ---- DDS --------------------------------------------------------------
namespace detail {
inline auto decode_dxt_color(std::span<const uint8> block, usize offset, std::vector<uint32> &out, uint32 w, uint32 h, uint32 x, uint32 y, bool force_four) -> void {
const auto c0 = static_cast<uint32>(block[offset]) | (static_cast<uint32>(block[offset + 1U]) << 8U);
const auto c1 = static_cast<uint32>(block[offset + 2U]) | (static_cast<uint32>(block[offset + 3U]) << 8U);
const auto hilo = [](uint32 c) -> std::array<uint8, 3> {
return {static_cast<uint8>(((c >> 11U) & 0x1FU) * 255U / 31U), static_cast<uint8>(((c >> 5U) & 0x3FU) * 255U / 63U),
static_cast<uint8>((c & 0x1FU) * 255U / 31U)};
};
const auto a = hilo(c0);
const auto b = hilo(c1);
std::array<uint32, 4> colors{};
colors[0] = argb(a[0], a[1], a[2]);
colors[1] = argb(b[0], b[1], b[2]);
if (c0 > c1 || force_four) {
colors[2] = argb(static_cast<uint8>((2U * a[0] + b[0]) / 3U), static_cast<uint8>((2U * a[1] + b[1]) / 3U), static_cast<uint8>((2U * a[2] + b[2]) / 3U));
colors[3] = argb(static_cast<uint8>((a[0] + 2U * b[0]) / 3U), static_cast<uint8>((a[1] + 2U * b[1]) / 3U), static_cast<uint8>((a[2] + 2U * b[2]) / 3U));
} else {
colors[2] = argb(static_cast<uint8>((a[0] + b[0]) / 2U), static_cast<uint8>((a[1] + b[1]) / 2U), static_cast<uint8>((a[2] + b[2]) / 2U));
colors[3] = 0x00000000U;
}
uint32 indices = 0;
for (uint32 k = 0; k < 4U; ++k) indices |= static_cast<uint32>(block[offset + 4U + k]) << (8U * k);
for (uint32 py = 0; py < 4U; ++py) {
for (uint32 px = 0; px < 4U; ++px) {
if (x + px >= w || y + py >= h) continue;
out[static_cast<usize>(y + py) * w + (x + px)] = colors[(indices >> (2U * (py * 4U + px))) & 3U];
}
}
}
}
/** Decode a DDS (DXT1/3/5 or uncompressed RGB) to an ARGB image; empty when unsupported. */
[[nodiscard]] inline auto decode_dds(std::span<const uint8> data) -> image {
if (data.size() < 128U || std::memcmp(data.data(), "DDS ", 4) != 0) return {};
const auto height = detail::u32(data, 12U);
const auto width = detail::u32(data, 16U);
const auto pfflags = detail::u32(data, 80U);
const auto bits = detail::u32(data, 88U);
const auto rmask = detail::u32(data, 92U);
const auto gmask = detail::u32(data, 96U);
const auto bmask = detail::u32(data, 100U);
const auto amask = detail::u32(data, 104U);
if (width == 0U || height == 0U || width > 8192U || height > 8192U) return {};
const auto fourcc = std::string_view{reinterpret_cast<const char *>(data.data() + 84U), 4U};
std::vector<uint32> pixels(static_cast<usize>(width) * height, 0xFF000000U);
if (fourcc == "DXT1" || fourcc == "DXT3" || fourcc == "DXT5") {
const auto stride = fourcc == "DXT1" ? 8U : 16U;
usize pos = 128U;
const auto blocks_x = (width + 3U) / 4U;
const auto blocks_y = (height + 3U) / 4U;
for (uint32 by = 0; by < blocks_y; ++by) {
for (uint32 bx = 0; bx < blocks_x; ++bx) {
if (pos + stride > data.size()) goto dds_done; // truncated: keep what we decoded
const auto block = data.subspan(pos, stride);
pos += stride;
const uint32 x = bx * 4U;
const uint32 y = by * 4U;
if (fourcc == "DXT1") {
detail::decode_dxt_color(block, 0U, pixels, width, height, x, y, false);
} else {
if (fourcc == "DXT3") {
for (uint32 py = 0; py < 4U; ++py) {
for (uint32 px = 0; px < 4U; ++px) {
const auto i = py * 4U + px;
const auto byte = block[i / 2U];
const auto alpha = static_cast<uint8>((i % 2U == 0U) ? (byte & 0x0FU) : (byte >> 4U));
const auto at = static_cast<usize>(y + py) * width + (x + px);
if (y + py < height && x + px < width) pixels[at] = (pixels[at] & 0x00FFFFFFU) | (static_cast<uint32>(alpha) * 17U << 24U);
}
}
} else { // DXT5
const auto a0 = block[0];
const auto a1 = block[1];
std::array<uint8, 8> palette{};
palette[0] = a0;
palette[1] = a1;
if (a0 > a1) {
for (uint32 i = 1U; i < 7U; ++i) palette[i + 1U] = static_cast<uint8>(((7U - i) * a0 + i * a1) / 7U);
} else {
for (uint32 i = 1U; i < 5U; ++i) palette[i + 1U] = static_cast<uint8>(((5U - i) * a0 + i * a1) / 5U);
palette[6] = 0U;
palette[7] = 255U;
}
std::uint64_t abits = 0;
for (uint32 k = 0; k < 6U; ++k) abits |= static_cast<std::uint64_t>(block[2U + k]) << (8U * k);
for (uint32 py = 0; py < 4U; ++py) {
for (uint32 px = 0; px < 4U; ++px) {
const auto i = py * 4U + px;
const auto alpha = palette[(abits >> (3U * i)) & 7U];
const auto at = static_cast<usize>(y + py) * width + (x + px);
if (y + py < height && x + px < width) pixels[at] = (pixels[at] & 0x00FFFFFFU) | (static_cast<uint32>(alpha) << 24U);
}
}
}
detail::decode_dxt_color(block, 8U, pixels, width, height, x, y, true);
}
}
}
} else if ((pfflags & 0x40U) != 0U && (bits == 16U || bits == 24U || bits == 32U)) {
const auto shift_of = [](uint32 mask) -> std::pair<uint32, uint32> {
if (mask == 0U) return {0U, 0U};
uint32 shift = 0;
while (((mask >> shift) & 1U) == 0U) ++shift;
uint32 size = 0;
while (((mask >> (shift + size)) & 1U) != 0U) ++size;
return {shift, size};
};
const auto channels = {shift_of(rmask), shift_of(gmask), shift_of(bmask), shift_of(amask)};
const auto bpp = bits / 8U;
const auto *raw = data.data() + 128U;
const auto available = data.size() > 128U ? (data.size() - 128U) / bpp : 0U;
for (usize i = 0; i < static_cast<usize>(width) * height && i < available; ++i) {
uint32 pixel = 0;
for (uint32 b = 0; b < bpp; ++b) pixel |= static_cast<uint32>(raw[i * bpp + b]) << (8U * b);
uint32 out = 0xFF000000U;
uint32 channel = 0;
for (const auto [shift, size]: channels) {
const auto maxv = size != 0U ? ((1U << size) - 1U) : 0U;
const auto value = maxv != 0U ? (((pixel >> shift) & maxv) * 255U / maxv) : 255U;
if (channel < 3U) {
out |= value << (16U - channel * 8U);
} else if (size != 0U) {
out = (out & 0x00FFFFFFU) | (value << 24U);
}
++channel;
}
pixels[i] = out;
}
} else {
return {};
}
dds_done:;
image result(width, height);
std::copy(pixels.begin(), pixels.end(), result.data());
return result;
}
/** Decode the DDS embedded in a compiled `Texture` instance; empty when none. */
[[nodiscard]] inline auto decode_texture(std::span<const uint8> instance) -> image {
const auto off = detail::u32(instance, 4U);
std::span<const uint8> dds{};
if (off > 0U && off < instance.size() && off + 4U <= instance.size() && std::memcmp(instance.data() + off, "DDS ", 4) == 0) {
dds = instance.subspan(off);
} else {
for (usize i = 0; i + 4U <= instance.size(); ++i) {
if (std::memcmp(instance.data() + i, "DDS ", 4) == 0) {
dds = instance.subspan(i);
break;
}
}
}
return dds.empty() ? image{} : decode_dds(dds);
}
/**
* The diffuse texture a mesh samples, or `nullptr`.
*
* A `W3DMesh` stores `FXShaderConstant`s at `+76` (count) / `+80` (items);
* a texture-valued constant (`0xA59096A6`) names its role and points at a
* 1-based index into the mesh's cross-asset references.
*/
[[nodiscard]] inline auto mesh_diffuse_texture(const asset_stream &stream, const asset_stream::asset &mesh_asset) -> const asset_stream::asset * {
const auto instance = stream.read_instance(mesh_asset);
if (instance.empty()) return nullptr;
const auto count = detail::u32(instance, 76U);
const auto items = detail::u32(instance, 80U);
for (uint32 i = 0; i < count; ++i) {
const auto constant = detail::u32(instance, items + static_cast<usize>(i) * 4U);
if (constant == 0U || detail::u32(instance, constant) != 0xA59096A6U) continue;
const auto name = asset_stream::lower(detail::cstr(instance, detail::u32(instance, constant + 8U), detail::i32(instance, constant + 4U)));
const bool is_diffuse = name.contains("diffuse") || name.contains("albedo") || name.contains("base") || name == "texture_0" || name == "texture0";
if (!is_diffuse) continue;
const auto reference = detail::u32(instance, constant + 12U);
if (reference >= 1U && reference <= mesh_asset.references.size()) {
const auto [type_id, instance_id] = mesh_asset.references[reference - 1U];
if (const auto *texture = stream.find(type_id, instance_id); texture != nullptr) return texture;
}
}
return nullptr;
}
// ---- scene ------------------------------------------------------------
/** A world-space triangle-soup vertex (position, normal, uv, texture layer). */
struct vertex {
float x = 0.0F;
float y = 0.0F;
float z = 0.0F;
float nx = 0.0F;
float ny = 0.0F;
float nz = 1.0F;
float u = 0.0F;
float v = 0.0F;
float layer = 0.0F;
};
/** Every model the map draws, flattened to world space and ready to upload. */
struct scene {
std::vector<vertex> vertices;
std::vector<uint32> indices;
std::vector<image> textures; ///< All `texture_size` square, ARGB.
uint32 texture_size = 0;
usize placed = 0;
usize missing = 0;
[[nodiscard]] auto empty() const -> bool { return indices.empty(); }
[[nodiscard]] auto triangle_count() const -> usize { return indices.size() / 3U; }
};
/** One placement request: an object type at a world position with a Z rotation. */
struct placement {
std::string type;
float x = 0.0F;
float y = 0.0F;
float z = 0.0F;
float angle = 0.0F;
float scale = 1.0F;
};
namespace detail {
/** Box-downscale `src` to a `size` square (averages, so atlas detail survives). */
[[nodiscard]] inline auto to_square(const image &src, uint32 size) -> image {
image out(size, size);
if (src.empty() || size == 0U) return out;
for (uint32 y = 0; y < size; ++y) {
const auto y0 = y * src.height() / size;
const auto y1 = std::max(y0 + 1U, (y + 1U) * src.height() / size);
for (uint32 x = 0; x < size; ++x) {
const auto x0 = x * src.width() / size;
const auto x1 = std::max(x0 + 1U, (x + 1U) * src.width() / size);
uint32 r = 0;
uint32 g = 0;
uint32 b = 0;
uint32 a = 0;
uint32 count = 0;
for (uint32 sy = y0; sy < y1; ++sy) {
for (uint32 sx = x0; sx < x1; ++sx) {
const auto texel = src.data()[static_cast<usize>(sy) * src.width() + sx];
r += (texel >> 16U) & 0xFFU;
g += (texel >> 8U) & 0xFFU;
b += texel & 0xFFU;
a += (texel >> 24U) & 0xFFU;
++count;
}
}
out.data()[static_cast<usize>(y) * size + x] =
argb(static_cast<uint8>(r / count), static_cast<uint8>(g / count), static_cast<uint8>(b / count), static_cast<uint8>(a / count));
}
}
return out;
}
}
/**
* Flatten every placement into one world-space scene.
*
* @param stream The map's compiled art stream.
* @param placements The objects to draw.
* @param ground_height Terrain height (world Z) at a world `(x, y)`, so
* objects sit on the relief instead of a flat plane.
* @param texture_size Edge length of the shared texture array (0 = auto).
*/
[[nodiscard]] inline auto build_scene(const asset_stream &stream, std::span<const placement> placements,
const std::function<float(float, float)> &ground_height = {}, uint32 texture_size = 128U) -> scene {
scene out;
out.texture_size = texture_size == 0U ? 128U : texture_size;
std::unordered_map<std::string, uint32> texture_layers;
const auto layer_for = [&](const asset_stream::asset *texture) -> uint32 {
if (texture == nullptr) return 0xFFFFFFFFU;
const auto key = asset_stream::lower(texture->name);
if (const auto it = texture_layers.find(key); it != texture_layers.end()) return it->second;
auto decoded = decode_texture(stream.read_instance(*texture));
if (decoded.empty()) return 0xFFFFFFFFU;
const auto layer = static_cast<uint32>(out.textures.size());
out.textures.push_back(detail::to_square(decoded, out.texture_size));
texture_layers.emplace(key, layer);
return layer;
};
// Bone world transforms per skeleton, cached by hierarchy name. Meshes are
// skinned; their vertices are stored in *bone space*, so a static render
// must rebuild the default pose and blend.
std::unordered_map<std::string, std::vector<bone_matrix>> skeletons;
const auto skeleton_for = [&](std::string_view mesh_instance) -> const std::vector<bone_matrix> * {
const auto dot = mesh_instance.find('.');
const auto base = std::string{mesh_instance.substr(0, dot)};
const auto key = asset_stream::lower(base);
if (const auto it = skeletons.find(key); it != skeletons.end()) return it->second.empty() ? nullptr : &it->second;
std::vector<bone_matrix> matrices;
if (const auto *asset = stream.find_name("W3DHierarchy:" + base); asset != nullptr) {
matrices = bone_world_matrices(decode_hierarchy(stream.read_instance(*asset)));
}
const auto it = skeletons.emplace(key, std::move(matrices)).first;
return it->second.empty() ? nullptr : &it->second;
};
for (const auto &item: placements) {
const auto meshes = stream.meshes_for(item.type);
if (meshes.empty()) {
++out.missing;
continue;
}
const auto cos_a = std::cos(item.angle);
const auto sin_a = std::sin(item.angle);
const auto base_z = (ground_height ? ground_height(item.x, item.y) : 0.0F) + item.z;
bool drawn = false;
for (const auto *mesh_asset: meshes) {
// Only opaque material parts are drawn; meshes with no diffuse
// role are effects (FX-light billboards, `DefaultW3D.fx`) or
// ambient helpers and would otherwise render as garbage.
const auto *diffuse = mesh_diffuse_texture(stream, *mesh_asset);
if (diffuse == nullptr) continue;
const auto geometry = decode_mesh(stream.read_instance(*mesh_asset));
if (geometry.indices.empty()) continue;
// Damage-fill shells (damaged interior/wreckage) are drawn on top
// of the main shell in our opaque pass and would show its
// interior texture (e.g. orange `CBBuilding_Wood`); skip them.
if (asset_stream::lower(geometry.shader).contains("damagefill")) continue;
drawn = true;
const auto layer = layer_for(diffuse);
const auto *skeleton = geometry.skinned() ? skeleton_for(mesh_asset->instance_name()) : nullptr;
const auto base_vertex = static_cast<uint32>(out.vertices.size());
const auto vertex_count = geometry.positions.size() / 3U;
for (usize i = 0; i < vertex_count; ++i) {
auto px = geometry.positions[i * 3U + 0U];
auto py = geometry.positions[i * 3U + 1U];
auto pz = geometry.positions[i * 3U + 2U];
auto nx = geometry.normals.empty() ? 0.0F : geometry.normals[i * 3U + 0U];
auto ny = geometry.normals.empty() ? 0.0F : geometry.normals[i * 3U + 1U];
auto nz = geometry.normals.empty() ? 1.0F : geometry.normals[i * 3U + 2U];
if (skeleton != nullptr) {
// The compiled shader binds one joint per vertex
// (`blendindices.x`), remapped through the mesh's bone
// table, then transforms by the joint's world matrix.
auto joint = static_cast<uint32>(geometry.blend_indices[i * 4U + 0U]);
if (joint < geometry.bone_remap.size()) joint = geometry.bone_remap[joint];
if (joint < skeleton->size()) {
const auto &m = (*skeleton)[joint];
const std::array<float, 3> p{px, py, pz};
const auto rp = detail::rotate(m.rotation, p);
const std::array<float, 3> n{nx, ny, nz};
const auto rn = detail::rotate(m.rotation, n);
px = rp[0] + m.translation[0];
py = rp[1] + m.translation[1];
pz = rp[2] + m.translation[2];
nx = rn[0];
ny = rn[1];
nz = rn[2];
}
}
vertex v;
v.x = item.x + (px * item.scale) * cos_a - (py * item.scale) * sin_a;
v.y = item.y + (px * item.scale) * sin_a + (py * item.scale) * cos_a;
v.z = base_z + pz * item.scale;
v.nx = nx * cos_a - ny * sin_a;
v.ny = nx * sin_a + ny * cos_a;
v.nz = nz;
if (!geometry.uvs.empty()) {
v.u = geometry.uvs[i * 2U + 0U];
v.v = geometry.uvs[i * 2U + 1U];
}
v.layer = layer == 0xFFFFFFFFU ? 0.0F : static_cast<float>(layer);
out.vertices.push_back(v);
}
for (const auto index: geometry.indices) out.indices.push_back(base_vertex + index);
}
if (drawn) {
++out.placed;
} else {
++out.missing;
}
}
return out;
}
}