Going-Commando/crates/gc-formats/src/tie.rs

//! Tie: instanced static props -- the environment sitting on the terrain.
//!
//! Spec: `docs/formats/tie.md`.
//!
//! A *class* is one prop shape (a crate, a pipe run, a railing); *instances*
//! place copies of it in the level. The class tables live in the core index and
//! the geometry sits in `core_data`, addressed by byte offset.
//!
//! # What is decoded here, and what is not
//!
//! The reference's class header is 0x80 bytes. RAC2 agrees with it only over
//! the first `0x48`: the per-LOD packet offsets, the three LOD distances and
//! `o_class` all check out on every class on every level. From `0x48` on, the
//! RAC1 fields do **not** describe RAC2 data -- `lod_info` reads back as noise,
//! and `packet_count` and `texture_count` are zero on every class examined.
//!
//! So this module decodes the part that is verified and stops there. Guessing
//! the remainder would repeat the pattern this project keeps running into: a
//! field order that produces plausible output and is wrong.
//!
//! Evidence for what is decoded, over 320 classes across five levels:
//!
//! * `o_class` at `0x44` equals the core index's class number: **320 of 320**.
//!   A coincidence is not a plausible explanation for that.
//! * the LOD distances are ordered `0 < near < mid < far`: 315 of 320. The
//!   five exceptions are not yet explained and are worth chasing.

use crate::buf::{Buf, Error, ErrorKind, Result};
use crate::level_core::ClassEntry;

/// Bytes of a tie class header that RAC2 and RAC1 agree on.
pub const TIE_CLASS_HEADER_MATCHED: usize = 0x48;

/// Value `TieClassHeader::packets[0]` takes on every one of the 1,864 classes.
///
/// It is a constant, not a block offset: the other two `packets` words vary
/// widely and land at different addresses in `core_data` on every class, while
/// this one is 256 everywhere. Read as an offset it would name a point just
/// past the header, which is exactly the kind of field that gets mistaken for
/// a pointer.
pub const TIE_PACKETS0: i32 = 256;

/// A tie class header, decoded as far as the two games agree.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct TieClassHeader {
    /// `0x00`: packet data offsets for LOD 0 (highest detail), 1 and 2.
    ///
    /// [`TIE_PACKETS0`] is constant across all 1,864 classes and is reported
    /// separately. `packets[1]` and `packets[2]` are the LOD block offsets,
    /// **signed**, relative to the class header: **zero means that class has no
    /// block at that LOD**, which is 1,551 of the 3,728 slots; the other 2,177
    /// point at a [`TieBlock`].
    ///
    /// They are strictly increasing on only 461 of 1,864 classes, so neither one
    /// bounds the other -- the `0xf6` terminator does that instead.
    pub packets: [i32; 3],
    /// `0x10`, `0x14`, `0x18`: LOD switch distances in world units.
    pub near_dist: f32,
    pub mid_dist: f32,
    pub far_dist: f32,
    /// `0x44`: the class's own number.
    ///
    /// Matches the core index's `o_class` for the class, which is what makes
    /// the header base trustworthy.
    pub o_class: i32,
}

impl TieClassHeader {
    /// Parse the agreed-on prefix. Needs `TIE_CLASS_HEADER_MATCHED` bytes.
    pub fn parse(bytes: &[u8]) -> Result<TieClassHeader> {
        let mut b = Buf::new(bytes);
        let mut packets = [0i32; 3];
        for slot in packets.iter_mut() {
            *slot = b.i32("tie packet offset")?;
        }
        // 0x0c: the 64 light-slot normals offset. Read past, not decoded.
        b.skip(4, "tie normals offset")?;
        let near_dist = b.f32("tie near distance")?;
        let mid_dist = b.f32("tie mid distance")?;
        let far_dist = b.f32("tie far distance")?;
        // 0x1c..0x44: the word TieProc reuses as a per-frame counter, the LOD
        // packet counts, the texture count, ad-gifs, the bounding sphere and
        // the class scale. None of it matches RAC2 yet.
        b.skip(0x44 - 0x1c, "tie class fields not decoded yet")?;
        let o_class = b.i32("tie class number")?;
        Ok(TieClassHeader {
            packets,
            near_dist,
            mid_dist,
            far_dist,
            o_class,
        })
    }

    /// The LOD this class draws at a view depth, or `None` past its far
    /// distance.
    pub fn lod_for_depth(&self, depth: f32) -> Option<usize> {
        if depth < self.near_dist {
            Some(0)
        } else if depth < self.mid_dist {
            Some(1)
        } else if depth < self.far_dist {
            Some(2)
        } else {
            None
        }
    }
}

/// Size of a tie LOD block's own header.
pub const TIE_BLOCK_HEADER_SIZE: usize = 0x10;

/// The four most common values of [`TieBlockHeader::packed_texture`].
///
/// Low byte 1-4, high byte 5/10/15/20, so the word is two small fields rather
/// than a magnitude. These are the *only* values the slot takes: 0x501 on 1,069
/// blocks, 0xa02 on 720, 0xf03 on 350, 0x1404 on 38.
pub const TIE_PACKED_TEXTURE_VALUES: [u16; 4] = [0x0501, 0x0a02, 0x0f03, 0x1404];

/// A tie LOD block's fixed 16-byte header.
///
/// Every figure here is measured over the **922** blocks where "the next
/// `packets` offset ends this block" holds, across all 27 levels. The other
/// 1,403 classes are unmeasured, not measured-and-different.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TieBlockHeader {
    /// `+0x00`: the byte offset from the start of the block to the vertex
    /// data -- that is, the length of the record table that follows this
    /// header. A multiple of 16 on all 2,177 blocks (42 distinct values, never
    /// zero), because the table holds 16-byte records: one block with this
    /// word at 0x10 has one record, one at 0x50 has five.
    ///
    /// This was read as an element *count* for a while, and the reading failed
    /// in a way worth recording: the block's payload length divided by it is a
    /// whole number on 0 of 922 blocks, at any record size and any header up
    /// to 512 bytes. A count of what is in the payload was never going to fit,
    /// because the word is not about the payload -- it is where the payload
    /// starts.
    ///
    /// Confirmed against the running game: each record's leading word is
    /// relocated at load to `table_address + word`, and the table's length is
    /// this word, on every block checked in RAM (16 of 16 on level 1).
    pub data_offset: u16,
    /// `+0x02`: zero on all 2,177 blocks.
    pub reserved_02: u16,
    /// `+0x04`: one of exactly [`TIE_PACKED_TEXTURE_VALUES`] on all 2,177 blocks.
    pub packed_texture: u16,
    /// `+0x06`: 16 distinct values, never zero.
    pub field_06: u16,
    /// `+0x08`: 180 distinct values, never zero.
    pub field_08: u16,
    /// `+0x0a`: 61 distinct values, never zero.
    pub field_0a: u16,
    /// `+0x0c`: 125 distinct values, never zero, clustered at 0x043f to 0x0443
    /// (~1,087 to 1,091). **Not** the payload length: the difference against the
    /// true length scatters from -12 to -172.
    pub field_0c: u16,
    /// `+0x0e`: 11 distinct values, zero on 2,162 of 2,177 -- the only field
    /// here that is not universal.
    pub reserved_0e: u16,
}

impl TieBlockHeader {
    /// Parse the block header, bounds-checked against `block`.
    pub fn parse(block: &[u8]) -> Result<Self> {
        let mut b = Buf::new(block);
        Ok(Self {
            data_offset: b.u16("tie block data offset")?,
            reserved_02: b.u16("tie block reserved")?,
            packed_texture: b.u16("tie block packed texture")?,
            field_06: b.u16("tie block field_06")?,
            field_08: b.u16("tie block field_08")?,
            field_0a: b.u16("tie block field_0a")?,
            field_0c: b.u16("tie block field_0c")?,
            reserved_0e: b.u16("tie block reserved_0e")?,
        })
    }

    /// True when every invariant measured across all 27 levels holds.
    ///
    /// Only the two that are universal: the data offset is a non-zero multiple
    /// of 16 and `reserved_02` is zero, both on all 2,177 blocks.
    pub fn invariants_hold(&self) -> bool {
        self.data_offset % 16 == 0 && self.data_offset != 0 && self.reserved_02 == 0
    }

    /// The number of 16-byte records in this block's table.
    pub fn record_count(&self) -> usize {
        self.data_offset as usize / TIE_RECORD_SIZE
    }

    /// The block's records, as byte slices of [`TIE_RECORD_SIZE`].
    ///
    /// Bounds-checked against `block`, which is the block starting at its
    /// header. **The header is the first record**: the table runs from the
    /// block start to `data_offset`, so a block with `data_offset == 0x50`
    /// holds five records, the first of which is the header itself and whose
    /// pointer names where the payload begins. A header whose table runs past
    /// the block yields nothing rather than a short slice.
    pub fn records<'b>(&self, block: &'b [u8]) -> impl Iterator<Item = &'b [u8]> + 'b {
        block
            .get(..self.record_count() * TIE_RECORD_SIZE)
            .unwrap_or(&[])
            .chunks_exact(TIE_RECORD_SIZE)
    }
}

/// One 16-byte tie block record.
///
/// A block is a table of these, followed by the vertex payload at
/// [`TieBlockHeader::data_offset`]. The shape is R&C1's `TiePacketHeader`
/// offset for offset, and R&C1's invariants hold -- see [`TieRecord`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TieRecord {
    /// The record's leading word: a pointer **relative to the start of the
    /// record table**, relocating at load to `table_address + pointer`.
    ///
    /// Confirmed on every block checked in PCSX2's RAM: the word becomes
    /// `table + word` with the other twelve bytes untouched.
    pub pointer: u32,
    /// `+0x04`: AD-GIF (material) blocks this packet references; 1..4 on disc.
    pub shader_count: u8,
    /// `+0x05`: quadwords of ad-gif data. Equals `5 * shader_count`.
    pub bfc_distance: u8,
    /// `+0x06`: 4..18 on disc.
    pub control_count: u8,
    /// `+0x07`: 1..5 on disc, never above `control_count`.
    pub control_size: u8,
    /// `+0x08`: 3..7 on disc, never above `vert_size`.
    pub vert_ofs: u8,
    /// `+0x09`: 4..60 on disc.
    pub vert_size: u8,
    /// `+0x0a`: equals `vert_size` on every record measured.
    pub color_ofs: u8,
    /// `+0x0b`: zero on every record measured.
    pub color_count: u8,
    /// `+0x0c`: 7..67 on disc, never below `scissor_size`.
    pub scissor_ofs: u8,
    /// `+0x0d`: 1..5 on disc.
    pub scissor_size: u8,
    /// `+0x0e`: zero on every record measured.
    pub strip_count: u8,
    /// `+0x0f`: zero on every record measured.
    pub strip_vertex_count: u8,
}

impl TieRecord {
    /// Parse one 16-byte record.
    pub fn parse(bytes: &[u8]) -> Result<TieRecord> {
        let mut b = Buf::new(bytes);
        Ok(TieRecord {
            pointer: b.u32("tie packet data offset")?,
            shader_count: b.u8("tie packet shader count")?,
            bfc_distance: b.u8("tie packet bfc distance")?,
            control_count: b.u8("tie packet control count")?,
            control_size: b.u8("tie packet control size")?,
            vert_ofs: b.u8("tie packet vertex offset")?,
            vert_size: b.u8("tie packet vertex size")?,
            color_ofs: b.u8("tie packet colour offset")?,
            color_count: b.u8("tie packet colour count")?,
            scissor_ofs: b.u8("tie packet scissor offset")?,
            scissor_size: b.u8("tie packet scissor size")?,
            strip_count: b.u8("tie packet strip count")?,
            strip_vertex_count: b.u8("tie packet strip vertex count")?,
        })
    }

    /// The twelve bytes after the pointer, verbatim.
    ///
    /// The comparison against the running game checks these are copied
    /// unchanged, so they are worth having as one value rather than a
    /// field-by-field walk.
    pub fn trailing_bytes(&self) -> [u8; 12] {
        [
            self.shader_count,
            self.bfc_distance,
            self.control_count,
            self.control_size,
            self.vert_ofs,
            self.vert_size,
            self.color_ofs,
            self.color_count,
            self.scissor_ofs,
            self.scissor_size,
            self.strip_count,
            self.strip_vertex_count,
        ]
    }

    /// True when every invariant that holds on **every** record measured holds.
    ///
    /// `bfc_distance == 5 * shader_count` is R&C1's documented rule for this
    /// field and it holds on all 15,219 records of all 2,177 blocks on all 27
    /// levels; that is what identifies the record as this struct rather than a
    /// resemblance. The three orderings are also universal.
    ///
    /// Relations that hold on most records but not all are deliberately *not*
    /// here -- a check that fails on a real record is worse than no check --
    /// and the disc test counts them instead. `color_ofs == vert_size` is the
    /// largest of those: 12,640 of 15,219.
    pub fn invariants_hold(&self) -> bool {
        self.bfc_distance == 5 * self.shader_count
            && self.vert_ofs <= self.vert_size
            && self.scissor_size <= self.scissor_ofs
            && self.control_size <= self.control_count
    }

    // A packet-length formula was here -- `(vert_ofs + vert_size +
    // scissor_ofs + scissor_size) * 16` -- and it was retracted. It was
    // measured on a block set found by reading `packets[2]` where `packets[1]`
    // was meant, over 61 pairs, and re-measured on the correct blocks it holds
    // on 0 of 13,042. The block set was the error, not the arithmetic: the
    // arithmetic agreed perfectly with a population that was not the tie
    // blocks at all. This is the same failure `AGENTS.md` rule 3 is about,
    // arriving through a wrong subscript instead of a wrong span.
}

/// Byte that terminates a tie LOD block's payload.
pub const TIE_BLOCK_TERMINATOR: u8 = 0xf6;

/// Bytes in one tie block record.
pub const TIE_RECORD_SIZE: usize = 0x10;

/// A tie LOD block: its header, its payload, and the zero padding that follows
/// the terminator.
///
/// **The blocks are self-delimiting.** Each ends with [`TIE_BLOCK_TERMINATOR`]
/// followed by zeros, on **2,177 of 2,177** blocks across all 27 levels. That is
/// what makes them parseable without trusting `packets[1]` to bound the block,
/// which it only does on 461 of 1,864 classes.
///
/// The terminator is not a coincidence of byte frequency: `0xf6` occurs at 0.83x
/// the uniform rate in payloads, i.e. slightly *less* often than chance, while
/// the last non-zero byte is `0xf6` on 909 of 922 blocks in the first subset
/// tested against **17 of 922** for the same blocks cut at a random point.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TieBlock<'a> {
    /// The 16-byte header.
    pub header: TieBlockHeader,
    /// Bytes from `0x10` up to and including the terminator.
    pub payload: &'a [u8],
    /// Zero bytes between the terminator and the next block.
    pub padding: &'a [u8],
}

/// Walk `region` from `offset` to the first `0xf6` that is followed by a zero,
/// and return the block there.
///
/// Returns `None` if no terminator is found before the end of the region.
pub fn parse_tie_block(region: &[u8], offset: usize) -> Option<TieBlock<'_>> {
    if offset.checked_add(TIE_BLOCK_HEADER_SIZE)? > region.len() {
        return None;
    }
    let header = TieBlockHeader::parse(&region[offset..]).ok()?;
    let start = offset + TIE_BLOCK_HEADER_SIZE;
    let mut i = start;
    while i + 1 < region.len() {
        if region[i] == TIE_BLOCK_TERMINATOR && region[i + 1] == 0 {
            let after = i + 1;
            let mut e = after;
            while e < region.len() && region[e] == 0 {
                e += 1;
            }
            return Some(TieBlock {
                header,
                payload: &region[start..after],
                padding: &region[after..e],
            });
        }
        i += 1;
    }
    None
}

/// A tie class resolved through both of its indirections.
///
/// The chain is two steps and both are easy to get wrong, so it is worth
/// stating plainly:
///
/// 1. The class **table** lives in the level's `core_index.bin`, not in
///    `core_data.bin`. On level 0 the tables chain as moby at `192`,
///    238 x 32 bytes later tie at `7808`, 74 x 32 later shrub at `10176`,
///    which closes. Reading the same table out of `core_data.bin` yields
///    plausible-looking garbage rather than an error.
/// 2. [`ClassEntry::offset_in_asset_wad`] addresses `core_data.bin`, despite the
///    field's name and despite `core_data.wad` existing alongside it.
///
/// With both handled, all **1,864** tie classes on the disc resolve to a header
/// whose `o_class` matches the class entry's own.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct ResolvedTieClass {
    /// The index entry this came from.
    pub entry: ClassEntry,
    /// Its index within the level's tie table.
    pub index: usize,
    /// The class header, at `offset_in_asset_wad` in `core_data`.
    pub header: TieClassHeader,
}

/// Resolve one tie class. `class_index` counts from zero.
///
/// `core_index` is the level's `core_index.bin`; `core_data` is the
/// decompressed `core_data.bin`. Both are bounds-checked.
pub fn resolve_tie_class(
    core_index: &[u8],
    core_data: &[u8],
    classes: crate::level_core::ArrayRange,
    class_index: usize,
) -> Result<ResolvedTieClass> {
    const ENTRY: usize = 32;
    let start = classes.offset as usize + class_index * ENTRY;
    if classes.offset < 0 || start.checked_add(ENTRY).is_none() || start + ENTRY > core_index.len()
    {
        return Err(Error {
            offset: start,
            context: "tie class table entry",
            kind: ErrorKind::OutOfBounds {
                needed: ENTRY,
                available: core_index.len().saturating_sub(start),
            },
        });
    }
    if class_index >= classes.count as usize {
        return Err(Error {
            offset: start,
            context: "tie class index",
            kind: ErrorKind::Invalid("class index past the table count"),
        });
    }
    let entry = ClassEntry::parse(&core_index[start..start + ENTRY])?;
    let base = entry.offset_in_asset_wad as usize;
    if base < TIE_CLASS_HEADER_MATCHED || base + TIE_CLASS_HEADER_MATCHED > core_data.len() {
        return Err(Error {
            offset: base,
            context: "tie class header",
            kind: ErrorKind::OutOfBounds {
                needed: TIE_CLASS_HEADER_MATCHED,
                available: core_data.len().saturating_sub(base),
            },
        });
    }
    let header = TieClassHeader::parse(&core_data[base..base + TIE_CLASS_HEADER_MATCHED])?;
    Ok(ResolvedTieClass {
        entry,
        index: class_index,
        header,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A header as measured on level 0's first tie class.
    fn header() -> Vec<u8> {
        let mut v = vec![0u8; TIE_CLASS_HEADER_MATCHED];
        for (i, x) in [256i32, 20_592, 26_048].iter().enumerate() {
            v[i * 4..i * 4 + 4].copy_from_slice(&x.to_le_bytes());
        }
        for (i, x) in [12.0f32, 25.0, 40.0].iter().enumerate() {
            let o = 0x10 + i * 4;
            v[o..o + 4].copy_from_slice(&x.to_le_bytes());
        }
        v[0x44..0x48].copy_from_slice(&2592i32.to_le_bytes());
        v
    }

    #[test]
    fn parses_the_measured_prefix() {
        let h = TieClassHeader::parse(&header()).unwrap();
        assert_eq!(h.packets, [256, 20_592, 26_048]);
        assert_eq!(h.near_dist, 12.0);
        assert_eq!(h.mid_dist, 25.0);
        assert_eq!(h.far_dist, 40.0);
        assert_eq!(h.o_class, 2592);
    }

    #[test]
    fn a_short_header_is_an_error_not_a_panic() {
        assert!(TieClassHeader::parse(&[0u8; 0x20]).is_err());
        assert!(TieClassHeader::parse(&[]).is_err());
    }

    #[test]
    fn lod_bands_follow_the_three_distances() {
        let h = TieClassHeader::parse(&header()).unwrap();
        assert_eq!(h.lod_for_depth(5.0), Some(0));
        assert_eq!(h.lod_for_depth(20.0), Some(1));
        assert_eq!(h.lod_for_depth(30.0), Some(2));
        assert_eq!(h.lod_for_depth(50.0), None);
    }

    /// A block whose data offset names a four-record table, the first of
    /// which is the header itself.
    fn block() -> Vec<u8> {
        let mut v = Vec::new();
        // data_offset 0x40 -- four records -- reserved 0, packed, then the
        // remaining fields.
        v.extend_from_slice(&0x40u16.to_le_bytes());
        v.extend_from_slice(&0u16.to_le_bytes());
        v.extend_from_slice(&0x0501u16.to_le_bytes());
        v.extend_from_slice(&0x1234u16.to_le_bytes());
        v.extend_from_slice(&0x5678u16.to_le_bytes());
        v.extend_from_slice(&0x9abcu16.to_le_bytes());
        v.extend_from_slice(&0x0442u16.to_le_bytes());
        v.extend_from_slice(&0u16.to_le_bytes());
        assert_eq!(v.len(), TIE_BLOCK_HEADER_SIZE);
        // Three packet headers, each a data pointer and R&C1's byte fields.
        for k in 0..3u32 {
            v.extend_from_slice(&(0x40 + k * 0x20).to_le_bytes());
            v.extend_from_slice(&[
                1, 5, // shader_count, bfc_distance = 5 * shader_count
                7, 2, // control_count, control_size
                4, 0x20, // vert_ofs, vert_size
                0x20, 0, // color_ofs (= vert_size), color_count
                0x30, 0x20, // scissor_ofs, scissor_size
                0, 0, // strip_count, strip_vertex_count
            ]);
        }
        v.extend_from_slice(&[0xa5u8; 8]);
        v
    }

    #[test]
    fn the_data_offset_bounds_the_record_table() {
        let b = block();
        let h = TieBlockHeader::parse(&b).unwrap();
        assert!(h.invariants_hold());
        assert_eq!(h.record_count(), 4);
        let records: Vec<_> = h.records(&b).collect();
        assert_eq!(records.len(), 4);
        // The first record is the header itself, pointing at the payload.
        let first = TieRecord::parse(records[0]).unwrap();
        assert_eq!(first.pointer, 0x40);
        // The fields follow R&C1's packet-header byte layout.
        let second = TieRecord::parse(records[1]).unwrap();
        assert_eq!(second.shader_count, 1);
        assert_eq!(second.bfc_distance, 5);
        assert!(second.invariants_hold());
    }

    #[test]
    fn a_table_running_past_the_block_yields_nothing() {
        let mut b = block();
        // Claim a table far longer than the block.
        b[0..2].copy_from_slice(&0x400u16.to_le_bytes());
        let h = TieBlockHeader::parse(&b).unwrap();
        assert_eq!(h.record_count(), 0x40);
        assert_eq!(h.records(&b).count(), 0, "must not slice past the block");
        // A 0x400 offset is still a multiple of 16, so the invariant holds even
        // though the table cannot exist.
        assert!(h.invariants_hold());
    }
}