//! Tie: instanced static props -- the environment sitting on the terrain. //! //! Spec: `docs/formats/tie.md`. //! //! A *class* is one prop shape (a crate, a pipe run, a railing); *instances* //! place copies of it in the level. The class tables live in the core index and //! the geometry sits in `core_data`, addressed by byte offset. //! //! # What is decoded here, and what is not //! //! The reference's class header is 0x80 bytes. RAC2 agrees with it only over //! the first `0x48`: the per-LOD packet offsets, the three LOD distances and //! `o_class` all check out on every class on every level. From `0x48` on, the //! RAC1 fields do **not** describe RAC2 data -- `lod_info` reads back as noise, //! and `packet_count` and `texture_count` are zero on every class examined. //! //! So this module decodes the part that is verified and stops there. Guessing //! the remainder would repeat the pattern this project keeps running into: a //! field order that produces plausible output and is wrong. //! //! Evidence for what is decoded, over 1,864 classes on all 27 levels: //! //! * `o_class` at `0x44` equals the core index's class number: **1,864 of 1,864**. //! A coincidence is not a plausible explanation for that. //! * the LOD distances are ordered `0 < near < mid < far` on most classes, but //! **80 of 1,864** break it in an exact pattern: `mid_dist` is exactly //! `8192.0` and `far_dist` is exactly `8190.0` in every one. Those classes //! have `far < mid`, so `lod_for_depth` can never return 2 (the `far` test is //! reached only after the `mid` test). Effectively two LOD tiers, not three. //! See `docs/plan/level-assets.md` for the test. use crate::buf::{Buf, Error, ErrorKind, Result}; use crate::level_core::ClassEntry; /// Bytes of a tie class header that RAC2 and RAC1 agree on. pub const TIE_CLASS_HEADER_MATCHED: usize = 0x48; /// Value `TieClassHeader::packets[0]` takes on every one of the 1,864 classes. /// /// It is a constant, not a block offset: the other two `packets` words vary /// widely and land at different addresses in `core_data` on every class, while /// this one is 256 everywhere. Read as an offset it would name a point just /// past the header, which is exactly the kind of field that gets mistaken for /// a pointer. pub const TIE_PACKETS0: i32 = 256; /// A tie class header, decoded as far as the two games agree. #[derive(Debug, Clone, Copy, PartialEq)] pub struct TieClassHeader { /// `0x00`: packet data offsets for LOD 0 (highest detail), 1 and 2. /// /// [`TIE_PACKETS0`] is constant across all 1,864 classes and is reported /// separately. `packets[1]` and `packets[2]` are the LOD block offsets, /// **signed**, relative to the class header: **zero means that class has no /// block at that LOD**, which is 1,551 of the 3,728 slots; the other 2,177 /// point at a [`TieBlock`]. /// /// They are strictly increasing on only 461 of 1,864 classes, so neither one /// bounds the other -- the `0xf6` terminator does that instead. pub packets: [i32; 3], /// `0x0c`: the record table's length of each LOD, as three `u8` counts /// followed by one unidentified byte. /// /// The word was read as R&C1's 64-light-slot *normals offset* and skipped, /// which was the right thing to do with the wrong information: the byte at /// `+0x0c` is not an offset at all. It is the **record count of the LOD 0 /// block**, `+0x0d` of LOD 1, `+0x0e` of LOD 2, verified on **1,864 of /// 1,864** classes on all 27 levels. R&C1 carries the same quadruple at /// `+0x20`; R&C2 has it here. /// /// `+0x0f` is not the shader (ad-gif) total, which disagrees on 1,827 of /// the classes measured; it is left unnamed rather than guessed at. pub packet_counts: [u8; 3], /// `0x0f`: unidentified. Not the shader total. pub unknown_0f: u8, /// `0x10`, `0x14`, `0x18`: LOD switch distances in world units. pub near_dist: f32, pub mid_dist: f32, pub far_dist: f32, /// `0x44`: the class's own number. /// /// Matches the core index's `o_class` for the class, which is what makes /// the header base trustworthy. pub o_class: i32, } impl TieClassHeader { /// Parse the agreed-on prefix. Needs `TIE_CLASS_HEADER_MATCHED` bytes. pub fn parse(bytes: &[u8]) -> Result { let mut b = Buf::new(bytes); let mut packets = [0i32; 3]; for slot in packets.iter_mut() { *slot = b.i32("tie packet offset")?; } // 0x0c: the three LOD record counts and one unidentified byte. let packet_counts = [ b.u8("tie LOD0 record count")?, b.u8("tie LOD1 record count")?, b.u8("tie LOD2 record count")?, ]; let unknown_0f = b.u8("tie class word 0x0f")?; let near_dist = b.f32("tie near distance")?; let mid_dist = b.f32("tie mid distance")?; let far_dist = b.f32("tie far distance")?; // 0x1c..0x44: the word TieProc reuses as a per-frame counter, the LOD // packet counts, the texture count, ad-gifs, the bounding sphere and // the class scale. None of it matches RAC2 yet. b.skip(0x44 - 0x1c, "tie class fields not decoded yet")?; let o_class = b.i32("tie class number")?; Ok(TieClassHeader { packets, packet_counts, unknown_0f, near_dist, mid_dist, far_dist, o_class, }) } /// The LOD this class draws at a view depth, or `None` past its far /// distance. pub fn lod_for_depth(&self, depth: f32) -> Option { if depth < self.near_dist { Some(0) } else if depth < self.mid_dist { Some(1) } else if depth < self.far_dist { Some(2) } else { None } } } /// Size of a tie LOD block's own header. pub const TIE_BLOCK_HEADER_SIZE: usize = 0x10; /// The four most common values of [`TieBlockHeader::packed_texture`]. /// /// Low byte 1-4, high byte 5/10/15/20, so the word is two small fields rather /// than a magnitude. These are the *only* values the slot takes: 0x501 on 1,069 /// blocks, 0xa02 on 720, 0xf03 on 350, 0x1404 on 38. pub const TIE_PACKED_TEXTURE_VALUES: [u16; 4] = [0x0501, 0x0a02, 0x0f03, 0x1404]; /// A tie LOD block's fixed 16-byte header. /// /// Every figure here is measured over the **922** blocks where "the next /// `packets` offset ends this block" holds, across all 27 levels. The other /// 1,403 classes are unmeasured, not measured-and-different. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TieBlockHeader { /// `+0x00`: the byte offset from the start of the block to the vertex /// data -- that is, the length of the record table that follows this /// header. A multiple of 16 on all 2,177 blocks (42 distinct values, never /// zero), because the table holds 16-byte records: one block with this /// word at 0x10 has one record, one at 0x50 has five. /// /// This was read as an element *count* for a while, and the reading failed /// in a way worth recording: the block's payload length divided by it is a /// whole number on 0 of 922 blocks, at any record size and any header up /// to 512 bytes. A count of what is in the payload was never going to fit, /// because the word is not about the payload -- it is where the payload /// starts. /// /// Confirmed against the running game: each record's leading word is /// relocated at load to `table_address + word`, and the table's length is /// this word, on every block checked in RAM (16 of 16 on level 1). pub data_offset: u16, /// `+0x02`: zero on all 2,177 blocks. pub reserved_02: u16, /// `+0x04`: one of exactly [`TIE_PACKED_TEXTURE_VALUES`] on all 2,177 blocks. pub packed_texture: u16, /// `+0x06`: 16 distinct values, never zero. pub field_06: u16, /// `+0x08`: 180 distinct values, never zero. pub field_08: u16, /// `+0x0a`: 61 distinct values, never zero. pub field_0a: u16, /// `+0x0c`: 125 distinct values, never zero, clustered at 0x043f to 0x0443 /// (~1,087 to 1,091). **Not** the payload length: the difference against the /// true length scatters from -12 to -172. pub field_0c: u16, /// `+0x0e`: 11 distinct values, zero on 2,162 of 2,177 -- the only field /// here that is not universal. pub reserved_0e: u16, } impl TieBlockHeader { /// Parse the block header, bounds-checked against `block`. pub fn parse(block: &[u8]) -> Result { let mut b = Buf::new(block); Ok(Self { data_offset: b.u16("tie block data offset")?, reserved_02: b.u16("tie block reserved")?, packed_texture: b.u16("tie block packed texture")?, field_06: b.u16("tie block field_06")?, field_08: b.u16("tie block field_08")?, field_0a: b.u16("tie block field_0a")?, field_0c: b.u16("tie block field_0c")?, reserved_0e: b.u16("tie block reserved_0e")?, }) } /// True when every invariant measured across all 27 levels holds. /// /// Only the two that are universal: the data offset is a non-zero multiple /// of 16 and `reserved_02` is zero, both on all 2,177 blocks. pub fn invariants_hold(&self) -> bool { self.data_offset % 16 == 0 && self.data_offset != 0 && self.reserved_02 == 0 } /// The number of 16-byte records in this block's table. pub fn record_count(&self) -> usize { self.data_offset as usize / TIE_RECORD_SIZE } /// The block's records, as byte slices of [`TIE_RECORD_SIZE`]. /// /// Bounds-checked against `block`, which is the block starting at its /// header. **The header is the first record**: the table runs from the /// block start to `data_offset`, so a block with `data_offset == 0x50` /// holds five records, the first of which is the header itself and whose /// pointer names where the payload begins. A header whose table runs past /// the block yields nothing rather than a short slice. pub fn records<'b>(&self, block: &'b [u8]) -> impl Iterator + 'b { block .get(..self.record_count() * TIE_RECORD_SIZE) .unwrap_or(&[]) .chunks_exact(TIE_RECORD_SIZE) } } /// One 16-byte tie block record. /// /// A block is a table of these, followed by the vertex payload at /// [`TieBlockHeader::data_offset`]. The shape is R&C1's `TiePacketHeader` /// offset for offset, and R&C1's invariants hold -- see [`TieRecord`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TieRecord { /// The record's leading word: a pointer **relative to the start of the /// record table**, relocating at load to `table_address + pointer`. /// /// Confirmed on every block checked in PCSX2's RAM: the word becomes /// `table + word` with the other twelve bytes untouched. pub pointer: u32, /// `+0x04`: AD-GIF (material) blocks this packet references; 1..4 on disc. pub shader_count: u8, /// `+0x05`: quadwords of ad-gif data. Equals `5 * shader_count`. pub bfc_distance: u8, /// `+0x06`: 4..18 on disc. pub control_count: u8, /// `+0x07`: 1..5 on disc, never above `control_count`. pub control_size: u8, /// `+0x08`: 3..7 on disc, never above `vert_size`. pub vert_ofs: u8, /// `+0x09`: 4..60 on disc. pub vert_size: u8, /// `+0x0a`: equals `vert_size` on every record measured. pub color_ofs: u8, /// `+0x0b`: zero on every record measured. pub color_count: u8, /// `+0x0c`: 7..67 on disc, never below `scissor_size`. pub scissor_ofs: u8, /// `+0x0d`: 1..5 on disc. pub scissor_size: u8, /// `+0x0e`: zero on every record measured. pub strip_count: u8, /// `+0x0f`: zero on every record measured. pub strip_vertex_count: u8, } impl TieRecord { /// Parse one 16-byte record. pub fn parse(bytes: &[u8]) -> Result { let mut b = Buf::new(bytes); Ok(TieRecord { pointer: b.u32("tie packet data offset")?, shader_count: b.u8("tie packet shader count")?, bfc_distance: b.u8("tie packet bfc distance")?, control_count: b.u8("tie packet control count")?, control_size: b.u8("tie packet control size")?, vert_ofs: b.u8("tie packet vertex offset")?, vert_size: b.u8("tie packet vertex size")?, color_ofs: b.u8("tie packet colour offset")?, color_count: b.u8("tie packet colour count")?, scissor_ofs: b.u8("tie packet scissor offset")?, scissor_size: b.u8("tie packet scissor size")?, strip_count: b.u8("tie packet strip count")?, strip_vertex_count: b.u8("tie packet strip vertex count")?, }) } /// The twelve bytes after the pointer, verbatim. /// /// The comparison against the running game checks these are copied /// unchanged, so they are worth having as one value rather than a /// field-by-field walk. pub fn trailing_bytes(&self) -> [u8; 12] { [ self.shader_count, self.bfc_distance, self.control_count, self.control_size, self.vert_ofs, self.vert_size, self.color_ofs, self.color_count, self.scissor_ofs, self.scissor_size, self.strip_count, self.strip_vertex_count, ] } /// True when every invariant that holds on **every** record measured holds. /// /// `bfc_distance == 5 * shader_count` is R&C1's documented rule for this /// field and it holds on all 15,219 records of all 2,177 blocks on all 27 /// levels; that is what identifies the record as this struct rather than a /// resemblance. The three orderings are also universal. /// /// Relations that hold on most records but not all are deliberately *not* /// here -- a check that fails on a real record is worse than no check -- /// and the disc test counts them instead. `color_ofs == vert_size` is the /// largest of those: 12,640 of 15,219. pub fn invariants_hold(&self) -> bool { self.bfc_distance == 5 * self.shader_count && self.vert_ofs <= self.vert_size && self.scissor_size <= self.scissor_ofs && self.control_size <= self.control_count } /// Quadwords of packet data: `vert_ofs + vert_size + scissor_size`. /// /// Holds on **12,927 of 12,927** record pairs whose `strip_count` is zero, /// and on **0 of 115** whose it is not -- the partition is exact, so the /// formula is safe to use when `strip_count == 0` and knows nothing /// otherwise. R&C1's `strip_count` is the strip count, and strips are /// variable-length data, so a non-zero count most likely adds a record /// count's worth of per-strip bytes this cannot see. pub fn packet_quadwords(&self) -> usize { self.vert_ofs as usize + self.vert_size as usize + self.scissor_size as usize } /// Whether [`Self::packet_quadwords`] is known to describe this record. /// /// `strip_count == 0` on every record the formula has been verified on; /// the 115 records where it is not decode to a fourth term of 0..19 /// quadwords that is not `strip_count` and is not established. pub fn packet_quadwords_are_known(&self) -> bool { self.strip_count == 0 } } /// Byte that terminates a tie LOD block's payload. pub const TIE_BLOCK_TERMINATOR: u8 = 0xf6; /// Bytes in one tie block record. pub const TIE_RECORD_SIZE: usize = 0x10; /// A tie LOD block: its header, its payload, and the zero padding that follows /// the terminator. /// /// **The blocks are self-delimiting.** Each ends with [`TIE_BLOCK_TERMINATOR`] /// followed by zeros, on **2,177 of 2,177** blocks across all 27 levels. That is /// what makes them parseable without trusting `packets[1]` to bound the block, /// which it only does on 461 of 1,864 classes. /// /// The terminator is not a coincidence of byte frequency: `0xf6` occurs at 0.83x /// the uniform rate in payloads, i.e. slightly *less* often than chance, while /// the last non-zero byte is `0xf6` on 909 of 922 blocks in the first subset /// tested against **17 of 922** for the same blocks cut at a random point. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TieBlock<'a> { /// The 16-byte header. pub header: TieBlockHeader, /// Bytes from `0x10` up to and including the terminator. /// /// **This is short.** The `0xf6` the walk stops at lies inside the last /// packet's data, an even 2 to 16 bytes before that packet's computed end, /// so the block's real extent runs past it. `tie_block_payload_divides_ /// into_contiguous_packets` measures this; finding a terminator packet /// data cannot imitate is its own task. pub payload: &'a [u8], /// Zero bytes between the terminator and the next block. pub padding: &'a [u8], } /// Walk `region` from `offset` to the first `0xf6` that is followed by a zero, /// and return the block there. /// /// Returns `None` if no terminator is found before the end of the region. pub fn parse_tie_block(region: &[u8], offset: usize) -> Option> { if offset.checked_add(TIE_BLOCK_HEADER_SIZE)? > region.len() { return None; } let header = TieBlockHeader::parse(®ion[offset..]).ok()?; let start = offset + TIE_BLOCK_HEADER_SIZE; let mut i = start; while i + 1 < region.len() { if region[i] == TIE_BLOCK_TERMINATOR && region[i + 1] == 0 { let after = i + 1; let mut e = after; while e < region.len() && region[e] == 0 { e += 1; } return Some(TieBlock { header, payload: ®ion[start..after], padding: ®ion[after..e], }); } i += 1; } None } /// A tie class resolved through both of its indirections. /// /// The chain is two steps and both are easy to get wrong, so it is worth /// stating plainly: /// /// 1. The class **table** lives in the level's `core_index.bin`, not in /// `core_data.bin`. On level 0 the tables chain as moby at `192`, /// 238 x 32 bytes later tie at `7808`, 74 x 32 later shrub at `10176`, /// which closes. Reading the same table out of `core_data.bin` yields /// plausible-looking garbage rather than an error. /// 2. [`ClassEntry::offset_in_asset_wad`] addresses `core_data.bin`, despite the /// field's name and despite `core_data.wad` existing alongside it. /// /// With both handled, all **1,864** tie classes on the disc resolve to a header /// whose `o_class` matches the class entry's own. #[derive(Debug, Clone, Copy, PartialEq)] pub struct ResolvedTieClass { /// The index entry this came from. pub entry: ClassEntry, /// Its index within the level's tie table. pub index: usize, /// The class header, at `offset_in_asset_wad` in `core_data`. pub header: TieClassHeader, } /// Resolve one tie class. `class_index` counts from zero. /// /// `core_index` is the level's `core_index.bin`; `core_data` is the /// decompressed `core_data.bin`. Both are bounds-checked. pub fn resolve_tie_class( core_index: &[u8], core_data: &[u8], classes: crate::level_core::ArrayRange, class_index: usize, ) -> Result { const ENTRY: usize = 32; let start = classes.offset as usize + class_index * ENTRY; if classes.offset < 0 || start.checked_add(ENTRY).is_none() || start + ENTRY > core_index.len() { return Err(Error { offset: start, context: "tie class table entry", kind: ErrorKind::OutOfBounds { needed: ENTRY, available: core_index.len().saturating_sub(start), }, }); } if class_index >= classes.count as usize { return Err(Error { offset: start, context: "tie class index", kind: ErrorKind::Invalid("class index past the table count"), }); } let entry = ClassEntry::parse(&core_index[start..start + ENTRY])?; let base = entry.offset_in_asset_wad as usize; if base < TIE_CLASS_HEADER_MATCHED || base + TIE_CLASS_HEADER_MATCHED > core_data.len() { return Err(Error { offset: base, context: "tie class header", kind: ErrorKind::OutOfBounds { needed: TIE_CLASS_HEADER_MATCHED, available: core_data.len().saturating_sub(base), }, }); } let header = TieClassHeader::parse(&core_data[base..base + TIE_CLASS_HEADER_MATCHED])?; Ok(ResolvedTieClass { entry, index: class_index, header, }) } #[cfg(test)] mod tests { use super::*; /// A header as measured on level 0's first tie class. fn header() -> Vec { let mut v = vec![0u8; TIE_CLASS_HEADER_MATCHED]; for (i, x) in [256i32, 20_592, 26_048].iter().enumerate() { v[i * 4..i * 4 + 4].copy_from_slice(&x.to_le_bytes()); } for (i, x) in [12.0f32, 25.0, 40.0].iter().enumerate() { let o = 0x10 + i * 4; v[o..o + 4].copy_from_slice(&x.to_le_bytes()); } v[0x44..0x48].copy_from_slice(&2592i32.to_le_bytes()); v } #[test] fn parses_the_measured_prefix() { let h = TieClassHeader::parse(&header()).unwrap(); assert_eq!(h.packets, [256, 20_592, 26_048]); assert_eq!(h.near_dist, 12.0); assert_eq!(h.mid_dist, 25.0); assert_eq!(h.far_dist, 40.0); assert_eq!(h.o_class, 2592); } #[test] fn a_short_header_is_an_error_not_a_panic() { assert!(TieClassHeader::parse(&[0u8; 0x20]).is_err()); assert!(TieClassHeader::parse(&[]).is_err()); } #[test] fn lod_bands_follow_the_three_distances() { let h = TieClassHeader::parse(&header()).unwrap(); assert_eq!(h.lod_for_depth(5.0), Some(0)); assert_eq!(h.lod_for_depth(20.0), Some(1)); assert_eq!(h.lod_for_depth(30.0), Some(2)); assert_eq!(h.lod_for_depth(50.0), None); } /// A block whose data offset names a four-record table, the first of /// which is the header itself. fn block() -> Vec { let mut v = Vec::new(); // data_offset 0x40 -- four records -- reserved 0, packed, then the // remaining fields. v.extend_from_slice(&0x40u16.to_le_bytes()); v.extend_from_slice(&0u16.to_le_bytes()); v.extend_from_slice(&0x0501u16.to_le_bytes()); v.extend_from_slice(&0x1234u16.to_le_bytes()); v.extend_from_slice(&0x5678u16.to_le_bytes()); v.extend_from_slice(&0x9abcu16.to_le_bytes()); v.extend_from_slice(&0x0442u16.to_le_bytes()); v.extend_from_slice(&0u16.to_le_bytes()); assert_eq!(v.len(), TIE_BLOCK_HEADER_SIZE); // Three packet headers, each a data pointer and R&C1's byte fields. for k in 0..3u32 { v.extend_from_slice(&(0x40 + k * 0x20).to_le_bytes()); v.extend_from_slice(&[ 1, 5, // shader_count, bfc_distance = 5 * shader_count 7, 2, // control_count, control_size 4, 0x20, // vert_ofs, vert_size 0x20, 0, // color_ofs (= vert_size), color_count 0x30, 0x20, // scissor_ofs, scissor_size 0, 0, // strip_count, strip_vertex_count ]); } v.extend_from_slice(&[0xa5u8; 8]); v } #[test] fn the_data_offset_bounds_the_record_table() { let b = block(); let h = TieBlockHeader::parse(&b).unwrap(); assert!(h.invariants_hold()); assert_eq!(h.record_count(), 4); let records: Vec<_> = h.records(&b).collect(); assert_eq!(records.len(), 4); // The first record is the header itself, pointing at the payload. let first = TieRecord::parse(records[0]).unwrap(); assert_eq!(first.pointer, 0x40); // The fields follow R&C1's packet-header byte layout. let second = TieRecord::parse(records[1]).unwrap(); assert_eq!(second.shader_count, 1); assert_eq!(second.bfc_distance, 5); assert!(second.invariants_hold()); } #[test] fn a_table_running_past_the_block_yields_nothing() { let mut b = block(); // Claim a table far longer than the block. b[0..2].copy_from_slice(&0x400u16.to_le_bytes()); let h = TieBlockHeader::parse(&b).unwrap(); assert_eq!(h.record_count(), 0x40); assert_eq!(h.records(&b).count(), 0, "must not slice past the block"); // A 0x400 offset is still a multiple of 16, so the invariant holds even // though the table cannot exist. assert!(h.invariants_hold()); } }