diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0f18791..b401307 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -385,3 +385,105 @@ fn level_geometry_decompresses() { core.len() ); } + +/// The terrain itself. This is the first test to touch real level content: it +/// parses the tfrags directory and decodes vertex positions from every patch. +/// +/// The gate is deliberately about *coverage*, not correctness of a sample: a +/// parser that works on one tfrag and silently returns `None` on the rest would +/// otherwise pass. +#[test] +fn tfrag_vertices_decode_for_every_level() { + require_disc!(); + // Opening the image also confirms the disc is readable, not merely present. + gc_formats::iso9660::IsoImage::open(std::path::Path::new( + &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), + )) + .expect("ISO should open"); + // Where the unpacked data lives. GC_EXTRACTED overrides; otherwise the + // default location used during development. + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!( + "skipped: unpack the disc first, e.g.\ + gc-extract unpack ~/Documents/gc-extracted\ + or point GC_EXTRACTED at an existing unpack" + ); + return; + } + + let mut levels = 0usize; + let mut total_verts = 0usize; + let mut total_tfrags = 0usize; + + for n in 0..27 { + let dir = data.join(format!("level{n}")); + let core_path = dir.join("core_data.bin"); + let index_path = dir.join("core_index.bin"); + if !core_path.exists() || !index_path.exists() { + continue; + } + let core = std::fs::read(&core_path).expect("core_data"); + let index = std::fs::read(&index_path).expect("core_index"); + + // The tfrags block starts at the core header's 0x08 field and runs to + // the next region that exists. + let region_start = + |off: usize| -> i32 { i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) }; + let start = region_start(0x08) as usize; + let end = [0x0c_usize, 0x10, 0x14] + .iter() + .map(|&o| region_start(o)) + .filter(|&v| v > 0) + .min() + .unwrap_or(core.len() as i32) as usize; + + let block = &core[start..end]; + let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header"); + assert_eq!(header.table_offset, 0x40, "level{n}: table offset"); + + let table_offset = header.table_offset as usize; + let mut decoded = 0usize; + let mut verts = 0usize; + + for i in 0..header.tfrag_count as usize { + let th = header.header(block, i).expect("tfrag header"); + let data_base = table_offset.wrapping_add(th.data as usize); + if data_base >= block.len() { + continue; + } + let slice = &block[data_base..]; + if let Some(p) = gc_formats::tfrag::tfrag_positions(slice, &th) { + decoded += 1; + verts += p.len(); + assert_eq!(p.len(), th.vert_count as usize); + } + } + + assert!(header.tfrag_count > 0, "level{n}: no tfrags"); + assert_eq!( + decoded, header.tfrag_count as usize, + "level{n}: only {decoded} of {} tfrags decoded", + header.tfrag_count + ); + levels += 1; + total_tfrags += header.tfrag_count as usize; + total_verts += verts; + } + + assert!(levels > 0, "no unpacked levels were found"); + assert!( + total_tfrags > 5000, + "expected thousands of tfrags, got {total_tfrags}" + ); + assert!( + total_verts > 100_000, + "expected 100k+ vertices, got {total_verts}" + ); + eprintln!("decoded {total_verts} vertices from {total_tfrags} tfrags across {levels} levels"); +} diff --git a/crates/gc-formats/src/lib.rs b/crates/gc-formats/src/lib.rs index 2545f3b..6b0e0b9 100644 --- a/crates/gc-formats/src/lib.rs +++ b/crates/gc-formats/src/lib.rs @@ -3,4 +3,5 @@ pub mod global_wad; pub mod iso9660; pub mod level_data; pub mod level_wad; +pub mod tfrag; pub mod wad; diff --git a/crates/gc-formats/src/tfrag.rs b/crates/gc-formats/src/tfrag.rs new file mode 100644 index 0000000..0d838cd --- /dev/null +++ b/crates/gc-formats/src/tfrag.rs @@ -0,0 +1,588 @@ +//! Terrain fragments: the `tfrags` block of a level's geometry. +//! +//! Spec: `docs/formats/tfrag.md`. +//! +//! A level's terrain is a set of **tfrags** — small patches of ground, each +//! with its own bounding sphere, its own vertex set and its own index strips, +//! organised into three levels of detail. The block is a directory followed by +//! one data block per tfrag. +//! +//! # Block layout +//! +//! ```text +//! +0x00 4 byte offset of the header table, from the start of the block +//! +0x04 4 number of tfrags +//! +0x08 4 float, LOD distance parameter +//! +0x10 .. padding, zero, up to the table +//! +table_offset TfragHeader[tfrag_count], 0x40 bytes each +//! tfrag 0's data block +//! tfrag 1's data block +//! ... +//! ``` +//! +//! A tfrag header's `data` field is a byte offset **relative to the start of the +//! header table**, not to the start of the block. Every other offset in the +//! record is relative to that tfrag's own data block. Two different bases in one +//! record is the sort of detail that produces silently wrong geometry. +//! +//! # What is recovered here +//! +//! Vertex **positions**: a V3_16 signed unpack inside the tfrag's VIF command +//! list, giving 6 bytes per vertex as three `s16` world coordinates. +//! +//! Not yet recovered: the vertex-info array (texture coordinates), the strip and +//! index arrays that define the triangles, and the ad-gif texture bindings. See +//! `docs/plan/tfrag.md`. + +use crate::buf::{Buf, Error, ErrorKind, Result}; + +/// Size of one tfrag header record. +pub const TFRAG_HEADER_SIZE: usize = 0x40; +/// Size of the block header before the padding. +pub const TFRAGS_BLOCK_HEADER_SIZE: usize = 0x10; + +/// The header at the start of a tfrags block. +#[derive(Debug, Clone, PartialEq)] +pub struct TfragsBlock { + /// Byte offset of the header table, relative to the start of the block. + pub table_offset: u32, + pub tfrag_count: u32, + /// LOD distance parameter. Preserved; semantics not confirmed. + pub lod_distance: f32, + /// The second unnamed word. Preserved. + pub reserved: u32, +} + +impl TfragsBlock { + /// Parse the block header from the start of the tfrags region. + pub fn parse(bytes: &[u8]) -> Result { + let mut b = Buf::new(bytes); + let table_offset = b.u32("tfrags table offset")?; + let tfrag_count = b.u32("tfrag count")?; + let lod_distance = b.f32("tfrags LOD distance")?; + let reserved = b.u32("tfrags reserved word")?; + Ok(TfragsBlock { + table_offset, + tfrag_count, + lod_distance, + reserved, + }) + } + + /// The tfrag header at index `i`, given the whole block. + pub fn header(&self, bytes: &[u8], i: usize) -> Result { + let off = self.table_offset as usize + i * TFRAG_HEADER_SIZE; + if off + TFRAG_HEADER_SIZE > bytes.len() { + return Err(Error { + offset: off, + context: "tfrag header table", + kind: ErrorKind::OutOfBounds { + needed: TFRAG_HEADER_SIZE, + available: bytes.len().saturating_sub(off), + }, + }); + } + TfragHeader::parse(&bytes[off..off + TFRAG_HEADER_SIZE]) + } + + /// Every tfrag header in the block. + pub fn headers(&self, bytes: &[u8]) -> Result> { + (0..self.tfrag_count as usize) + .map(|i| self.header(bytes, i)) + .collect() + } +} + +/// One tfrag's header record. +#[derive(Debug, Clone, PartialEq)] +pub struct TfragHeader { + /// Bounding sphere: centre xyz and radius w. The centre is a VU1 memory + /// address on this game rather than a world position, so it is not usable + /// as a world-space bound without further work. + pub bsphere: [f32; 4], + /// Byte offset of this tfrag's data block, **relative to the table**. + pub data: i32, + pub lod_2_ofs: u16, + pub shared_ofs: u16, + pub lod_1_ofs: u16, + pub lod_0_ofs: u16, + /// Offset of the ad-gif texture payload inside the common VIF list. + pub tex_ofs: u16, + /// Offset of the per-vertex RGBA array. + pub rgba_ofs: u16, + pub common_size: u8, + pub lod_2_size: u8, + pub lod_1_size: u8, + pub lod_0_size: u8, + pub texture_count: u8, + pub rgba_size: u8, + pub msphere_ofs: u16, + pub light_ofs: u16, + pub light_end_ofs: u16, + pub cube_ofs: u16, + /// Occlusion / visibility index for this tfrag. + pub occl_index: u16, + /// Total vertex count. `u8`, so at most 255 positions per tfrag. + pub vert_count: u8, + /// Triangle count at LOD 0. + pub tri_count: u8, +} + +impl TfragHeader { + /// Parse one 0x40-byte header record. + pub fn parse(bytes: &[u8]) -> Result { + let mut b = Buf::new(bytes); + let mut bsphere = [0f32; 4]; + for slot in bsphere.iter_mut() { + *slot = b.f32("tfrag bounding sphere")?; + } + let data = b.i32("tfrag data offset")?; + let lod_2_ofs = b.u16("tfrag lod_2_ofs")?; + let shared_ofs = b.u16("tfrag shared_ofs")?; + let lod_1_ofs = b.u16("tfrag lod_1_ofs")?; + let lod_0_ofs = b.u16("tfrag lod_0_ofs")?; + let tex_ofs = b.u16("tfrag tex_ofs")?; + let rgba_ofs = b.u16("tfrag rgba_ofs")?; + // 0x20..0x2d are single bytes; 0x2d..0x2e then a u16, so read in order. + let common_size = b.u8("tfrag common_size")?; + let lod_2_size = b.u8("tfrag lod_2_size")?; + let lod_1_size = b.u8("tfrag lod_1_size")?; + let lod_0_size = b.u8("tfrag lod_0_size")?; + let _lod_2_rgba = b.u8("tfrag lod_2_rgba_count")?; + let _lod_1_rgba = b.u8("tfrag lod_1_rgba_count")?; + let _lod_0_rgba = b.u8("tfrag lod_0_rgba_count")?; + let _base_only = b.u8("tfrag base_only")?; + let texture_count = b.u8("tfrag texture_count")?; + let rgba_size = b.u8("tfrag rgba_size")?; + let _rgba_verts_loc = b.u8("tfrag rgba_verts_loc")?; + let _occl_index_stash = b.u8("tfrag occl_index_stash")?; + let _msphere_count = b.u8("tfrag msphere_count")?; + let _flags = b.u8("tfrag flags")?; + let msphere_ofs = b.u16("tfrag msphere_ofs")?; + let light_ofs = b.u16("tfrag light_ofs")?; + let light_end_ofs = b.u16("tfrag light_end_ofs")?; + let _dir_lights_one = b.u8("tfrag dir_lights_one")?; + let _dir_lights_upd = b.u8("tfrag dir_lights_upd")?; + let _point_lights = b.u16("tfrag point_lights")?; + let cube_ofs = b.u16("tfrag cube_ofs")?; + let occl_index = b.u16("tfrag occl_index")?; + let vert_count = b.u8("tfrag vert_count")?; + let tri_count = b.u8("tfrag tri_count")?; + let _mip_dist = b.u16("tfrag mip_dist")?; + + Ok(TfragHeader { + bsphere, + data, + lod_2_ofs, + shared_ofs, + lod_1_ofs, + lod_0_ofs, + tex_ofs, + rgba_ofs, + common_size, + lod_2_size, + lod_1_size, + lod_0_size, + texture_count, + rgba_size, + msphere_ofs, + light_ofs, + light_end_ofs, + cube_ofs, + occl_index, + vert_count, + tri_count, + }) + } +} + +/// A VIF code word decoded into its fields. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VifCode { + pub irq: bool, + /// Command field, bits 30..24. + pub cmd: u8, + /// Payload word count, bits 23..16. Zero means 256. + pub num: u16, + /// Immediate, bits 15..0. + pub imm: u16, +} + +impl VifCode { + /// Re-encode into a code word. Used by tests to build packets. + pub fn to_word(&self) -> u32 { + ((self.irq as u32) << 31) + | ((self.cmd as u32) << 24) + | ((self.num as u32) << 16) + | self.imm as u32 + } + + pub fn decode(word: u32) -> VifCode { + VifCode { + irq: (word >> 31) & 1 == 1, + cmd: ((word >> 24) & 0x7F) as u8, + num: ((word >> 16) & 0xFF) as u16, + imm: (word & 0xFFFF) as u16, + } + } + + /// Is this an UNPACK? Recognised by the top two bits of the command. + pub fn is_unpack(&self) -> bool { + (self.cmd & 0b110_0000) == 0b110_0000 + } + + /// Number of components minus one: 0..3 = 1..4 components. + pub fn vn(&self) -> u8 { + (self.cmd >> 2) & 3 + } + + /// Component width: 0 = 32-bit, 1 = 16-bit, 2 = 8-bit, 3 = 5-bit. + pub fn vl(&self) -> u8 { + self.cmd & 3 + } + + /// Bytes per unpacked element on disk. + /// + /// The whole vector format is encoded in the command byte: `vn` gives the + /// component count and `vl` the component width. Nothing about the format + /// lives in the immediate. + pub fn element_size(&self) -> usize { + ((32usize >> self.vl()) * (self.vn() as usize + 1)) / 8 + } + + /// The PS2 vector format name, for diagnostics. + pub fn format_name(&self) -> String { + const NAMES: [&str; 4] = ["V1", "V2", "V3", "V4"]; + const WIDTHS: [&str; 4] = ["32", "16", "8", "5"]; + if self.is_unpack() { + format!( + "{}_{}", + NAMES[self.vn() as usize], + WIDTHS[self.vl() as usize] + ) + } else { + "not an unpack".to_string() + } + } + + /// Effective element count: a `num` of zero means 256. + pub fn count(&self) -> usize { + if self.num == 0 { + 256 + } else { + self.num as usize + } + } + + /// Inline payload size in bytes, not counting the 4-byte code word. + /// + /// An unpack's payload is padded up to 4 bytes. Only a few non-unpack + /// commands carry inline data; everything else is a bare code word. Getting + /// this wrong desynchronises the walk and yields plausible garbage, so the + /// set is enumerated rather than assumed empty. + pub fn payload_size(&self) -> usize { + if self.is_unpack() { + (self.count() * self.element_size() + 3) & !3 + } else { + match self.cmd { + STMASK => 4, + STROW | STCOL => 16, + MPG => self.num as usize * 8, + DIRECT | DIRECTHL => { + if self.imm == 0 { + 65536 * 16 + } else { + self.imm as usize * 16 + } + } + _ => 0, + } + } + } + + /// Total packet size in bytes, including the 4-byte code word. + pub fn packet_size(&self) -> usize { + 4 + self.payload_size() + } +} + +/// VIF command bytes for non-unpack commands that carry inline data. +pub const STMASK: u8 = 0x20; +pub const STROW: u8 = 0x30; +pub const STCOL: u8 = 0x31; +pub const MPG: u8 = 0x4a; +pub const DIRECT: u8 = 0x50; +pub const DIRECTHL: u8 = 0x51; + +/// The VIF lists that make up a tfrag's data block, in the order the game +/// transfers them. +/// +/// The four offsets do not run in ascending order: the LOD-2 list sits *before* +/// the common list. Each is a byte range from the start of the tfrag's data +/// block. Positions appear in the common list and the two LOD-0 lists only. +fn position_lists(h: &TfragHeader) -> [(usize, usize); 3] { + // The LOD-0-only list starts here rather than at an offset of its own. + let lod0_start = h.shared_ofs as usize + h.lod_1_size as usize * 0x10; + [ + (h.shared_ofs as usize, h.lod_1_ofs as usize), + (h.lod_0_ofs as usize, lod0_start), + (lod0_start, h.rgba_ofs as usize), + ] +} + +/// Collect the V3_16 position payloads from one byte range. +fn positions_in_list(block: &[u8], start: usize, end: usize, out: &mut Vec<[i32; 3]>) -> bool { + if start >= end || end > block.len() { + // Empty lists and inverted ranges (a level with no LOD 2) are normal. + return start >= end || end > block.len(); + } + let mut pos = start; + while pos + 4 <= end { + let code = u32::from_le_bytes([block[pos], block[pos + 1], block[pos + 2], block[pos + 3]]); + let packet = VifCode::decode(code); + if pos + packet.packet_size() > end { + return false; + } + if packet.is_unpack() && packet.vl() == 1 && packet.vn() == 2 { + let base = pos + 4; + for v in 0..packet.count() { + let o = base + v * 6; + if o + 6 > end { + return false; + } + out.push([ + i16::from_le_bytes([block[o], block[o + 1]]) as i32, + i16::from_le_bytes([block[o + 2], block[o + 3]]) as i32, + i16::from_le_bytes([block[o + 4], block[o + 5]]) as i32, + ]); + } + } + pos += packet.packet_size(); + } + true +} + +/// Vertex positions of a tfrag, in game order. +/// +/// `block` is the tfrag's data block, starting at its own `data` offset; +/// the offsets in `header` are relative to that. `vert_count` covers all three +/// position tiers (common, LOD 0+1 and LOD 0), so positions are gathered from +/// each list in transfer order. +/// +/// Returns `None` if the lists cannot be walked cleanly or do not yield +/// exactly `vert_count` positions: a wrong count means the walk desynchronised, +/// and returning partial geometry would hide that. +pub fn tfrag_positions(block: &[u8], header: &TfragHeader) -> Option> { + if header.vert_count == 0 { + return Some(Vec::new()); + } + let mut out = Vec::with_capacity(header.vert_count as usize); + for (start, end) in position_lists(header) { + if !positions_in_list(block, start, end, &mut out) { + return None; + } + } + if out.len() != header.vert_count as usize { + return None; + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A block header as measured on level 0. + fn block_header() -> Vec { + let mut v = vec![0u8; TFRAGS_BLOCK_HEADER_SIZE]; + v[0x00..0x04].copy_from_slice(&0x40u32.to_le_bytes()); + v[0x04..0x08].copy_from_slice(&833u32.to_le_bytes()); + v[0x08..0x0c].copy_from_slice(&32.0f32.to_le_bytes()); + v + } + + #[test] + fn parses_the_measured_block_header() { + let h = TfragsBlock::parse(&block_header()).unwrap(); + assert_eq!(h.table_offset, 0x40); + assert_eq!(h.tfrag_count, 833); + assert_eq!(h.lod_distance, 32.0); + } + + #[test] + fn parses_a_tfrag_header() { + let mut v = vec![0u8; TFRAG_HEADER_SIZE]; + v[0x00..0x10].copy_from_slice(&[0u8; 16]); + v[0x10..0x14].copy_from_slice(&53_312i32.to_le_bytes()); + v[0x14..0x16].copy_from_slice(&0u16.to_le_bytes()); + v[0x16..0x18].copy_from_slice(&208u16.to_le_bytes()); + v[0x1c..0x1e].copy_from_slice(&256u16.to_le_bytes()); + v[0x3c] = 75; + v[0x3d] = 62; + + let h = TfragHeader::parse(&v).unwrap(); + assert_eq!(h.data, 53_312); + assert_eq!(h.shared_ofs, 208); + assert_eq!(h.tex_ofs, 256); + assert_eq!(h.vert_count, 75); + assert_eq!(h.tri_count, 62); + } + + #[test] + fn header_table_access_is_bounds_checked() { + let mut bytes = block_header(); + bytes.resize(0x40 + 4 * TFRAG_HEADER_SIZE, 0); + let block = TfragsBlock::parse(&bytes).unwrap(); + block.header(&bytes, 3).unwrap(); + assert!(block.header(&bytes, 99).is_err()); + } + + #[test] + fn vif_code_decoding() { + // 0x694b_8022, measured in level 0's first tfrag: a V3_16 unpack of 75 + // positions to VU address 2. + let c = VifCode::decode(0x694b_8022); + assert!(c.is_unpack()); + assert_eq!((c.cmd, c.num, c.imm), (0x69, 0x4b, 0x8022)); + assert_eq!((c.vn(), c.vl()), (2, 1)); + assert_eq!(c.element_size(), 6); + assert_eq!(c.format_name(), "V3_16"); + // 75 * 6 = 450, padded up to 452, plus the code word. + assert_eq!(c.payload_size(), 452); + assert_eq!(c.packet_size(), 456); + } + + #[test] + fn vector_formats_come_from_the_command_byte() { + // (code word, vn, vl, bytes per element) + for (word, vn, vl, size) in [ + (0x6900_8009u32, 2u8, 1u8, 6usize), // V3_16 + (0x6c00_0000, 3, 0, 16), // V4_32 + (0x6d00_c012, 3, 1, 8), // V4_16 + (0x6e00_0000, 3, 2, 4), // V4_8 + (0x6a00_0000, 2, 2, 3), // V3_8 + (0x6800_0000, 2, 0, 12), // V3_32 + ] { + let c = VifCode::decode(word); + assert_eq!( + (c.vn(), c.vl(), c.element_size()), + (vn, vl, size), + "{word:#010x}" + ); + } + } + + #[test] + fn vif_num_zero_means_256() { + assert_eq!(VifCode::decode(0x6e00_0000).count(), 256); + assert_eq!(VifCode::decode(0x6e00_0000).payload_size(), 256 * 4); + } + + #[test] + fn only_some_non_unpack_commands_carry_payload() { + // STROW and STCOL move a quadword each; a bare code word moves none. + assert_eq!(VifCode::decode(0x3000_0000).payload_size(), 16); + assert_eq!(VifCode::decode(0x3100_0000).payload_size(), 16); + assert_eq!(VifCode::decode(0x2000_0000).payload_size(), 4); + assert_eq!(VifCode::decode(0x4a02_0000).payload_size(), 16); // MPG num=2 + assert_eq!(VifCode::decode(0x5000_0001).payload_size(), 16); // DIRECT imm=1 + assert_eq!(VifCode::decode(0x0500_0001).payload_size(), 0); // STMOD + assert_eq!(VifCode::decode(0x1400_0000).payload_size(), 0); // MSCAL + } + + /// Build a data block holding a single V3_16 unpack in the common list. + fn synthetic_tfrag(verts: &[[i16; 3]]) -> (Vec, TfragHeader) { + let mut block = Vec::new(); + let n = verts.len() as u16; + // NUM is 8 bits, so a 256-element unpack has to encode zero. + let num = if n == 256 { 0u16 } else { n }; + block.extend_from_slice( + &VifCode { + irq: false, + cmd: 0x69, + num, + imm: 0x8009, + } + .to_word() + .to_le_bytes(), + ); + for v in verts { + for c in v { + block.extend_from_slice(&c.to_le_bytes()); + } + } + // The next list starts after the padded packet, not after the raw + // vertices, so an odd element count advances the boundary. + let common_end = block.len().div_ceil(4) * 4; + block.resize(128, 0); + + let mut h = TfragHeader::parse(&[0u8; TFRAG_HEADER_SIZE]).unwrap(); + h.vert_count = verts.len() as u8; + h.shared_ofs = 0; + h.lod_1_ofs = common_end as u16; + h.lod_0_ofs = common_end as u16; + // The LOD-0-only list starts on a quadword boundary at or after the + // common list, so round the span up rather than truncating it. + h.lod_1_size = common_end.div_ceil(16) as u8; + h.rgba_ofs = 128; + (block, h) + } + + #[test] + fn finds_positions_in_a_synthetic_block() { + let verts = [ + [99, -3795, -5203], + [1123, -3795, -5203], + [-715, -3072, -5203], + [1123, -3072, -5203], + ]; + let (block, header) = synthetic_tfrag(&verts); + let pos = tfrag_positions(&block, &header).expect("positions should be found"); + assert_eq!(pos.len(), 4); + for (got, want) in pos.iter().zip(verts.iter()) { + assert_eq!(*got, [want[0] as i32, want[1] as i32, want[2] as i32]); + } + } + + #[test] + fn an_odd_count_is_padded_to_four_bytes() { + // 3 elements = 18 bytes, padded to 20, so the next code word sits at + // +24 rather than +22. + let verts = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]; + let (block, header) = synthetic_tfrag(&verts); + let c = VifCode::decode(u32::from_le_bytes(block[0..4].try_into().unwrap())); + assert_eq!(c.payload_size(), 20); + let pos = tfrag_positions(&block, &header).expect("positions should be found"); + assert_eq!(pos.len(), 3); + } + + #[test] + fn a_desynchronised_walk_is_reported_rather_than_trusted() { + let verts = [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]; + let (block, header) = synthetic_tfrag(&verts); + // Claim more vertices than the block actually holds. + let mut bad = header; + bad.vert_count = 40; + assert!(tfrag_positions(&block, &bad).is_none()); + } + + #[test] + fn a_tfrag_with_no_unpack_returns_none() { + let block = vec![0u8; 128]; + let mut header = TfragHeader::parse(&[0u8; TFRAG_HEADER_SIZE]).unwrap(); + header.vert_count = 4; + header.shared_ofs = 0; + header.lod_1_ofs = 64; + header.lod_0_ofs = 64; + header.rgba_ofs = 128; + assert!(tfrag_positions(&block, &header).is_none()); + } + + #[test] + fn zero_vertex_count_gives_an_empty_result() { + let block = vec![0u8; 16]; + let mut header = TfragHeader::parse(&[0u8; TFRAG_HEADER_SIZE]).unwrap(); + header.vert_count = 0; + assert_eq!(tfrag_positions(&block, &header), Some(Vec::new())); + } +} diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index 5350301..5eb14f4 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -205,6 +205,9 @@ Next: rather than guess the header further, identify `tfrag` by its own structure. Terrain is the largest subsystem and the one that decides whether anything draws, so it is the right next target even though it is the harder one. +*(Done — see "tfrag terrain" below. Identifying tfrag by its own structure +worked where the class-table guess did not.)* + ## `data`: the real target, untouched 6,484 sectors on level 0, down to 2,102 on level 24 — 4.2 to 21.9 MB. This is @@ -237,4 +240,80 @@ hypothesis, test it against **all 27 levels at once**, and require an invariant that holds everywhere. The sector/byte ambiguity in Phase 1 survived because it happened to be correct on the one container checked by hand. The level header was fixed in a single pass because the tiling check failed immediately on -levels 1, 2, 4, 7, 8, 11, 19 and 20. \ No newline at end of file +levels 1, 2, 4, 7, 8, 11, 19 and 20. +## tfrag terrain: block structure and vertex positions + +Settled. `gc-formats::tfrag` parses the block, and a gated test decodes every +patch on all 27 levels. + +### Block and patch framing + +The tfrags block opens with a 12-byte header: `table_offset` (always `0x40`), +`tfrag_count`, and an `f32` LOD distance (`32.0`). The patch table follows at +`table_offset`, `tfrag_count` records of `0x40` bytes each. On level 0 that is +833 patches. + +A patch header's `data` offset is **relative to `table_offset`**, not to the +start of the block. Getting this wrong happens to work for the first patch and +silently produces garbage afterwards. + +### Positions live in three lists, not one + +`vert_count` is the *total* across all three position tiers. Within a patch's +data block the VIF lists do not run in ascending offset order — the LOD-2 list +sits *before* the common list — and the LOD-0-only list has no offset of its +own, starting instead at `shared_ofs + lod_1_size * 16`. The ranges, in transfer +order, are: + +| List | Range (from the patch data block) | Holds | +|---|---|---| +| common | `shared_ofs .. lod_1_ofs` | VU header, ad-gifs, vertex info, origin, positions | +| LOD 2 | `lod_2_ofs .. shared_ofs` | strips, indices | +| LOD 1 | `lod_1_ofs .. lod_0_ofs` | strips, indices | +| LOD 0+1 | `lod_0_ofs .. shared_ofs + lod_1_size*16` | parents, vertex info, positions | +| LOD 0 | `shared_ofs + lod_1_size*16 .. rgba_ofs` | positions, strips, indices, parents | + +Scanning only the common list and expecting `num == vert_count` finds positions +in roughly a quarter of patches. The other three quarters keep most of their +positions in the LOD lists. That was the first wrong turn and it looked +plausible: every patch it did match matched *exactly*. + +### The VIF1 code word + +Two rules had to be right before any of this worked, and both were wrong on the +first attempt. + +**The vector format lives in the command byte, not the immediate.** Bits 2-3 of +the command are `vn` (component count minus one) and bits 0-1 are `vl` (0 = +32-bit, 1 = 16-bit, 2 = 8-bit, 3 = 5-bit), so + + element_size = ((32 >> vl) * (vn + 1)) / 8 + +Terrain positions are `vn=2, vl=1` — command `0x69` — at 6 bytes each. Reading +the format out of the immediate's low nibble, which is the more commonly quoted +layout, yields 12 or 16 bytes and quietly misaligns everything after it. + +**Unpack payloads pad to 4 bytes, and several non-unpack commands carry inline +data.** An unpack advances `(count * element_size + 3) & !3`. `STROW`/`STCOL` +carry 16 bytes, `STMASK` 4, `MPG` `num * 8`, `DIRECT`/`DIRECTHL` `imm * 16`; +every other non-unpack is a bare 4-byte code word. Ignoring the first four +desynchronises the walk, and a desynchronised walk still finds code words that +look like positions — it just finds the wrong number of them. + +The 4-byte padding is easy to misread. With 75 elements the payload is 450 +bytes and the following word lands at +452, where `0x01000404` (STCYCL) reads +perfectly plausibly. The word at +450 is `0x04040000` (ITOP), which is the real +one. Taking the plausible-looking reading is how this was briefly wrong. + +### Verification + +Requiring every patch to yield exactly its own `vert_count` is the invariant +that caught both errors, and it is the check the parser is built around: a +mismatch means the walk desynchronised, so `tfrag_positions` returns `None` +rather than partial geometry. Across all 27 levels: 18,102 patches, +821,439 vertices, zero mismatches. + +### Still open + +Strips and triangle indices, the RGBA vertex-info arrays, texture binding, and +the LOD/morph/cull rules in `TfragProc`. Positions alone are not drawable. diff --git a/docs/roadmap.md b/docs/roadmap.md index 8a0f37c..52c9cc5 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -20,7 +20,7 @@ early, while they are still cheap to retire. header, and the `data` extent is parsed as a container of byte ranges. The geometry decompresses: level 0 yields 16,969,296 bytes of level content from a 9.7 MB stream. -- [ ] Next: the geometry directory's class tables, then `tfrag` terrain. +- [ ] Next: tfrag triangle topology, then the geometry directory's class tables. The disc being the exact build the reference projects pin is the single biggest favourable fact so far: no disc conversion, no regional timing work, @@ -114,9 +114,16 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [x] Region layout of `core_data` established: tfrag, occlusion, sky, collision and `textures_base` are offsets, strictly increasing on all 27 levels. Terrain is `[0, occlusion)` and carries inline texels. -- [ ] **tfrag vertex format: not recovered.** The 64-byte record framing and the - scale-8 fixed-point reading were both tested and disproven; see - `plan/level-assets.md` for what was ruled out. +- [x] **tfrag vertex format: recovered.** The tfrags block header gives a table + offset (`0x40`) and a patch count; each patch is a 0x40-byte header whose + `data` offset is relative to the table, not to the block. Positions are + V3_16 payloads inside a VIF1 command list, split across three lists per + patch (common, LOD 0+1, LOD 0). The whole vector format is encoded in the + command byte and every element size follows from it. Verified on all 27 + levels: **18,102 patches, 821,439 vertices**, each patch yielding exactly + its own `vert_count`. See `plan/level-assets.md` and `gc-formats::tfrag`. +- [ ] tfrag topology: strips, triangle indices and the RGBA vertex-info arrays, + which is what turns positions into drawable geometry. - [ ] `gameplay` — begins `WAD` + a per-level byte, and does *not* decode with the WAD LZ77 decoder. All 27 levels fail identically, so the input is wrong rather than the decoder. Ruled out: bit order, header length, the