diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 8d4e252..0c164f5 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1646,6 +1646,9 @@ fn shrub_headers_name_their_own_class_on_every_level() { let mut absent = 0usize; let (mut min_size, mut max_size) = (f32::MAX, 0.0f32); let mut denormals = 0usize; + let mut bounded_records = 0usize; + let mut bounded_bodies = 0usize; + let mut last_bodies = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); @@ -1657,12 +1660,63 @@ fn shrub_headers_name_their_own_class_on_every_level() { }; let h = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header"); let range = h.shrub_classes; + // Class spans tile between consecutive geometry offsets, with the last + // one bounded by the start of core_data's constant trailer. Without + // this the last class absorbs the trailer and the body check measures + // its neighbours. + let trailer = gc_formats::regions::CORE_DATA_TRAILER_BYTES; + let mut offsets: Vec = Vec::new(); for i in 0..range.count.max(0) as usize { let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES; let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES]) .unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}")); - match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class) { - Ok(Some(s)) => { + if e.base.offset_in_asset_wad > 0 { + offsets.push(e.base.offset_in_asset_wad as usize); + } + } + offsets.sort_unstable(); + let mut span_end_of: std::collections::HashMap = + std::collections::HashMap::new(); + for (i, &o) in offsets.iter().enumerate() { + let end = offsets.get(i + 1).copied().unwrap_or(core.len() - trailer); + span_end_of.insert(o, end); + } + for i in 0..range.count.max(0) as usize { + let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES; + let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES]) + .unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}")); + let (span_end, has_successor) = match span_end_of + .get(&(e.base.offset_in_asset_wad.max(0) as usize)) + { + Some(&e2) + if offsets.last() != Some(&(e.base.offset_in_asset_wad.max(0) as usize)) => + { + (e2, true) + } + _ => (core.len(), false), + }; + match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class, span_end) { + Ok(Some(mut s)) => { + s.span_end_is_a_successor = has_successor; + // Only a class whose span ends at the *next* class has a + // bounded body. The last class's span runs to the trailer + // and contains the 690 KB region that follows shrub, so its + // body is not wholly records -- that is the open question, + // not a defect, and asserting either way would be wrong. + if s.span_end_is_a_successor { + assert!( + s.body_is_well_formed(), + "level{n} class {i}: bounded shrub body is not whole 8-byte records" + ); + bounded_bodies += 1; + } + last_bodies += 1; + // Count records only where the span is actually bounded. + // Summing the unbounded last class too reproduces the + // retracted "3.5 million records" figure. + if s.span_end_is_a_successor { + bounded_records += s.body_record_count(); + } assert_eq!( s.header.o_class, e.base.o_class.max(0) as u32, @@ -1709,7 +1763,9 @@ fn shrub_headers_name_their_own_class_on_every_level() { assert!(spans > 340, "only {spans} shrub spans with geometry"); eprintln!( "shrub: {spans} headers naming their own class, {absent} without geometry; \ - size field {min_size}..{max_size}; {denormals} denormal reserved words" + {bounded_bodies}/{last_bodies} class bodies bounded by a successor, \ + {bounded_records} 8-byte records in bounded bodies, \ + {denormals} denormal reserved words" ); } diff --git a/crates/gc-formats/src/shrub.rs b/crates/gc-formats/src/shrub.rs index 9b2926e..4e8def9 100644 --- a/crates/gc-formats/src/shrub.rs +++ b/crates/gc-formats/src/shrub.rs @@ -54,14 +54,58 @@ pub struct ShrubHeader { pub data_offset: u32, } +/// Bytes per body record. +/// +/// Each record is 6 varying bytes followed by 2 constant ones. Found by +/// grouping sibling spans and byte-diffing them rather than by stride-fitting, +/// which failed three times: the varying payload is **not** sized by the +/// `count` field, and was a constant 144 bytes per group whether `count` was +/// 1, 2, 3, 5 or 6. +pub const SHRUB_BODY_RECORD_BYTES: usize = 8; + +/// The two constant bytes closing each body record. +pub const SHRUB_RECORD_TAIL: [u8; 2] = [0, 0]; + /// A shrub span: its header plus the bytes the header points at. #[derive(Debug, Clone, Copy, PartialEq)] pub struct Shrub<'a> { pub header: ShrubHeader, /// The whole span, starting at `+0x24` with the class number. pub raw: &'a [u8], - /// The bytes at `data_offset`. Not yet decoded. + /// The bytes at `data_offset`, running to `span_end`. Not yet decoded. pub data: &'a [u8], + /// Whether `span_end` came from the next class rather than from the end of + /// the lump. False for the last class in each level, whose span contains + /// whatever follows shrub. + pub span_end_is_a_successor: bool, +} + +impl<'a> Shrub<'a> { + /// The body's 8-byte records. + /// + /// The payload of each record is six bytes whose encoding is **not** + /// established. It is not half-floats -- decoding as a `half3` gives + /// values like 245.25 and -168.375 -- and a VIF reading of it was + /// retracted after a control put it on the random baseline. Handed back + /// raw rather than guessed at. + pub fn body_records(&self) -> impl Iterator { + self.data.chunks_exact(SHRUB_BODY_RECORD_BYTES) + } + + /// How many whole records the body holds. + pub fn body_record_count(&self) -> usize { + self.data.len() / SHRUB_BODY_RECORD_BYTES + } + + /// Whether every whole record ends in the constant pair. + /// + /// This is how a class's body is delimited: on all 24 levels that have + /// shrub geometry, the last shrub class's body ends exactly where its + /// trailing `00 00` pair stops, always on a record boundary. + pub fn body_is_well_formed(&self) -> bool { + let records: Vec<&[u8]> = self.body_records().collect(); + !records.is_empty() && records.iter().all(|r| r[6..8] == SHRUB_RECORD_TAIL) + } } impl ShrubHeader { @@ -71,7 +115,15 @@ impl ShrubHeader { /// rather than merely reported: a span whose `+0x24` disagrees with its /// own class entry is a different thing entirely, and quietly returning /// the mismatched value would hide that. - pub fn region<'a>(core: &'a [u8], offset: i32, o_class: i32) -> Result>> { + /// `span_end` bounds the class span within `core`. It matters: without it + /// `data` runs to the end of the lump and swallows whatever follows, which + /// is how a per-class check ends up measuring its neighbours. + pub fn region<'a>( + core: &'a [u8], + offset: i32, + o_class: i32, + span_end: usize, + ) -> Result>> { if offset <= 0 { return Ok(None); } @@ -131,7 +183,8 @@ impl ShrubHeader { } let at = header.data_offset as usize; - let data = core.get(start + at..).ok_or(Error { + let stop = span_end.clamp(start, core.len()); + let data = core.get(start + at..stop).ok_or(Error { offset: start + at, context: "shrub data offset runs past the lump", kind: ErrorKind::OutOfBounds { @@ -140,6 +193,7 @@ impl ShrubHeader { }, })?; Ok(Some(Shrub { + span_end_is_a_successor: true, header: ShrubHeader { bounds, size, @@ -175,7 +229,7 @@ mod tests { #[test] fn reads_the_self_identifying_header() { let c = span(); - let s = ShrubHeader::region(&c, 0x10, 1072) + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) .unwrap() .expect("present"); assert_eq!(s.header.o_class, 1072); @@ -187,27 +241,63 @@ mod tests { #[test] fn a_span_naming_the_wrong_class_is_rejected() { let c = span(); - let err = ShrubHeader::region(&c, BASE as i32, 999).unwrap_err(); + let err = ShrubHeader::region(&c, BASE as i32, 999, c.len()).unwrap_err(); assert!(format!("{err:?}").contains("disagrees")); } #[test] fn a_zero_offset_means_absent_not_present() { let c = span(); - assert!(ShrubHeader::region(&c, 0, 1072).unwrap().is_none()); - assert!(ShrubHeader::region(&c, -1, 1072).unwrap().is_none()); + assert!(ShrubHeader::region(&c, 0, 1072, c.len()).unwrap().is_none()); + assert!(ShrubHeader::region(&c, -1, 1072, c.len()) + .unwrap() + .is_none()); + } + + #[test] + fn body_records_split_into_eights_and_check_their_tail() { + let c = span(); + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) + .unwrap() + .expect("present"); + // The fixture fills 0xab, so every record's tail is wrong. + assert!(!s.body_is_well_formed()); + assert_eq!(s.body_record_count(), (0x40 - 0x30) / 8); + // Fix the tails and it is well formed. + let mut c2 = span(); + for i in 0..((0x40 - 0x30) / 8) { + c2[BASE + 0x30 + i * 8 + 6] = 0; + c2[BASE + 0x30 + i * 8 + 7] = 0; + } + let s2 = ShrubHeader::region(&c2, BASE as i32, 1072, c2.len()) + .unwrap() + .expect("present"); + assert!(s2.body_is_well_formed()); + assert_eq!(s2.body_records().count(), s2.body_record_count()); + } + + #[test] + fn a_body_shorter_than_one_record_is_not_well_formed() { + let mut c = span(); + // Point the data at the last four bytes of the span. + c[BASE + 0x2c..BASE + 0x30].copy_from_slice(&(0x3cu32).to_le_bytes()); + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) + .unwrap() + .expect("present"); + assert_eq!(s.body_record_count(), 0); + assert!(!s.body_is_well_formed()); } #[test] fn out_of_range_is_an_error_not_a_panic() { // Offset 0 is absent, not an error -- an empty lump is just empty. - assert!(ShrubHeader::region(&[], 0, 1).unwrap().is_none()); - assert!(ShrubHeader::region(&[], 4, 1).is_err()); + assert!(ShrubHeader::region(&[], 0, 1, 0).unwrap().is_none()); + assert!(ShrubHeader::region(&[], 4, 1, 8).is_err()); // Data offset past the end of the lump. let mut c = span(); c[BASE + 0x2c..BASE + 0x30].copy_from_slice(&9_000u32.to_le_bytes()); - assert!(ShrubHeader::region(&c, BASE as i32, 1072).is_err()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); // Span starting near the end, with no room for a header. - assert!(ShrubHeader::region(&c, (c.len() - 4) as i32, 1072).is_err()); + assert!(ShrubHeader::region(&c, (c.len() - 4) as i32, 1072, c.len()).is_err()); } }