Expose the shrub body as 8-byte records, bounded per class

Turns the differential finding into code. Each shrub body record is 6 varying bytes plus 2 constant ones; body_records() hands them back raw and body_is_well_formed() checks the trailing pair. ShrubHeader::region now takes an explicit span_end. It has to: without it the data slice runs to the end of the lump, so the last class in each level absorbs whatever follows shrub and a per-class check ends up measuring its neighbours. That is the same unbounded-span mistake as before, caught this time by a failing assertion rather than by reading the docs afterwards. The disc test asserts well-formedness only for the 334 of 358 classes whose span ends at the next class. The last class in each level has no successor, so its span contains the 690 KB region after shrub and its body is not wholly records -- that is the open question, and asserting either way would be wrong. Shrub::span_end_is_a_successor records which case a span is. Records are summed only across bounded bodies. Summing the last class too reproduces the "3.5 million records" figure retracted two commits ago, to within rounding -- which is a useful check on how easy that mistake is to repeat by accident. 8,938 records across the 334 bounded bodies; 16 denormal reserved words; 358 headers naming their own class.
2026-10-11 03:26:06 +02:00
parent ed7e568231
commit 69b81af3b2
2 changed files with 160 additions and 14 deletions
@@ -1646,6 +1646,9 @@ fn shrub_headers_name_their_own_class_on_every_level() {
    let mut absent = 0usize;
    let (mut min_size, mut max_size) = (f32::MAX, 0.0f32);
    let mut denormals = 0usize;
+    let mut bounded_records = 0usize;
+    let mut bounded_bodies = 0usize;
+    let mut last_bodies = 0usize;

    for n in 0..27 {
        let dir = data.join(format!("level{n}"));
@@ -1657,12 +1660,63 @@ fn shrub_headers_name_their_own_class_on_every_level() {
        };
        let h = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
        let range = h.shrub_classes;
+        // Class spans tile between consecutive geometry offsets, with the last
+        // one bounded by the start of core_data's constant trailer. Without
+        // this the last class absorbs the trailer and the body check measures
+        // its neighbours.
+        let trailer = gc_formats::regions::CORE_DATA_TRAILER_BYTES;
+        let mut offsets: Vec<usize> = Vec::new();
        for i in 0..range.count.max(0) as usize {
            let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
            let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
                .unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
-            match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class) {
-                Ok(Some(s)) => {
+            if e.base.offset_in_asset_wad > 0 {
+                offsets.push(e.base.offset_in_asset_wad as usize);
+            }
+        }
+        offsets.sort_unstable();
+        let mut span_end_of: std::collections::HashMap<usize, usize> =
+            std::collections::HashMap::new();
+        for (i, &o) in offsets.iter().enumerate() {
+            let end = offsets.get(i + 1).copied().unwrap_or(core.len() - trailer);
+            span_end_of.insert(o, end);
+        }
+        for i in 0..range.count.max(0) as usize {
+            let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
+            let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
+                .unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
+            let (span_end, has_successor) = match span_end_of
+                .get(&(e.base.offset_in_asset_wad.max(0) as usize))
+            {
+                Some(&e2)
+                    if offsets.last() != Some(&(e.base.offset_in_asset_wad.max(0) as usize)) =>
+                {
+                    (e2, true)
+                }
+                _ => (core.len(), false),
+            };
+            match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class, span_end) {
+                Ok(Some(mut s)) => {
+                    s.span_end_is_a_successor = has_successor;
+                    // Only a class whose span ends at the *next* class has a
+                    // bounded body. The last class's span runs to the trailer
+                    // and contains the 690 KB region that follows shrub, so its
+                    // body is not wholly records -- that is the open question,
+                    // not a defect, and asserting either way would be wrong.
+                    if s.span_end_is_a_successor {
+                        assert!(
+                            s.body_is_well_formed(),
+                            "level{n} class {i}: bounded shrub body is not whole 8-byte records"
+                        );
+                        bounded_bodies += 1;
+                    }
+                    last_bodies += 1;
+                    // Count records only where the span is actually bounded.
+                    // Summing the unbounded last class too reproduces the
+                    // retracted "3.5 million records" figure.
+                    if s.span_end_is_a_successor {
+                        bounded_records += s.body_record_count();
+                    }
                    assert_eq!(
                        s.header.o_class,
                        e.base.o_class.max(0) as u32,
@@ -1709,7 +1763,9 @@ fn shrub_headers_name_their_own_class_on_every_level() {
    assert!(spans > 340, "only {spans} shrub spans with geometry");
    eprintln!(
        "shrub: {spans} headers naming their own class, {absent} without geometry; \
-         size field {min_size}..{max_size}; {denormals} denormal reserved words"
+         {bounded_bodies}/{last_bodies} class bodies bounded by a successor, \
+         {bounded_records} 8-byte records in bounded bodies, \
+         {denormals} denormal reserved words"
    );
 }

@@ -54,14 +54,58 @@ pub struct ShrubHeader {
    pub data_offset: u32,
 }

+/// Bytes per body record.
+///
+/// Each record is 6 varying bytes followed by 2 constant ones. Found by
+/// grouping sibling spans and byte-diffing them rather than by stride-fitting,
+/// which failed three times: the varying payload is **not** sized by the
+/// `count` field, and was a constant 144 bytes per group whether `count` was
+/// 1, 2, 3, 5 or 6.
+pub const SHRUB_BODY_RECORD_BYTES: usize = 8;
+
+/// The two constant bytes closing each body record.
+pub const SHRUB_RECORD_TAIL: [u8; 2] = [0, 0];
+
 /// A shrub span: its header plus the bytes the header points at.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct Shrub<'a> {
    pub header: ShrubHeader,
    /// The whole span, starting at `+0x24` with the class number.
    pub raw: &'a [u8],
-    /// The bytes at `data_offset`. Not yet decoded.
+    /// The bytes at `data_offset`, running to `span_end`. Not yet decoded.
    pub data: &'a [u8],
+    /// Whether `span_end` came from the next class rather than from the end of
+    /// the lump. False for the last class in each level, whose span contains
+    /// whatever follows shrub.
+    pub span_end_is_a_successor: bool,
+}
+
+impl<'a> Shrub<'a> {
+    /// The body's 8-byte records.
+    ///
+    /// The payload of each record is six bytes whose encoding is **not**
+    /// established. It is not half-floats -- decoding as a `half3` gives
+    /// values like 245.25 and -168.375 -- and a VIF reading of it was
+    /// retracted after a control put it on the random baseline. Handed back
+    /// raw rather than guessed at.
+    pub fn body_records(&self) -> impl Iterator<Item = &'a [u8]> {
+        self.data.chunks_exact(SHRUB_BODY_RECORD_BYTES)
+    }
+
+    /// How many whole records the body holds.
+    pub fn body_record_count(&self) -> usize {
+        self.data.len() / SHRUB_BODY_RECORD_BYTES
+    }
+
+    /// Whether every whole record ends in the constant pair.
+    ///
+    /// This is how a class's body is delimited: on all 24 levels that have
+    /// shrub geometry, the last shrub class's body ends exactly where its
+    /// trailing `00 00` pair stops, always on a record boundary.
+    pub fn body_is_well_formed(&self) -> bool {
+        let records: Vec<&[u8]> = self.body_records().collect();
+        !records.is_empty() && records.iter().all(|r| r[6..8] == SHRUB_RECORD_TAIL)
+    }
 }

 impl ShrubHeader {
@@ -71,7 +115,15 @@ impl ShrubHeader {
    /// rather than merely reported: a span whose `+0x24` disagrees with its
    /// own class entry is a different thing entirely, and quietly returning
    /// the mismatched value would hide that.
-    pub fn region<'a>(core: &'a [u8], offset: i32, o_class: i32) -> Result<Option<Shrub<'a>>> {
+    /// `span_end` bounds the class span within `core`. It matters: without it
+    /// `data` runs to the end of the lump and swallows whatever follows, which
+    /// is how a per-class check ends up measuring its neighbours.
+    pub fn region<'a>(
+        core: &'a [u8],
+        offset: i32,
+        o_class: i32,
+        span_end: usize,
+    ) -> Result<Option<Shrub<'a>>> {
        if offset <= 0 {
            return Ok(None);
        }
@@ -131,7 +183,8 @@ impl ShrubHeader {
        }

        let at = header.data_offset as usize;
-        let data = core.get(start + at..).ok_or(Error {
+        let stop = span_end.clamp(start, core.len());
+        let data = core.get(start + at..stop).ok_or(Error {
            offset: start + at,
            context: "shrub data offset runs past the lump",
            kind: ErrorKind::OutOfBounds {
@@ -140,6 +193,7 @@ impl ShrubHeader {
            },
        })?;
        Ok(Some(Shrub {
+            span_end_is_a_successor: true,
            header: ShrubHeader {
                bounds,
                size,
@@ -175,7 +229,7 @@ mod tests {
    #[test]
    fn reads_the_self_identifying_header() {
        let c = span();
-        let s = ShrubHeader::region(&c, 0x10, 1072)
+        let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len())
            .unwrap()
            .expect("present");
        assert_eq!(s.header.o_class, 1072);
@@ -187,27 +241,63 @@ mod tests {
    #[test]
    fn a_span_naming_the_wrong_class_is_rejected() {
        let c = span();
-        let err = ShrubHeader::region(&c, BASE as i32, 999).unwrap_err();
+        let err = ShrubHeader::region(&c, BASE as i32, 999, c.len()).unwrap_err();
        assert!(format!("{err:?}").contains("disagrees"));
    }

    #[test]
    fn a_zero_offset_means_absent_not_present() {
        let c = span();
-        assert!(ShrubHeader::region(&c, 0, 1072).unwrap().is_none());
-        assert!(ShrubHeader::region(&c, -1, 1072).unwrap().is_none());
+        assert!(ShrubHeader::region(&c, 0, 1072, c.len()).unwrap().is_none());
+        assert!(ShrubHeader::region(&c, -1, 1072, c.len())
+            .unwrap()
+            .is_none());
+    }
+
+    #[test]
+    fn body_records_split_into_eights_and_check_their_tail() {
+        let c = span();
+        let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len())
+            .unwrap()
+            .expect("present");
+        // The fixture fills 0xab, so every record's tail is wrong.
+        assert!(!s.body_is_well_formed());
+        assert_eq!(s.body_record_count(), (0x40 - 0x30) / 8);
+        // Fix the tails and it is well formed.
+        let mut c2 = span();
+        for i in 0..((0x40 - 0x30) / 8) {
+            c2[BASE + 0x30 + i * 8 + 6] = 0;
+            c2[BASE + 0x30 + i * 8 + 7] = 0;
+        }
+        let s2 = ShrubHeader::region(&c2, BASE as i32, 1072, c2.len())
+            .unwrap()
+            .expect("present");
+        assert!(s2.body_is_well_formed());
+        assert_eq!(s2.body_records().count(), s2.body_record_count());
+    }
+
+    #[test]
+    fn a_body_shorter_than_one_record_is_not_well_formed() {
+        let mut c = span();
+        // Point the data at the last four bytes of the span.
+        c[BASE + 0x2c..BASE + 0x30].copy_from_slice(&(0x3cu32).to_le_bytes());
+        let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len())
+            .unwrap()
+            .expect("present");
+        assert_eq!(s.body_record_count(), 0);
+        assert!(!s.body_is_well_formed());
    }

    #[test]
    fn out_of_range_is_an_error_not_a_panic() {
        // Offset 0 is absent, not an error -- an empty lump is just empty.
-        assert!(ShrubHeader::region(&[], 0, 1).unwrap().is_none());
-        assert!(ShrubHeader::region(&[], 4, 1).is_err());
+        assert!(ShrubHeader::region(&[], 0, 1, 0).unwrap().is_none());
+        assert!(ShrubHeader::region(&[], 4, 1, 8).is_err());
        // Data offset past the end of the lump.
        let mut c = span();
        c[BASE + 0x2c..BASE + 0x30].copy_from_slice(&9_000u32.to_le_bytes());
-        assert!(ShrubHeader::region(&c, BASE as i32, 1072).is_err());
+        assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err());
        // Span starting near the end, with no room for a header.
-        assert!(ShrubHeader::region(&c, (c.len() - 4) as i32, 1072).is_err());
+        assert!(ShrubHeader::region(&c, (c.len() - 4) as i32, 1072, c.len()).is_err());
    }
 }