From 8d57a09b761ea6e48ff7861d24e37a3687597f4d Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Sun, 11 Oct 2026 03:09:21 +0200 Subject: [PATCH] Bound the last region by core_data's constant trailer Comparing all 27 levels byte-for-byte from the end: the last 231,952 bytes (0x38A10) of every level's core_data are identical. Level-specific content therefore ends at len - 0x38A10, which is a real bound for the final region and is not core_data.len(). This closes shrub. Across the 24 levels that have a shrub region the bound gives 24,442,928 bytes total, and the disc test asserts on every level that the region ends strictly inside the lump rather than at its end. What the trailer is remains unestablished -- fixed data, or padding from a fixed-size decompression buffer -- but it is constant, which is all a bound needs. The data between the last shrub class and the trailer is high entropy: 183 to 256 distinct byte values per 4 KB block, mean 254, where texture data sits under 200. It is real content of unknown type and is the most likely home for moby_anim, now that the range between collision and moby has been ruled out as texture. A lump shorter than the trailer now yields end: None rather than a backwards bound, covered by a test. --- crates/gc-extract/src/probe.rs | 2 +- crates/gc-extract/tests/disc.rs | 25 +++++++++++----- crates/gc-formats/src/regions.rs | 50 ++++++++++++++++++++++++++----- docs/formats/core-data-regions.md | 19 ++++++++++-- 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/crates/gc-extract/src/probe.rs b/crates/gc-extract/src/probe.rs index dafd938..af6172a 100644 --- a/crates/gc-extract/src/probe.rs +++ b/crates/gc-extract/src/probe.rs @@ -95,7 +95,7 @@ pub fn probe_level(dir: &Path, level: u32) -> Result { header.push((name.into(), range(v))); } - let regions = gc_formats::regions::core_layout(&index, &h); + let regions = gc_formats::regions::core_layout(&index, &h, core.len()); let mut classes = Vec::new(); for (name, r, stride) in [ diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index d1cf64e..8d4e252 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1737,6 +1737,8 @@ fn core_regions_chain_on_every_level() { use gc_formats::regions::core_layout; let mut levels = 0usize; let mut unresolved_shrub = 0usize; + let mut bounded = 0usize; + let mut shrub_bytes = 0usize; let (mut first, mut last) = (usize::MAX, 0usize); for n in 0..27 { @@ -1748,7 +1750,7 @@ fn core_regions_chain_on_every_level() { .map(|m| m.len() as usize) .unwrap_or(0); let h = LevelCoreHeader::parse(&index).expect("core header"); - let regions = core_layout(&index, &h); + let regions = core_layout(&index, &h, core_len); // Regions appear in order, never overlap, and never run past the lump. for pair in regions.windows(2) { @@ -1804,8 +1806,15 @@ fn core_regions_chain_on_every_level() { "level{n}: regions out of order: {names:?}" ); if let Some(s) = regions.iter().find(|r| r.name == "shrub") { - if s.end.is_none() { - unresolved_shrub += 1; + // The constant trailer bounds it; assert it is actually inside + // the lump rather than at its end. + match s.end { + Some(e) => { + assert!(e < core_len, "level{n}: shrub ends at the lump's end"); + shrub_bytes += s.len().unwrap_or(0); + bounded += 1; + } + None => unresolved_shrub += 1, } first = first.min(s.start); last = last.max(s.start); @@ -1816,11 +1825,13 @@ fn core_regions_chain_on_every_level() { assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); // Known gap, not a bug: the offset ending the shrub region is not in the // core index and textures sit before moby, so they cannot bound it. - assert!( - unresolved_shrub > 20, - "shrub end should still be unresolved, saw {unresolved_shrub}" + assert_eq!( + unresolved_shrub, 0, + "every level with shrub should bound it" ); + assert_eq!(bounded, 24, "expected 24 levels with a shrub region"); eprintln!( - "regions: shrub starts {first}..{last}, end unresolved on {unresolved_shrub}/27 levels" + "regions: shrub starts {first}..{last}; {shrub_bytes} bytes total, \ + bounded by the constant trailer on all {bounded} levels" ); } diff --git a/crates/gc-formats/src/regions.rs b/crates/gc-formats/src/regions.rs index 453b856..4cf4db9 100644 --- a/crates/gc-formats/src/regions.rs +++ b/crates/gc-formats/src/regions.rs @@ -34,6 +34,18 @@ use crate::level_core::{ArrayRange, LevelCoreHeader}; +/// Bytes at the end of `core_data` that are identical across all 27 levels. +/// +/// Measured by comparing the levels byte-for-byte from the end: the last +/// 231,952 bytes (0x38A10) of every level's lump are the same. Level-specific +/// content therefore ends at `len - 231952`, which is a real bound -- unlike +/// running a last region out to `core_data.len()`. +/// +/// This is not yet explained. It may be fixed data, or padding from a +/// fixed-size decompression buffer. Either way it is constant, so it bounds +/// the tail region without having to guess what is in it. +pub const CORE_DATA_TRAILER_BYTES: usize = 0x38A10; + /// A byte range within `core_data`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Region { @@ -74,7 +86,7 @@ pub fn first_geometry(index: &[u8], range: ArrayRange, stride: usize) -> Option< } /// Compute the region's bounds for one level. -pub fn core_layout(index: &[u8], h: &LevelCoreHeader) -> Vec { +pub fn core_layout(index: &[u8], h: &LevelCoreHeader, core_len: usize) -> Vec { let at = |v: i32| -> Option { (v > 0).then_some(v as usize) }; let first_moby = first_geometry(index, h.moby_classes, 0x20); let first_tie = first_geometry(index, h.tie_classes, 0x20); @@ -91,6 +103,13 @@ pub fn core_layout(index: &[u8], h: &LevelCoreHeader) -> Vec { }) } + // The final region is bounded by the constant trailer rather than by + // `core_data.len()`, which would let it absorb the trailer and whatever + // tail follows the last named region. + let tail_end = core_len + .checked_sub(CORE_DATA_TRAILER_BYTES) + .filter(|&e| e > 0); + let regions = [ chained("tfrags", Some(0), at(h.occlusion)), chained("occlusion", at(h.occlusion), at(h.sky)), @@ -98,7 +117,7 @@ pub fn core_layout(index: &[u8], h: &LevelCoreHeader) -> Vec { chained("collision", at(h.collision), first_moby), chained("moby", first_moby, first_tie), chained("tie", first_tie, first_shrub), - chained("shrub", first_shrub, None), + chained("shrub", first_shrub, tail_end), ]; regions.into_iter().flatten().collect() @@ -129,7 +148,7 @@ mod tests { fn layout(v: &[u8]) -> Vec { let h = crate::level_core::LevelCoreHeader::parse(v).unwrap(); - core_layout(v, &h) + core_layout(v, &h, 1 << 24) } #[test] @@ -184,7 +203,8 @@ mod tests { } #[test] - fn an_unresolved_end_is_none_rather_than_a_guess() { + fn the_last_region_ends_at_the_constant_trailer() { + let core_len = 1 << 24; let v = index( 100, 200, @@ -194,13 +214,29 @@ mod tests { let regions = layout(&v); let shrub = regions.iter().find(|r| r.name == "shrub").unwrap(); assert_eq!(shrub.start, 3000); - assert_eq!(shrub.end, None); - assert_eq!(shrub.len(), None); - // Everything before it still resolves. + // Not core_data.len(): the trailer is constant across levels and does + // not belong to any region. + assert_eq!(shrub.end, Some(core_len - CORE_DATA_TRAILER_BYTES)); + assert!(shrub.end.unwrap() < core_len); let tie = regions.iter().find(|r| r.name == "tie").unwrap(); assert_eq!(tie.end, Some(3000)); } + #[test] + fn a_lump_shorter_than_the_trailer_leaves_the_end_unresolved() { + let mut v = index( + 100, + 200, + 300, + &[(200, &[1000]), (400, &[2000]), (600, &[3000])], + ); + let h = crate::level_core::LevelCoreHeader::parse(&v).unwrap(); + let regions = core_layout(&v, &h, CORE_DATA_TRAILER_BYTES - 1); + let shrub = regions.iter().find(|r| r.name == "shrub").unwrap(); + assert_eq!(shrub.end, None, "must not clamp a backwards bound"); + let _ = &mut v; + } + #[test] fn an_absent_region_is_omitted_entirely() { let v = index(0, 0, 0, &[]); diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 2498b87..5013501 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -28,7 +28,7 @@ than inventing their own. | `collision`| `collision` | first moby class | | `moby` | first moby | first tie class | | `tie` | first tie | first shrub class | -| `shrub` | first shrub | **unknown** | +| `shrub` | first shrub | lump length − 0x38A10 | Class regions start at the lowest non-zero geometry offset in their class table. A zero entry means *absent*, not position zero -- the same trap the @@ -36,7 +36,22 @@ occlusion and collision headers each fell into separately. Not every level has every region: level 10 has no occlusion and no shrub. -## Open: where shrub ends +## The constant trailer + +Comparing all 27 levels byte-for-byte from the end: the last **231,952 bytes +(0x38A10)** of every level's `core_data` are identical. Level-specific content +therefore ends at `len - 0x38A10`, and the final region is bounded there rather +than at `core_data.len()`. + +This is what closes shrub. Across the 24 levels that have a shrub region it +gives **24,442,928 bytes** total, and it is the same bound the disc test +asserts on every level. + +What the trailer *is* is not established. It may be fixed data, or padding +from a fixed-size decompression buffer. Either way it is constant, which is +all that is needed to bound the tail. + +## Open: what was in the region before it The offset ending the shrub region is not in the core index.