From cb38b8c2798fa24fa93ca2bbd683961e1dcea976 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Sun, 11 Oct 2026 03:02:02 +0200 Subject: [PATCH] Compute core_data region bounds once, from successor boundaries Region spans were being re-derived per format while probing, usually running from one known offset to the next with the last span running to core_data.len(). On level 0 that gave the final shrub class a 925,008-byte span absorbing most of the lump's tail, and every aggregate computed over it -- record counts, value distributions, "this looks random" -- was silently wrong. Nothing errored, which is what made it survive two commits and a correction. core_layout computes the layout once: each region ends where the next one begins, class regions start at the lowest non-zero geometry offset in their table, and a region whose successor is unknown gets end: None rather than a clamped guess. Zero class entries mean absent, not position zero -- the same trap occlusion and collision each fell into separately. The disc test asserts what the module is actually for: that the unknown is visible. Shrub's end is unresolved on the 24 levels that have a shrub region, and that is asserted rather than hidden. It also found that not every level has every region -- level 10 has neither occlusion nor shrub -- so the test checks ordering against the canonical sequence instead of demanding an identical set. Still open: where the shrub region ends. Texture texels cannot bound it; they sit before moby, not after shrub. See docs/formats/core-data-regions.md. --- crates/gc-extract/tests/disc.rs | 112 ++++++++++++++ crates/gc-formats/src/lib.rs | 1 + crates/gc-formats/src/regions.rs | 233 ++++++++++++++++++++++++++++++ docs/formats/core-data-regions.md | 50 +++++++ 4 files changed, 396 insertions(+) create mode 100644 crates/gc-formats/src/regions.rs create mode 100644 docs/formats/core-data-regions.md diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 4dc788b..d1cf64e 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1712,3 +1712,115 @@ fn shrub_headers_name_their_own_class_on_every_level() { size field {min_size}..{max_size}; {denormals} denormal reserved words" ); } + +/// Region bounds derived from successor boundaries, on every level. +/// +/// The point is not that these are correct -- shrub's end is still unknown -- +/// but that the unknown is visible. Every region here is closed by the next +/// region's start rather than by `core_data.len()`, so no span silently +/// absorbs the tail of the lump the way the last shrub class was doing. +#[test] +fn core_regions_chain_on_every_level() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::regions::core_layout; + let mut levels = 0usize; + let mut unresolved_shrub = 0usize; + let (mut first, mut last) = (usize::MAX, 0usize); + + for n in 0..27 { + let dir = data.join(format!("level{n}")); + let Ok(index) = std::fs::read(dir.join("core_index.bin")) else { + continue; + }; + let core_len = std::fs::metadata(dir.join("core_data.bin")) + .map(|m| m.len() as usize) + .unwrap_or(0); + let h = LevelCoreHeader::parse(&index).expect("core header"); + let regions = core_layout(&index, &h); + + // Regions appear in order, never overlap, and never run past the lump. + for pair in regions.windows(2) { + assert!( + pair[0].start <= pair[1].start, + "level{n}: {} vs {}", + pair[0].name, + pair[1].name + ); + if let Some(end) = pair[0].end { + assert!( + end <= pair[1].start, + "level{n}: {} ends after {}", + pair[0].name, + pair[1].name + ); + } + } + for r in ®ions { + assert!( + r.start < core_len, + "level{n}: {} starts past the lump", + r.name + ); + if let Some(end) = r.end { + assert!(end <= core_len, "level{n}: {} ends past the lump", r.name); + } + } + + // Regions appear in the canonical order, but not every level has every + // region -- level 10 has no occlusion and no shrub at all. + const ORDER: [&str; 7] = [ + "tfrags", + "occlusion", + "sky", + "collision", + "moby", + "tie", + "shrub", + ]; + let names: Vec<_> = regions.iter().map(|r| r.name).collect(); + let ranks: Vec = names + .iter() + .map(|name| { + ORDER + .iter() + .position(|o| o == name) + .unwrap_or_else(|| panic!("level{n}: unknown region {name}")) + }) + .collect(); + assert!( + ranks.windows(2).all(|w| w[0] < w[1]), + "level{n}: regions out of order: {names:?}" + ); + if let Some(s) = regions.iter().find(|r| r.name == "shrub") { + if s.end.is_none() { + unresolved_shrub += 1; + } + first = first.min(s.start); + last = last.max(s.start); + } + levels += 1; + } + + assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); + // Known gap, not a bug: the offset ending the shrub region is not in the + // core index and textures sit before moby, so they cannot bound it. + assert!( + unresolved_shrub > 20, + "shrub end should still be unresolved, saw {unresolved_shrub}" + ); + eprintln!( + "regions: shrub starts {first}..{last}, end unresolved on {unresolved_shrub}/27 levels" + ); +} diff --git a/crates/gc-formats/src/lib.rs b/crates/gc-formats/src/lib.rs index 165a22c..772fa87 100644 --- a/crates/gc-formats/src/lib.rs +++ b/crates/gc-formats/src/lib.rs @@ -7,6 +7,7 @@ pub mod level_data; pub mod level_wad; pub mod moby; pub mod occlusion; +pub mod regions; pub mod shrub; pub mod sound_bank; pub mod texture; diff --git a/crates/gc-formats/src/regions.rs b/crates/gc-formats/src/regions.rs new file mode 100644 index 0000000..453b856 --- /dev/null +++ b/crates/gc-formats/src/regions.rs @@ -0,0 +1,233 @@ +//! Region bounds for a level's `core_data`. +//! +//! Spec: `docs/formats/core-data-regions.md`. +//! +//! This exists because of a specific failure. Region spans were being derived +//! per-format during probing, usually by running from one known offset to the +//! next, and usually with the last span running to `core_data.len()`. On level +//! 0 that made the final shrub class a 925,008-byte span absorbing most of the +//! lump's tail. Aggregates computed over that -- record counts, value +//! distributions -- were silently wrong, and nothing errored. +//! +//! So bounds are computed once, here, from successor boundaries, and every +//! caller uses these rather than inventing its own. Where a bound is not yet +//! known it is `None`, not a guess: [`Region::end`] being `None` means +//! "unresolved", and [`core_layout`] refuses to close a region whose successor +//! it cannot find. +//! +//! Measured on the disc: +//! +//! | region | start | end | +//! |------------|--------------|-------------------------| +//! | tfrags | 0 | occlusion | +//! | occlusion | occlusion | sky | +//! | sky | sky | collision | +//! | collision | collision | first moby class | +//! | moby | first moby | first tie class | +//! | tie | first tie | first shrub class | +//! | shrub | first shrub | **unresolved** | +//! +//! The shrub end is the open item. Texture texels sit *before* moby, not after +//! shrub, so they cannot bound it: on level 0 the texel range is 3539328 to +//! 5326208 while the last shrub class starts at 16044288. The ~925 KB after +//! the last shrub class is real, non-zero data that is not accounted for yet. + +use crate::level_core::{ArrayRange, LevelCoreHeader}; + +/// A byte range within `core_data`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Region { + pub name: &'static str, + pub start: usize, + /// Exclusive end, or `None` when the successor boundary is not known. + pub end: Option, +} + +impl Region { + pub fn len(&self) -> Option { + self.end.map(|e| e.saturating_sub(self.start)) + } + + pub fn is_empty(&self) -> bool { + self.len() == Some(0) + } +} + +/// The lowest geometry offset in a class table, ignoring absent entries. +/// +/// Absent entries are zero, and treating zero as an offset would drag the +/// region's start to zero -- the same "zero means absent, not position" trap +/// that the occlusion and collision headers both fell into. +pub fn first_geometry(index: &[u8], range: ArrayRange, stride: usize) -> Option { + let count = range.count.max(0) as usize; + let base = range.offset.max(0) as usize; + let mut best: Option = None; + for i in 0..count { + let at = base + i * stride; + let chunk = index.get(at..at + 4)?; + let off = i32::from_le_bytes(chunk.try_into().expect("4 bytes")); + if off > 0 { + best = Some(best.map_or(off as usize, |b: usize| b.min(off as usize))); + } + } + best +} + +/// Compute the region's bounds for one level. +pub fn core_layout(index: &[u8], h: &LevelCoreHeader) -> Vec { + let at = |v: i32| -> Option { (v > 0).then_some(v as usize) }; + let first_moby = first_geometry(index, h.moby_classes, 0x20); + let first_tie = first_geometry(index, h.tie_classes, 0x20); + let first_shrub = first_geometry(index, h.shrub_classes, 0x30); + + // Each region ends where the next one begins. `chained` keeps `None` + // propagating rather than silently closing a region against a gap. + fn chained(name: &'static str, start: Option, next: Option) -> Option { + let start = start?; + Some(Region { + name, + start, + end: next.filter(|&n| n > start), + }) + } + + let regions = [ + chained("tfrags", Some(0), at(h.occlusion)), + chained("occlusion", at(h.occlusion), at(h.sky)), + chained("sky", at(h.sky), at(h.collision)), + chained("collision", at(h.collision), first_moby), + chained("moby", first_moby, first_tie), + chained("tie", first_tie, first_shrub), + chained("shrub", first_shrub, None), + ]; + + regions.into_iter().flatten().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// One index buffer carrying both the header fields and the class tables, + /// so a test cannot accidentally pair a header with unrelated class data. + fn index(occlusion: i32, sky: i32, collision: i32, classes: &[(usize, &[i32])]) -> Vec { + let mut v = vec![0u8; 4096]; + v[0x0c..0x10].copy_from_slice(&occlusion.to_le_bytes()); + v[0x10..0x14].copy_from_slice(&sky.to_le_bytes()); + v[0x14..0x18].copy_from_slice(&collision.to_le_bytes()); + let bases = [0x18usize, 0x20, 0x28]; + for (slot, (offset, vals)) in classes.iter().enumerate() { + v[bases[slot]..bases[slot] + 4].copy_from_slice(&(vals.len() as i32).to_le_bytes()); + v[bases[slot] + 4..bases[slot] + 8].copy_from_slice(&(*offset as i32).to_le_bytes()); + for (i, g) in vals.iter().enumerate() { + let at = offset + i * 0x20; + v[at..at + 4].copy_from_slice(&g.to_le_bytes()); + } + } + v + } + + fn layout(v: &[u8]) -> Vec { + let h = crate::level_core::LevelCoreHeader::parse(v).unwrap(); + core_layout(v, &h) + } + + #[test] + fn regions_chain_and_do_not_overlap() { + let v = index( + 2_117_888, + 2_192_576, + 2_615_232, + &[(200, &[3000]), (400, &[4000]), (600, &[5000])], + ); + let regions = layout(&v); + let names: Vec<_> = regions.iter().map(|r| r.name).collect(); + assert_eq!( + names, + [ + "tfrags", + "occlusion", + "sky", + "collision", + "moby", + "tie", + "shrub" + ] + ); + assert_eq!(regions[0].start, 0); + assert_eq!(regions[1].start, 2_117_888); + assert_eq!(regions[1].end, Some(2_192_576)); + assert_eq!(regions[4].start, 3000); + assert_eq!(regions[4].end, Some(4000)); + for pair in regions.windows(2) { + if let Some(end) = pair[0].end { + assert!( + end <= pair[1].start, + "{:?} ends after {:?} starts", + pair[0], + pair[1] + ); + } + } + } + + #[test] + fn a_zero_class_offset_is_absent_not_position_zero() { + let v = index( + 100, + 200, + 300, + &[(200, &[0, 3000]), (400, &[4000]), (600, &[5000])], + ); + let moby = layout(&v).into_iter().find(|r| r.name == "moby").unwrap(); + assert_eq!(moby.start, 3000); + } + + #[test] + fn an_unresolved_end_is_none_rather_than_a_guess() { + let v = index( + 100, + 200, + 300, + &[(200, &[1000]), (400, &[2000]), (600, &[3000])], + ); + let regions = layout(&v); + let shrub = regions.iter().find(|r| r.name == "shrub").unwrap(); + assert_eq!(shrub.start, 3000); + assert_eq!(shrub.end, None); + assert_eq!(shrub.len(), None); + // Everything before it still resolves. + let tie = regions.iter().find(|r| r.name == "tie").unwrap(); + assert_eq!(tie.end, Some(3000)); + } + + #[test] + fn an_absent_region_is_omitted_entirely() { + let v = index(0, 0, 0, &[]); + let regions = layout(&v); + // Only tfrags can be placed at all, and only with an unknown end. + assert_eq!(regions.len(), 1, "{regions:?}"); + assert_eq!(regions[0].name, "tfrags"); + assert_eq!(regions[0].start, 0); + assert_eq!(regions[0].end, None); + } + + #[test] + fn a_gap_does_not_close_a_region() { + // Sky starts before occlusion ends: nonsense, so neither may be closed + // against the other. + let v = index( + 500, + 400, + 600, + &[(200, &[3000]), (400, &[4000]), (600, &[5000])], + ); + let regions = layout(&v); + // occlusion at 500 cannot be closed against sky at 400, so it stays + // open rather than being clamped into a backwards range. + let occ = regions.iter().find(|r| r.name == "occlusion").unwrap(); + assert_eq!(occ.start, 500); + assert_eq!(occ.end, None); + assert_eq!(occ.len(), None); + } +} diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md new file mode 100644 index 0000000..e109a24 --- /dev/null +++ b/docs/formats/core-data-regions.md @@ -0,0 +1,50 @@ +# core_data region bounds + +How a level's `core_data` lump divides up, and how each region is bounded. + +## The rule + +A region's end is the **next region's start**, never `core_data.len()`. The +last region in a layout is the exception, and when its successor is unknown the +end is `None` rather than a guess. + +This exists because region bounds were being re-derived per format during +probing, usually by running from one known offset to the next with the last +span running to the end of the lump. On level 0 that made the final shrub class +a 925,008-byte span absorbing most of the lump's tail. Every aggregate computed +over it -- record counts, value distributions, "this data looks random" -- was +silently wrong, and nothing errored. + +`gc_formats::regions::core_layout` computes these once. Callers use it rather +than inventing their own. + +## Layout + +| region | start | end | +|------------|----------------|----------------------| +| `tfrags` | 0 | `occlusion` | +| `occlusion`| `occlusion` | `sky` | +| `sky` | `sky` | `collision` | +| `collision`| `collision` | first moby class | +| `moby` | first moby | first tie class | +| `tie` | first tie | first shrub class | +| `shrub` | first shrub | **unknown** | + +Class regions start at the lowest non-zero geometry offset in their class +table. A zero entry means *absent*, not position zero -- the same trap the +occlusion and collision headers each fell into separately. + +Not every level has every region: level 10 has no occlusion and no shrub. + +## Open: where shrub ends + +The offset ending the shrub region is not in the core index. + +Texture texels do **not** bound it. They sit before moby, not after shrub: on +level 0 the texel range is 3539328..5326208 while the last shrub class starts +at 16044288, leaving ~925 KB of real, non-zero data after the last shrub class +that nothing yet accounts for. + +`core_layout` reports `end: None` for shrub on the 24 levels that have one, and +the disc test asserts it, so the gap stays visible rather than being papered +over by clamping to the end of the lump.