Add gc-extract probe to dump structures through the real parser

Reading these formats by unpacking raw offsets out of core_index.bin has
cost real time. The index interleaves single-word region offsets with
count/offset pairs, so miscounting one shifts every field after it: that
produced a page of implausible collision headers on all 27 levels, sent
me chasing a tie-breakoff that turned out not to exist, and nearly had me
"fix" a stride bug in code that was already right.

probe dumps a level's core header and all three class tables through
LevelCoreHeader, so a field name printed is the field name the parser
uses. It also reports how many entries carry geometry inside core_data,
which is what pins the class table strides rather than guessing at them:
shrub reads 7/7 sane at 0x30 against 4/7 read at 0x20.

Reading a structure by hand in a shell still works; this just stops it
being the default, and stops the offset arithmetic being re-derived every
time a new region is investigated.

crates/gc-extract/src/lib.rs

@@ -15,9 +15,11 @@
 //! outside the repository. See `AGENTS.md`.
 
 pub mod identify;
+pub mod probe;
 pub mod unpack;
 pub mod verify;
 
 pub use identify::{identify, DiscInfo};
+pub use probe::probe_level;
 pub use unpack::unpack;
 pub use verify::{verify, VerifyReport};

crates/gc-extract/src/main.rs

@@ -26,6 +26,7 @@ fn main() -> ExitCode {
         // byte range of an already-unpacked file with the WAD decoder.
         Some("wad") => run_wad(&args[1..]),
         Some("dump-texture") => run_dump_texture(&args[1..]),
+        Some("probe") => run_probe(&args[1..]),
         Some("--help") | Some("-h") | Some("help") | None => {
             usage();
             ExitCode::SUCCESS
@@ -66,6 +67,12 @@ fn usage() {
   verify <out-dir> <table.tsv>
       Check <out-dir> against a previously written table.
 
+  probe <unpacked-dir> <level> [max-rows]
+      Dump a level's core header and class tables through the same parser the
+      rest of the project uses. Reach for this before reading core_index.bin
+      by hand: the index interleaves single-word region offsets with
+      count/offset pairs, and miscounting one shifts every field after it.
+
   dump-texture <unpacked-dir> <level> <index> [mip] <out.ppm>
       Decode one tfrag texture to a binary PPM, so a decode can be eyeballed
       against the disc rather than trusted. Textures are 8-bit indexed; a wrong
@@ -314,6 +321,73 @@ fn run_dump_texture(args: &[String]) -> ExitCode {
     ExitCode::SUCCESS
 }
 
+fn run_probe(args: &[String]) -> ExitCode {
+    if args.len() < 2 {
+        eprintln!("usage: gc-extract probe <unpacked-dir> <level> [max-rows]");
+        return ExitCode::FAILURE;
+    }
+    let dir = Path::new(&args[0]);
+    let level: u32 = match args[1].parse() {
+        Ok(v) => v,
+        Err(e) => {
+            eprintln!("bad level: {e}");
+            return ExitCode::FAILURE;
+        }
+    };
+    let max_rows: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(8);
+    let p = match gc_extract::probe_level(dir, level) {
+        Ok(p) => p,
+        Err(e) => {
+            eprintln!("error: {e:#}");
+            return ExitCode::FAILURE;
+        }
+    };
+
+    println!(
+        "level{level}: core_data {} bytes, core_index {} bytes",
+        p.core_len, p.index_len
+    );
+    println!("core header");
+    println!("{}", "-".repeat(64));
+    for (k, v) in &p.header {
+        println!("  {k:<22} {v:>14}");
+    }
+    for t in &p.classes {
+        let sane = t.entries.iter().filter(|e| e.in_core(p.core_len)).count();
+        println!();
+        println!(
+            "{} classes: {} entries at stride 0x{:x}, {sane} with geometry in core_data",
+            t.name,
+            t.entries.len(),
+            t.stride
+        );
+        println!("{}", "-".repeat(64));
+        println!(
+            "  {:>4}  {:>12}  {:>8}  {:>10}  {:>10}",
+            "idx", "geom_offset", "o_class", "unknown_8", "unknown_c"
+        );
+        for e in t.entries.iter().take(max_rows) {
+            println!(
+                "  {:>4}  {:>12}  {:>8}  {:>10}  {:>10}{}",
+                e.index,
+                e.geometry_offset,
+                e.o_class,
+                e.unknown_8,
+                e.unknown_c,
+                if e.in_core(p.core_len) {
+                    ""
+                } else {
+                    "   <- outside core_data"
+                }
+            );
+        }
+        if t.entries.len() > max_rows {
+            println!("  ... {} more", t.entries.len() - max_rows);
+        }
+    }
+    ExitCode::SUCCESS
+}
+
 fn run_wad(args: &[String]) -> ExitCode {
     if args.len() < 4 {
         eprintln!("usage: gc-extract wad <file> <offset> <size> <out>");

crates/gc-extract/src/probe.rs

@@ -0,0 +1,185 @@
+//! A structure dumper for reverse engineering.
+//!
+//! This exists because of a specific, repeated failure. Probing these formats
+//! by unpacking raw offsets out of `core_index.bin` got `collision` read as
+//! `sky` on all 27 levels, produced a page of implausible collision headers,
+//! and led to "fixing" a stride bug that did not exist. The core index
+//! interleaves single-word region offsets with count/offset **pairs**, so every
+//! field after a pair shifts if you miscount it.
+//!
+//! Everything here goes through the same `gc_formats` types the rest of the
+//! project uses, so a field name printed here is the field name the parser
+//! uses. Reading a structure by hand in a shell remains possible; this just
+//! stops it being the default.
+
+use std::path::Path;
+
+use anyhow::{Context, Result};
+
+/// Byte size of a moby or tie class entry.
+pub const CLASS_ENTRY_BYTES: usize = 0x20;
+/// Byte size of a shrub class entry, which carries billboard fields the other
+/// two do not.
+pub const SHRUB_CLASS_ENTRY_BYTES: usize = 0x30;
+
+/// One level's core header plus its three class tables.
+pub struct LevelProbe {
+    pub level: u32,
+    pub core_len: usize,
+    pub index_len: usize,
+    /// Rendered field-by-field dump of the core header.
+    pub header: Vec<(String, String)>,
+    /// The three class tables, in the order the header lists them.
+    pub classes: Vec<ClassTable>,
+}
+
+/// One class table, with its entries already sliced at the right stride.
+pub struct ClassTable {
+    pub name: &'static str,
+    pub stride: usize,
+    pub entries: Vec<ClassRow>,
+}
+
+/// One class table row.
+#[derive(Clone, Copy)]
+pub struct ClassRow {
+    pub index: usize,
+    pub geometry_offset: i32,
+    pub o_class: i32,
+    pub unknown_8: i32,
+    pub unknown_c: i32,
+    /// The entry's raw bytes, so a probe can look at what the struct drops.
+    pub raw: [u8; SHRUB_CLASS_ENTRY_BYTES],
+}
+
+impl ClassRow {
+    /// Whether this row's geometry offset lands inside `core_data`.
+    pub fn in_core(&self, core_len: usize) -> bool {
+        self.geometry_offset > 0 && (self.geometry_offset as usize) < core_len
+    }
+}
+
+/// Read a level's core index and dump its header and class tables.
+pub fn probe_level(dir: &Path, level: u32) -> Result<LevelProbe> {
+    let level_dir = dir.join(format!("level{level}"));
+    let core = std::fs::read(level_dir.join("core_data.bin"))
+        .with_context(|| format!("reading {}", level_dir.join("core_data.bin").display()))?;
+    let index = std::fs::read(level_dir.join("core_index.bin"))
+        .with_context(|| format!("reading {}", level_dir.join("core_index.bin").display()))?;
+
+    let h = gc_formats::level_core::LevelCoreHeader::parse(&index)
+        .context("parsing the core header")?;
+
+    let range = |v: i32| {
+        if v < 0 {
+            "(absent)".to_string()
+        } else {
+            v.to_string()
+        }
+    };
+    let mut header: Vec<(String, String)> = Vec::new();
+    header.push(("gs_ram count".into(), h.gs_ram.count.to_string()));
+    header.push(("gs_ram offset".into(), h.gs_ram.offset.to_string()));
+    for (name, v) in [
+        ("tfrags", h.tfrags),
+        ("occlusion", h.occlusion),
+        ("sky", h.sky),
+        ("collision", h.collision),
+        ("textures_base", h.textures_base),
+        ("assets compressed", h.assets_compressed_size),
+        ("assets decompressed", h.assets_decompressed_size),
+    ] {
+        header.push((name.into(), range(v)));
+    }
+
+    let mut classes = Vec::new();
+    for (name, r, stride) in [
+        ("moby", h.moby_classes, CLASS_ENTRY_BYTES),
+        ("tie", h.tie_classes, CLASS_ENTRY_BYTES),
+        ("shrub", h.shrub_classes, SHRUB_CLASS_ENTRY_BYTES),
+    ] {
+        classes.push(ClassTable {
+            name,
+            stride,
+            entries: read_class_table(&index, r, stride)?,
+        });
+    }
+
+    Ok(LevelProbe {
+        level,
+        core_len: core.len(),
+        index_len: index.len(),
+        header,
+        classes,
+    })
+}
+
+fn read_class_table(
+    index: &[u8],
+    r: gc_formats::level_core::ArrayRange,
+    stride: usize,
+) -> Result<Vec<ClassRow>> {
+    let base = r.offset.max(0) as usize;
+    let count = r.count.max(0) as usize;
+    let mut out = Vec::with_capacity(count);
+    for i in 0..count {
+        let at = base + i * stride;
+        let mut raw = [0u8; SHRUB_CLASS_ENTRY_BYTES];
+        let Some(chunk) = index.get(at..at + stride) else {
+            break;
+        };
+        raw[..stride].copy_from_slice(chunk);
+        out.push(ClassRow {
+            index: i,
+            geometry_offset: gc_formats::level_core::ClassEntry::parse(chunk)
+                .map(|e| e.offset_in_asset_wad)
+                .unwrap_or(0),
+            o_class: gc_formats::level_core::ClassEntry::parse(chunk)
+                .map(|e| e.o_class)
+                .unwrap_or(0),
+            unknown_8: gc_formats::level_core::ClassEntry::parse(chunk)
+                .map(|e| e.unknown_8)
+                .unwrap_or(0),
+            unknown_c: gc_formats::level_core::ClassEntry::parse(chunk)
+                .map(|e| e.unknown_c)
+                .unwrap_or(0),
+            raw,
+        });
+    }
+    Ok(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn class_rows_report_whether_geometry_is_in_core() {
+        let r = ClassRow {
+            index: 0,
+            geometry_offset: 100,
+            o_class: 1,
+            unknown_8: 0,
+            unknown_c: 0,
+            raw: [0; SHRUB_CLASS_ENTRY_BYTES],
+        };
+        assert!(r.in_core(200));
+        assert!(!r.in_core(50));
+        assert!(!ClassRow {
+            geometry_offset: 0,
+            ..r
+        }
+        .in_core(200));
+    }
+
+    #[test]
+    fn a_table_past_the_index_stops_rather_than_panicking() {
+        let index = vec![0u8; 16];
+        let r = gc_formats::level_core::ArrayRange {
+            count: 100,
+            offset: 8,
+        };
+        let rows = read_class_table(&index, r, CLASS_ENTRY_BYTES).unwrap();
+        assert!(rows.len() < 100, "should stop at the end of the index");
+    }
+}

docs/plan/level-assets.md

@@ -870,6 +870,15 @@ plausible floats varies from 1 to 6 across spans, with 822 spans having 6,
 455 having 1 and 373 having 2. So there is more header past `0x48`, and it
 is not a single array of distances.
 
+### Probe tool
+
+`gc-extract probe <unpacked-dir> <level> [max-rows]` dumps the core header and
+all three class tables through `LevelCoreHeader`, which is the point. The
+table above and the shrub offset below were both established by hand first and
+one of them nearly sent me to fix a bug that was not there. The tool also
+reports how many entries carry geometry inside `core_data`, which is how the
+shrub stride was pinned: 7/7 at 0x30 against 4/7 read at 0x20.
+
 Note the LOD distances already decoded sit *before* this point (12.0, 25.0,
 40.0 at `+0x10`), and the floats after `0x48` are a different population --
 negative values, and a repeated leading 40.0.