From f6bbbf8d5f37b3af3d0e0035d1761c953f6db8d6 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Sun, 11 Oct 2026 02:36:37 +0200 Subject: [PATCH] Add gc-extract probe to dump structures through the real parser Reading these formats by unpacking raw offsets out of core_index.bin has cost real time. The index interleaves single-word region offsets with count/offset pairs, so miscounting one shifts every field after it: that produced a page of implausible collision headers on all 27 levels, sent me chasing a tie-breakoff that turned out not to exist, and nearly had me "fix" a stride bug in code that was already right. probe dumps a level's core header and all three class tables through LevelCoreHeader, so a field name printed is the field name the parser uses. It also reports how many entries carry geometry inside core_data, which is what pins the class table strides rather than guessing at them: shrub reads 7/7 sane at 0x30 against 4/7 read at 0x20. Reading a structure by hand in a shell still works; this just stops it being the default, and stops the offset arithmetic being re-derived every time a new region is investigated. --- crates/gc-extract/src/lib.rs | 2 + crates/gc-extract/src/main.rs | 74 +++++++++++++ crates/gc-extract/src/probe.rs | 185 +++++++++++++++++++++++++++++++++ docs/plan/level-assets.md | 9 ++ 4 files changed, 270 insertions(+) create mode 100644 crates/gc-extract/src/probe.rs diff --git a/crates/gc-extract/src/lib.rs b/crates/gc-extract/src/lib.rs index d2a16fd..c0c0811 100644 --- a/crates/gc-extract/src/lib.rs +++ b/crates/gc-extract/src/lib.rs @@ -15,9 +15,11 @@ //! outside the repository. See `AGENTS.md`. pub mod identify; +pub mod probe; pub mod unpack; pub mod verify; pub use identify::{identify, DiscInfo}; +pub use probe::probe_level; pub use unpack::unpack; pub use verify::{verify, VerifyReport}; diff --git a/crates/gc-extract/src/main.rs b/crates/gc-extract/src/main.rs index 99b5ced..a8b770f 100644 --- a/crates/gc-extract/src/main.rs +++ b/crates/gc-extract/src/main.rs @@ -26,6 +26,7 @@ fn main() -> ExitCode { // byte range of an already-unpacked file with the WAD decoder. Some("wad") => run_wad(&args[1..]), Some("dump-texture") => run_dump_texture(&args[1..]), + Some("probe") => run_probe(&args[1..]), Some("--help") | Some("-h") | Some("help") | None => { usage(); ExitCode::SUCCESS @@ -66,6 +67,12 @@ fn usage() { verify Check against a previously written table. + probe [max-rows] + Dump a level's core header and class tables through the same parser the + rest of the project uses. Reach for this before reading core_index.bin + by hand: the index interleaves single-word region offsets with + count/offset pairs, and miscounting one shifts every field after it. + dump-texture [mip] Decode one tfrag texture to a binary PPM, so a decode can be eyeballed against the disc rather than trusted. Textures are 8-bit indexed; a wrong @@ -314,6 +321,73 @@ fn run_dump_texture(args: &[String]) -> ExitCode { ExitCode::SUCCESS } +fn run_probe(args: &[String]) -> ExitCode { + if args.len() < 2 { + eprintln!("usage: gc-extract probe [max-rows]"); + return ExitCode::FAILURE; + } + let dir = Path::new(&args[0]); + let level: u32 = match args[1].parse() { + Ok(v) => v, + Err(e) => { + eprintln!("bad level: {e}"); + return ExitCode::FAILURE; + } + }; + let max_rows: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(8); + let p = match gc_extract::probe_level(dir, level) { + Ok(p) => p, + Err(e) => { + eprintln!("error: {e:#}"); + return ExitCode::FAILURE; + } + }; + + println!( + "level{level}: core_data {} bytes, core_index {} bytes", + p.core_len, p.index_len + ); + println!("core header"); + println!("{}", "-".repeat(64)); + for (k, v) in &p.header { + println!(" {k:<22} {v:>14}"); + } + for t in &p.classes { + let sane = t.entries.iter().filter(|e| e.in_core(p.core_len)).count(); + println!(); + println!( + "{} classes: {} entries at stride 0x{:x}, {sane} with geometry in core_data", + t.name, + t.entries.len(), + t.stride + ); + println!("{}", "-".repeat(64)); + println!( + " {:>4} {:>12} {:>8} {:>10} {:>10}", + "idx", "geom_offset", "o_class", "unknown_8", "unknown_c" + ); + for e in t.entries.iter().take(max_rows) { + println!( + " {:>4} {:>12} {:>8} {:>10} {:>10}{}", + e.index, + e.geometry_offset, + e.o_class, + e.unknown_8, + e.unknown_c, + if e.in_core(p.core_len) { + "" + } else { + " <- outside core_data" + } + ); + } + if t.entries.len() > max_rows { + println!(" ... {} more", t.entries.len() - max_rows); + } + } + ExitCode::SUCCESS +} + fn run_wad(args: &[String]) -> ExitCode { if args.len() < 4 { eprintln!("usage: gc-extract wad "); diff --git a/crates/gc-extract/src/probe.rs b/crates/gc-extract/src/probe.rs new file mode 100644 index 0000000..0694a06 --- /dev/null +++ b/crates/gc-extract/src/probe.rs @@ -0,0 +1,185 @@ +//! A structure dumper for reverse engineering. +//! +//! This exists because of a specific, repeated failure. Probing these formats +//! by unpacking raw offsets out of `core_index.bin` got `collision` read as +//! `sky` on all 27 levels, produced a page of implausible collision headers, +//! and led to "fixing" a stride bug that did not exist. The core index +//! interleaves single-word region offsets with count/offset **pairs**, so every +//! field after a pair shifts if you miscount it. +//! +//! Everything here goes through the same `gc_formats` types the rest of the +//! project uses, so a field name printed here is the field name the parser +//! uses. Reading a structure by hand in a shell remains possible; this just +//! stops it being the default. + +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Byte size of a moby or tie class entry. +pub const CLASS_ENTRY_BYTES: usize = 0x20; +/// Byte size of a shrub class entry, which carries billboard fields the other +/// two do not. +pub const SHRUB_CLASS_ENTRY_BYTES: usize = 0x30; + +/// One level's core header plus its three class tables. +pub struct LevelProbe { + pub level: u32, + pub core_len: usize, + pub index_len: usize, + /// Rendered field-by-field dump of the core header. + pub header: Vec<(String, String)>, + /// The three class tables, in the order the header lists them. + pub classes: Vec, +} + +/// One class table, with its entries already sliced at the right stride. +pub struct ClassTable { + pub name: &'static str, + pub stride: usize, + pub entries: Vec, +} + +/// One class table row. +#[derive(Clone, Copy)] +pub struct ClassRow { + pub index: usize, + pub geometry_offset: i32, + pub o_class: i32, + pub unknown_8: i32, + pub unknown_c: i32, + /// The entry's raw bytes, so a probe can look at what the struct drops. + pub raw: [u8; SHRUB_CLASS_ENTRY_BYTES], +} + +impl ClassRow { + /// Whether this row's geometry offset lands inside `core_data`. + pub fn in_core(&self, core_len: usize) -> bool { + self.geometry_offset > 0 && (self.geometry_offset as usize) < core_len + } +} + +/// Read a level's core index and dump its header and class tables. +pub fn probe_level(dir: &Path, level: u32) -> Result { + let level_dir = dir.join(format!("level{level}")); + let core = std::fs::read(level_dir.join("core_data.bin")) + .with_context(|| format!("reading {}", level_dir.join("core_data.bin").display()))?; + let index = std::fs::read(level_dir.join("core_index.bin")) + .with_context(|| format!("reading {}", level_dir.join("core_index.bin").display()))?; + + let h = gc_formats::level_core::LevelCoreHeader::parse(&index) + .context("parsing the core header")?; + + let range = |v: i32| { + if v < 0 { + "(absent)".to_string() + } else { + v.to_string() + } + }; + let mut header: Vec<(String, String)> = Vec::new(); + header.push(("gs_ram count".into(), h.gs_ram.count.to_string())); + header.push(("gs_ram offset".into(), h.gs_ram.offset.to_string())); + for (name, v) in [ + ("tfrags", h.tfrags), + ("occlusion", h.occlusion), + ("sky", h.sky), + ("collision", h.collision), + ("textures_base", h.textures_base), + ("assets compressed", h.assets_compressed_size), + ("assets decompressed", h.assets_decompressed_size), + ] { + header.push((name.into(), range(v))); + } + + let mut classes = Vec::new(); + for (name, r, stride) in [ + ("moby", h.moby_classes, CLASS_ENTRY_BYTES), + ("tie", h.tie_classes, CLASS_ENTRY_BYTES), + ("shrub", h.shrub_classes, SHRUB_CLASS_ENTRY_BYTES), + ] { + classes.push(ClassTable { + name, + stride, + entries: read_class_table(&index, r, stride)?, + }); + } + + Ok(LevelProbe { + level, + core_len: core.len(), + index_len: index.len(), + header, + classes, + }) +} + +fn read_class_table( + index: &[u8], + r: gc_formats::level_core::ArrayRange, + stride: usize, +) -> Result> { + let base = r.offset.max(0) as usize; + let count = r.count.max(0) as usize; + let mut out = Vec::with_capacity(count); + for i in 0..count { + let at = base + i * stride; + let mut raw = [0u8; SHRUB_CLASS_ENTRY_BYTES]; + let Some(chunk) = index.get(at..at + stride) else { + break; + }; + raw[..stride].copy_from_slice(chunk); + out.push(ClassRow { + index: i, + geometry_offset: gc_formats::level_core::ClassEntry::parse(chunk) + .map(|e| e.offset_in_asset_wad) + .unwrap_or(0), + o_class: gc_formats::level_core::ClassEntry::parse(chunk) + .map(|e| e.o_class) + .unwrap_or(0), + unknown_8: gc_formats::level_core::ClassEntry::parse(chunk) + .map(|e| e.unknown_8) + .unwrap_or(0), + unknown_c: gc_formats::level_core::ClassEntry::parse(chunk) + .map(|e| e.unknown_c) + .unwrap_or(0), + raw, + }); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn class_rows_report_whether_geometry_is_in_core() { + let r = ClassRow { + index: 0, + geometry_offset: 100, + o_class: 1, + unknown_8: 0, + unknown_c: 0, + raw: [0; SHRUB_CLASS_ENTRY_BYTES], + }; + assert!(r.in_core(200)); + assert!(!r.in_core(50)); + assert!(!ClassRow { + geometry_offset: 0, + ..r + } + .in_core(200)); + } + + #[test] + fn a_table_past_the_index_stops_rather_than_panicking() { + let index = vec![0u8; 16]; + let r = gc_formats::level_core::ArrayRange { + count: 100, + offset: 8, + }; + let rows = read_class_table(&index, r, CLASS_ENTRY_BYTES).unwrap(); + assert!(rows.len() < 100, "should stop at the end of the index"); + } +} diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index 9f5b56d..4df1776 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -870,6 +870,15 @@ plausible floats varies from 1 to 6 across spans, with 822 spans having 6, 455 having 1 and 373 having 2. So there is more header past `0x48`, and it is not a single array of distances. +### Probe tool + +`gc-extract probe [max-rows]` dumps the core header and +all three class tables through `LevelCoreHeader`, which is the point. The +table above and the shrub offset below were both established by hand first and +one of them nearly sent me to fix a bug that was not there. The tool also +reports how many entries carry geometry inside `core_data`, which is how the +shrub stride was pinned: 7/7 at 0x30 against 4/7 read at 0x20. + Note the LOD distances already decoded sit *before* this point (12.0, 25.0, 40.0 at `+0x10`), and the floats after `0x48` are a different population -- negative values, and a repeated leading 40.0.