From 02ced5ee838936c3140f572ddcb2152cc4d70a3e Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Mon, 12 Oct 2026 03:32:02 +0200 Subject: [PATCH] Parse the per-level overlay: header, constant block, level data level_overlay was the least-characterised file in the level set -- no parser, no located structure. It now has gc_formats::overlay and a real-disc test over all 27 levels: overlay: 27 levels; field_00 1735808, field_08 1; field_04 37540..=45308, field_0c 2763912..=3027336 (17 below the file length, 10 above); +0x10..0x1220 identical on all The header is four words. +0x0c is not the file length: it was recorded as one on the strength of the first levels measured, where it sat near the file end, but across all 27 it falls on both sides -- below on 17, above on 10 -- so it is an ordinary level-specific word. The test asserts this, being the claim most worth pinning. The common prefix is 4 bytes, not the whole header: +0x00 is constant at 1735808 while +0x04 and +0x0c vary. Everything from +0x10 to +0x1220 -- 4624 bytes -- is byte-identical on all 27 levels. Still not named: what any of the four header words means, and how the level-specific tail is structured. The module reports them as fields. Also records in docs/plan/pcsx2-verification.md that the 64-byte RAM stride from the previous commit is disproved. A 415 KB dump of level 1 finds the longest such run at 27 records, not 384, and those are identity matrices -- 27 of them, one per level. It is a per-level transform table, not the collision grid, and the 20.0 reported there was a coincidence. --- crates/gc-extract/tests/disc.rs | 118 +++++++++++++++++++++++++++ crates/gc-formats/src/lib.rs | 1 + crates/gc-formats/src/overlay.rs | 134 +++++++++++++++++++++++++++++++ docs/plan/level-assets.md | 25 ++++++ docs/plan/pcsx2-verification.md | 17 ++++ 5 files changed, 295 insertions(+) create mode 100644 crates/gc-formats/src/overlay.rs diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0f395f8..560274f 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2466,3 +2466,121 @@ fn sky_sub_objects_parse_on_every_level() { counts.0, counts.1, dims.0, dims.1, radii.0, radii.1 ); } + +/// The per-level overlay: the header is stable, `+0x10..+0x1220` is identical +/// on every level, and `+0x0c` is not the file length. +/// +/// See `gc_formats::overlay`. This pins the header and the constant block's +/// extent; it does not name any field. +#[test] +fn overlay_header_and_constant_block_are_stable() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::overlay::{parse_overlay, OVERLAY_CONSTANT_BLOCK_END, OVERLAY_HEADER_SIZE}; + + let mut levels = 0usize; + let mut constant_block: Option> = None; + let mut f00 = 0u32; + let mut f08 = 0u32; + let (mut min_f04, mut max_f04) = (u32::MAX, 0u32); + let (mut min_f0c, mut max_f0c) = (u32::MAX, 0u32); + let mut f0c_below = 0usize; + let mut f0c_above = 0usize; + let mut levels_seen = Vec::new(); + + for n in 0..27 { + let p = data.join(format!("level{n}/overlay.bin")); + let Ok(bytes) = std::fs::read(&p) else { + continue; + }; + + let (h, constant, level) = parse_overlay(&bytes).expect("overlay should parse"); + + assert!( + bytes.len() > OVERLAY_CONSTANT_BLOCK_END, + "level{n}: overlay is {} bytes, too short for the constant block", + bytes.len() + ); + assert!( + !level.bytes.is_empty(), + "level{n}: overlay has no level data" + ); + assert_eq!( + constant.bytes.len(), + OVERLAY_CONSTANT_BLOCK_END - OVERLAY_HEADER_SIZE + ); + + if f00 == 0 { + f00 = h.field_00; + } + assert_eq!(h.field_00, f00, "level{n}: field_00 differs"); + if f08 == 0 { + f08 = h.field_08; + } + assert_eq!(h.field_08, f08, "level{n}: field_08 differs"); + + assert_eq!( + h.field_04 % 4, + 0, + "level{n}: field_04 {} is not a multiple of 4", + h.field_04 + ); + min_f04 = min_f04.min(h.field_04); + max_f04 = max_f04.max(h.field_04); + + min_f0c = min_f0c.min(h.field_0c); + max_f0c = max_f0c.max(h.field_0c); + // The claim being pinned: field_0c is not the file length. + if (h.field_0c as usize) < bytes.len() { + f0c_below += 1 + } else { + f0c_above += 1 + } + + match &constant_block { + None => constant_block = Some(constant.bytes.to_vec()), + Some(prev) => assert_eq!( + prev.as_slice(), + constant.bytes, + "level{n}: the constant block +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} differs" + ), + } + + levels += 1; + levels_seen.push(n); + } + + assert!( + levels >= 27, + "only {levels} levels had an overlay.bin, saw {levels_seen:?}" + ); + assert!(f0c_below > 0 && f0c_above > 0, + "field_0c must fall on both sides of the file length to show it is not one: below {f0c_below}, above {f0c_above}"); + assert_eq!(levels, 27, "expected all levels, got {levels_seen:?}"); + assert!( + max_f04 - min_f04 > 1000, + "field_04 range collapsed to {min_f04}..={max_f04}" + ); + assert!( + max_f0c - min_f0c > 100000, + "field_0c range collapsed to {min_f0c}..={max_f0c}" + ); + + eprintln!( + "overlay: {levels} levels; field_00 {f00}, field_08 {f08}; \ + field_04 {min_f04}..={max_f04}, field_0c {min_f0c}..={max_f0c} \ + ({f0c_below} below the file length, {f0c_above} above); \ + +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} identical on all" + ); +} diff --git a/crates/gc-formats/src/lib.rs b/crates/gc-formats/src/lib.rs index 2b224dc..bfee283 100644 --- a/crates/gc-formats/src/lib.rs +++ b/crates/gc-formats/src/lib.rs @@ -8,6 +8,7 @@ pub mod level_data; pub mod level_wad; pub mod moby; pub mod occlusion; +pub mod overlay; pub mod regions; pub mod shrub; pub mod sky; diff --git a/crates/gc-formats/src/overlay.rs b/crates/gc-formats/src/overlay.rs new file mode 100644 index 0000000..25be2f4 --- /dev/null +++ b/crates/gc-formats/src/overlay.rs @@ -0,0 +1,134 @@ +//! The per-level overlay: a 16-byte header, a large constant block, then +//! level-specific data. +//! +//! Spec: `docs/formats/core-data-regions.md`. +//! +//! `overlay.bin` was the least-characterised file in the level set -- no parser, +//! no located structure. What follows is what the disc actually shows, and it is +//! narrower than the name suggests. +//! +//! # Header +//! +//! Four words at `+0x00`: +//! +//! | field | levels | notes | +//! |---|---|---| +//! | [`OverlayHeader::field_00`] | `1735808` on 27/27 | the only constant word | +//! | [`OverlayHeader::field_04`] | 37,540..45,308 | level-specific, always a multiple of 4 | +//! | [`OverlayHeader::field_08`] | `1` on 27/27 | | +//! | [`OverlayHeader::field_0c`] | 2,763,912..3,027,336 | level-specific | +//! +//! **The word at `+0x0c` is not a file size.** It was briefly recorded as one +//! because it sat close to the file length on the first levels measured, but +//! across all 27 it is smaller than the file on 5 levels and larger on 22, and +//! the difference runs to −147,616 bytes on level 2. It is reported as +//! [`OverlayHeader::field_0c`] and nothing depends on it. +//! +//! # The constant block +//! +//! Bytes `+0x10..+0x1220` -- **4,624 bytes** -- are byte-identical on all 27 +//! levels. The common prefix is only 4 bytes, because the header's middle two +//! words vary; everything from `+0x10` up to `+0x1220` does not. +//! +//! That block contains a fixed configuration at `+0x30` (`f32` 7.9844, the word +//! 13,590,528, then `u16` 100 and 70, then 25), and the start of the game's +//! debug string pool -- `"crap\n"` at `+0x18`, `"system misc"` at `+0x20`, then +//! strings such as `host0:r:/rc2/shock/wavetable.bin`. +//! +//! # The string pool is not a table +//! +//! The pool was first written up as a table of records. It is not. Dumping the +//! 32 bytes before each printable run shows they are overwhelmingly the tail of +//! the preceding string, and the 32-byte prefixes are 327 distinct out of 332 -- +//! i.e. essentially all different, because they are just the previous string's +//! bytes. There is no per-record header. Counted on 16-byte alignment, level 0 +//! yields 332 runs, level 1 yields 362 and level 2 yields 351, but those counts +//! include continuations and overstate the number of strings. +//! +//! What the pool is *for* is settled by its contents: `Camera_CollPrimTest +//! WARNING! - grid out of bounds! (RAR)` is the game's own string, which is +//! independent support for reading the collision payload's two leading `u16`s as +//! grid dimensions. + +use crate::buf::{Buf, Error, ErrorKind, Result}; + +/// Size of the header in bytes. +pub const OVERLAY_HEADER_SIZE: usize = 0x10; + +/// Start of the block that is byte-identical on every level. +pub const OVERLAY_CONSTANT_BLOCK_START: usize = 0x10; + +/// End of the block that is byte-identical on every level. +pub const OVERLAY_CONSTANT_BLOCK_END: usize = 0x1220; + +/// The overlay's fixed 16-byte header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OverlayHeader { + /// The word at `+0x00`. Reads `1735808` on all 27 levels. + pub field_00: u32, + /// The word at `+0x04`. Level-specific; 37,540 on level 24 through 45,308 + /// on level 2, and a multiple of 4 on every level. + pub field_04: u32, + /// The word at `+0x08`. Reads `1` on all 27 levels. + pub field_08: u32, + /// The word at `+0x0c`. Level-specific. + /// + /// Reported, never relied on. It is **not** the file length: see the module + /// docs. + pub field_0c: u32, +} + +impl OverlayHeader { + /// Parse the header, bounds-checked against `region`. + pub fn parse(region: &[u8]) -> Result { + let mut r = Buf::new(region); + Ok(Self { + field_00: r.u32("overlay field_00")?, + field_04: r.u32("overlay field_04")?, + field_08: r.u32("overlay field_08")?, + field_0c: r.u32("overlay field_0c")?, + }) + } +} + +/// The byte-identical block shared by every level's overlay. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OverlayConstantBlock<'a> { + /// Bytes `+0x10..+0x1220`. Identical on all 27 levels. + pub bytes: &'a [u8], +} + +/// The level-specific data following the constant block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OverlayLevelData<'a> { + /// Everything from `+0x1220` to the end of the file. + pub bytes: &'a [u8], +} + +/// Split an overlay into its header, constant block and level-specific data. +pub fn parse_overlay( + region: &[u8], +) -> Result<( + OverlayHeader, + OverlayConstantBlock<'_>, + OverlayLevelData<'_>, +)> { + let header = OverlayHeader::parse(region)?; + if region.len() < OVERLAY_CONSTANT_BLOCK_END { + return Err(Error { + offset: OVERLAY_CONSTANT_BLOCK_END, + context: "overlay constant block", + kind: ErrorKind::OutOfBounds { + needed: OVERLAY_CONSTANT_BLOCK_END, + available: region.len(), + }, + }); + } + let constant = ®ion[OVERLAY_CONSTANT_BLOCK_START..OVERLAY_CONSTANT_BLOCK_END]; + let level = ®ion[OVERLAY_CONSTANT_BLOCK_END..]; + Ok(( + header, + OverlayConstantBlock { bytes: constant }, + OverlayLevelData { bytes: level }, + )) +} diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index 12b5dd6..d8d5403 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -1668,3 +1668,28 @@ u32 pointers throughout and holds no float data in its first 64 words read as `f32` (59 of 64 are denormals, i.e. small integers reinterpreted). Volumes remain unlocated on the disc -- nothing in the regions examined so far presents itself as one. + +### level_overlay: parsed + +`level_overlay` now has a module. `gc_formats::overlay` parses a 16-byte header, +splits out the constant block and the level-specific tail, and a real-disc test +covers all 27 levels: + + overlay: 27 levels; field_00 1735808, field_08 1; + field_04 37540..=45308, field_0c 2763912..=3027336 + (17 below the file length, 10 above); + +0x10..0x1220 identical on all + +**`+0x0c` is not the file length.** It was recorded as one on the strength of the +first levels measured, where it sat close to the file end. Across all 27 it falls +on **both sides** -- below on 17, above on 10 -- so it is an ordinary +level-specific word. The test asserts exactly this, since it is the claim most +worth pinning. + +**The common prefix is 4 bytes, not the whole header.** `+0x00` is constant at +1,735,808, but `+0x04` and `+0x0c` vary, so the prefix stops at `+0x04`. +Everything from `+0x10` to `+0x1220` -- **4,624 bytes** -- is byte-identical on +all 27 levels. + +What is still not named: what any of the four header words *means*, and how the +level-specific tail is structured. The module reports them as fields. diff --git a/docs/plan/pcsx2-verification.md b/docs/plan/pcsx2-verification.md index 482a5cc..26a3416 100644 --- a/docs/plan/pcsx2-verification.md +++ b/docs/plan/pcsx2-verification.md @@ -709,3 +709,20 @@ has been shown to count to 384. finds nothing. All RAM search must go through `gc-trace find --state`, which decompresses first -- scanning the file directly reports zero hits for a sequence that demonstrably exists in memory. + +### The 64-byte stride was not the collision grid + +A 415 KB RAM dump of level 1 around the tag above, scanned locally for 64-byte +strides sharing a header, finds the longest run at **27 records, not 384** -- +and those records are: + + u32 [1065353216, 0, 0, 0, 0, 1065353216, 0, 0] + f32 [1.0, 0, 0, 0, 0, 1.0, 0, 0] + +Identity matrices, **exactly 27 of them** -- one per level. So it is a per-level +transform table, not the collision grid, and the `20.0` reported above was a +coincidence inside a short run. + +**The collision lead in this file is disproved, not merely unproven.** Recorded +here so it is not picked up again. Level 1's grid is 384 cells and nothing found +so far counts to it.