Parse the per-level overlay: header, constant block, level data

level_overlay was the least-characterised file in the level set -- no parser, no
located structure. It now has gc_formats::overlay and a real-disc test over all 27
levels:

  overlay: 27 levels; field_00 1735808, field_08 1;
  field_04 37540..=45308, field_0c 2763912..=3027336
  (17 below the file length, 10 above);
  +0x10..0x1220 identical on all

The header is four words. +0x0c is not the file length: it was recorded as one on
the strength of the first levels measured, where it sat near the file end, but
across all 27 it falls on both sides -- below on 17, above on 10 -- so it is an
ordinary level-specific word. The test asserts this, being the claim most worth
pinning.

The common prefix is 4 bytes, not the whole header: +0x00 is constant at 1735808
while +0x04 and +0x0c vary. Everything from +0x10 to +0x1220 -- 4624 bytes -- is
byte-identical on all 27 levels.

Still not named: what any of the four header words means, and how the
level-specific tail is structured. The module reports them as fields.

Also records in docs/plan/pcsx2-verification.md that the 64-byte RAM stride from
the previous commit is disproved. A 415 KB dump of level 1 finds the longest such
run at 27 records, not 384, and those are identity matrices -- 27 of them, one per
level. It is a per-level transform table, not the collision grid, and the 20.0
reported there was a coincidence.

crates/gc-extract/tests/disc.rs

@@ -2466,3 +2466,121 @@ fn sky_sub_objects_parse_on_every_level() {
         counts.0, counts.1, dims.0, dims.1, radii.0, radii.1
     );
 }
+
+/// The per-level overlay: the header is stable, `+0x10..+0x1220` is identical
+/// on every level, and `+0x0c` is not the file length.
+///
+/// See `gc_formats::overlay`. This pins the header and the constant block's
+/// extent; it does not name any field.
+#[test]
+fn overlay_header_and_constant_block_are_stable() {
+    require_disc!();
+
+    let data = std::env::var_os("GC_EXTRACTED")
+        .map(PathBuf::from)
+        .unwrap_or_else(|| {
+            let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
+            PathBuf::from(home).join("Documents/gc-extracted")
+        });
+    if !data.exists() {
+        eprintln!("skipped: unpack the disc first");
+        return;
+    }
+
+    use gc_formats::overlay::{parse_overlay, OVERLAY_CONSTANT_BLOCK_END, OVERLAY_HEADER_SIZE};
+
+    let mut levels = 0usize;
+    let mut constant_block: Option<Vec<u8>> = None;
+    let mut f00 = 0u32;
+    let mut f08 = 0u32;
+    let (mut min_f04, mut max_f04) = (u32::MAX, 0u32);
+    let (mut min_f0c, mut max_f0c) = (u32::MAX, 0u32);
+    let mut f0c_below = 0usize;
+    let mut f0c_above = 0usize;
+    let mut levels_seen = Vec::new();
+
+    for n in 0..27 {
+        let p = data.join(format!("level{n}/overlay.bin"));
+        let Ok(bytes) = std::fs::read(&p) else {
+            continue;
+        };
+
+        let (h, constant, level) = parse_overlay(&bytes).expect("overlay should parse");
+
+        assert!(
+            bytes.len() > OVERLAY_CONSTANT_BLOCK_END,
+            "level{n}: overlay is {} bytes, too short for the constant block",
+            bytes.len()
+        );
+        assert!(
+            !level.bytes.is_empty(),
+            "level{n}: overlay has no level data"
+        );
+        assert_eq!(
+            constant.bytes.len(),
+            OVERLAY_CONSTANT_BLOCK_END - OVERLAY_HEADER_SIZE
+        );
+
+        if f00 == 0 {
+            f00 = h.field_00;
+        }
+        assert_eq!(h.field_00, f00, "level{n}: field_00 differs");
+        if f08 == 0 {
+            f08 = h.field_08;
+        }
+        assert_eq!(h.field_08, f08, "level{n}: field_08 differs");
+
+        assert_eq!(
+            h.field_04 % 4,
+            0,
+            "level{n}: field_04 {} is not a multiple of 4",
+            h.field_04
+        );
+        min_f04 = min_f04.min(h.field_04);
+        max_f04 = max_f04.max(h.field_04);
+
+        min_f0c = min_f0c.min(h.field_0c);
+        max_f0c = max_f0c.max(h.field_0c);
+        // The claim being pinned: field_0c is not the file length.
+        if (h.field_0c as usize) < bytes.len() {
+            f0c_below += 1
+        } else {
+            f0c_above += 1
+        }
+
+        match &constant_block {
+            None => constant_block = Some(constant.bytes.to_vec()),
+            Some(prev) => assert_eq!(
+                prev.as_slice(),
+                constant.bytes,
+                "level{n}: the constant block +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} differs"
+            ),
+        }
+
+        levels += 1;
+        levels_seen.push(n);
+    }
+
+    assert!(
+        levels >= 27,
+        "only {levels} levels had an overlay.bin, saw {levels_seen:?}"
+    );
+    assert!(f0c_below > 0 && f0c_above > 0,
+        "field_0c must fall on both sides of the file length to show it is not one: below {f0c_below}, above {f0c_above}");
+    assert_eq!(levels, 27, "expected all levels, got {levels_seen:?}");
+    assert!(
+        max_f04 - min_f04 > 1000,
+        "field_04 range collapsed to {min_f04}..={max_f04}"
+    );
+    assert!(
+        max_f0c - min_f0c > 100000,
+        "field_0c range collapsed to {min_f0c}..={max_f0c}"
+    );
+
+    eprintln!(
+        "overlay: {levels} levels; field_00 {f00}, field_08 {f08}; \
+         field_04 {min_f04}..={max_f04}, field_0c {min_f0c}..={max_f0c} \
+         ({f0c_below} below the file length, {f0c_above} above); \
+         +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} identical on all"
+    );
+}

crates/gc-formats/src/lib.rs

@@ -8,6 +8,7 @@ pub mod level_data;
 pub mod level_wad;
 pub mod moby;
 pub mod occlusion;
+pub mod overlay;
 pub mod regions;
 pub mod shrub;
 pub mod sky;

crates/gc-formats/src/overlay.rs

@@ -0,0 +1,134 @@
+//! The per-level overlay: a 16-byte header, a large constant block, then
+//! level-specific data.
+//!
+//! Spec: `docs/formats/core-data-regions.md`.
+//!
+//! `overlay.bin` was the least-characterised file in the level set -- no parser,
+//! no located structure. What follows is what the disc actually shows, and it is
+//! narrower than the name suggests.
+//!
+//! # Header
+//!
+//! Four words at `+0x00`:
+//!
+//! | field | levels | notes |
+//! |---|---|---|
+//! | [`OverlayHeader::field_00`] | `1735808` on 27/27 | the only constant word |
+//! | [`OverlayHeader::field_04`] | 37,540..45,308 | level-specific, always a multiple of 4 |
+//! | [`OverlayHeader::field_08`] | `1` on 27/27 | |
+//! | [`OverlayHeader::field_0c`] | 2,763,912..3,027,336 | level-specific |
+//!
+//! **The word at `+0x0c` is not a file size.** It was briefly recorded as one
+//! because it sat close to the file length on the first levels measured, but
+//! across all 27 it is smaller than the file on 5 levels and larger on 22, and
+//! the difference runs to −147,616 bytes on level 2. It is reported as
+//! [`OverlayHeader::field_0c`] and nothing depends on it.
+//!
+//! # The constant block
+//!
+//! Bytes `+0x10..+0x1220` -- **4,624 bytes** -- are byte-identical on all 27
+//! levels. The common prefix is only 4 bytes, because the header's middle two
+//! words vary; everything from `+0x10` up to `+0x1220` does not.
+//!
+//! That block contains a fixed configuration at `+0x30` (`f32` 7.9844, the word
+//! 13,590,528, then `u16` 100 and 70, then 25), and the start of the game's
+//! debug string pool -- `"crap\n"` at `+0x18`, `"system misc"` at `+0x20`, then
+//! strings such as `host0:r:/rc2/shock/wavetable.bin`.
+//!
+//! # The string pool is not a table
+//!
+//! The pool was first written up as a table of records. It is not. Dumping the
+//! 32 bytes before each printable run shows they are overwhelmingly the tail of
+//! the preceding string, and the 32-byte prefixes are 327 distinct out of 332 --
+//! i.e. essentially all different, because they are just the previous string's
+//! bytes. There is no per-record header. Counted on 16-byte alignment, level 0
+//! yields 332 runs, level 1 yields 362 and level 2 yields 351, but those counts
+//! include continuations and overstate the number of strings.
+//!
+//! What the pool is *for* is settled by its contents: `Camera_CollPrimTest
+//! WARNING! - grid out of bounds! (RAR)` is the game's own string, which is
+//! independent support for reading the collision payload's two leading `u16`s as
+//! grid dimensions.
+
+use crate::buf::{Buf, Error, ErrorKind, Result};
+
+/// Size of the header in bytes.
+pub const OVERLAY_HEADER_SIZE: usize = 0x10;
+
+/// Start of the block that is byte-identical on every level.
+pub const OVERLAY_CONSTANT_BLOCK_START: usize = 0x10;
+
+/// End of the block that is byte-identical on every level.
+pub const OVERLAY_CONSTANT_BLOCK_END: usize = 0x1220;
+
+/// The overlay's fixed 16-byte header.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct OverlayHeader {
+    /// The word at `+0x00`. Reads `1735808` on all 27 levels.
+    pub field_00: u32,
+    /// The word at `+0x04`. Level-specific; 37,540 on level 24 through 45,308
+    /// on level 2, and a multiple of 4 on every level.
+    pub field_04: u32,
+    /// The word at `+0x08`. Reads `1` on all 27 levels.
+    pub field_08: u32,
+    /// The word at `+0x0c`. Level-specific.
+    ///
+    /// Reported, never relied on. It is **not** the file length: see the module
+    /// docs.
+    pub field_0c: u32,
+}
+
+impl OverlayHeader {
+    /// Parse the header, bounds-checked against `region`.
+    pub fn parse(region: &[u8]) -> Result<Self> {
+        let mut r = Buf::new(region);
+        Ok(Self {
+            field_00: r.u32("overlay field_00")?,
+            field_04: r.u32("overlay field_04")?,
+            field_08: r.u32("overlay field_08")?,
+            field_0c: r.u32("overlay field_0c")?,
+        })
+    }
+}
+
+/// The byte-identical block shared by every level's overlay.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct OverlayConstantBlock<'a> {
+    /// Bytes `+0x10..+0x1220`. Identical on all 27 levels.
+    pub bytes: &'a [u8],
+}
+
+/// The level-specific data following the constant block.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct OverlayLevelData<'a> {
+    /// Everything from `+0x1220` to the end of the file.
+    pub bytes: &'a [u8],
+}
+
+/// Split an overlay into its header, constant block and level-specific data.
+pub fn parse_overlay(
+    region: &[u8],
+) -> Result<(
+    OverlayHeader,
+    OverlayConstantBlock<'_>,
+    OverlayLevelData<'_>,
+)> {
+    let header = OverlayHeader::parse(region)?;
+    if region.len() < OVERLAY_CONSTANT_BLOCK_END {
+        return Err(Error {
+            offset: OVERLAY_CONSTANT_BLOCK_END,
+            context: "overlay constant block",
+            kind: ErrorKind::OutOfBounds {
+                needed: OVERLAY_CONSTANT_BLOCK_END,
+                available: region.len(),
+            },
+        });
+    }
+    let constant = &region[OVERLAY_CONSTANT_BLOCK_START..OVERLAY_CONSTANT_BLOCK_END];
+    let level = &region[OVERLAY_CONSTANT_BLOCK_END..];
+    Ok((
+        header,
+        OverlayConstantBlock { bytes: constant },
+        OverlayLevelData { bytes: level },
+    ))
+}

docs/plan/level-assets.md

@@ -1668,3 +1668,28 @@ u32 pointers throughout and holds no float data in its first 64 words read as
 `f32` (59 of 64 are denormals, i.e. small integers reinterpreted). Volumes remain
 unlocated on the disc -- nothing in the regions examined so far presents itself
 as one.
+
+### level_overlay: parsed
+
+`level_overlay` now has a module. `gc_formats::overlay` parses a 16-byte header,
+splits out the constant block and the level-specific tail, and a real-disc test
+covers all 27 levels:
+
+    overlay: 27 levels; field_00 1735808, field_08 1;
+    field_04 37540..=45308, field_0c 2763912..=3027336
+    (17 below the file length, 10 above);
+    +0x10..0x1220 identical on all
+
+**`+0x0c` is not the file length.** It was recorded as one on the strength of the
+first levels measured, where it sat close to the file end. Across all 27 it falls
+on **both sides** -- below on 17, above on 10 -- so it is an ordinary
+level-specific word. The test asserts exactly this, since it is the claim most
+worth pinning.
+
+**The common prefix is 4 bytes, not the whole header.** `+0x00` is constant at
+1,735,808, but `+0x04` and `+0x0c` vary, so the prefix stops at `+0x04`.
+Everything from `+0x10` to `+0x1220` -- **4,624 bytes** -- is byte-identical on
+all 27 levels.
+
+What is still not named: what any of the four header words *means*, and how the
+level-specific tail is structured. The module reports them as fields.

docs/plan/pcsx2-verification.md

@@ -709,3 +709,20 @@ has been shown to count to 384.
 finds nothing. All RAM search must go through `gc-trace find --state`, which
 decompresses first -- scanning the file directly reports zero hits for a
 sequence that demonstrably exists in memory.
+
+### The 64-byte stride was not the collision grid
+
+A 415 KB RAM dump of level 1 around the tag above, scanned locally for 64-byte
+strides sharing a header, finds the longest run at **27 records, not 384** --
+and those records are:
+
+    u32 [1065353216, 0, 0, 0, 0, 1065353216, 0, 0]
+    f32 [1.0,       0, 0, 0, 0, 1.0,       0, 0]
+
+Identity matrices, **exactly 27 of them** -- one per level. So it is a per-level
+transform table, not the collision grid, and the `20.0` reported above was a
+coincidence inside a short run.
+
+**The collision lead in this file is disproved, not merely unproven.** Recorded
+here so it is not picked up again. Level 1's grid is 384 cells and nothing found
+so far counts to it.