Parse the per-level overlay: header, constant block, level data
level_overlay was the least-characterised file in the level set -- no parser, no located structure. It now has gc_formats::overlay and a real-disc test over all 27 levels: overlay: 27 levels; field_00 1735808, field_08 1; field_04 37540..=45308, field_0c 2763912..=3027336 (17 below the file length, 10 above); +0x10..0x1220 identical on all The header is four words. +0x0c is not the file length: it was recorded as one on the strength of the first levels measured, where it sat near the file end, but across all 27 it falls on both sides -- below on 17, above on 10 -- so it is an ordinary level-specific word. The test asserts this, being the claim most worth pinning. The common prefix is 4 bytes, not the whole header: +0x00 is constant at 1735808 while +0x04 and +0x0c vary. Everything from +0x10 to +0x1220 -- 4624 bytes -- is byte-identical on all 27 levels. Still not named: what any of the four header words means, and how the level-specific tail is structured. The module reports them as fields. Also records in docs/plan/pcsx2-verification.md that the 64-byte RAM stride from the previous commit is disproved. A 415 KB dump of level 1 finds the longest such run at 27 records, not 384, and those are identity matrices -- 27 of them, one per level. It is a per-level transform table, not the collision grid, and the 20.0 reported there was a coincidence.
This commit is contained in:
@@ -2466,3 +2466,121 @@ fn sky_sub_objects_parse_on_every_level() {
|
||||
counts.0, counts.1, dims.0, dims.1, radii.0, radii.1
|
||||
);
|
||||
}
|
||||
|
||||
/// The per-level overlay: the header is stable, `+0x10..+0x1220` is identical
|
||||
/// on every level, and `+0x0c` is not the file length.
|
||||
///
|
||||
/// See `gc_formats::overlay`. This pins the header and the constant block's
|
||||
/// extent; it does not name any field.
|
||||
#[test]
|
||||
fn overlay_header_and_constant_block_are_stable() {
|
||||
require_disc!();
|
||||
|
||||
let data = std::env::var_os("GC_EXTRACTED")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| {
|
||||
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
||||
PathBuf::from(home).join("Documents/gc-extracted")
|
||||
});
|
||||
if !data.exists() {
|
||||
eprintln!("skipped: unpack the disc first");
|
||||
return;
|
||||
}
|
||||
|
||||
use gc_formats::overlay::{parse_overlay, OVERLAY_CONSTANT_BLOCK_END, OVERLAY_HEADER_SIZE};
|
||||
|
||||
let mut levels = 0usize;
|
||||
let mut constant_block: Option<Vec<u8>> = None;
|
||||
let mut f00 = 0u32;
|
||||
let mut f08 = 0u32;
|
||||
let (mut min_f04, mut max_f04) = (u32::MAX, 0u32);
|
||||
let (mut min_f0c, mut max_f0c) = (u32::MAX, 0u32);
|
||||
let mut f0c_below = 0usize;
|
||||
let mut f0c_above = 0usize;
|
||||
let mut levels_seen = Vec::new();
|
||||
|
||||
for n in 0..27 {
|
||||
let p = data.join(format!("level{n}/overlay.bin"));
|
||||
let Ok(bytes) = std::fs::read(&p) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let (h, constant, level) = parse_overlay(&bytes).expect("overlay should parse");
|
||||
|
||||
assert!(
|
||||
bytes.len() > OVERLAY_CONSTANT_BLOCK_END,
|
||||
"level{n}: overlay is {} bytes, too short for the constant block",
|
||||
bytes.len()
|
||||
);
|
||||
assert!(
|
||||
!level.bytes.is_empty(),
|
||||
"level{n}: overlay has no level data"
|
||||
);
|
||||
assert_eq!(
|
||||
constant.bytes.len(),
|
||||
OVERLAY_CONSTANT_BLOCK_END - OVERLAY_HEADER_SIZE
|
||||
);
|
||||
|
||||
if f00 == 0 {
|
||||
f00 = h.field_00;
|
||||
}
|
||||
assert_eq!(h.field_00, f00, "level{n}: field_00 differs");
|
||||
if f08 == 0 {
|
||||
f08 = h.field_08;
|
||||
}
|
||||
assert_eq!(h.field_08, f08, "level{n}: field_08 differs");
|
||||
|
||||
assert_eq!(
|
||||
h.field_04 % 4,
|
||||
0,
|
||||
"level{n}: field_04 {} is not a multiple of 4",
|
||||
h.field_04
|
||||
);
|
||||
min_f04 = min_f04.min(h.field_04);
|
||||
max_f04 = max_f04.max(h.field_04);
|
||||
|
||||
min_f0c = min_f0c.min(h.field_0c);
|
||||
max_f0c = max_f0c.max(h.field_0c);
|
||||
// The claim being pinned: field_0c is not the file length.
|
||||
if (h.field_0c as usize) < bytes.len() {
|
||||
f0c_below += 1
|
||||
} else {
|
||||
f0c_above += 1
|
||||
}
|
||||
|
||||
match &constant_block {
|
||||
None => constant_block = Some(constant.bytes.to_vec()),
|
||||
Some(prev) => assert_eq!(
|
||||
prev.as_slice(),
|
||||
constant.bytes,
|
||||
"level{n}: the constant block +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} differs"
|
||||
),
|
||||
}
|
||||
|
||||
levels += 1;
|
||||
levels_seen.push(n);
|
||||
}
|
||||
|
||||
assert!(
|
||||
levels >= 27,
|
||||
"only {levels} levels had an overlay.bin, saw {levels_seen:?}"
|
||||
);
|
||||
assert!(f0c_below > 0 && f0c_above > 0,
|
||||
"field_0c must fall on both sides of the file length to show it is not one: below {f0c_below}, above {f0c_above}");
|
||||
assert_eq!(levels, 27, "expected all levels, got {levels_seen:?}");
|
||||
assert!(
|
||||
max_f04 - min_f04 > 1000,
|
||||
"field_04 range collapsed to {min_f04}..={max_f04}"
|
||||
);
|
||||
assert!(
|
||||
max_f0c - min_f0c > 100000,
|
||||
"field_0c range collapsed to {min_f0c}..={max_f0c}"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"overlay: {levels} levels; field_00 {f00}, field_08 {f08}; \
|
||||
field_04 {min_f04}..={max_f04}, field_0c {min_f0c}..={max_f0c} \
|
||||
({f0c_below} below the file length, {f0c_above} above); \
|
||||
+0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} identical on all"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ pub mod level_data;
|
||||
pub mod level_wad;
|
||||
pub mod moby;
|
||||
pub mod occlusion;
|
||||
pub mod overlay;
|
||||
pub mod regions;
|
||||
pub mod shrub;
|
||||
pub mod sky;
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
//! The per-level overlay: a 16-byte header, a large constant block, then
|
||||
//! level-specific data.
|
||||
//!
|
||||
//! Spec: `docs/formats/core-data-regions.md`.
|
||||
//!
|
||||
//! `overlay.bin` was the least-characterised file in the level set -- no parser,
|
||||
//! no located structure. What follows is what the disc actually shows, and it is
|
||||
//! narrower than the name suggests.
|
||||
//!
|
||||
//! # Header
|
||||
//!
|
||||
//! Four words at `+0x00`:
|
||||
//!
|
||||
//! | field | levels | notes |
|
||||
//! |---|---|---|
|
||||
//! | [`OverlayHeader::field_00`] | `1735808` on 27/27 | the only constant word |
|
||||
//! | [`OverlayHeader::field_04`] | 37,540..45,308 | level-specific, always a multiple of 4 |
|
||||
//! | [`OverlayHeader::field_08`] | `1` on 27/27 | |
|
||||
//! | [`OverlayHeader::field_0c`] | 2,763,912..3,027,336 | level-specific |
|
||||
//!
|
||||
//! **The word at `+0x0c` is not a file size.** It was briefly recorded as one
|
||||
//! because it sat close to the file length on the first levels measured, but
|
||||
//! across all 27 it is smaller than the file on 5 levels and larger on 22, and
|
||||
//! the difference runs to −147,616 bytes on level 2. It is reported as
|
||||
//! [`OverlayHeader::field_0c`] and nothing depends on it.
|
||||
//!
|
||||
//! # The constant block
|
||||
//!
|
||||
//! Bytes `+0x10..+0x1220` -- **4,624 bytes** -- are byte-identical on all 27
|
||||
//! levels. The common prefix is only 4 bytes, because the header's middle two
|
||||
//! words vary; everything from `+0x10` up to `+0x1220` does not.
|
||||
//!
|
||||
//! That block contains a fixed configuration at `+0x30` (`f32` 7.9844, the word
|
||||
//! 13,590,528, then `u16` 100 and 70, then 25), and the start of the game's
|
||||
//! debug string pool -- `"crap\n"` at `+0x18`, `"system misc"` at `+0x20`, then
|
||||
//! strings such as `host0:r:/rc2/shock/wavetable.bin`.
|
||||
//!
|
||||
//! # The string pool is not a table
|
||||
//!
|
||||
//! The pool was first written up as a table of records. It is not. Dumping the
|
||||
//! 32 bytes before each printable run shows they are overwhelmingly the tail of
|
||||
//! the preceding string, and the 32-byte prefixes are 327 distinct out of 332 --
|
||||
//! i.e. essentially all different, because they are just the previous string's
|
||||
//! bytes. There is no per-record header. Counted on 16-byte alignment, level 0
|
||||
//! yields 332 runs, level 1 yields 362 and level 2 yields 351, but those counts
|
||||
//! include continuations and overstate the number of strings.
|
||||
//!
|
||||
//! What the pool is *for* is settled by its contents: `Camera_CollPrimTest
|
||||
//! WARNING! - grid out of bounds! (RAR)` is the game's own string, which is
|
||||
//! independent support for reading the collision payload's two leading `u16`s as
|
||||
//! grid dimensions.
|
||||
|
||||
use crate::buf::{Buf, Error, ErrorKind, Result};
|
||||
|
||||
/// Size of the header in bytes.
|
||||
pub const OVERLAY_HEADER_SIZE: usize = 0x10;
|
||||
|
||||
/// Start of the block that is byte-identical on every level.
|
||||
pub const OVERLAY_CONSTANT_BLOCK_START: usize = 0x10;
|
||||
|
||||
/// End of the block that is byte-identical on every level.
|
||||
pub const OVERLAY_CONSTANT_BLOCK_END: usize = 0x1220;
|
||||
|
||||
/// The overlay's fixed 16-byte header.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct OverlayHeader {
|
||||
/// The word at `+0x00`. Reads `1735808` on all 27 levels.
|
||||
pub field_00: u32,
|
||||
/// The word at `+0x04`. Level-specific; 37,540 on level 24 through 45,308
|
||||
/// on level 2, and a multiple of 4 on every level.
|
||||
pub field_04: u32,
|
||||
/// The word at `+0x08`. Reads `1` on all 27 levels.
|
||||
pub field_08: u32,
|
||||
/// The word at `+0x0c`. Level-specific.
|
||||
///
|
||||
/// Reported, never relied on. It is **not** the file length: see the module
|
||||
/// docs.
|
||||
pub field_0c: u32,
|
||||
}
|
||||
|
||||
impl OverlayHeader {
|
||||
/// Parse the header, bounds-checked against `region`.
|
||||
pub fn parse(region: &[u8]) -> Result<Self> {
|
||||
let mut r = Buf::new(region);
|
||||
Ok(Self {
|
||||
field_00: r.u32("overlay field_00")?,
|
||||
field_04: r.u32("overlay field_04")?,
|
||||
field_08: r.u32("overlay field_08")?,
|
||||
field_0c: r.u32("overlay field_0c")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The byte-identical block shared by every level's overlay.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct OverlayConstantBlock<'a> {
|
||||
/// Bytes `+0x10..+0x1220`. Identical on all 27 levels.
|
||||
pub bytes: &'a [u8],
|
||||
}
|
||||
|
||||
/// The level-specific data following the constant block.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct OverlayLevelData<'a> {
|
||||
/// Everything from `+0x1220` to the end of the file.
|
||||
pub bytes: &'a [u8],
|
||||
}
|
||||
|
||||
/// Split an overlay into its header, constant block and level-specific data.
|
||||
pub fn parse_overlay(
|
||||
region: &[u8],
|
||||
) -> Result<(
|
||||
OverlayHeader,
|
||||
OverlayConstantBlock<'_>,
|
||||
OverlayLevelData<'_>,
|
||||
)> {
|
||||
let header = OverlayHeader::parse(region)?;
|
||||
if region.len() < OVERLAY_CONSTANT_BLOCK_END {
|
||||
return Err(Error {
|
||||
offset: OVERLAY_CONSTANT_BLOCK_END,
|
||||
context: "overlay constant block",
|
||||
kind: ErrorKind::OutOfBounds {
|
||||
needed: OVERLAY_CONSTANT_BLOCK_END,
|
||||
available: region.len(),
|
||||
},
|
||||
});
|
||||
}
|
||||
let constant = ®ion[OVERLAY_CONSTANT_BLOCK_START..OVERLAY_CONSTANT_BLOCK_END];
|
||||
let level = ®ion[OVERLAY_CONSTANT_BLOCK_END..];
|
||||
Ok((
|
||||
header,
|
||||
OverlayConstantBlock { bytes: constant },
|
||||
OverlayLevelData { bytes: level },
|
||||
))
|
||||
}
|
||||
@@ -1668,3 +1668,28 @@ u32 pointers throughout and holds no float data in its first 64 words read as
|
||||
`f32` (59 of 64 are denormals, i.e. small integers reinterpreted). Volumes remain
|
||||
unlocated on the disc -- nothing in the regions examined so far presents itself
|
||||
as one.
|
||||
|
||||
### level_overlay: parsed
|
||||
|
||||
`level_overlay` now has a module. `gc_formats::overlay` parses a 16-byte header,
|
||||
splits out the constant block and the level-specific tail, and a real-disc test
|
||||
covers all 27 levels:
|
||||
|
||||
overlay: 27 levels; field_00 1735808, field_08 1;
|
||||
field_04 37540..=45308, field_0c 2763912..=3027336
|
||||
(17 below the file length, 10 above);
|
||||
+0x10..0x1220 identical on all
|
||||
|
||||
**`+0x0c` is not the file length.** It was recorded as one on the strength of the
|
||||
first levels measured, where it sat close to the file end. Across all 27 it falls
|
||||
on **both sides** -- below on 17, above on 10 -- so it is an ordinary
|
||||
level-specific word. The test asserts exactly this, since it is the claim most
|
||||
worth pinning.
|
||||
|
||||
**The common prefix is 4 bytes, not the whole header.** `+0x00` is constant at
|
||||
1,735,808, but `+0x04` and `+0x0c` vary, so the prefix stops at `+0x04`.
|
||||
Everything from `+0x10` to `+0x1220` -- **4,624 bytes** -- is byte-identical on
|
||||
all 27 levels.
|
||||
|
||||
What is still not named: what any of the four header words *means*, and how the
|
||||
level-specific tail is structured. The module reports them as fields.
|
||||
|
||||
@@ -709,3 +709,20 @@ has been shown to count to 384.
|
||||
finds nothing. All RAM search must go through `gc-trace find --state`, which
|
||||
decompresses first -- scanning the file directly reports zero hits for a
|
||||
sequence that demonstrably exists in memory.
|
||||
|
||||
### The 64-byte stride was not the collision grid
|
||||
|
||||
A 415 KB RAM dump of level 1 around the tag above, scanned locally for 64-byte
|
||||
strides sharing a header, finds the longest run at **27 records, not 384** --
|
||||
and those records are:
|
||||
|
||||
u32 [1065353216, 0, 0, 0, 0, 1065353216, 0, 0]
|
||||
f32 [1.0, 0, 0, 0, 0, 1.0, 0, 0]
|
||||
|
||||
Identity matrices, **exactly 27 of them** -- one per level. So it is a per-level
|
||||
transform table, not the collision grid, and the `20.0` reported above was a
|
||||
coincidence inside a short run.
|
||||
|
||||
**The collision lead in this file is disproved, not merely unproven.** Recorded
|
||||
here so it is not picked up again. Level 1's grid is 384 cells and nothing found
|
||||
so far counts to it.
|
||||
|
||||
Reference in New Issue
Block a user