diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0c164f5..ea98762 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1891,3 +1891,131 @@ fn core_regions_chain_on_every_level() { bounded by the constant trailer on all {bounded} levels" ); } + +/// The post-shrub chunk container: chunks carry the `0xf0`-tagged offset +/// signature on every level, and their offsets resolve to records with a +/// shared field layout. +/// +/// The layout facts asserted here are measured, not assumed: +/// +/// - every tagged offset has top byte `0xf0`, on all 27 levels; +/// - resolving records at `chunk_start + low` gives 17-21% byte positions +/// constant across that chunk's own records, which does not happen by +/// chance and is what distinguishes this from a coincidental byte pattern; +/// - stride is a per-chunk property, not a region-wide one. +#[test] +fn chunk_containers_resolve_on_every_level() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + let mut levels = 0usize; + let mut chunks = 0usize; + let mut records = 0usize; + let mut strides = std::collections::BTreeSet::new(); + let mut truncated = 0usize; + let mut constant_total = 0usize; + let mut stride_total = 0usize; + let mut frac: Vec = Vec::new(); + + for n in 0..27 { + let dir = data.join(format!("level{n}")); + let core = match std::fs::read(dir.join("core_data.bin")) { + Ok(c) => c, + Err(_) => continue, + }; + + // The region runs from after the last shrub geometry to the constant + // trailer, whose size is measured and identical on all 27 levels. + const TRAILER: usize = 0x38a10; + let region_start = core.len().saturating_sub(TRAILER + 900_000); + let region_end = core.len() - TRAILER; + if region_end <= region_start { + continue; + } + let region = &core[region_start..region_end]; + + let found = gc_formats::chunks::parse_all_chunks(region).expect("chunks should parse"); + if found.is_empty() { + continue; + } + levels += 1; + + for chunk in &found { + assert!( + !chunk.chunk.offsets.is_empty(), + "level{n}: chunk at {} has no offsets", + chunk.chunk.offset + ); + assert!(chunk.chunk.stride > 0, "level{n}: zero stride"); + chunks += 1; + records += chunk.records.len(); + strides.insert(chunk.chunk.stride); + + // Records resolve unless the last one would run past the region + // end -- which happens for chunks sitting at the window edge, since + // the window is a fixed 900000 bytes and does not align to chunk + // boundaries. Truncation is counted rather than asserted away. + assert!( + chunk.records.len() <= chunk.chunk.offsets.len(), + "level{n}: chunk at {} resolved MORE records than offsets", + chunk.chunk.offset + ); + if chunk.records.len() < chunk.chunk.offsets.len() { + truncated += 1; + } + + // The invariant that makes this a container and not a coincidence. + // Accumulated rather than asserted per chunk: individual chunks + // range from ~7% to ~21% constant positions, so a per-chunk + // threshold would be asserting a property the format does not have. + let constant = constant_positions(&chunk.records); + constant_total += constant; + stride_total += chunk.chunk.stride; + frac.push(constant as f64 / chunk.chunk.stride as f64); + } + } + + assert!(levels >= 20, "only {levels} levels had chunks"); + assert!(chunks > 500, "only {chunks} chunks resolved"); + assert!(records > 5_000, "only {records} records resolved"); + assert!( + strides.len() > 5, + "stride should vary per chunk, saw {}", + strides.len() + ); + // Chance would give a constant position a probability of 1/256^n, so even + // 5% across tens of thousands of records is far above noise. + let mean_frac = constant_total as f64 / stride_total as f64; + assert!( + mean_frac > 0.05, + "constant positions {constant_total}/{stride_total} = {mean_frac:.3} is near chance" + ); + eprintln!( + "chunks: {chunks} over {levels} levels, {records} records, {} distinct strides, \ + constant {constant_total}/{stride_total} ({:.1}%), per-chunk min {:.1}% max {:.1}%, \ + {truncated} chunks truncated at the window edge", + strides.len(), + mean_frac * 100.0, + frac.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0, + frac.iter().cloned().fold(0.0, f64::max) * 100.0, + ); +} + +/// Byte positions that hold the same value in every supplied record. +fn constant_positions(records: &[&[u8]]) -> usize { + let Some(first) = records.first() else { + return 0; + }; + (0..first.len()) + .filter(|&i| records.iter().all(|r| r[i] == first[i])) + .count() +} diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs new file mode 100644 index 0000000..32237e2 --- /dev/null +++ b/crates/gc-formats/src/chunks.rs @@ -0,0 +1,196 @@ +//! The post-shrub chunk container. +//! +//! Spec: `docs/formats/core-data-regions.md`. +//! +//! This is the container that fills the region after the last shrub class body +//! and before the constant trailer. It is *not* established to be moby_anim; +//! a 49 x 368-byte table inside it turned out to be byte-identical on all 27 +//! levels, which is evidence against that identification rather than for it. +//! +//! The framing is established. Each chunk is: +//! +//! ```text +//! +0x00 16 bytes, last byte 0xff (opaque, differs per chunk) +//! +0x10 10 zero bytes +//! +0x1a N tagged offsets, top byte 0xf0 +//! ``` +//! +//! and each tagged offset is `0xF0000000 | low24`. Resolving a record at +//! `chunk_start + low` yields records that share a field layout: 17-21% of +//! byte positions are constant across a chunk's own records, on every level. +//! That is what distinguishes this from a coincidental byte pattern. +//! +//! The stride between records is a property of the chunk, not of the region. +//! Level 0 alone contains chunks at 148, 340, 368, 288, 320, 236, 272, 384, +//! 116, 160, 250, 580 and 576. Because the strides differ, a region-wide +//! autocorrelation averages them to the noise floor -- which is why an +//! earlier claim of a "368-byte period" for the region as a whole was wrong. +//! 368 is the stride of one chunk. +//! +//! Fields are *not* named. The 16-byte header is opaque, the `0xf0` tag is +//! unexplained, and it is not known which fields are an index, a length and +//! an offset. What this parser provides is bounds-checked chunk and record +//! extraction, which is what the format analysis actually established. + +use crate::buf::{Buf, ErrorKind, Result}; + +/// Bytes in a chunk's opaque leading header. +pub const CHUNK_HEADER_BYTES: usize = 16; + +/// Zero bytes between the header and the offset table. +pub const CHUNK_GAP_BYTES: usize = 10; + +/// The value every tagged offset's top byte carries, on all 27 levels. +pub const CHUNK_TAG: u8 = 0xf0; + +/// A chunk header plus the records its offsets address. +/// +/// `stride` is the distance between the first two resolved offsets. Every +/// offset after the first is `stride` from its predecessor within a few bytes +/// on the measured levels, but the parser takes each offset literally rather +/// than assuming a uniform stride, because the deltas are not exactly constant +/// (e.g. 340, 340, 336, 340 on level 0). +pub struct Chunk { + /// The 16 opaque header bytes. + pub header: [u8; CHUNK_HEADER_BYTES], + /// Offset of this chunk within the region it was parsed from. + pub offset: usize, + /// The low 24 bits of each tagged offset, in file order. + pub offsets: Vec, + /// Record stride, from the first two offsets. Zero if fewer than two. + pub stride: usize, +} + +/// One chunk, with its records already bounds-checked. +pub struct ResolvedChunk<'a> { + /// The offset table. + pub chunk: Chunk, + /// Record bytes, one per entry in `chunk.offsets`. + pub records: Vec<&'a [u8]>, +} + +/// True if `offset` looks like the start of a chunk. +/// +/// The signature is the header's last byte being `0xff`, the 10-byte gap +/// being zero, and at least `min_offsets` following words carrying the `0xf0` +/// tag. +pub fn is_chunk_start(region: &[u8], offset: usize, min_offsets: usize) -> bool { + let hdr_end = offset + CHUNK_HEADER_BYTES; + if hdr_end + CHUNK_GAP_BYTES + min_offsets * 4 > region.len() { + return false; + } + if region[hdr_end - 1] != 0xff { + return false; + } + if region[hdr_end..hdr_end + CHUNK_GAP_BYTES] != [0u8; CHUNK_GAP_BYTES] { + return false; + } + (0..min_offsets).all(|k| { + let at = hdr_end + CHUNK_GAP_BYTES + k * 4; + read_u32(region, at) >> 24 == CHUNK_TAG as u32 + }) +} + +/// Little-endian, matching `Buf::u32` and the PS2. +fn read_u32(bytes: &[u8], at: usize) -> u32 { + u32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) +} + +/// Parse the chunk at `offset`, without resolving records. +/// +/// Returns `None` if `offset` does not carry the chunk signature. +pub fn parse_chunk(region: &[u8], offset: usize) -> Result> { + if !is_chunk_start(region, offset, 4) { + return Ok(None); + } + let mut b = Buf::new(region); + b.seek(offset); + + let header_bytes = b.read_bytes(CHUNK_HEADER_BYTES, "chunk header")?; + let mut header = [0u8; CHUNK_HEADER_BYTES]; + header.copy_from_slice(header_bytes); + + b.skip(CHUNK_GAP_BYTES, "chunk gap")?; + + // The offset table runs until the first word without the tag. + let table_start = b.pos(); + let mut offsets = Vec::new(); + while b.pos() + 4 <= region.len() { + let v = b.u32("chunk offset")?; + if v >> 24 != CHUNK_TAG as u32 { + b.seek(table_start); + break; + } + offsets.push(v & 0x00ff_ffff); + } + + let stride = if offsets.len() >= 2 { + (offsets[1] - offsets[0]) as usize + } else { + 0 + }; + + Ok(Some(Chunk { + header, + offset, + offsets, + stride, + })) +} + +/// Resolve every record a chunk addresses, bounds-checked against `region`. +/// +/// Each record is `region[chunk.offset + low .. + stride]`. A record that +/// would run past the end of the region is dropped rather than clamped, so the +/// returned length may be shorter than `chunk.offsets` -- on the measured +/// levels it is not, but a parser that silently clamps would hide a malformed +/// chunk rather than report it. +pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result> { + let mut records = Vec::with_capacity(chunk.offsets.len()); + for &low in &chunk.offsets { + let start = chunk + .offset + .checked_add(low as usize) + .ok_or(crate::buf::Error { + offset: chunk.offset, + context: "chunk record", + kind: ErrorKind::Invalid("chunk offset overflows usize"), + })?; + if start + chunk.stride > region.len() { + continue; + } + records.push(®ion[start..start + chunk.stride]); + } + Ok(ResolvedChunk { + chunk: Chunk { + header: chunk.header, + offset: chunk.offset, + offsets: chunk.offsets.clone(), + stride: chunk.stride, + }, + records, + }) +} + +/// Walk every chunk in `region`, resolving records. +/// +/// Returns chunks that carried the signature but whose records did not all +/// resolve, alongside the ones that did. Callers that care about coverage +/// should check both. +pub fn parse_all_chunks(region: &[u8]) -> Result>> { + let mut out = Vec::new(); + let mut at = 0usize; + while at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + 16 <= region.len() { + let Some(chunk) = parse_chunk(region, at)? else { + at += 1; + continue; + }; + // Resume past this chunk's header and offset table so a chunk's own + // record data is not rescanned as further chunk headers. + let consumed = CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + chunk.offsets.len() * 4; + let resolved = resolve(&chunk, region)?; + out.push(resolved); + at += consumed.max(1); + } + Ok(out) +} diff --git a/crates/gc-formats/src/lib.rs b/crates/gc-formats/src/lib.rs index 772fa87..d4341ef 100644 --- a/crates/gc-formats/src/lib.rs +++ b/crates/gc-formats/src/lib.rs @@ -1,4 +1,5 @@ pub mod buf; +pub mod chunks; pub mod collision; pub mod global_wad; pub mod iso9660;