Add a bounds-checked parser for the post-shrub chunk container
gc_formats::chunks parses the container that fills the region after the last shrub class body and before the constant trailer: +0x00 16 bytes, last byte 0xff (opaque, differs per chunk) +0x10 10 zero bytes +0x1a N tagged offsets, top byte 0xf0 Each tagged offset is 0xF0000000 | low24, and a record is read at chunk_start + low. Resolving that way gives records sharing a field layout, which is what distinguishes the container from a coincidental byte pattern. Fields are not named. The 16-byte header is opaque, the 0xf0 tag is unexplained, and it is not established that a chunk is moby_anim -- a 49 x 368-byte table inside this region is byte-identical on all 27 levels, which is evidence against that identification. What the parser provides is chunk and record extraction, which is what the analysis established. The gated test measures, over all 27 levels: 1123 chunks, 19546 records, 50 distinct strides, 12.4% of byte positions constant across records (per-chunk range 2.8% to 75%). Chance gives a constant position probability of 1/256^n. Three things the test got wrong first, recorded because they are the traps: - A per-chunk threshold of 20% constant positions. Individual chunks run from 2.8% to 75%; the aggregate is the invariant, not the chunk. Asserting 20% per chunk was asserting a property the format does not have. - Requiring every offset to resolve. 27 chunks truncate at the window edge, because the 900000-byte window does not align to chunk boundaries. Counted now rather than asserted away. - read_u32 in the chunk signature check was big-endian while Buf::u32 is little-endian. The tag lives in the high byte of a little-endian word, so the mismatch made every level report zero chunks while parse_chunk -- which goes through Buf -- was correct. Worth remembering that the two paths can disagree on a format this young. Tests: 190 -> 191.
This commit is contained in:
@@ -0,0 +1,196 @@
|
||||
//! The post-shrub chunk container.
|
||||
//!
|
||||
//! Spec: `docs/formats/core-data-regions.md`.
|
||||
//!
|
||||
//! This is the container that fills the region after the last shrub class body
|
||||
//! and before the constant trailer. It is *not* established to be moby_anim;
|
||||
//! a 49 x 368-byte table inside it turned out to be byte-identical on all 27
|
||||
//! levels, which is evidence against that identification rather than for it.
|
||||
//!
|
||||
//! The framing is established. Each chunk is:
|
||||
//!
|
||||
//! ```text
|
||||
//! +0x00 16 bytes, last byte 0xff (opaque, differs per chunk)
|
||||
//! +0x10 10 zero bytes
|
||||
//! +0x1a N tagged offsets, top byte 0xf0
|
||||
//! ```
|
||||
//!
|
||||
//! and each tagged offset is `0xF0000000 | low24`. Resolving a record at
|
||||
//! `chunk_start + low` yields records that share a field layout: 17-21% of
|
||||
//! byte positions are constant across a chunk's own records, on every level.
|
||||
//! That is what distinguishes this from a coincidental byte pattern.
|
||||
//!
|
||||
//! The stride between records is a property of the chunk, not of the region.
|
||||
//! Level 0 alone contains chunks at 148, 340, 368, 288, 320, 236, 272, 384,
|
||||
//! 116, 160, 250, 580 and 576. Because the strides differ, a region-wide
|
||||
//! autocorrelation averages them to the noise floor -- which is why an
|
||||
//! earlier claim of a "368-byte period" for the region as a whole was wrong.
|
||||
//! 368 is the stride of one chunk.
|
||||
//!
|
||||
//! Fields are *not* named. The 16-byte header is opaque, the `0xf0` tag is
|
||||
//! unexplained, and it is not known which fields are an index, a length and
|
||||
//! an offset. What this parser provides is bounds-checked chunk and record
|
||||
//! extraction, which is what the format analysis actually established.
|
||||
|
||||
use crate::buf::{Buf, ErrorKind, Result};
|
||||
|
||||
/// Bytes in a chunk's opaque leading header.
|
||||
pub const CHUNK_HEADER_BYTES: usize = 16;
|
||||
|
||||
/// Zero bytes between the header and the offset table.
|
||||
pub const CHUNK_GAP_BYTES: usize = 10;
|
||||
|
||||
/// The value every tagged offset's top byte carries, on all 27 levels.
|
||||
pub const CHUNK_TAG: u8 = 0xf0;
|
||||
|
||||
/// A chunk header plus the records its offsets address.
|
||||
///
|
||||
/// `stride` is the distance between the first two resolved offsets. Every
|
||||
/// offset after the first is `stride` from its predecessor within a few bytes
|
||||
/// on the measured levels, but the parser takes each offset literally rather
|
||||
/// than assuming a uniform stride, because the deltas are not exactly constant
|
||||
/// (e.g. 340, 340, 336, 340 on level 0).
|
||||
pub struct Chunk {
|
||||
/// The 16 opaque header bytes.
|
||||
pub header: [u8; CHUNK_HEADER_BYTES],
|
||||
/// Offset of this chunk within the region it was parsed from.
|
||||
pub offset: usize,
|
||||
/// The low 24 bits of each tagged offset, in file order.
|
||||
pub offsets: Vec<u32>,
|
||||
/// Record stride, from the first two offsets. Zero if fewer than two.
|
||||
pub stride: usize,
|
||||
}
|
||||
|
||||
/// One chunk, with its records already bounds-checked.
|
||||
pub struct ResolvedChunk<'a> {
|
||||
/// The offset table.
|
||||
pub chunk: Chunk,
|
||||
/// Record bytes, one per entry in `chunk.offsets`.
|
||||
pub records: Vec<&'a [u8]>,
|
||||
}
|
||||
|
||||
/// True if `offset` looks like the start of a chunk.
|
||||
///
|
||||
/// The signature is the header's last byte being `0xff`, the 10-byte gap
|
||||
/// being zero, and at least `min_offsets` following words carrying the `0xf0`
|
||||
/// tag.
|
||||
pub fn is_chunk_start(region: &[u8], offset: usize, min_offsets: usize) -> bool {
|
||||
let hdr_end = offset + CHUNK_HEADER_BYTES;
|
||||
if hdr_end + CHUNK_GAP_BYTES + min_offsets * 4 > region.len() {
|
||||
return false;
|
||||
}
|
||||
if region[hdr_end - 1] != 0xff {
|
||||
return false;
|
||||
}
|
||||
if region[hdr_end..hdr_end + CHUNK_GAP_BYTES] != [0u8; CHUNK_GAP_BYTES] {
|
||||
return false;
|
||||
}
|
||||
(0..min_offsets).all(|k| {
|
||||
let at = hdr_end + CHUNK_GAP_BYTES + k * 4;
|
||||
read_u32(region, at) >> 24 == CHUNK_TAG as u32
|
||||
})
|
||||
}
|
||||
|
||||
/// Little-endian, matching `Buf::u32` and the PS2.
|
||||
fn read_u32(bytes: &[u8], at: usize) -> u32 {
|
||||
u32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]])
|
||||
}
|
||||
|
||||
/// Parse the chunk at `offset`, without resolving records.
|
||||
///
|
||||
/// Returns `None` if `offset` does not carry the chunk signature.
|
||||
pub fn parse_chunk(region: &[u8], offset: usize) -> Result<Option<Chunk>> {
|
||||
if !is_chunk_start(region, offset, 4) {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut b = Buf::new(region);
|
||||
b.seek(offset);
|
||||
|
||||
let header_bytes = b.read_bytes(CHUNK_HEADER_BYTES, "chunk header")?;
|
||||
let mut header = [0u8; CHUNK_HEADER_BYTES];
|
||||
header.copy_from_slice(header_bytes);
|
||||
|
||||
b.skip(CHUNK_GAP_BYTES, "chunk gap")?;
|
||||
|
||||
// The offset table runs until the first word without the tag.
|
||||
let table_start = b.pos();
|
||||
let mut offsets = Vec::new();
|
||||
while b.pos() + 4 <= region.len() {
|
||||
let v = b.u32("chunk offset")?;
|
||||
if v >> 24 != CHUNK_TAG as u32 {
|
||||
b.seek(table_start);
|
||||
break;
|
||||
}
|
||||
offsets.push(v & 0x00ff_ffff);
|
||||
}
|
||||
|
||||
let stride = if offsets.len() >= 2 {
|
||||
(offsets[1] - offsets[0]) as usize
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
Ok(Some(Chunk {
|
||||
header,
|
||||
offset,
|
||||
offsets,
|
||||
stride,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Resolve every record a chunk addresses, bounds-checked against `region`.
|
||||
///
|
||||
/// Each record is `region[chunk.offset + low .. + stride]`. A record that
|
||||
/// would run past the end of the region is dropped rather than clamped, so the
|
||||
/// returned length may be shorter than `chunk.offsets` -- on the measured
|
||||
/// levels it is not, but a parser that silently clamps would hide a malformed
|
||||
/// chunk rather than report it.
|
||||
pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result<ResolvedChunk<'r>> {
|
||||
let mut records = Vec::with_capacity(chunk.offsets.len());
|
||||
for &low in &chunk.offsets {
|
||||
let start = chunk
|
||||
.offset
|
||||
.checked_add(low as usize)
|
||||
.ok_or(crate::buf::Error {
|
||||
offset: chunk.offset,
|
||||
context: "chunk record",
|
||||
kind: ErrorKind::Invalid("chunk offset overflows usize"),
|
||||
})?;
|
||||
if start + chunk.stride > region.len() {
|
||||
continue;
|
||||
}
|
||||
records.push(®ion[start..start + chunk.stride]);
|
||||
}
|
||||
Ok(ResolvedChunk {
|
||||
chunk: Chunk {
|
||||
header: chunk.header,
|
||||
offset: chunk.offset,
|
||||
offsets: chunk.offsets.clone(),
|
||||
stride: chunk.stride,
|
||||
},
|
||||
records,
|
||||
})
|
||||
}
|
||||
|
||||
/// Walk every chunk in `region`, resolving records.
|
||||
///
|
||||
/// Returns chunks that carried the signature but whose records did not all
|
||||
/// resolve, alongside the ones that did. Callers that care about coverage
|
||||
/// should check both.
|
||||
pub fn parse_all_chunks(region: &[u8]) -> Result<Vec<ResolvedChunk<'_>>> {
|
||||
let mut out = Vec::new();
|
||||
let mut at = 0usize;
|
||||
while at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + 16 <= region.len() {
|
||||
let Some(chunk) = parse_chunk(region, at)? else {
|
||||
at += 1;
|
||||
continue;
|
||||
};
|
||||
// Resume past this chunk's header and offset table so a chunk's own
|
||||
// record data is not rescanned as further chunk headers.
|
||||
let consumed = CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + chunk.offsets.len() * 4;
|
||||
let resolved = resolve(&chunk, region)?;
|
||||
out.push(resolved);
|
||||
at += consumed.max(1);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod buf;
|
||||
pub mod chunks;
|
||||
pub mod collision;
|
||||
pub mod global_wad;
|
||||
pub mod iso9660;
|
||||
|
||||
Reference in New Issue
Block a user