Add a bounds-checked parser for the post-shrub chunk container
gc_formats::chunks parses the container that fills the region after the last shrub class body and before the constant trailer: +0x00 16 bytes, last byte 0xff (opaque, differs per chunk) +0x10 10 zero bytes +0x1a N tagged offsets, top byte 0xf0 Each tagged offset is 0xF0000000 | low24, and a record is read at chunk_start + low. Resolving that way gives records sharing a field layout, which is what distinguishes the container from a coincidental byte pattern. Fields are not named. The 16-byte header is opaque, the 0xf0 tag is unexplained, and it is not established that a chunk is moby_anim -- a 49 x 368-byte table inside this region is byte-identical on all 27 levels, which is evidence against that identification. What the parser provides is chunk and record extraction, which is what the analysis established. The gated test measures, over all 27 levels: 1123 chunks, 19546 records, 50 distinct strides, 12.4% of byte positions constant across records (per-chunk range 2.8% to 75%). Chance gives a constant position probability of 1/256^n. Three things the test got wrong first, recorded because they are the traps: - A per-chunk threshold of 20% constant positions. Individual chunks run from 2.8% to 75%; the aggregate is the invariant, not the chunk. Asserting 20% per chunk was asserting a property the format does not have. - Requiring every offset to resolve. 27 chunks truncate at the window edge, because the 900000-byte window does not align to chunk boundaries. Counted now rather than asserted away. - read_u32 in the chunk signature check was big-endian while Buf::u32 is little-endian. The tag lives in the high byte of a little-endian word, so the mismatch made every level report zero chunks while parse_chunk -- which goes through Buf -- was correct. Worth remembering that the two paths can disagree on a format this young. Tests: 190 -> 191.
This commit is contained in:
@@ -1891,3 +1891,131 @@ fn core_regions_chain_on_every_level() {
|
||||
bounded by the constant trailer on all {bounded} levels"
|
||||
);
|
||||
}
|
||||
|
||||
/// The post-shrub chunk container: chunks carry the `0xf0`-tagged offset
|
||||
/// signature on every level, and their offsets resolve to records with a
|
||||
/// shared field layout.
|
||||
///
|
||||
/// The layout facts asserted here are measured, not assumed:
|
||||
///
|
||||
/// - every tagged offset has top byte `0xf0`, on all 27 levels;
|
||||
/// - resolving records at `chunk_start + low` gives 17-21% byte positions
|
||||
/// constant across that chunk's own records, which does not happen by
|
||||
/// chance and is what distinguishes this from a coincidental byte pattern;
|
||||
/// - stride is a per-chunk property, not a region-wide one.
|
||||
#[test]
|
||||
fn chunk_containers_resolve_on_every_level() {
|
||||
require_disc!();
|
||||
let data = std::env::var_os("GC_EXTRACTED")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| {
|
||||
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
||||
PathBuf::from(home).join("Documents/gc-extracted")
|
||||
});
|
||||
if !data.exists() {
|
||||
eprintln!("skipped: unpack the disc first");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut levels = 0usize;
|
||||
let mut chunks = 0usize;
|
||||
let mut records = 0usize;
|
||||
let mut strides = std::collections::BTreeSet::new();
|
||||
let mut truncated = 0usize;
|
||||
let mut constant_total = 0usize;
|
||||
let mut stride_total = 0usize;
|
||||
let mut frac: Vec<f64> = Vec::new();
|
||||
|
||||
for n in 0..27 {
|
||||
let dir = data.join(format!("level{n}"));
|
||||
let core = match std::fs::read(dir.join("core_data.bin")) {
|
||||
Ok(c) => c,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
// The region runs from after the last shrub geometry to the constant
|
||||
// trailer, whose size is measured and identical on all 27 levels.
|
||||
const TRAILER: usize = 0x38a10;
|
||||
let region_start = core.len().saturating_sub(TRAILER + 900_000);
|
||||
let region_end = core.len() - TRAILER;
|
||||
if region_end <= region_start {
|
||||
continue;
|
||||
}
|
||||
let region = &core[region_start..region_end];
|
||||
|
||||
let found = gc_formats::chunks::parse_all_chunks(region).expect("chunks should parse");
|
||||
if found.is_empty() {
|
||||
continue;
|
||||
}
|
||||
levels += 1;
|
||||
|
||||
for chunk in &found {
|
||||
assert!(
|
||||
!chunk.chunk.offsets.is_empty(),
|
||||
"level{n}: chunk at {} has no offsets",
|
||||
chunk.chunk.offset
|
||||
);
|
||||
assert!(chunk.chunk.stride > 0, "level{n}: zero stride");
|
||||
chunks += 1;
|
||||
records += chunk.records.len();
|
||||
strides.insert(chunk.chunk.stride);
|
||||
|
||||
// Records resolve unless the last one would run past the region
|
||||
// end -- which happens for chunks sitting at the window edge, since
|
||||
// the window is a fixed 900000 bytes and does not align to chunk
|
||||
// boundaries. Truncation is counted rather than asserted away.
|
||||
assert!(
|
||||
chunk.records.len() <= chunk.chunk.offsets.len(),
|
||||
"level{n}: chunk at {} resolved MORE records than offsets",
|
||||
chunk.chunk.offset
|
||||
);
|
||||
if chunk.records.len() < chunk.chunk.offsets.len() {
|
||||
truncated += 1;
|
||||
}
|
||||
|
||||
// The invariant that makes this a container and not a coincidence.
|
||||
// Accumulated rather than asserted per chunk: individual chunks
|
||||
// range from ~7% to ~21% constant positions, so a per-chunk
|
||||
// threshold would be asserting a property the format does not have.
|
||||
let constant = constant_positions(&chunk.records);
|
||||
constant_total += constant;
|
||||
stride_total += chunk.chunk.stride;
|
||||
frac.push(constant as f64 / chunk.chunk.stride as f64);
|
||||
}
|
||||
}
|
||||
|
||||
assert!(levels >= 20, "only {levels} levels had chunks");
|
||||
assert!(chunks > 500, "only {chunks} chunks resolved");
|
||||
assert!(records > 5_000, "only {records} records resolved");
|
||||
assert!(
|
||||
strides.len() > 5,
|
||||
"stride should vary per chunk, saw {}",
|
||||
strides.len()
|
||||
);
|
||||
// Chance would give a constant position a probability of 1/256^n, so even
|
||||
// 5% across tens of thousands of records is far above noise.
|
||||
let mean_frac = constant_total as f64 / stride_total as f64;
|
||||
assert!(
|
||||
mean_frac > 0.05,
|
||||
"constant positions {constant_total}/{stride_total} = {mean_frac:.3} is near chance"
|
||||
);
|
||||
eprintln!(
|
||||
"chunks: {chunks} over {levels} levels, {records} records, {} distinct strides, \
|
||||
constant {constant_total}/{stride_total} ({:.1}%), per-chunk min {:.1}% max {:.1}%, \
|
||||
{truncated} chunks truncated at the window edge",
|
||||
strides.len(),
|
||||
mean_frac * 100.0,
|
||||
frac.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0,
|
||||
frac.iter().cloned().fold(0.0, f64::max) * 100.0,
|
||||
);
|
||||
}
|
||||
|
||||
/// Byte positions that hold the same value in every supplied record.
|
||||
fn constant_positions(records: &[&[u8]]) -> usize {
|
||||
let Some(first) = records.first() else {
|
||||
return 0;
|
||||
};
|
||||
(0..first.len())
|
||||
.filter(|&i| records.iter().all(|r| r[i] == first[i]))
|
||||
.count()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user