diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index ba3b8fe..a032f02 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -4414,3 +4414,98 @@ fn most_chunk_signatures_are_level_independent() { assert_eq!(shared, 65, "the shared/level-specific split moved"); assert_eq!(specific.len(), 9, "level-specific signature count moved"); } + +/// The signature `[75 42 ef f2 71 b5]` produces three chunks per level, and one +/// of the three has genuinely variable-length records. +/// +/// The parser takes a single stride per chunk. For the 92-byte chunk that is +/// wrong: its record offsets step by `92, 88, 84, 92, 92, 92, 84, 88`. Every +/// record *starts* correctly, because `resolve` uses the offset table, but each +/// is sliced at 92 bytes -- so the two 88-byte and three 84-byte records +/// over-read by 4 and 8 bytes and pick up the head of the next record. +/// +/// So "this signature has strides {76, 92, 104}" is a misdescription. It has +/// three chunk shapes, one of which is not fixed-stride at all. Asserted with +/// the exact delta sequences so a future single-stride assumption cannot survive +/// quietly. +#[test] +fn chunk_type_75_42_ef_has_one_variable_length_chunk_per_level() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::{chunk_region_start, parse_all_chunks}; + use gc_formats::level_core::LevelCoreHeader; + use std::collections::BTreeMap; + + let target: [u8; 6] = [0x75, 0x42, 0xef, 0xf2, 0x71, 0xb5]; + let mut shapes: BTreeMap, usize> = BTreeMap::new(); + let mut mixed = 0usize; + let mut chunks = 0usize; + for lv in 0..27usize { + let (Ok(idx), Ok(core)) = ( + std::fs::read(data.join(format!("level{lv}/core_index.bin"))), + std::fs::read(data.join(format!("level{lv}/core_data.bin"))), + ) else { + continue; + }; + let _ = LevelCoreHeader::parse(&idx).expect("core header"); + let scan = chunk_region_start(&core, 0) + .or_else(|| chunk_region_start(&core, 0x1000)) + .unwrap_or(0); + let Ok(found) = parse_all_chunks(&core[scan..]) else { + continue; + }; + for rc in found { + if rc.chunk.header[..6] != target[..] { + continue; + } + chunks += 1; + let deltas: Vec = rc.chunk.offsets.windows(2).map(|w| w[1] - w[0]).collect(); + if !deltas.windows(2).all(|w| w[0] == w[1]) { + mixed += 1; + } + *shapes.entry(deltas).or_default() += 1; + } + } + + assert_eq!(chunks, 81, "chunk count for this signature moved"); + assert_eq!(mixed, 27, "the number of variable-length chunks moved"); + + let uniform76: Vec = vec![76; 8]; + let variable: Vec = vec![92, 88, 84, 92, 92, 92, 84, 88]; + let uniform104: Vec = vec![104; 8]; + assert_eq!( + shapes.get(&uniform76).copied(), + Some(27), + "the 76-byte chunk shape changed" + ); + assert_eq!( + shapes.get(&variable).copied(), + Some(27), + "the variable-length chunk's delta sequence changed" + ); + assert_eq!( + shapes.get(&uniform104).copied(), + Some(27), + "the 104-byte chunk shape changed" + ); + assert_eq!(shapes.len(), 3, "a fourth chunk shape appeared"); + + // the defect, stated as a measurement rather than a comment + let over = variable.iter().filter(|&&d| d < 92).count(); + eprintln!( + "chunk [75 42 ef]: 81 chunks; 27 variable-length, deltas {variable:?}; \ + {over} of 8 records shorter than the 92 the parser slices" + ); + assert_eq!(over, 4, "the count of short records changed"); +} diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 3b37d2a..b84c809 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -2387,3 +2387,52 @@ levels**, and zero byte positions vary across levels. Had the u32 distinct counts been the only measurement taken, the conclusion would have been "one value per level" -- which is the opposite of the truth, and would have described a per-level parameter table that does not exist. + +## A parser defect: one chunk type is not fixed-stride + +`[75 42 ef f2 71 b5]` -- the signature that is both level-specific and +variable-length -- produces **three chunks per level**, 81 in total. Their record +offsets step like this: + + stride 76 deltas [76, 76, 76, 76, 76, 76, 76, 76] uniform + stride 92 deltas [92, 88, 84, 92, 92, 92, 84, 88] NOT uniform + stride 104 deltas [104, 104, 104, 104, 104, 104, ...] uniform + +Identical on all 27 levels -- 27 chunks of each shape, three shapes, no others. + +So the description "this signature has strides {76, 92, 104}" is a +**misdescription**. It has three chunk shapes, one of which is not fixed-stride +at all. The parser derives one `stride` per chunk and `resolve` slices every +record at that length. + +What that costs, precisely: + +* Records are read at the **offset table's** positions, so every record *starts* + correctly. There is no misalignment. +* But each is sliced at **92** bytes. Two of the eight are 88 bytes and two are + 84, so **four records over-read by 4 and 8 bytes** and carry the head of the + next record inside them. + +Bounds checking holds -- the slices are inside the region -- so this is a +correctness bug in the data, not a safety one. It is exactly the kind that +produces plausible-looking fields that are offset by four bytes, which is how it +would have gone unnoticed had the stride table not been collected per signature. + +It also explains why this signature was flagged earlier as breaking the 8-byte +alignment rule: its records genuinely are not a uniform multiple. + +**This is a defect in the parser, not in the data**, and the right fix is for +`resolve` to use the offset deltas rather than a single stride for a chunk whose +deltas are not uniform. Recorded here first, because the assertion in +`chunk_type_75_ef_has_one_variable_length_chunk_per_level` pins the delta +sequences -- if the data ever does become uniform, the test fails and asks for +the model to be revisited rather than the other way round. + +### A near-miss that nearly hid this + +The leading `u16` of these records appeared to step by exactly `0x50` (80) for +all nine records of the 76-byte chunk -- a tidy, meaningful-looking pattern that +would have been easy to report as "records are indexed by 80". Reading the +records at their real offsets shows the lead values are `b1a5, a529, 047e, +a529, 04f0, a529, 0562, a529, 05d4` -- no arithmetic progression at all. The +apparent 80-byte step came from comparing the wrong bytes. diff --git a/docs/roadmap.md b/docs/roadmap.md index 7924a07..7e43d77 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -477,7 +477,14 @@ Phase 1 hands over the extents; this phase works out what each one contains. rather than level content -- asserted. Consequence for method: the 65 cannot be decoded by comparing levels, because there is nothing to compare. `[75 42 ef]` is both level-specific *and* the three-stride type, so it is - where per-type decoding should start. The body + where per-type decoding should start -- and decoding it found a **parser + defect**: it makes three chunks per level, one of which has record offsets + stepping `92, 88, 84, 92, 92, 92, 84, 88` on every level. `resolve` slices + every record at the chunk's single stride, so four of the eight records + over-read by 4 and 8 bytes and include the next record's head. Records + still *start* correctly and bounds checking holds, so it is a data-correctness + bug, not a safety one. The "strides {76, 92, 104}" description is a + misdescription of three chunk shapes, one of them variable. The body is dense (74% of `f32` reads plausible against a 49% random control) with an index-like tail -- 12 of 163 `u16` slots in the largest type stay below `0x1000` -- but **no count field** exists in the first 64 body bytes at any