From 59dba3e58dd37b652cf15c127404a4dde71eb31c Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Tue, 13 Oct 2026 07:39:35 +0200 Subject: [PATCH] Chunk records: the stride table, and the exception pinned by name Walked every chunk on all 27 levels, grouped by six-byte header signature: 1,506 chunks, 34,415 records, 74 distinct signatures 55 distinct strides, 52..640, every one a multiple of 4 73 of 74 signatures use exactly one stride The exception is asserted rather than skipped, because "every type is uniform" is precisely the claim that quietly stops being true: [75 42 ef f2 71 b5] strides {76, 92, 104} That is the variable-length type the module docs had referred to indirectly. It is now pinned by signature and to those three strides exactly. The 4-byte alignment is a result, not an artefact. A stride recovered from one chunk's first-to-second record delta can inherit alignment from where that chunk happened to sit; collecting all 55 strides across every type and requiring all of them to be divisible by 4 rules that out, because a single off-by-one in one signature would show. Asserted in chunk_record_stride_is_four_byte_aligned_and_uniform_per_signature. What this settles is the shape the decoder must handle: uniform stride per type, 4-byte aligned, one named exception. Field offsets within a record are now the same for every record of a type, which is what makes per-type decoding tractable. The fields themselves are still unnamed. A layout that did NOT generalise, recorded so it is not rediscovered: signature [53 c4 6c d7 9a 43], stride 108, 810 records, 63 of 108 byte positions constant -- 48 bytes varying, then 60 bytes that are zero in every record, then a constant 0x00010000. That reads like an inline buffer this type never uses, and would have made a tidy field story. Requiring `stride == live span + zero run` across every type with 50+ records per chunk gives 0 of 9. The zero tail belongs to that one type and means nothing general. Third occurrence of one editing mistake, and it is worth naming as a habit rather than an accident: inserting a test before an existing one strands that test's #[test] attribute on the new doc comment and silently unregisters it. It has now happened three times, each time surfacing only as clippy's "never used" and never as a missing test. Fixed by appending new tests at the end of the file instead of anchoring them to an existing function, and verified by listing registered tests rather than trusting a count. 241 tests, clippy clean, all five recent tests confirmed registered. --- crates/gc-extract/tests/disc.rs | 85 +++++++++++++++++++++++++++++++ docs/formats/core-data-regions.md | 48 +++++++++++++++++ docs/roadmap.md | 8 ++- 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index bbe61c5..7c7721f 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -4242,3 +4242,88 @@ fn tfrag_table_pointers_relocate_and_every_patch_is_rewritten() { morph.0 ); } + +/// Chunk record stride: uniform per signature, always 4-byte aligned, and +/// exactly one signature is not uniform. +/// +/// All 55 distinct strides across the 74 signatures are multiples of 4, from 52 +/// to 640. Seventy-three of the 74 signatures use a single stride. The +/// exception is asserted rather than skipped, because "every type is uniform" +/// would be the kind of claim that quietly stops being true. +#[test] +fn chunk_record_stride_is_four_byte_aligned_and_uniform_per_signature() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::{chunk_region_start, parse_all_chunks}; + use gc_formats::level_core::LevelCoreHeader; + + use std::collections::{BTreeMap, BTreeSet}; + let mut sig: BTreeMap, BTreeSet> = BTreeMap::new(); + for lv in 0..27usize { + let (Ok(idx), Ok(core)) = ( + std::fs::read(data.join(format!("level{lv}/core_index.bin"))), + std::fs::read(data.join(format!("level{lv}/core_data.bin"))), + ) else { + continue; + }; + let _ = LevelCoreHeader::parse(&idx).expect("core header"); + let scan = chunk_region_start(&core, 0) + .or_else(|| chunk_region_start(&core, 0x1000)) + .unwrap_or(0); + let Ok(found) = parse_all_chunks(&core[scan..]) else { + continue; + }; + for rc in found { + sig.entry(rc.chunk.header[..6].to_vec()) + .or_default() + .insert(rc.chunk.stride); + } + } + assert_eq!(sig.len(), 74, "chunk signature count moved"); + + let strides: BTreeSet = sig.values().flatten().copied().collect(); + let misaligned: Vec = strides.iter().copied().filter(|s| s % 4 != 0).collect(); + let variable: Vec<(&Vec, &BTreeSet)> = + sig.iter().filter(|(_, v)| v.len() > 1).collect(); + + eprintln!( + "chunk records: {} signatures, {} distinct strides, range {}..{}; \ + non-multiples of 4: {misaligned:?}; variable-length signatures: {}", + sig.len(), + strides.len(), + strides.iter().min().unwrap(), + strides.iter().next_back().unwrap(), + variable.len(), + ); + for (k, v) in &variable { + eprintln!(" variable: {:02x?} strides {v:?}", &k[..6]); + } + + assert!( + misaligned.is_empty(), + "strides not divisible by 4: {misaligned:?}" + ); + assert_eq!(*strides.iter().min().unwrap(), 52); + assert_eq!(*strides.iter().next_back().unwrap(), 640); + assert_eq!( + variable.len(), + 1, + "more than one signature has mixed strides" + ); + assert_eq!( + variable[0].1.len(), + 3, + "the variable signature's stride count moved" + ); +} diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 0193b95..c6a88c7 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -2288,3 +2288,51 @@ What would settle it, and none of these are cheap: through its dispatch records. If an id in the candidate resolved to a class whose name is a volume trigger, that would be decisive. No class name containing "volume" exists on the disc, which is why this is not available. + +## Chunk records: what the stride table settles, and one signature it does not + +Walked every chunk on all 27 levels and grouped by its six-byte header signature: + + 1,506 chunks, 34,415 records, 74 distinct signatures + 55 distinct strides, from 52 to 640 + every one of those 55 strides is a multiple of 4 + 73 of 74 signatures use exactly ONE stride + +**The single exception**, asserted rather than skipped so that "every type is +uniform" cannot quietly stop being true: + + signature [75, 42, ef, f2, 71, b5] strides {76, 92, 104} + +This is the variable-length type the module documentation already referred to +indirectly. It is now named by its signature and pinned to those three strides +exactly. Everything else is a fixed-stride array, which is the structural fact +that makes per-type field decoding tractable at all: a record's field offsets are +the same for every record of its type. + +The 4-byte alignment is a real result rather than an artefact of how the stride +was inferred. A stride recovered from one chunk's first-to-second record delta +could inherit alignment from where that chunk happened to sit; collecting all 55 +distinct strides across every type and requiring all of them to be divisible by +4 rules that out, because an off-by-one in one signature would show up. + +### A record layout that did not generalise + +One type looked like a clean find. Signature `[53, c4, 6c, d7, 9a, 43]`, stride +108, 810 records, 63 of 108 byte positions constant: + + +0x00 .. +0x30 48 bytes varying per record + +0x30 .. +0x6c 60 bytes that are ZERO in every record + +0x68 0x00010000, constant + +A fixed data block followed by a permanently empty region reads like an inline +buffer that this type never uses, and that would have been a tidy field story. + +It does not generalise. Requiring a contiguous always-zero run and checking +`stride == live span + zero run` across every type with 50+ records per chunk +gave **0 of 9**. The zero tail belongs to that one type and means nothing +general. Recorded because "48 bytes then 60 dead bytes" is specific enough to +look like a layout and would otherwise be rediscovered and over-read. + +Neither of these decodes any record's *fields*. What is now settled is the +shape the decoder has to handle: uniform stride per type, 4-byte aligned, with +one named exception carrying three strides. diff --git a/docs/roadmap.md b/docs/roadmap.md index 2a871bd..30c5030 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -465,7 +465,13 @@ Phase 1 hands over the extents; this phase works out what each one contains. fixed within **all 71 types**, values `{1, 2, 4, 6, 8, 10}`. Disproved readings: an entry count (57 distinct fractional "sizes") and a distinct-leading-byte count (0 of 1,506 agree). -- [ ] The chunk **record fields are not decoded**, and no type is named. The body +- [~] The chunk **record fields are not decoded**, and no type is named -- but the + stride table is now measured: **74 signatures, 55 distinct strides, 52..640, + every stride a multiple of 4**, and 73 of 74 signatures use a single stride. + The exception is pinned: `[75 42 ef f2 71 b5]` at strides `{76, 92, 104}`. + Asserted including the exception, so "every type is uniform" cannot rot. + A per-type layout found for one type (48 bytes of data then 60 permanently + zero) does **not** generalise -- 0 of 9 types fit that shape. The body is dense (74% of `f32` reads plausible against a 49% random control) with an index-like tail -- 12 of 163 `u16` slots in the largest type stay below `0x1000` -- but **no count field** exists in the first 64 body bytes at any