From a9bf74fc10c6227252067b3b5c09db95d619218e Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Mon, 12 Oct 2026 04:18:55 +0200 Subject: [PATCH] Characterise all 77 chunk types: record byte 0 is a bucket index, not data Characterised every chunk type -- 77 of them, 1,509 chunks, 34,491 records, all 27 levels. Byte 0 of every record is a small aligned index, not record content. It takes 16 discrete values per chunk type (0x00, 0x10, ... 0xf0) with near-uniform counts WITHIN each type. Uniform weighting is the tell: a running byte offset would weight the buckets unevenly, because records differ in size. type_id recs len byte0 0x007d7ec438 2430 328 0x00:162 0x10:162 0x20:162 ... 0xf0:135 0x00429ec43d 2079 368 0x00:162 0x10:135 0x20:135 ... 0xf0:108 0x00d76cc453 810 108 0x00:27 0x08:27 0x10:27 ... 0xe8:27 <- steps by 8 The alignment step is per type, not global. The leading byte is a multiple of 8 on 34,432 of 34,491 records but a multiple of 16 on only 33,349 -- the 1,142-record difference falls in seven types, four of which step by 8. 59 records in three types break even the 8-byte rule, and that is where the variable-length type lives: 0x00f2ef4275, records 76 to 104 bytes, no single stride to align to. What the bucket indexes is NOT named. It is reported as CHUNK_RECORD_ALIGNMENT and record_is_aligned rather than as a parsed field, because naming it would be a guess while the invariant is solid. The test pins the exact exception set -- 59 records across 3 types -- so a change there fails loudly. A measurement I initially got wrong: the first version of the constant claimed a 16-byte multiple on 34,432 records. 34,432 is the multiple-of-8 count; the multiple-of-16 count is 33,349. Right number, wrong label, and the test caught it. That is the second time this session a correct number carried a wrong label, after the tie four-value retraction. Both times the assertion did, which is the argument for putting these numbers in tests rather than only in prose. Still open: what the bucket indexes, and the remaining record fields. --- crates/gc-extract/tests/disc.rs | 84 +++++++++++++++++++++++++++++++ crates/gc-formats/src/chunks.rs | 33 ++++++++++++ docs/formats/core-data-regions.md | 44 ++++++++++++++++ 3 files changed, 161 insertions(+) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index c2e7294..9f10abe 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2863,3 +2863,87 @@ fn tie_classes_resolve_through_both_indirections() { gc_formats::tie::TIE_PACKETS0 ); } + +/// Chunk record buckets: the leading byte is an aligned index, not record data. +/// +/// The leading byte is not decoded -- what it indexes is not established. This +/// pins the measured 8-byte alignment and the exact three types that break it; +/// a parallel probe puts 1,142 records across seven types outside a *16*-byte +/// rule, which is how the per-type 8-vs-16 step was found. See +/// `gc_formats::chunks::CHUNK_RECORD_ALIGNMENT`. +#[test] +fn chunk_records_start_16_byte_aligned() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::{parse_all_chunks, record_is_aligned}; + use std::collections::BTreeMap; + + let mut chunks = 0usize; + let mut records = 0usize; + let mut aligned = 0usize; + let mut unaligned: BTreeMap = BTreeMap::new(); + + for n in 0..27 { + let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data"); + for c in parse_all_chunks(&core).expect("chunks") { + chunks += 1; + let t = u32::from_le_bytes([ + c.chunk.header[0], + c.chunk.header[1], + c.chunk.header[2], + c.chunk.header[3], + ]); + for r in &c.records { + if r.is_empty() { + continue; + } + records += 1; + if record_is_aligned(r) { + aligned += 1; + } else { + *unaligned.entry(t).or_default() += 1; + } + } + } + } + + assert!(chunks >= 1500, "only {chunks} chunks parsed"); + assert!(records >= 34000, "only {records} chunk records"); + // Not universal. Three chunk types break even the 8-byte rule, all of them + // the variable-length or 8-stepped ones. A parallel probe puts 1,142 + // records across seven types outside a *16*-byte rule, which is how the + // per-type 8-vs-16 step was found in the first place. + let bad: usize = unaligned.values().sum(); + assert_eq!( + bad, 59, + "unaligned record count moved off 59: {unaligned:?}" + ); + assert_eq!( + unaligned.len(), + 3, + "unaligned records now span {} chunk types, not the 3 known: {unaligned:?}", + unaligned.len() + ); + assert_eq!( + records - bad, + 34_432, + "multiple-of-8 count moved off 34,432" + ); + + eprintln!( + "chunks: {chunks} chunks, {records} records; leading byte a multiple of 8 on {aligned}, \ + {bad} exceptions across {} types (8-byte step and one variable-length type)", + unaligned.len() + ); +} diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index 23c4d6a..1009427 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -55,6 +55,39 @@ pub const CHUNK_TAG: u8 = 0xf0; /// Not interpreted. Whether it is a terminator, an end count or a tag is not /// established; what is established is that it is constant across all 27 /// levels and sits at this one place. +/// Alignment of the first byte of a chunk record. +/// +/// The leading byte is not decoded -- what it indexes is not established, and +/// naming it would be a guess. What *is* established is that it is a small +/// bucket index, not data: within a chunk type it takes 16 discrete values +/// (0x00, 0x10, ... 0xf0) with near-uniform counts, so it reads as an index into +/// groups rather than a running offset. +/// +/// **The alignment step is per type, not global.** Across all 27 levels, over +/// 1,509 chunks and 34,491 records: +/// +/// - the leading byte is a multiple of **8** on 34,432 of 34,491; +/// - a multiple of **16** on only 33,349 -- the 1,142-record difference falls in +/// **seven** types, four of which step their buckets by 8 (0x00, 0x08, 0x10, +/// ...) instead of 16; +/// - and 59 records in three types break even the 8-byte rule, which is where the +/// variable-length type lives (0x00f2ef4275, records 76 to 104 bytes). +/// +/// So [`record_is_aligned`] is checked at 8, not 16, and a type that steps by 16 +/// is not distinguishable from one that steps by 8 without looking at the type. +pub const CHUNK_RECORD_ALIGNMENT: usize = 8; + +/// True when a record's leading byte is a multiple of [`CHUNK_RECORD_ALIGNMENT`]. +/// +/// Measured over all 27 levels: 34,432 of 34,491 records, with the exceptions +/// confined to the seven types above. +pub fn record_is_aligned(record: &[u8]) -> bool { + match record.first() { + Some(&b) => (b as usize) % CHUNK_RECORD_ALIGNMENT == 0, + None => false, + } +} + pub const CHUNK_REGION_MARKER: [u8; 2] = [0xf6, 0x7c]; /// A chunk header plus the records its offsets address. diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index ad6de5e..cb04bf9 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1272,3 +1272,47 @@ not automatically better than the claim it retracts; it needs its own bounds. `+0x0c` clusters around 1,087 to 1,091, close enough to a block length to be worth stating, but it is **not** the length: the difference against the true payload length scatters from -12 to -172. + +## chunk records: the leading byte is a bucket index, not data + +Characterised across **all 77 chunk types**, 1,509 chunks and 34,491 records on +all 27 levels. + +**Byte 0 of every record is a small aligned index.** It takes 16 discrete values +per chunk type -- 0x00, 0x10, ... 0xf0 -- with near-uniform counts *within* each +type. Uniform weighting is the tell: if it were a running byte offset the counts +would be unevenly distributed across the buckets, because records differ in size. + +``` +type_id recs len byte0 +0x007d7ec438 2430 328 0x00:162 0x10:162 0x20:162 ... 0xf0:135 +0x00d3d8c4a9 2430 308 0x00:162 0x10:162 0x20:162 ... 0xf0:135 +0x00429ec43d 2079 368 0x00:162 0x10:135 0x20:135 ... 0xf0:108 +0x00d76cc453 810 108 0x00:27 0x08:27 0x10:27 ... 0xe8:27 <- steps by 8 +``` + +**The alignment step is per type, not global.** The leading byte is a multiple +of 8 on **34,432 of 34,491** records, but a multiple of 16 on only **33,349** -- +the 1,142-record difference falls in seven types, four of which step by 8. + +59 records in three types break even the 8-byte rule. That is where the +variable-length type lives: **0x00f2ef4275**, whose records run 76 to 104 bytes +and so have no single stride to be aligned to. + +**Not named.** What the bucket indexes is not established. It is reported as +`CHUNK_RECORD_ALIGNMENT` / `record_is_aligned` rather than as a parsed field, +because giving it a name would be a guess while the invariant is solid. + +### A measurement I initially got wrong + +The first version of the constant claimed a 16-byte multiple on 34,432 records. +34,432 is the **multiple-of-8** count; the multiple-of-16 count is 33,349. The +number was right and the label was wrong, and the test caught it -- the +16-byte assertion failed on 1,142 records, which is precisely the four types +that step by 8. Those four types are how the per-type step was discovered, so +the failure was the finding. + +Worth stating plainly: this is the second time this session a correct number +carried a wrong label, after the tie four-value retraction. Both times the test +was the thing that caught it, which is the argument for asserting the number in +a test at all rather than only in prose.