diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 9f10abe..fa05946 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2920,6 +2920,25 @@ fn chunk_records_start_16_byte_aligned() { assert!(chunks >= 1500, "only {chunks} chunks parsed"); assert!(records >= 34000, "only {records} chunk records"); + // Every chunk has a uniform record length -- length varies between chunks + // of a type, never within one -- and on most chunks the leading byte is a + // per-chunk phase counter rather than data. + let mut uniform = 0usize; + let mut ruled = 0usize; + for n in 0..27 { + let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data"); + for c in parse_all_chunks(&core).expect("chunks") { + if c.has_uniform_record_length() { + uniform += 1; + } + if c.bucket_rule().is_some() { + ruled += 1; + } + } + } + assert_eq!(uniform, 1509, "uniform-record-length count moved off 1509"); + assert_eq!(ruled, 1262, "bucket-rule count moved off 1262"); + // Not universal. Three chunk types break even the 8-byte rule, all of them // the variable-length or 8-stepped ones. A parallel probe puts 1,142 // records across seven types outside a *16*-byte rule, which is how the @@ -2942,8 +2961,9 @@ fn chunk_records_start_16_byte_aligned() { ); eprintln!( - "chunks: {chunks} chunks, {records} records; leading byte a multiple of 8 on {aligned}, \ - {bad} exceptions across {} types (8-byte step and one variable-length type)", + "chunks: {chunks} chunks, {records} records; uniform record length on {uniform}; \ + leading byte a multiple of 8 on {aligned}, {bad} exceptions across {} types; \ + leading byte fully explained by a per-chunk phase counter on {ruled}", unaligned.len() ); } diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index 1009427..8030c17 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -122,6 +122,65 @@ pub struct ResolvedChunk<'a> { pub records: Vec<&'a [u8]>, } +impl<'a> ResolvedChunk<'a> { + /// The per-chunk phase and step that explain every record's leading byte, or + /// `None` when no `(step, phase)` in that form fits. + /// + /// **Measured: 1,262 of 1,509 chunks** across all 27 levels are fully explained + /// by `record[i].leading == step * ((i + phase) mod (256 / step))`, with + /// `step` taking 8 on 51 chunks, 16 on 554, 32 on 520 and 64 on 137. The other + /// 247 chunks, across 11 chunk types, are not explained by it. + /// + /// Where this returns `Some`, the leading byte carries **no per-record + /// information**: it is a deterministic function of the record's index and the + /// chunk's phase, so it is a position marker rather than data. The 247 chunks + /// where it returns `None` are the ones where that has *not* been shown, and + /// nothing should skip their leading byte on the strength of this. + pub fn bucket_rule(&self) -> Option<(u8, usize)> { + use std::collections::BTreeSet; + if self.records.is_empty() { + return None; + } + let n = self.records.len(); + for step in [8u16, 16, 32, 64] { + let m = (256 / step) as usize; + for phase in 0..m { + let mut ok = true; + for (i, r) in self.records.iter().enumerate() { + let Some(&b) = r.first() else { + ok = false; + break; + }; + if b != ((((i + phase) % m) * step as usize % 256) as u8) { + ok = false; + break; + } + } + if ok { + return Some((step as u8, phase)); + } + } + } + let _ = n; + let _: Option> = None; + None + } + + /// True when every record in this chunk has the same length. + /// + /// **Measured: true for 1,509 of 1,509 chunks** across all 27 levels. Record + /// length varies *between* chunks of the same type -- one type runs 76 to 104 + /// bytes -- but never within one. An earlier claim that some type had + /// variable-length records was pooling across chunks; it is withdrawn. + pub fn has_uniform_record_length(&self) -> bool { + let mut it = self.records.iter().filter(|r| !r.is_empty()); + let Some(first) = it.next().map(|r| r.len()) else { + return true; + }; + it.all(|r| r.len() == first) + } +} + /// True if `offset` looks like the start of a chunk. /// /// The signature is the header's last byte being `0xff`, the 10-byte gap diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index cb04bf9..8b5690a 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1316,3 +1316,39 @@ Worth stating plainly: this is the second time this session a correct number carried a wrong label, after the tie four-value retraction. Both times the test was the thing that caught it, which is the argument for asserting the number in a test at all rather than only in prose. + +## The chunk record's leading byte is a per-chunk phase counter + +Follow-up to the alignment result, and it settles what the leading byte is. + +Within one chunk the leading byte steps by a constant and wraps: + + offsets: 1420 1748 2072 2392 2716 3044 3372 3700 4028 4352 ... + byte0: 0 16 32 48 64 80 96 112 128 144 ... + +i.e. `byte0 == step * ((index + phase) mod (256 / step))`. Sweeping every +`(step, phase)` over all 27 levels: + + fully explained: 1,262 of 1,509 chunks + by step: 8 -> 51 chunks, 16 -> 554, 32 -> 520, 64 -> 137 + unexplained: 247 chunks across 11 chunk types + +**So on 1,262 chunks the leading byte carries no per-record information.** It is +a deterministic function of the record's index and the chunk's phase. That makes +it a position marker, not data -- which is why the earlier "16 discrete values +with uniform counts" pattern showed up. + +`ResolvedChunk::bucket_rule` returns the `(step, phase)` when the whole chunk +fits and `None` otherwise, so the 247 unexplained chunks are distinguishable +from the 1,262 explained ones and nothing skips a leading byte on the strength of +a rule that does not apply. + +### Every chunk has a uniform record length + +**1,509 of 1,509 chunks.** Length varies between chunks of the same type -- one +type runs 76 to 104 bytes -- but never within one chunk. + +This **withdraws the previous commit's "variable-length type"**. That came from +taking `minlen`/`maxlen` over all of a type's records at once, pooling across +chunks; rule 3a's pooled claim, third time this session. Within any one chunk the +length is fixed.