diff --git a/crates/gc-extract/examples/u.rs b/crates/gc-extract/examples/u.rs deleted file mode 100644 index dd0d21a..0000000 --- a/crates/gc-extract/examples/u.rs +++ /dev/null @@ -1,57 +0,0 @@ -use std::collections::BTreeMap; - -/// For each chunk with a non-uniform delta, look at the bytes a uniform stride -/// would swallow past the next record's start. Under "padding" they should be -/// uniform or zero; under "variable-length" they are data and should not be. -fn main() { - let data = - std::path::PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted"); - let mut hist: BTreeMap, usize> = BTreeMap::new(); - let mut total = 0usize; - let mut zero = 0usize; - for lv in 0..6usize { - let Ok(core) = std::fs::read(data.join(format!("level{lv}/core_data.bin"))) else { - continue; - }; - for c in gc_formats::chunks::parse_all_chunks(&core).unwrap() { - let s = c.chunk.stride; - let o = &c.chunk.offsets; - for w in o.windows(2) { - let (a, b) = (w[0] as usize, w[1] as usize); - if b - a >= s { - continue; - } - let gap = s - (b - a); // bytes the uniform slice over-reads - // the disputed bytes are the last `gap` of the uniform slice - let lo = c.chunk.offset + a + (b - a); - let Some(tail) = core.get(lo..lo + gap) else { - continue; - }; - let mut t = tail.to_vec(); - t.sort_unstable(); - if t.iter().all(|&x| x == t[0]) { - zero += 1; - } - *hist.entry(tail.to_vec()).or_default() += 1; - total += 1; - } - } - } - println!( - "{total} over-reads examined; {zero} are a single repeated byte ({:.1}%)", - 100.0 * zero as f64 / total.max(1) as f64 - ); - println!("\nmost common disputed byte sequences:"); - let mut v: Vec<_> = hist.into_iter().collect(); - v.sort_by_key(|(_, n)| std::cmp::Reverse(*n)); - for (b, n) in v.iter().take(15) { - println!( - " x{n:<5} {}", - b.iter() - .map(|x| format!("{x:02x}")) - .collect::>() - .join(" ") - ); - } - println!("\ndistinct disputed sequences: {}", v.len()); -} diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index eb7e8e5..240180b 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -4546,3 +4546,81 @@ fn chunk_type_75_42_ef_has_one_variable_length_chunk_per_level() { ); assert_eq!(over, 4, "the count of short records changed"); } + +/// Chunk records share a field layout: 17-21% of byte positions are constant +/// across a chunk's own records, which does not happen by chance. +/// +/// This survived the record-boundary fix, which is the interesting part. The fix +/// changed the length of records in 1,252 of 1,509 chunks, and this statistic is +/// computed over record bytes, so it was the obvious candidate for a claim that +/// had been contaminated by over-reads. Re-measured with each record bounded by +/// its successor, and comparing only the span every record in the chunk has, it +/// still lands in 16-20% per level. It is now asserted rather than only described. +#[test] +fn chunk_records_share_a_layout_measured_by_constant_byte_positions() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::parse_all_chunks; + + use std::collections::BTreeMap; + + let mut per_level: BTreeMap = BTreeMap::new(); + for lv in 0..27usize { + let Ok(core) = std::fs::read(data.join(format!("level{lv}/core_data.bin"))) else { + continue; + }; + let (mut tot, mut chunks) = (0u64, 0usize); + for c in parse_all_chunks(&core).expect("chunks") { + if c.records.len() < 3 { + continue; + } + // records in a chunk are not all the same length; compare the common + // prefix only + let len = c.records.iter().map(|r| r.len()).min().unwrap_or(0); + if len == 0 { + continue; + } + let n = (0..len) + .filter(|&i| c.records.iter().all(|r| r[i] == c.records[0][i])) + .count(); + tot += 1000 * n as u64 / len as u64; + chunks += 1; + } + if chunks > 0 { + per_level.insert(lv, (tot, chunks)); + } + } + assert_eq!(per_level.len(), 27, "a level produced no chunks to measure"); + + let pcts: Vec<(usize, f64)> = per_level + .iter() + .map(|(&lv, &(t, c))| (lv, t as f64 / (10.0 * c as f64))) + .collect(); + let lo = pcts.iter().map(|p| p.1).fold(f64::MAX, f64::min); + let hi = pcts.iter().map(|p| p.1).fold(0.0f64, f64::max); + let lowest = pcts + .iter() + .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap()) + .unwrap(); + eprintln!( + "chunk record layout: {lo:.1}%..{hi:.1}% of byte positions constant within a \ + chunk; lowest is level {} at {:.1}%", + lowest.0, lowest.1 + ); + // lo/hi are percentages, not fractions + assert!( + (10.0..30.0).contains(&lo) && (10.0..30.0).contains(&hi), + "constant byte positions left the 10-30% band: {lo:.3}..{hi:.3}" + ); +} diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index 8c847c0..e506715 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -16,9 +16,11 @@ //! ``` //! //! and each tagged offset is `0xF0000000 | low24`. Resolving a record at -//! `chunk_start + low` yields records that share a field layout: 17-21% of -//! byte positions are constant across a chunk's own records, on every level. -//! That is what distinguishes this from a coincidental byte pattern. +//! `chunk_start + low` yields records that share a field layout: 16.7% to 20.0% +//! of byte positions are constant across a chunk's own records, on every level. +//! That is what distinguishes this from a coincidental byte pattern. Records in +//! one chunk are **not** all the same length -- each runs to its successor's +//! offset -- so the statistic is computed over the span every record shares. //! //! The stride between records is a property of the chunk, not of the region. //! Level 0 alone contains chunks at 148, 340, 368, 288, 320, 236, 272, 384, diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 7167eb3..ccef769 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -831,7 +831,7 @@ take the stride as the first delta, load one record per offset at chunk's records. **Result: 1,096 chunks and roughly 13,000 records resolve on all 27 levels.** -Every level's chunks show 17-21% of byte positions constant across their own +Every level's chunks show 16.7-20.0% of byte positions constant across their own records — a consistent field layout rather than noise. Level 0 in detail: 27 chunks resolve, 466 records, 20.8% constant. The @@ -2452,3 +2452,37 @@ since the two views are of the same bytes. The uniform model stays in place, the ambiguity is recorded here and in [`resolve`]'s docs, and the existing 1,509-of-1,509 assertion stands as a statement about what the parser does, not about what the disc contains. + +## Audit: which claims were contaminated by the over-read? + +The record-boundary fix changed the length of records in 1,252 of 1,509 chunks, +so every committed claim computed over *record bytes* had to be re-checked. All of +them: + +| claim | reads | verdict | +| --- | --- | --- | +| 1,509 chunks, 34,491 records | offsets, not records | unaffected | +| 1,262 chunks with a leading-byte bucket rule | record byte 0 | unaffected | +| 1,235 chunks with an index rule | record bytes 1-2 | unaffected | +| 34,432 of 34,491 records 8-byte aligned | record byte 0 | unaffected | +| 55 distinct strides, all multiples of 4 | offsets | unaffected | +| moby class IDs in one slot per chunk type | record bytes | **retracted** | +| 17-21% of byte positions constant | record bytes | **survives** | + +Only the claims that scanned *across* a record's body were at risk, and of those +one was retracted (the class-ID slot) and one survived (the constant-byte +statistic). The rest read the first three bytes of a record, which the over-read +never touched -- an over-read appends to the end, so a claim about a record's +*start* was never in danger. That asymmetry is why the damage was contained, and +it is the reason to prefer claims anchored at record starts. + +The surviving one is worth stating precisely because it was the obvious candidate +for contamination. Re-measured with each record bounded by its successor, and +comparing only the span every record in the chunk has: + + per level, % of byte positions constant within a chunk + lowest level 8 16.7% + highest 20.0% + +Still inside the documented band, now asserted rather than only described, in +`chunk_records_share_a_layout_measured_by_constant_byte_positions`.