diff --git a/crates/gc-extract/examples/u.rs b/crates/gc-extract/examples/u.rs new file mode 100644 index 0000000..dd0d21a --- /dev/null +++ b/crates/gc-extract/examples/u.rs @@ -0,0 +1,57 @@ +use std::collections::BTreeMap; + +/// For each chunk with a non-uniform delta, look at the bytes a uniform stride +/// would swallow past the next record's start. Under "padding" they should be +/// uniform or zero; under "variable-length" they are data and should not be. +fn main() { + let data = + std::path::PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted"); + let mut hist: BTreeMap, usize> = BTreeMap::new(); + let mut total = 0usize; + let mut zero = 0usize; + for lv in 0..6usize { + let Ok(core) = std::fs::read(data.join(format!("level{lv}/core_data.bin"))) else { + continue; + }; + for c in gc_formats::chunks::parse_all_chunks(&core).unwrap() { + let s = c.chunk.stride; + let o = &c.chunk.offsets; + for w in o.windows(2) { + let (a, b) = (w[0] as usize, w[1] as usize); + if b - a >= s { + continue; + } + let gap = s - (b - a); // bytes the uniform slice over-reads + // the disputed bytes are the last `gap` of the uniform slice + let lo = c.chunk.offset + a + (b - a); + let Some(tail) = core.get(lo..lo + gap) else { + continue; + }; + let mut t = tail.to_vec(); + t.sort_unstable(); + if t.iter().all(|&x| x == t[0]) { + zero += 1; + } + *hist.entry(tail.to_vec()).or_default() += 1; + total += 1; + } + } + } + println!( + "{total} over-reads examined; {zero} are a single repeated byte ({:.1}%)", + 100.0 * zero as f64 / total.max(1) as f64 + ); + println!("\nmost common disputed byte sequences:"); + let mut v: Vec<_> = hist.into_iter().collect(); + v.sort_by_key(|(_, n)| std::cmp::Reverse(*n)); + for (b, n) in v.iter().take(15) { + println!( + " x{n:<5} {}", + b.iter() + .map(|x| format!("{x:02x}")) + .collect::>() + .join(" ") + ); + } + println!("\ndistinct disputed sequences: {}", v.len()); +} diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index a032f02..eb7e8e5 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2577,7 +2577,11 @@ fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { return 0; }; - (0..first.len()) + // Records in one chunk are NOT all the same length: the offset deltas vary, + // and `resolve` measures each record against its successor. Compare only the + // span every record has, or this indexes past the short ones. + let len = records.iter().map(|r| r.len()).min().unwrap_or(0); + (0..len) .filter(|&i| records.iter().all(|r| r[i] == first[i])) .count() } @@ -3151,10 +3155,14 @@ fn chunk_records_start_16_byte_aligned() { assert!(chunks >= 1500, "only {chunks} chunks parsed"); assert!(records >= 34000, "only {records} chunk records"); - // Every chunk has a uniform record length -- length varies between chunks - // of a type, never within one -- and on most chunks the leading byte is a - // per-chunk phase counter rather than data. + // Record length varies BETWEEN chunks of a type, and -- contrary to what + // this test used to assert -- also WITHIN a chunk, on 1,252 of them. The old + // "1,509 of 1,509 uniform" was true only because `resolve` sliced every + // record at the chunk's first delta, which the test was then confirming back + // to itself. On most chunks the leading byte is a per-chunk phase counter + // rather than data. let mut uniform = 0usize; + let mut ragged_chunks = 0usize; let mut ruled = 0usize; let mut indexed = 0usize; for n in 0..27 { @@ -3162,6 +3170,8 @@ fn chunk_records_start_16_byte_aligned() { for c in parse_all_chunks(&core).expect("chunks") { if c.has_uniform_record_length() { uniform += 1; + } else { + ragged_chunks += 1; } if c.bucket_rule().is_some() { ruled += 1; @@ -3171,7 +3181,12 @@ fn chunk_records_start_16_byte_aligned() { } } } - assert_eq!(uniform, 1509, "uniform-record-length count moved off 1509"); + assert_eq!(uniform, 257, "uniform-record-length count moved off 257"); + assert_eq!( + uniform + ragged_chunks, + 1509, + "chunk total moved; add the ragged counter" + ); assert_eq!(ruled, 1262, "bucket-rule count moved off 1262"); assert_eq!(indexed, 1235, "index-rule count moved off 1235"); @@ -3207,9 +3222,20 @@ fn chunk_records_start_16_byte_aligned() { /// Moby class IDs live in one body slot per chunk type, and the slot varies. /// -/// Measured over all 27 levels: the best slot per type carries 5 to 42 distinct -/// valid moby class IDs at 153x to 289x enrichment over the chance rate. The -/// `distinct >= 4` filter is load-bearing -- without it the top slots are +/// **Largely retracted.** This used to assert that >= 20 chunk types carry a +/// slot of moby class IDs, at 153x to 289x enrichment over chance. +/// +/// It was measuring over-read records. `resolve` sliced every record at the +/// chunk's first offset delta, which on 1,252 of 1,509 chunks ran 4 to 8 bytes +/// past a record's real end -- four bytes of the *next* record, glued on. That +/// reads as extra fields and lifts a slot's apparent hit rate. With record +/// boundaries measured against each record's successor, and ragged chunks +/// skipped, only 2 types show a varying class-ID slot. +/// +/// What survives is narrow and is asserted as such: 2 chunk types show a slot +/// that varies in the manner of a class ID, and exactly 1 of those still clears +/// 100x enrichment over chance. The +/// `distinct >= 4` filter is still load-bearing -- without it the top slots are /// constants that read as 305x because any value below the moby count counts as /// a hit. See `docs/formats/core-data-regions.md`. #[test] @@ -3295,16 +3321,27 @@ fn moby_class_ids_sit_in_one_slot_per_chunk_type() { } } - assert!( - best.len() >= 20, - "only {} chunk types carry a varying class-ID slot", - best.len() - ); + // RETRACTED. This asserted >= 20 types while every record was being sliced + // at the chunk's first delta, which over-read 4 to 8 bytes past the true end + // of a record on 1,252 of 1,509 chunks. Records with four bytes of the next + // one glued to the end score as though they carried extra fields. With the + // boundaries fixed, and this test skipping ragged chunks, only 2 types show + // a varying class-ID slot -- so "one class-ID slot per chunk type" was a + // measurement of the parser's over-read, not of the disc. let strong: Vec<_> = best.values().filter(|(_, r, _)| *r >= 100.0).collect(); assert!( - strong.len() >= 20, - "only {} types reach 100x enrichment", - strong.len() + !strong.is_empty(), + "no type reaches 100x enrichment -- was 20 before the record-boundary fix" + ); + assert_eq!(strong.len(), 1, "strong-enrichment type count moved"); + assert!( + best.len() >= 2, + "only {} chunk types carry a varying class-ID slot (was >= 20)", + best.len() + ); + assert!( + best.len() < 20, + "class-ID slots are back at >= 20 types; revisit the retraction" ); let mut all: Vec<_> = best diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index ae85ce5..8c847c0 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -103,12 +103,12 @@ pub const CHUNK_REGION_MARKER: [u8; 2] = [0xf6, 0x7c]; /// A chunk header plus the records its offsets address. /// /// `stride` is the distance between the first two resolved offsets, and nothing -/// more. The parser takes each offset literally rather than assuming a uniform -/// stride, because the deltas are not constant. On level 0 some chunks step -/// 340, 340, 336, 340; on all 27 levels signature `[75 42 ef f2 71 b5]` steps -/// 92, 88, 84, 92, 92, 92, 84, 88, which is a real variation in record length -/// and not a rounding artifact. Record *lengths* come from the deltas, in -/// [`resolve`]. +/// more. The deltas within a chunk are **not** constant, and the variation is +/// real: on level 0 some chunks step 340, 340, 336, 340, and on all 27 levels +/// signature `[75 42 ef f2 71 b5]` steps 92, 88, 84, 92, 92, 92, 84, 88. Taking +/// every record at `stride` would overlap four of those eight with their own +/// successors, which an explicit offset table cannot express. Record *lengths* +/// come from the deltas; see [`resolve`]. pub struct Chunk { /// The 16 opaque header bytes. pub header: [u8; CHUNK_HEADER_BYTES], @@ -391,18 +391,26 @@ pub fn parse_chunk(region: &[u8], offset: usize) -> Result> { /// Resolve every record a chunk addresses, bounds-checked against `region`. /// -/// Each record is `region[chunk.offset + low .. + stride]`, and `stride` is the -/// chunk's first offset delta. +/// A record runs from its own offset to the **next** record's offset, not to +/// `chunk.offset + stride`. /// -/// **This may slice past a record's real end.** 1,252 of the 1,509 chunks have -/// offsets that are not uniformly spaced -- the deviations are word-aligned -/// multiples of 4, mostly +/-4 -- so for those the last few bytes of a slice can -/// be the head of the next record. Whether that means the records are genuinely -/// variable-length, or that the offset table has inter-record padding, **cannot -/// be told from the offsets alone**: both models produce exactly this table. -/// Deciding between them needs a field whose end is independently observable, so -/// until that exists this function takes the uniform reading and the ambiguity -/// is recorded rather than guessed at. See `docs/formats/core-data-regions.md`. +/// This is not a stylistic choice. `stride` is the *first* offset delta, and on +/// 1,252 of the 1,509 measured chunks later deltas are smaller -- the deviations +/// are word-aligned multiples of 4, most exactly +/-4. Slicing every record at +/// the first delta would therefore make some records run past the start of their +/// own successor. On the measured `92, 88, 84, 92, 92, 92, 84, 88` chunk, four +/// of the eight records overlap the next one that way, and records are addressed +/// by an explicit offset table, so they cannot overlap. The only coherent +/// reading is that a record ends where the next begins. +/// +/// Slicing at `stride` was not wrong in a way that would error; it handed four +/// records 4 to 8 bytes of the following record, which is the kind of defect +/// that surfaces much later as fields that look meaningful and are quietly off +/// by four bytes. +/// +/// The last record has no successor to measure against and falls back to +/// `stride`; on the measured levels the final delta is itself uniform, so this +/// is a fallback rather than a silent guess. /// /// A record that would run past the end of the region is dropped rather than /// clamped, so the returned length may be shorter than `chunk.offsets` -- on the @@ -410,7 +418,7 @@ pub fn parse_chunk(region: &[u8], offset: usize) -> Result> { /// malformed chunk rather than report it. pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result> { let mut records = Vec::with_capacity(chunk.offsets.len()); - for &low in &chunk.offsets { + for (i, &low) in chunk.offsets.iter().enumerate() { let start = chunk .offset .checked_add(low as usize) @@ -419,10 +427,14 @@ pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result> context: "chunk record", kind: ErrorKind::Invalid("chunk offset overflows usize"), })?; - if start + chunk.stride > region.len() { + let len = match chunk.offsets.get(i + 1) { + Some(&next) => (next - low) as usize, + None => chunk.stride, + }; + if start + len > region.len() { continue; } - records.push(®ion[start..start + chunk.stride]); + records.push(®ion[start..start + len]); } Ok(ResolvedChunk { chunk: Chunk { diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index d8bd83d..7167eb3 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -967,77 +967,37 @@ correlating. **Still open:** what the 64 types mean, which record fields are index/length/offset, and whether any type is moby_anim. -## Chunk records reference moby class IDs at 38x chance +## RETRACTED: chunk records referencing moby class IDs at 38x chance -**Range measured:** level 0, the 900,000 bytes before the trailer; 28 chunks, -475 records, 67,780 `u16` slots. Reference set: the 238 distinct `o_class` -values from level 0's moby class table, read through `LevelCoreHeader`. +This section previously claimed that chunk records reference moby class IDs, with +one body slot per chunk type, at 38x-289x enrichment over chance, and that >= 20 +chunk types showed it. -**Method:** scan every `u16` in every record; count how many are in the moby -class set. Compare against the rate expected if hits were chance, at -238/65536 per slot. +**That was a measurement of the parser, not of the disc.** -**Result: 9,493 hits against 246.1 expected -- 38.6x excess.** +`resolve` sliced every record at the chunk's *first* offset delta. On 1,252 of +1,509 chunks the later deltas are smaller -- word-aligned multiples of 4, most +exactly +/-4 -- so those slices ran past the record's real end and carried **4 to +8 bytes of the following record**. A record with the head of the next one glued +to its tail scores as though it carried extra fields, and a slot scan across +those bytes lifts apparent hit rates without anything being true. -The hits are not spread. They concentrate in a contiguous run of `u16` slots: +With record boundaries measured against each record's successor, and ragged +chunks skipped, what survives is: -| slot | hits | | slot | hits | -|---|---|---|---|---| -| 138 | 188 | | 134 | 173 | -| 135 | 182 | | 136 | 168 | -| 137 | 180 | | 129 | 139 | -| | | | 130 | 131 | + chunk types with a varying class-ID-like slot 2 (was >= 20) + of those, clearing 100x enrichment 1 (was >= 20) -Slots 129 through 140 -- twelve consecutive `u16`, that is bytes 258 through -280 of the record -- carry 105 to 188 hits each, and nothing else in the record -comes close except a separate cluster at slot 38. +So the general claim -- one class-ID slot per chunk type, across the container -- +does not hold and is withdrawn. One chunk type still shows a slot that varies in +the manner of a class ID at 100x enrichment over chance. That is asserted, and +the test also asserts the count has *not* returned to 20, so the retraction +cannot quietly lapse back into the claim. -**What this looks like.** A contiguous twelve-value array in which most entries -are moby class identifiers is what a model or instance definition listing the -mobies it references would look like. Every chunk contributes in proportion to -its record count: the stride-340 chunk with 77 records has 1,451 hits, the -stride-368 chunk 1,384, the stride-148 chunk with 10 records has 254. +The `distinct >= 4` filter this section introduced is still load-bearing and is +unaffected: without it the top slots are constants that read as 305x, because any +value below the moby count counts as a hit. -**What this does not show.** Enrichment is not identification. A 38x excess -over chance is strong evidence that these values are moby class IDs rather -than incidental small integers, but it does not establish that the containing -chunk is `moby_anim` -- an instance table, a model list and an animation table -would all reference class IDs. The format is still unnamed, and so is the -meaning of slots 129-140. - -**Correction to the above: the slot offsets were a pooling artifact.** - -The entry above reported slots 129 through 140. Repeating the scan per chunk -type shows that is wrong: **every chunk type places its class array at a -different offset.** - -| type (header prefix) | top hit slots | -|---|---| -| `aac4` | 129-134 | -| `82c4` | 104-109 | -| `4ec4` | 106-111 | -| `b8c4` | 120-125 | -| `9943` | 82-87 | -| `79c4` | 111-115 | -| `3245` | 24-29 | - -Slots 129-140 were simply the type with the largest stride, dominating a scan -that pooled all types together. The previous entry flagged exactly this as -unverified, and the check confirmed the flag. - -**What survives the correction.** In every one of the 26 types the top hits -form a **contiguous run of consecutive `u16` slots**. That is the real -invariant, and it is per-type: each chunk type has its own class array at its -own offset, of its own length. - -So the corrected statement is: chunk records contain a contiguous array of -moby class IDs, and where that array sits depends on the chunk's type header. -This is consistent with each type being a distinct record layout, which is -what the header-implies-one-stride result already indicated -- but the class -array is a per-type field, not a common one at a fixed offset. - -The moby reference itself is unaffected: 38.6x excess measured across all -records stands, since it does not depend on which slots were counted. ## Shrub body records on level 0: six varying bytes, two constant @@ -1825,8 +1785,8 @@ likely to be lost. Audit against `crates/gc-extract/tests/disc.rs`: packets[0] == 256 on 1,864 of 1,864 same test chunk record length uniform per chunk tie-classes test + chunk_records_ start_16_byte_aligned - moby class IDs in one body slot per type moby_class_ids_sit_in_one_slot_ - per_chunk_type + moby class IDs in one body slot per type RETRACTED -- see above; the test + now asserts 2 types, 1 strong collision payload essentially verbatim collision_payload_is_verbatim_ in_ram (GC_SSTATE-gated) tfrag pointers relocate, patches rewrite tfrag_table_pointers_relocate_