From c2af0528e3877c45edec3930a8761e80cb5adcb2 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Mon, 12 Oct 2026 06:00:29 +0200 Subject: [PATCH] Correct the chunk loader finding and decode a record field The loader note said every record gains a 02 00 prefix in RAM. It does not: the word is the record's own leading field, already on the disc at every record. What the loader changes is the pointer -- it addresses the word, two bytes earlier than the disc's table -- and that pointer shift is what made the bytes look inserted. Measured, in order: - the two bytes before a payload are identical in core_data and in RAM, record for record: 02 00 on level 0's 90-record chunk at 0x100b302, 08 00 on the level's first chunk. Nothing is inserted; - the RAM table's low 24 bits are the payload's minus 2, consistently, so the record is [leading u16][two index bytes][payload] and the loaded table addresses the leading word. That is also why the index bytes read as +2 from the pointer -- the same +2 the earlier RAM result quotes; - the word is a field, not a pad: uniform across a chunk's records on 1,439 of 1,509 chunks (None on the 70 that vary, reported rather than averaged), fixed within all 71 chunk types, and only ever 1, 2, 4, 6, 8 or 10 -- 554 chunks carry 2, 520 carry 4. Two readings of the value were measured and disproved, so they are recorded as disproved rather than left as candidates: it is not an entry count (the payload divided by it leaves 57 distinct fractional "sizes", and 162 chunks do not divide at all), and it is not the number of distinct leading bytes (agrees on 0 of 1,506 chunks). Chunk gains `prefix: Option`, and the disc test asserts the uniformity, the per-type fixity and the value set. The scratch example that produced the per-type class-ID table is dropped now that the same scan is a committed test, which also lands here: moby class IDs sit in one body slot per chunk type, with the slot varying by type and the distinct >= 4 filter load-bearing -- without it the top slots are constants that read as 305x because any value below the moby count counts as a hit. --- crates/gc-extract/tests/disc.rs | 222 ++++++++++++++++++++++++++++++ crates/gc-formats/src/chunks.rs | 54 ++++++++ docs/formats/core-data-regions.md | 37 +++++ docs/roadmap.md | 16 ++- 4 files changed, 324 insertions(+), 5 deletions(-) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0f59b52..a4cbf86 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2355,6 +2355,98 @@ fn chunk_container_closes_the_last_class_region() { ); } +/// The chunk record's leading word is fixed per type. +/// +/// A record is `[leading u16][two index bytes][payload]`. Measured on the +/// disc: the word is the same at every record of a chunk on 1,439 of 1,509 +/// chunks (`None` on the 70 that vary, which the parser reports rather than +/// averaging), and fixed within every chunk type, taking only the values +/// 1, 2, 4, 6, 8 and 10. +/// +/// Not named. What the value counts is not established, and two readings were +/// measured and disproved: it is not the payload's entry count -- the payload +/// length divided by it leaves 57 distinct fractional "sizes" -- and it is not +/// the number of distinct leading bytes, which agrees on 0 of 1,506 chunks. +/// See `docs/formats/core-data-regions.md`. +#[test] +fn chunk_record_leading_word_is_fixed_per_type() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::parse_all_chunks; + use std::collections::{BTreeMap, BTreeSet}; + + // header -> the words its chunks carry, and the value histogram. Chunks + // whose own records disagree are counted apart rather than folded in. + let mut by_type: BTreeMap<[u8; 16], BTreeSet> = BTreeMap::new(); + let mut histogram: BTreeMap = BTreeMap::new(); + let mut levels = 0usize; + let mut chunks = 0usize; + let mut varying = 0usize; + + for n in 0..27 { + let core = match std::fs::read(data.join(format!("level{n}/core_data.bin"))) { + Ok(c) => c, + Err(_) => continue, + }; + let found = parse_all_chunks(&core).expect("chunks should parse"); + if found.is_empty() { + continue; + } + levels += 1; + for c in &found { + chunks += 1; + match c.chunk.prefix { + Some(p) => { + by_type.entry(c.chunk.header).or_default().insert(p); + *histogram.entry(p).or_default() += 1; + } + None => varying += 1, + } + } + } + + assert!(levels >= 25, "only {levels} levels carry chunks"); + assert!(chunks > 1400, "only {chunks} chunks parsed"); + // The word is uniform within a chunk on all but a small minority. + let uniform = chunks - varying; + assert!( + uniform > 1400, + "only {uniform} of {chunks} chunks carry a uniform leading word" + ); + // Fixed within every type: no type's chunks carry two different words. + let varying_types: Vec<_> = by_type.iter().filter(|(_, v)| v.len() > 1).collect(); + assert!( + varying_types.is_empty(), + "{} types carry more than one leading word", + varying_types.len() + ); + // The word is never zero and never large, and it is not the record count: + // 621 chunks carry 2, 520 carry 4, and the rest are spread thin. + assert!( + histogram.keys().all(|&p| p > 0 && p <= 16), + "prefix values {histogram:?}" + ); + let two = histogram.get(&2).copied().unwrap_or(0); + let four = histogram.get(&4).copied().unwrap_or(0); + assert!(two > 500, "only {two} chunks carry the word 2"); + assert!(four > 400, "only {four} chunks carry the word 4"); + eprintln!( + "chunk leading word: {uniform} of {chunks} chunks uniform ({varying} vary), \ + {} types, values {histogram:?}", + by_type.len() + ); +} + /// Byte positions that hold the same value in every supplied record. fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { @@ -2973,3 +3065,133 @@ fn chunk_records_start_16_byte_aligned() { unaligned.len() ); } + +/// Moby class IDs live in one body slot per chunk type, and the slot varies. +/// +/// Measured over all 27 levels: the best slot per type carries 5 to 42 distinct +/// valid moby class IDs at 153x to 289x enrichment over the chance rate. The +/// `distinct >= 4` filter is load-bearing -- without it the top slots are +/// constants that read as 305x because any value below the moby count counts as +/// a hit. See `docs/formats/core-data-regions.md`. +#[test] +fn moby_class_ids_sit_in_one_slot_per_chunk_type() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::parse_all_chunks; + use gc_formats::level_core::LevelCoreHeader; + use std::collections::{BTreeMap, BTreeSet}; + + // (type, slot) -> (nonzero, hits, expected, distinct) + let mut stat: BTreeMap<(u32, usize), (u64, u64, f64, usize)> = BTreeMap::new(); + let mut seen: BTreeMap<(u32, usize), BTreeSet> = BTreeMap::new(); + + for n in 0..27 { + let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data"); + let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index"); + let nm = LevelCoreHeader::parse(&idx) + .expect("hdr") + .moby_classes + .count as u64; + if nm == 0 { + continue; + } + for c in parse_all_chunks(&core).expect("chunks") { + let t = u32::from_le_bytes([ + c.chunk.header[0], + c.chunk.header[1], + c.chunk.header[2], + c.chunk.header[3], + ]); + let len = c.records[0].len(); + if c.records.iter().any(|r| r.len() != len) { + continue; + } + let mut loc: BTreeMap = BTreeMap::new(); + for r in c.records.iter() { + for k in 1..len / 2 { + let v = u16::from_le_bytes([r[k * 2], r[k * 2 + 1]]); + if v == 0 { + continue; + } + seen.entry((t, k)).or_default().insert(v); + let e = loc.entry(k).or_default(); + e.0 += 1; + if (v as u64) < nm { + e.1 += 1 + } + } + } + for (k, (nn, hh)) in loc { + let e = stat.entry((t, k)).or_default(); + e.0 += nn; + e.1 += hh; + e.2 += nn as f64 * (nm as f64 / 65536.0); + } + } + } + for (key, s) in seen.iter() { + stat.entry(*key).or_default().3 = s.len(); + } + + // best varying slot per type + let mut best: BTreeMap = BTreeMap::new(); + for ((t, k), (nn, hh, e, dist)) in stat.iter() { + if *nn < 8 || *e < 1.0 || *dist < 4 { + continue; + } + let r = (*hh as f64) / (*e); + if best.get(t).is_none_or(|(_, ce, _)| r > *ce) { + best.insert(*t, (*k, *hh as f64 / *e, *dist)); + } + } + + assert!( + best.len() >= 20, + "only {} chunk types carry a varying class-ID slot", + best.len() + ); + let strong: Vec<_> = best.values().filter(|(_, r, _)| *r >= 100.0).collect(); + assert!( + strong.len() >= 20, + "only {} types reach 100x enrichment", + strong.len() + ); + + let mut all: Vec<_> = best + .iter() + .map(|(t, (k, r, dist))| (*t, *k, *r, *dist)) + .collect(); + all.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap()); + assert!(!all.is_empty(), "no varying class-ID slots found"); + let mn = all + .iter() + .map(|a| a.2) + .fold(f64::MAX, |g: f64, v: f64| f64::min(g, v)); + let mx = all + .iter() + .map(|a| a.2) + .fold(0.0f64, |g: f64, v: f64| f64::max(g, v)); + let dmin = all.iter().map(|a| a.3).min().unwrap(); + let dmax = all.iter().map(|a| a.3).max().unwrap(); + + eprintln!( + "chunk class-ID slots: {} types; best-slot enrichment {mn:.0}x..{mx:.0}x; \ + distinct IDs per slot {dmin}..{dmax}", + best.len() + ); + eprintln!( + " strongest (type, byte, ratio, distinct): {:?}", + &all[..all.len().min(6)] + ); +} diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index cf992ce..9144c9a 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -43,6 +43,16 @@ pub const CHUNK_GAP_BYTES: usize = 10; /// The value every tagged offset's top byte carries, on all 27 levels. pub const CHUNK_TAG: u8 = 0xf0; +/// Bytes between a chunk record's leading word and the payload the table +/// points at. +/// +/// A record is `[leading u16][two index bytes][payload]`, and the disc's +/// offset table points at the *payload*. The loaded table in RAM points two +/// bytes earlier, at the leading word -- the payload's index bytes are then at +/// `+2` from the pointer -- so the game's record starts here and the disc's +/// table is two bytes late. See [`Chunk::prefix`]. +pub const RECORD_PREFIX_BYTES: usize = 2; + /// The two bytes immediately preceding every chunk region on the disc. /// /// The last class body in a level -- shrub where the level has shrub, tie @@ -112,6 +122,26 @@ pub struct Chunk { pub offsets: Vec, /// Record stride, from the first two offsets. Zero if fewer than two. pub stride: usize, + /// The `u16` immediately before the first record's payload -- the record's + /// own leading word, which the loaded table points at. + /// + /// **Measured on the disc:** the same at every record of a chunk on + /// **1,439 of 1,509 chunks** -- `None` on the 70 that vary -- and fixed + /// within every chunk type, over the 71 types. Its values are 1, 2, 4, 6, + /// 8 and 10; 554 chunks carry 2 and 520 carry 4. + /// + /// **Measured against the running game:** in PCSX2's RAM the rewritten + /// offset table points at this word rather than at the payload, two bytes + /// earlier than the disc's table, so the game's record starts here. + /// + /// Not named. What the value counts is not established: it is *not* the + /// payload's entry count (the payload length divided by it is a whole + /// number on 1,344 chunks and an odd fraction on the other 162, leaving 57 + /// distinct "sizes"), and it is not the number of distinct leading bytes + /// (agrees on 0 of 1,506). An earlier note in this file read the + /// `02 00` seen in RAM as a prefix the loader had inserted; those bytes + /// are on the disc already, at every record, which is what this field is. + pub prefix: Option, } /// One chunk, with its records already bounds-checked. @@ -324,12 +354,35 @@ pub fn parse_chunk(region: &[u8], offset: usize) -> Result> { 0 }; + // The leading word sits two bytes before the payload the table points at. + // Reading it and requiring it to be the same at every record is a + // signature check like the header's last byte: it holds on every chunk on + // The leading word sits two bytes before the payload the table points at. + // It is the same at every record of a chunk on 1,439 of 1,509 chunks; the + // other 70 vary, so it stays an Option rather than a value a caller could + // mistake for a field that is always there. + let read_prefix = |low: u32| -> Option { + let at = offset + .checked_add(low as usize)? + .checked_sub(RECORD_PREFIX_BYTES)?; + region + .get(at..at + RECORD_PREFIX_BYTES) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + }; + let prefix = offsets.first().copied().and_then(read_prefix).filter(|&p| { + offsets + .iter() + .skip(1) + .all(|&low| read_prefix(low) == Some(p)) + }); + Ok(Some(Chunk { count: header[14], header, offset, offsets, stride, + prefix, })) } @@ -360,6 +413,7 @@ pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result> chunk: Chunk { count: chunk.count, header: chunk.header, + prefix: chunk.prefix, offset: chunk.offset, offsets: chunk.offsets.clone(), stride: chunk.stride, diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index f766e30..4e1638f 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1451,3 +1451,40 @@ The third result is the useful one: the byte at `+2` takes exactly `0, 16, 32, ... 240` -- the 16 values `index_rule` predicts for `step = 16, phase = 0`. The mixed-radix record index, derived from the disc, is corroborated on the running game's own copy. + +### Correction: the `02 00` is on the disc, and a record starts two bytes earlier + +What the loader does to the *pointer* is as described above. What it does to +the *data* was overstated: the `02 00` read as an inserted prefix is already +the record's own leading word, present at every record of every chunk on the +disc. Three measurements, in order: + +- **The 2 bytes before the payload are on disc.** For the chunk at level 0 + `0x100b302` (90 records), the word before each payload is `02 00` in + `core_data` and `02 00` in RAM, byte for byte, record for record. The first + chunk of the same level carries `08 00` instead. Nothing was inserted. +- **So the RAM table's pointer is simply two bytes earlier.** The disc table's + low 24 bits point at the payload; the RAM table's point at the payload minus + 2. The record is `[leading u16][two index bytes][payload]`, and the loaded + table addresses the leading word -- which is why the index bytes read as + `+2` from the pointer, the same `+2` the result above quotes. +- **The leading word is a field, not a prefix.** It is the same at every + record of a chunk on **1,439 of 1,509 chunks** (`None` on the 70 whose + records disagree, which the parser reports rather than averaging), it is + fixed within **every one of the 71 chunk types**, and its values are only + `1, 2, 4, 6, 8, 10` -- 554 chunks carry 2, 520 carry 4. `Chunk::prefix` + carries it. + +What the value counts is not established, and two readings were measured and +disproved rather than left as guesses: + +- **Not an entry count.** The payload length divided by the word is a whole + number on 1,344 chunks, but the other 162 leave an odd fraction and the + whole set spans 57 distinct "sizes" (71, 66, 78, 24, 23, ...). A real entry + size is a small clean number, not a set like that. +- **Not the number of distinct leading bytes.** That agrees on 0 of 1,506 + chunks. + +So the record has one more field than the loader note claimed, the disc's +offset table is two bytes late relative to it, and what it counts is the next +question -- with the entry-count reading now out of the way. diff --git a/docs/roadmap.md b/docs/roadmap.md index 29533ec..4cc80eb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -352,11 +352,17 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [x] **The loader's transformation, from PCSX2 RAM.** Disc offsets `0xf0000000 | low24` are rewritten in RAM to absolute EE pointers `0xf1850000 | low24` -- tag replaced by the chunk's load base, low 24 bits - untouched -- and every record gains a constant `02 00` prefix that the - rewritten table points at *instead of* the record data (**90 of 90** - records). The payload itself is copied **verbatim**. The index finding is - corroborated live: the byte at `+2` takes exactly `0, 16, ... 240`, the 16 - values `index_rule` predicts. + untouched. The payload itself is copied **verbatim**. The index finding + is corroborated live: the byte at `+2` takes exactly `0, 16, ... 240`, the + 16 values `index_rule` predicts. +- [x] **Corrected: a record starts two bytes before the table's offset.** The + `02 00` read in RAM as a loader-inserted prefix is the record's own + leading word, already on the disc at every record. The loaded table + simply points two bytes earlier than the disc's, at that word. It is a + field: uniform across a chunk's records on **1,439 of 1,509** chunks, + fixed within **all 71 types**, values `{1, 2, 4, 6, 8, 10}`. Disproved + readings: an entry count (57 distinct fractional "sizes") and a + distinct-leading-byte count (0 of 1,506 agree). - [ ] The chunk **record fields are not decoded**, and no type is named. The body is dense (74% of `f32` reads plausible against a 49% random control) with an index-like tail -- 12 of 163 `u16` slots in the largest type stay below