diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0f59b52..a4cbf86 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2355,6 +2355,98 @@ fn chunk_container_closes_the_last_class_region() { ); } +/// The chunk record's leading word is fixed per type. +/// +/// A record is `[leading u16][two index bytes][payload]`. Measured on the +/// disc: the word is the same at every record of a chunk on 1,439 of 1,509 +/// chunks (`None` on the 70 that vary, which the parser reports rather than +/// averaging), and fixed within every chunk type, taking only the values +/// 1, 2, 4, 6, 8 and 10. +/// +/// Not named. What the value counts is not established, and two readings were +/// measured and disproved: it is not the payload's entry count -- the payload +/// length divided by it leaves 57 distinct fractional "sizes" -- and it is not +/// the number of distinct leading bytes, which agrees on 0 of 1,506 chunks. +/// See `docs/formats/core-data-regions.md`. +#[test] +fn chunk_record_leading_word_is_fixed_per_type() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::parse_all_chunks; + use std::collections::{BTreeMap, BTreeSet}; + + // header -> the words its chunks carry, and the value histogram. Chunks + // whose own records disagree are counted apart rather than folded in. + let mut by_type: BTreeMap<[u8; 16], BTreeSet> = BTreeMap::new(); + let mut histogram: BTreeMap = BTreeMap::new(); + let mut levels = 0usize; + let mut chunks = 0usize; + let mut varying = 0usize; + + for n in 0..27 { + let core = match std::fs::read(data.join(format!("level{n}/core_data.bin"))) { + Ok(c) => c, + Err(_) => continue, + }; + let found = parse_all_chunks(&core).expect("chunks should parse"); + if found.is_empty() { + continue; + } + levels += 1; + for c in &found { + chunks += 1; + match c.chunk.prefix { + Some(p) => { + by_type.entry(c.chunk.header).or_default().insert(p); + *histogram.entry(p).or_default() += 1; + } + None => varying += 1, + } + } + } + + assert!(levels >= 25, "only {levels} levels carry chunks"); + assert!(chunks > 1400, "only {chunks} chunks parsed"); + // The word is uniform within a chunk on all but a small minority. + let uniform = chunks - varying; + assert!( + uniform > 1400, + "only {uniform} of {chunks} chunks carry a uniform leading word" + ); + // Fixed within every type: no type's chunks carry two different words. + let varying_types: Vec<_> = by_type.iter().filter(|(_, v)| v.len() > 1).collect(); + assert!( + varying_types.is_empty(), + "{} types carry more than one leading word", + varying_types.len() + ); + // The word is never zero and never large, and it is not the record count: + // 621 chunks carry 2, 520 carry 4, and the rest are spread thin. + assert!( + histogram.keys().all(|&p| p > 0 && p <= 16), + "prefix values {histogram:?}" + ); + let two = histogram.get(&2).copied().unwrap_or(0); + let four = histogram.get(&4).copied().unwrap_or(0); + assert!(two > 500, "only {two} chunks carry the word 2"); + assert!(four > 400, "only {four} chunks carry the word 4"); + eprintln!( + "chunk leading word: {uniform} of {chunks} chunks uniform ({varying} vary), \ + {} types, values {histogram:?}", + by_type.len() + ); +} + /// Byte positions that hold the same value in every supplied record. fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { @@ -2973,3 +3065,133 @@ fn chunk_records_start_16_byte_aligned() { unaligned.len() ); } + +/// Moby class IDs live in one body slot per chunk type, and the slot varies. +/// +/// Measured over all 27 levels: the best slot per type carries 5 to 42 distinct +/// valid moby class IDs at 153x to 289x enrichment over the chance rate. The +/// `distinct >= 4` filter is load-bearing -- without it the top slots are +/// constants that read as 305x because any value below the moby count counts as +/// a hit. See `docs/formats/core-data-regions.md`. +#[test] +fn moby_class_ids_sit_in_one_slot_per_chunk_type() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::parse_all_chunks; + use gc_formats::level_core::LevelCoreHeader; + use std::collections::{BTreeMap, BTreeSet}; + + // (type, slot) -> (nonzero, hits, expected, distinct) + let mut stat: BTreeMap<(u32, usize), (u64, u64, f64, usize)> = BTreeMap::new(); + let mut seen: BTreeMap<(u32, usize), BTreeSet> = BTreeMap::new(); + + for n in 0..27 { + let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data"); + let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index"); + let nm = LevelCoreHeader::parse(&idx) + .expect("hdr") + .moby_classes + .count as u64; + if nm == 0 { + continue; + } + for c in parse_all_chunks(&core).expect("chunks") { + let t = u32::from_le_bytes([ + c.chunk.header[0], + c.chunk.header[1], + c.chunk.header[2], + c.chunk.header[3], + ]); + let len = c.records[0].len(); + if c.records.iter().any(|r| r.len() != len) { + continue; + } + let mut loc: BTreeMap = BTreeMap::new(); + for r in c.records.iter() { + for k in 1..len / 2 { + let v = u16::from_le_bytes([r[k * 2], r[k * 2 + 1]]); + if v == 0 { + continue; + } + seen.entry((t, k)).or_default().insert(v); + let e = loc.entry(k).or_default(); + e.0 += 1; + if (v as u64) < nm { + e.1 += 1 + } + } + } + for (k, (nn, hh)) in loc { + let e = stat.entry((t, k)).or_default(); + e.0 += nn; + e.1 += hh; + e.2 += nn as f64 * (nm as f64 / 65536.0); + } + } + } + for (key, s) in seen.iter() { + stat.entry(*key).or_default().3 = s.len(); + } + + // best varying slot per type + let mut best: BTreeMap = BTreeMap::new(); + for ((t, k), (nn, hh, e, dist)) in stat.iter() { + if *nn < 8 || *e < 1.0 || *dist < 4 { + continue; + } + let r = (*hh as f64) / (*e); + if best.get(t).is_none_or(|(_, ce, _)| r > *ce) { + best.insert(*t, (*k, *hh as f64 / *e, *dist)); + } + } + + assert!( + best.len() >= 20, + "only {} chunk types carry a varying class-ID slot", + best.len() + ); + let strong: Vec<_> = best.values().filter(|(_, r, _)| *r >= 100.0).collect(); + assert!( + strong.len() >= 20, + "only {} types reach 100x enrichment", + strong.len() + ); + + let mut all: Vec<_> = best + .iter() + .map(|(t, (k, r, dist))| (*t, *k, *r, *dist)) + .collect(); + all.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap()); + assert!(!all.is_empty(), "no varying class-ID slots found"); + let mn = all + .iter() + .map(|a| a.2) + .fold(f64::MAX, |g: f64, v: f64| f64::min(g, v)); + let mx = all + .iter() + .map(|a| a.2) + .fold(0.0f64, |g: f64, v: f64| f64::max(g, v)); + let dmin = all.iter().map(|a| a.3).min().unwrap(); + let dmax = all.iter().map(|a| a.3).max().unwrap(); + + eprintln!( + "chunk class-ID slots: {} types; best-slot enrichment {mn:.0}x..{mx:.0}x; \ + distinct IDs per slot {dmin}..{dmax}", + best.len() + ); + eprintln!( + " strongest (type, byte, ratio, distinct): {:?}", + &all[..all.len().min(6)] + ); +} diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index cf992ce..9144c9a 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -43,6 +43,16 @@ pub const CHUNK_GAP_BYTES: usize = 10; /// The value every tagged offset's top byte carries, on all 27 levels. pub const CHUNK_TAG: u8 = 0xf0; +/// Bytes between a chunk record's leading word and the payload the table +/// points at. +/// +/// A record is `[leading u16][two index bytes][payload]`, and the disc's +/// offset table points at the *payload*. The loaded table in RAM points two +/// bytes earlier, at the leading word -- the payload's index bytes are then at +/// `+2` from the pointer -- so the game's record starts here and the disc's +/// table is two bytes late. See [`Chunk::prefix`]. +pub const RECORD_PREFIX_BYTES: usize = 2; + /// The two bytes immediately preceding every chunk region on the disc. /// /// The last class body in a level -- shrub where the level has shrub, tie @@ -112,6 +122,26 @@ pub struct Chunk { pub offsets: Vec, /// Record stride, from the first two offsets. Zero if fewer than two. pub stride: usize, + /// The `u16` immediately before the first record's payload -- the record's + /// own leading word, which the loaded table points at. + /// + /// **Measured on the disc:** the same at every record of a chunk on + /// **1,439 of 1,509 chunks** -- `None` on the 70 that vary -- and fixed + /// within every chunk type, over the 71 types. Its values are 1, 2, 4, 6, + /// 8 and 10; 554 chunks carry 2 and 520 carry 4. + /// + /// **Measured against the running game:** in PCSX2's RAM the rewritten + /// offset table points at this word rather than at the payload, two bytes + /// earlier than the disc's table, so the game's record starts here. + /// + /// Not named. What the value counts is not established: it is *not* the + /// payload's entry count (the payload length divided by it is a whole + /// number on 1,344 chunks and an odd fraction on the other 162, leaving 57 + /// distinct "sizes"), and it is not the number of distinct leading bytes + /// (agrees on 0 of 1,506). An earlier note in this file read the + /// `02 00` seen in RAM as a prefix the loader had inserted; those bytes + /// are on the disc already, at every record, which is what this field is. + pub prefix: Option, } /// One chunk, with its records already bounds-checked. @@ -324,12 +354,35 @@ pub fn parse_chunk(region: &[u8], offset: usize) -> Result> { 0 }; + // The leading word sits two bytes before the payload the table points at. + // Reading it and requiring it to be the same at every record is a + // signature check like the header's last byte: it holds on every chunk on + // The leading word sits two bytes before the payload the table points at. + // It is the same at every record of a chunk on 1,439 of 1,509 chunks; the + // other 70 vary, so it stays an Option rather than a value a caller could + // mistake for a field that is always there. + let read_prefix = |low: u32| -> Option { + let at = offset + .checked_add(low as usize)? + .checked_sub(RECORD_PREFIX_BYTES)?; + region + .get(at..at + RECORD_PREFIX_BYTES) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + }; + let prefix = offsets.first().copied().and_then(read_prefix).filter(|&p| { + offsets + .iter() + .skip(1) + .all(|&low| read_prefix(low) == Some(p)) + }); + Ok(Some(Chunk { count: header[14], header, offset, offsets, stride, + prefix, })) } @@ -360,6 +413,7 @@ pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result> chunk: Chunk { count: chunk.count, header: chunk.header, + prefix: chunk.prefix, offset: chunk.offset, offsets: chunk.offsets.clone(), stride: chunk.stride, diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index f766e30..4e1638f 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1451,3 +1451,40 @@ The third result is the useful one: the byte at `+2` takes exactly `0, 16, 32, ... 240` -- the 16 values `index_rule` predicts for `step = 16, phase = 0`. The mixed-radix record index, derived from the disc, is corroborated on the running game's own copy. + +### Correction: the `02 00` is on the disc, and a record starts two bytes earlier + +What the loader does to the *pointer* is as described above. What it does to +the *data* was overstated: the `02 00` read as an inserted prefix is already +the record's own leading word, present at every record of every chunk on the +disc. Three measurements, in order: + +- **The 2 bytes before the payload are on disc.** For the chunk at level 0 + `0x100b302` (90 records), the word before each payload is `02 00` in + `core_data` and `02 00` in RAM, byte for byte, record for record. The first + chunk of the same level carries `08 00` instead. Nothing was inserted. +- **So the RAM table's pointer is simply two bytes earlier.** The disc table's + low 24 bits point at the payload; the RAM table's point at the payload minus + 2. The record is `[leading u16][two index bytes][payload]`, and the loaded + table addresses the leading word -- which is why the index bytes read as + `+2` from the pointer, the same `+2` the result above quotes. +- **The leading word is a field, not a prefix.** It is the same at every + record of a chunk on **1,439 of 1,509 chunks** (`None` on the 70 whose + records disagree, which the parser reports rather than averaging), it is + fixed within **every one of the 71 chunk types**, and its values are only + `1, 2, 4, 6, 8, 10` -- 554 chunks carry 2, 520 carry 4. `Chunk::prefix` + carries it. + +What the value counts is not established, and two readings were measured and +disproved rather than left as guesses: + +- **Not an entry count.** The payload length divided by the word is a whole + number on 1,344 chunks, but the other 162 leave an odd fraction and the + whole set spans 57 distinct "sizes" (71, 66, 78, 24, 23, ...). A real entry + size is a small clean number, not a set like that. +- **Not the number of distinct leading bytes.** That agrees on 0 of 1,506 + chunks. + +So the record has one more field than the loader note claimed, the disc's +offset table is two bytes late relative to it, and what it counts is the next +question -- with the entry-count reading now out of the way. diff --git a/docs/roadmap.md b/docs/roadmap.md index 29533ec..4cc80eb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -352,11 +352,17 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [x] **The loader's transformation, from PCSX2 RAM.** Disc offsets `0xf0000000 | low24` are rewritten in RAM to absolute EE pointers `0xf1850000 | low24` -- tag replaced by the chunk's load base, low 24 bits - untouched -- and every record gains a constant `02 00` prefix that the - rewritten table points at *instead of* the record data (**90 of 90** - records). The payload itself is copied **verbatim**. The index finding is - corroborated live: the byte at `+2` takes exactly `0, 16, ... 240`, the 16 - values `index_rule` predicts. + untouched. The payload itself is copied **verbatim**. The index finding + is corroborated live: the byte at `+2` takes exactly `0, 16, ... 240`, the + 16 values `index_rule` predicts. +- [x] **Corrected: a record starts two bytes before the table's offset.** The + `02 00` read in RAM as a loader-inserted prefix is the record's own + leading word, already on the disc at every record. The loaded table + simply points two bytes earlier than the disc's, at that word. It is a + field: uniform across a chunk's records on **1,439 of 1,509** chunks, + fixed within **all 71 types**, values `{1, 2, 4, 6, 8, 10}`. Disproved + readings: an entry count (57 distinct fractional "sizes") and a + distinct-leading-byte count (0 of 1,506 agree). - [ ] The chunk **record fields are not decoded**, and no type is named. The body is dense (74% of `f32` reads plausible against a 49% random control) with an index-like tail -- 12 of 163 `u16` slots in the largest type stay below