diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 70fe212..3a8285b 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -7006,3 +7006,144 @@ fn overlay_section_index_agrees_with_the_flat_blob() { eprintln!("{checked} words agree; 0x2ee138 verified"); } + +/// `moby_classes.offset` is enriched over its neighbours, on correct bytes. +/// +/// This is `moby_classes_offset_enrichment_survives_set_size_matching` redone +/// through `overlay_bytes_at` instead of the displaced section index, after +/// `cd16485` established the flat memory image is the correct mapping. +/// +/// It survives. 391 of 2,313 level-0 functions carry the immediate `0x00c0` +/// against a best neighbour of 15, and at matched immediate-set size the ratio +/// runs 11x to 45x. The displaced version measured 394 and 13x-72x, so the +/// uniform displacement shifted windows without materially changing their +/// contents -- which is why the frequency result held and the disassembly did not. +/// +/// Bounds: level 0 only; extents from the decomp, "level 0" meaning first listed +/// place is level 00; 128-byte size floor. Presence of an immediate is not proof +/// the chunk is read there -- only that the constant is used. +#[test] +fn moby_classes_enrichment_survives_the_correct_mapping() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted") + }); + let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv"); + if !data.exists() || !dcomp.exists() { + eprintln!("skipped"); + return; + } + let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap(); + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::overlay_reloc::overlay_bytes_at; + use std::collections::{BTreeMap, BTreeSet}; + + let txt = std::fs::read_to_string(dcomp).unwrap(); + let mut funcs: Vec<(u32, u32)> = Vec::new(); + for line in txt.lines() { + if line.starts_with('#') || line.trim().is_empty() { + continue; + } + let f: Vec<&str> = line.split('\t').collect(); + if f.len() < 6 { + continue; + } + let size: u32 = match f[2].parse() { + Ok(v) => v, + Err(_) => continue, + }; + for pl in f[5].split(',') { + let mut it = pl.split(':'); + if it.next() == Some("00") { + if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) { + funcs.push((a, size)); + break; + } + } + } + } + funcs.sort_unstable(); + + let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap(); + let h = LevelCoreHeader::parse(&idx).unwrap(); + let base = h.moby_classes.offset as u16; + let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06); + + let mut sets: Vec> = Vec::new(); + for (start, size) in &funcs { + if *size < 128 { + continue; + } + let b = match overlay_bytes_at(&ov, *start) { + Some(b) => b, + None => continue, + }; + let n = (*size as usize).min(b.len()) & !3; + let mut set = BTreeSet::new(); + for w in b[..n].chunks_exact(4) { + let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]); + if has_imm((i >> 26) as u8) { + set.insert((i & 0xffff) as u16); + } + } + if !set.is_empty() { + sets.push(set); + } + } + let hit = |v: u16| sets.iter().filter(|s| s.contains(&v)).count(); + eprintln!("F functions kept {} (>=128b, decodable window)", sets.len()); + eprintln!("F moby_classes {base:#06x} -> {}", hit(base)); + assert!(sets.len() > 2_000, "only {} functions", sets.len()); + let self_hits = hit(base); + assert!( + self_hits > 100, + "only {self_hits} functions carry {base:#06x}" + ); + let mut nb = Vec::new(); + for d in -8i32..=8i32 { + if d != 0 { + let v = (base as i32 + d) as u16; + nb.push((v, hit(v))); + } + } + nb.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); + let vals: Vec = nb.iter().map(|(v, c)| format!("{v:#06x}:{c}")).collect(); + eprintln!("F neighbours {}", vals.join(" ")); + // bucket-matched, as before + let mut tot: BTreeMap = BTreeMap::new(); + let mut bh: BTreeMap<(usize, u16), usize> = BTreeMap::new(); + for s in &sets { + let b = (s.len() / 16) * 16; + *tot.entry(b).or_default() += 1; + for d in -8i32..=8i32 { + let v = (base as i32 + d) as u16; + if s.contains(&v) { + *bh.entry((b, v)).or_default() += 1; + } + } + } + for (b, n) in &tot { + if *n < 20 { + continue; + } + let c0 = bh.get(&(*b, base)).copied().unwrap_or(0); + let mut best = 0usize; + for d in -8i32..=8i32 { + if d != 0 { + let v = (base as i32 + d) as u16; + best = best.max(bh.get(&(*b, v)).copied().unwrap_or(0)); + } + } + eprintln!( + "F bucket {b:5} n={n:4} c0={c0:4} best_neighbour={best:4} ratio={:.1}", + c0 as f64 / best.max(1) as f64 + ); + assert!( + c0 > best * 5, + "bucket {b}: {c0} vs neighbour {best} -- enrichment gone at this set size" + ); + } +} diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index e47c726..b2fa364 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -2499,3 +2499,45 @@ The honest next step is to redo the frequency measurement with `overlay_bytes_at and the decomp extents, and to accept that it may well produce nothing -- the `0xc0` enrichment was a property of displaced windows, and there is no reason to expect it to survive. + + +### The frequency result survives; the disassembly does not + +Redone through `overlay_bytes_at`: + + functions kept 2,313 + moby_classes 0x00c0 391 (displaced: 394) + best neighbour (0x00c8) 15 (displaced: 4) + + bucket n 0xc0 best neighbour ratio (displaced ratio) + 0-15 1,033 73 2 36.5x 72.0x + 16-31 759 118 4 29.5x 29.8x + 32-47 283 89 2 44.5x 34.7x + 48-63 118 41 3 13.7x 19.5x + 64-79 55 32 2 16.0x 15.5x + 80-95 37 22 2 11.0x 13.0x + +**It survives, and the reason is worth stating because it separates the two +results.** The displacement was uniform -- every window shifted by the same amount +-- so a function's *contents* barely changed and a frequency count over a window is +robust to it. A disassembly is not: it depends on the exact alignment of the bytes, +which is precisely what was wrong. Hence 394 -> 391 with the same shape, while the +`0x3a8060` walk evaporated entirely. + +So the correct summary of this investigation is not "it was all wrong". It is: + +- **the frequency result is real**, and is now measured on verified bytes with a + test that will fail if the enrichment goes; +- **the disassembly result was never real**, and the whole `jal` chain built on it + is retracted; +- **no loader has been located**, and 0x2ee138 is VU microcode. + +The lesson is about which measurements survive a mistake and which do not, not +about avoiding mistakes. A count over a window tolerates a constant displacement; a +decode does not. Sixteen probes leaned on the kind of measurement that does not, +and reported every one with the same confidence as the kind that does. + +**Next:** with the mapping verified and the enrichment confirmed, the ~391 functions +are a legitimate shortlist -- but the disassembly that would consume them must go +through `tools/overlay-anchors`, whose output has now been checked against a word +the build pins.