From 36d8d2b00c65590255849697fd1c0a65ccf9b0e9 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Wed, 14 Oct 2026 06:23:49 +0200 Subject: [PATCH] The frequency enrichment survives the redo; the disassembly does not Redone through overlay_bytes_at, the verified flat mapping: functions kept 2,313 moby_classes 0x00c0 391 (displaced: 394) best neighbour (0x00c8) 15 (displaced: 4) bucket n 0xc0 best neighbour ratio (displaced) 0-15 1,033 73 2 36.5x 72.0x 16-31 759 118 4 29.5x 29.8x 32-47 283 89 2 44.5x 34.7x 48-63 118 41 3 13.7x 19.5x 64-79 55 32 2 16.0x 15.5x 80-95 37 22 2 11.0x 13.0x The reason it survives is worth stating, because it separates the two results. The displacement was uniform -- every window shifted by the same amount -- so a function's contents barely changed, and a frequency count over a window is robust to that. A disassembly is not: it depends on the exact byte alignment, which is what was wrong. Hence 394 becoming 391 with the same shape, while the 0x3a8060 walk and the 16-byte stride evaporated entirely. So the investigation was not "all wrong". Precisely: - the frequency enrichment is real, now on verified bytes, with a test that fails if it goes; - the disassembly was never real, and the whole jal chain built on it is retracted; - no loader has been located, and 0x2ee138 is VU microcode. THE LESSON IS ABOUT WHICH MEASUREMENTS SURVIVE A MISTAKE A count over a window tolerates a constant displacement. A decode does not. Sixteen probes leaned on the kind that does not tolerate it, and reported each with the same confidence as the kind that does. Nothing about the numbers distinguished them at the time -- that is the whole difficulty, and why the verification in cd16485 was the missing step rather than a formality. Next: the ~391 functions are a legitimate shortlist, but consuming them means disassembly through tools/overlay-anchors, whose output is now checked against a word the build pins. 286 passing with the disc present (23 + 19 + 66 + 156 + 22; disc binary 34.89s), clippy clean. --- crates/gc-extract/tests/disc.rs | 141 ++++++++++++++++++++++++++++++++ docs/plan/level-assets.md | 42 ++++++++++ 2 files changed, 183 insertions(+) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 70fe212..3a8285b 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -7006,3 +7006,144 @@ fn overlay_section_index_agrees_with_the_flat_blob() { eprintln!("{checked} words agree; 0x2ee138 verified"); } + +/// `moby_classes.offset` is enriched over its neighbours, on correct bytes. +/// +/// This is `moby_classes_offset_enrichment_survives_set_size_matching` redone +/// through `overlay_bytes_at` instead of the displaced section index, after +/// `cd16485` established the flat memory image is the correct mapping. +/// +/// It survives. 391 of 2,313 level-0 functions carry the immediate `0x00c0` +/// against a best neighbour of 15, and at matched immediate-set size the ratio +/// runs 11x to 45x. The displaced version measured 394 and 13x-72x, so the +/// uniform displacement shifted windows without materially changing their +/// contents -- which is why the frequency result held and the disassembly did not. +/// +/// Bounds: level 0 only; extents from the decomp, "level 0" meaning first listed +/// place is level 00; 128-byte size floor. Presence of an immediate is not proof +/// the chunk is read there -- only that the constant is used. +#[test] +fn moby_classes_enrichment_survives_the_correct_mapping() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted") + }); + let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv"); + if !data.exists() || !dcomp.exists() { + eprintln!("skipped"); + return; + } + let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap(); + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::overlay_reloc::overlay_bytes_at; + use std::collections::{BTreeMap, BTreeSet}; + + let txt = std::fs::read_to_string(dcomp).unwrap(); + let mut funcs: Vec<(u32, u32)> = Vec::new(); + for line in txt.lines() { + if line.starts_with('#') || line.trim().is_empty() { + continue; + } + let f: Vec<&str> = line.split('\t').collect(); + if f.len() < 6 { + continue; + } + let size: u32 = match f[2].parse() { + Ok(v) => v, + Err(_) => continue, + }; + for pl in f[5].split(',') { + let mut it = pl.split(':'); + if it.next() == Some("00") { + if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) { + funcs.push((a, size)); + break; + } + } + } + } + funcs.sort_unstable(); + + let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap(); + let h = LevelCoreHeader::parse(&idx).unwrap(); + let base = h.moby_classes.offset as u16; + let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06); + + let mut sets: Vec> = Vec::new(); + for (start, size) in &funcs { + if *size < 128 { + continue; + } + let b = match overlay_bytes_at(&ov, *start) { + Some(b) => b, + None => continue, + }; + let n = (*size as usize).min(b.len()) & !3; + let mut set = BTreeSet::new(); + for w in b[..n].chunks_exact(4) { + let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]); + if has_imm((i >> 26) as u8) { + set.insert((i & 0xffff) as u16); + } + } + if !set.is_empty() { + sets.push(set); + } + } + let hit = |v: u16| sets.iter().filter(|s| s.contains(&v)).count(); + eprintln!("F functions kept {} (>=128b, decodable window)", sets.len()); + eprintln!("F moby_classes {base:#06x} -> {}", hit(base)); + assert!(sets.len() > 2_000, "only {} functions", sets.len()); + let self_hits = hit(base); + assert!( + self_hits > 100, + "only {self_hits} functions carry {base:#06x}" + ); + let mut nb = Vec::new(); + for d in -8i32..=8i32 { + if d != 0 { + let v = (base as i32 + d) as u16; + nb.push((v, hit(v))); + } + } + nb.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); + let vals: Vec = nb.iter().map(|(v, c)| format!("{v:#06x}:{c}")).collect(); + eprintln!("F neighbours {}", vals.join(" ")); + // bucket-matched, as before + let mut tot: BTreeMap = BTreeMap::new(); + let mut bh: BTreeMap<(usize, u16), usize> = BTreeMap::new(); + for s in &sets { + let b = (s.len() / 16) * 16; + *tot.entry(b).or_default() += 1; + for d in -8i32..=8i32 { + let v = (base as i32 + d) as u16; + if s.contains(&v) { + *bh.entry((b, v)).or_default() += 1; + } + } + } + for (b, n) in &tot { + if *n < 20 { + continue; + } + let c0 = bh.get(&(*b, base)).copied().unwrap_or(0); + let mut best = 0usize; + for d in -8i32..=8i32 { + if d != 0 { + let v = (base as i32 + d) as u16; + best = best.max(bh.get(&(*b, v)).copied().unwrap_or(0)); + } + } + eprintln!( + "F bucket {b:5} n={n:4} c0={c0:4} best_neighbour={best:4} ratio={:.1}", + c0 as f64 / best.max(1) as f64 + ); + assert!( + c0 > best * 5, + "bucket {b}: {c0} vs neighbour {best} -- enrichment gone at this set size" + ); + } +} diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index e47c726..b2fa364 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -2499,3 +2499,45 @@ The honest next step is to redo the frequency measurement with `overlay_bytes_at and the decomp extents, and to accept that it may well produce nothing -- the `0xc0` enrichment was a property of displaced windows, and there is no reason to expect it to survive. + + +### The frequency result survives; the disassembly does not + +Redone through `overlay_bytes_at`: + + functions kept 2,313 + moby_classes 0x00c0 391 (displaced: 394) + best neighbour (0x00c8) 15 (displaced: 4) + + bucket n 0xc0 best neighbour ratio (displaced ratio) + 0-15 1,033 73 2 36.5x 72.0x + 16-31 759 118 4 29.5x 29.8x + 32-47 283 89 2 44.5x 34.7x + 48-63 118 41 3 13.7x 19.5x + 64-79 55 32 2 16.0x 15.5x + 80-95 37 22 2 11.0x 13.0x + +**It survives, and the reason is worth stating because it separates the two +results.** The displacement was uniform -- every window shifted by the same amount +-- so a function's *contents* barely changed and a frequency count over a window is +robust to it. A disassembly is not: it depends on the exact alignment of the bytes, +which is precisely what was wrong. Hence 394 -> 391 with the same shape, while the +`0x3a8060` walk evaporated entirely. + +So the correct summary of this investigation is not "it was all wrong". It is: + +- **the frequency result is real**, and is now measured on verified bytes with a + test that will fail if the enrichment goes; +- **the disassembly result was never real**, and the whole `jal` chain built on it + is retracted; +- **no loader has been located**, and 0x2ee138 is VU microcode. + +The lesson is about which measurements survive a mistake and which do not, not +about avoiding mistakes. A count over a window tolerates a constant displacement; a +decode does not. Sixteen probes leaned on the kind of measurement that does not, +and reported every one with the same confidence as the kind that does. + +**Next:** with the mapping verified and the enrichment confirmed, the ~391 functions +are a legitimate shortlist -- but the disassembly that would consume them must go +through `tools/overlay-anchors`, whose output has now been checked against a word +the build pins.