The frequency enrichment survives the redo; the disassembly does not
Redone through overlay_bytes_at, the verified flat mapping:
functions kept 2,313
moby_classes 0x00c0 391 (displaced: 394)
best neighbour (0x00c8) 15 (displaced: 4)
bucket n 0xc0 best neighbour ratio (displaced)
0-15 1,033 73 2 36.5x 72.0x
16-31 759 118 4 29.5x 29.8x
32-47 283 89 2 44.5x 34.7x
48-63 118 41 3 13.7x 19.5x
64-79 55 32 2 16.0x 15.5x
80-95 37 22 2 11.0x 13.0x
The reason it survives is worth stating, because it separates the two results. The
displacement was uniform -- every window shifted by the same amount -- so a
function's contents barely changed, and a frequency count over a window is robust
to that. A disassembly is not: it depends on the exact byte alignment, which is
what was wrong. Hence 394 becoming 391 with the same shape, while the 0x3a8060
walk and the 16-byte stride evaporated entirely.
So the investigation was not "all wrong". Precisely:
- the frequency enrichment is real, now on verified bytes, with a test that fails
if it goes;
- the disassembly was never real, and the whole jal chain built on it is retracted;
- no loader has been located, and 0x2ee138 is VU microcode.
THE LESSON IS ABOUT WHICH MEASUREMENTS SURVIVE A MISTAKE
A count over a window tolerates a constant displacement. A decode does not. Sixteen
probes leaned on the kind that does not tolerate it, and reported each with the
same confidence as the kind that does. Nothing about the numbers distinguished them
at the time -- that is the whole difficulty, and why the verification in cd16485
was the missing step rather than a formality.
Next: the ~391 functions are a legitimate shortlist, but consuming them means
disassembly through tools/overlay-anchors, whose output is now checked against a
word the build pins.
286 passing with the disc present (23 + 19 + 66 + 156 + 22; disc binary 34.89s),
clippy clean.
This commit is contained in:
@@ -7006,3 +7006,144 @@ fn overlay_section_index_agrees_with_the_flat_blob() {
|
||||
|
||||
eprintln!("{checked} words agree; 0x2ee138 verified");
|
||||
}
|
||||
|
||||
/// `moby_classes.offset` is enriched over its neighbours, on correct bytes.
|
||||
///
|
||||
/// This is `moby_classes_offset_enrichment_survives_set_size_matching` redone
|
||||
/// through `overlay_bytes_at` instead of the displaced section index, after
|
||||
/// `cd16485` established the flat memory image is the correct mapping.
|
||||
///
|
||||
/// It survives. 391 of 2,313 level-0 functions carry the immediate `0x00c0`
|
||||
/// against a best neighbour of 15, and at matched immediate-set size the ratio
|
||||
/// runs 11x to 45x. The displaced version measured 394 and 13x-72x, so the
|
||||
/// uniform displacement shifted windows without materially changing their
|
||||
/// contents -- which is why the frequency result held and the disassembly did not.
|
||||
///
|
||||
/// Bounds: level 0 only; extents from the decomp, "level 0" meaning first listed
|
||||
/// place is level 00; 128-byte size floor. Presence of an immediate is not proof
|
||||
/// the chunk is read there -- only that the constant is used.
|
||||
#[test]
|
||||
fn moby_classes_enrichment_survives_the_correct_mapping() {
|
||||
require_disc!();
|
||||
let data = std::env::var_os("GC_EXTRACTED")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| {
|
||||
PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted")
|
||||
});
|
||||
let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv");
|
||||
if !data.exists() || !dcomp.exists() {
|
||||
eprintln!("skipped");
|
||||
return;
|
||||
}
|
||||
let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap();
|
||||
use gc_formats::level_core::LevelCoreHeader;
|
||||
use gc_formats::overlay_reloc::overlay_bytes_at;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
let txt = std::fs::read_to_string(dcomp).unwrap();
|
||||
let mut funcs: Vec<(u32, u32)> = Vec::new();
|
||||
for line in txt.lines() {
|
||||
if line.starts_with('#') || line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let f: Vec<&str> = line.split('\t').collect();
|
||||
if f.len() < 6 {
|
||||
continue;
|
||||
}
|
||||
let size: u32 = match f[2].parse() {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for pl in f[5].split(',') {
|
||||
let mut it = pl.split(':');
|
||||
if it.next() == Some("00") {
|
||||
if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) {
|
||||
funcs.push((a, size));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
funcs.sort_unstable();
|
||||
|
||||
let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap();
|
||||
let h = LevelCoreHeader::parse(&idx).unwrap();
|
||||
let base = h.moby_classes.offset as u16;
|
||||
let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06);
|
||||
|
||||
let mut sets: Vec<BTreeSet<u16>> = Vec::new();
|
||||
for (start, size) in &funcs {
|
||||
if *size < 128 {
|
||||
continue;
|
||||
}
|
||||
let b = match overlay_bytes_at(&ov, *start) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let n = (*size as usize).min(b.len()) & !3;
|
||||
let mut set = BTreeSet::new();
|
||||
for w in b[..n].chunks_exact(4) {
|
||||
let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]);
|
||||
if has_imm((i >> 26) as u8) {
|
||||
set.insert((i & 0xffff) as u16);
|
||||
}
|
||||
}
|
||||
if !set.is_empty() {
|
||||
sets.push(set);
|
||||
}
|
||||
}
|
||||
let hit = |v: u16| sets.iter().filter(|s| s.contains(&v)).count();
|
||||
eprintln!("F functions kept {} (>=128b, decodable window)", sets.len());
|
||||
eprintln!("F moby_classes {base:#06x} -> {}", hit(base));
|
||||
assert!(sets.len() > 2_000, "only {} functions", sets.len());
|
||||
let self_hits = hit(base);
|
||||
assert!(
|
||||
self_hits > 100,
|
||||
"only {self_hits} functions carry {base:#06x}"
|
||||
);
|
||||
let mut nb = Vec::new();
|
||||
for d in -8i32..=8i32 {
|
||||
if d != 0 {
|
||||
let v = (base as i32 + d) as u16;
|
||||
nb.push((v, hit(v)));
|
||||
}
|
||||
}
|
||||
nb.sort_by_key(|(_, c)| std::cmp::Reverse(*c));
|
||||
let vals: Vec<String> = nb.iter().map(|(v, c)| format!("{v:#06x}:{c}")).collect();
|
||||
eprintln!("F neighbours {}", vals.join(" "));
|
||||
// bucket-matched, as before
|
||||
let mut tot: BTreeMap<usize, usize> = BTreeMap::new();
|
||||
let mut bh: BTreeMap<(usize, u16), usize> = BTreeMap::new();
|
||||
for s in &sets {
|
||||
let b = (s.len() / 16) * 16;
|
||||
*tot.entry(b).or_default() += 1;
|
||||
for d in -8i32..=8i32 {
|
||||
let v = (base as i32 + d) as u16;
|
||||
if s.contains(&v) {
|
||||
*bh.entry((b, v)).or_default() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (b, n) in &tot {
|
||||
if *n < 20 {
|
||||
continue;
|
||||
}
|
||||
let c0 = bh.get(&(*b, base)).copied().unwrap_or(0);
|
||||
let mut best = 0usize;
|
||||
for d in -8i32..=8i32 {
|
||||
if d != 0 {
|
||||
let v = (base as i32 + d) as u16;
|
||||
best = best.max(bh.get(&(*b, v)).copied().unwrap_or(0));
|
||||
}
|
||||
}
|
||||
eprintln!(
|
||||
"F bucket {b:5} n={n:4} c0={c0:4} best_neighbour={best:4} ratio={:.1}",
|
||||
c0 as f64 / best.max(1) as f64
|
||||
);
|
||||
assert!(
|
||||
c0 > best * 5,
|
||||
"bucket {b}: {c0} vs neighbour {best} -- enrichment gone at this set size"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user