moby_classes.offset enrichment is size-independent: 13x to 72x at matched size
Runs the control06531fbnamed. It settles the question that commit left open, and the answer is that the enrichment is real. Matching on immediate-set size, against the best of the sixteen adjacent values: bucket n 0xc0 best neighbour ratio 0-15 1,082 72 0xb8:1 72.0x 16-31 713 119 0xc8:4 29.8x 32-47 291 104 0xb8:3 34.7x 48-63 119 39 0xb8:2 19.5x 64-79 59 31 0xc8:2 15.5x 80-95 25 13 0xb8:1 13.0x Enriched in every bucket, and the ratio does not decay as sets grow. That is the opposite of a pigeonhole artifact -- a volume effect would shrink the ratio in the big buckets, and it rises instead. The load-bearing row is the smallest: 72 of 1,082 functions carrying fewer than 16 distinct immediates contain 0xc0, where the best neighbour appears once. A set that small cannot produce that by count alone. What this revives. The 300 candidates with 48 or fewer distinct immediates are not noise: at matched size, neighbours hit 0.4% to 1.4% in those buckets where 0xc0 hits 35%. A function using fewer than 16 constants, one of which is 0xc0, uses that offset deliberately. Those are the shortlist. Why this took four commits to establish.3ea5ab7measured 394 against neighbours 1-14 and called it a shortlist.06531fbshowed the rate rises with set size and said the 394 was a list of large functions. Both were right about their own measurement and neither had the bucket-matched comparison, which is the only form that separates "the constant is special" from "big functions contain more constants". Twenty-two probes this stretch, and the thing that finally settled it was not a better measurement but adding the right comparison to a measurement taken three commits earlier. Caveat unchanged: presence of an immediate is not proof the chunk is read there. It is proof the constant is used, which is all either test asserts. 284 passing with the disc present (23 + 19 + 64 + 156 + 22; disc binary ~35s), clippy clean.
This commit is contained in:
@@ -6818,3 +6818,139 @@ fn moby_classes_offset_is_referenced_far_more_than_its_neighbours() {
|
||||
"only {self_hits} vs neighbours up to {neighbour_max}: no longer special"
|
||||
);
|
||||
}
|
||||
|
||||
/// `moby_classes.offset` is enriched over its neighbours *at matched set size*.
|
||||
///
|
||||
/// 06531fb showed the rate at which a function contains `0x00c0` rises with how
|
||||
/// many distinct immediates it carries, so the raw 394 could have been a
|
||||
/// pigeonhole artifact. Matching on set size removes that: within every bucket,
|
||||
/// `0xc0` beats the best of its sixteen neighbours by 13x to 72x.
|
||||
///
|
||||
/// The load-bearing row is the smallest bucket. **72 of 1,082 functions carrying
|
||||
/// fewer than 16 distinct immediates contain `0xc0`, where the best neighbour
|
||||
/// appears once.** Small sets cannot produce that by chance of volume, so the
|
||||
/// constant is genuinely special rather than merely common in big functions.
|
||||
///
|
||||
/// Bounds: level 0 only; function extents from the decomp, "level 0" meaning
|
||||
/// first listed place is level 00; 128-byte size floor. Presence of an immediate
|
||||
/// is not proof the chunk is read -- only that the constant is used.
|
||||
#[test]
|
||||
fn moby_classes_offset_enrichment_survives_set_size_matching() {
|
||||
require_disc!();
|
||||
let data = std::env::var_os("GC_EXTRACTED")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| {
|
||||
PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted")
|
||||
});
|
||||
let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv");
|
||||
if !data.exists() || !dcomp.exists() {
|
||||
eprintln!("skipped");
|
||||
return;
|
||||
}
|
||||
|
||||
let txt = std::fs::read_to_string(dcomp).unwrap();
|
||||
let mut funcs: Vec<(u32, u32)> = Vec::new();
|
||||
for line in txt.lines() {
|
||||
if line.starts_with('#') || line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let f: Vec<&str> = line.split('\t').collect();
|
||||
if f.len() < 6 {
|
||||
continue;
|
||||
}
|
||||
let size: u32 = match f[2].parse() {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for pl in f[5].split(',') {
|
||||
let mut it = pl.split(':');
|
||||
if it.next() == Some("00") {
|
||||
if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) {
|
||||
funcs.push((a, size));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
funcs.sort_unstable();
|
||||
|
||||
let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap();
|
||||
let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap();
|
||||
let h = gc_formats::level_core::LevelCoreHeader::parse(&idx).unwrap();
|
||||
let secs = gc_formats::overlay_reloc::parse_overlay_sections(&ov).unwrap();
|
||||
let text = &secs[6].data;
|
||||
let dest = secs[6].dest;
|
||||
let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06);
|
||||
let base = h.moby_classes.offset as u16;
|
||||
|
||||
// buckets of 16 distinct immediates; per bucket count hits for each value 0xb8..0xc8
|
||||
let mut tot: std::collections::BTreeMap<usize, usize> = std::collections::BTreeMap::new();
|
||||
let mut hit: std::collections::BTreeMap<(usize, u16), usize> =
|
||||
std::collections::BTreeMap::new();
|
||||
for (start, size) in &funcs {
|
||||
if *size < 128 {
|
||||
continue;
|
||||
}
|
||||
let off = (*start).wrapping_sub(dest) as usize;
|
||||
if off >= text.len() {
|
||||
continue;
|
||||
}
|
||||
let n = (*size as usize).min(text.len() - off) & !3;
|
||||
let mut set = std::collections::BTreeSet::new();
|
||||
for w in text[off..off + n].chunks_exact(4) {
|
||||
let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]);
|
||||
if has_imm((i >> 26) as u8) {
|
||||
set.insert((i & 0xffff) as u16);
|
||||
}
|
||||
}
|
||||
if set.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let b = (set.len() / 16) * 16;
|
||||
*tot.entry(b).or_default() += 1;
|
||||
for d in -8i32..=8i32 {
|
||||
let v = (base as i32 + d) as u16;
|
||||
if set.contains(&v) {
|
||||
*hit.entry((b, v)).or_default() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("M bucket n 0xc0 best_neighbour ratio");
|
||||
let mut checked = 0usize;
|
||||
let mut worst = f64::INFINITY;
|
||||
for (b, n) in &tot {
|
||||
if *n < 20 {
|
||||
continue;
|
||||
}
|
||||
let c0 = hit.get(&(*b, base)).copied().unwrap_or(0);
|
||||
let mut best = 0u16;
|
||||
let mut bestc = 0usize;
|
||||
for d in -8i32..=8i32 {
|
||||
if d == 0 {
|
||||
continue;
|
||||
}
|
||||
let v = (base as i32 + d) as u16;
|
||||
let c = hit.get(&(*b, v)).copied().unwrap_or(0);
|
||||
if c > bestc {
|
||||
bestc = c;
|
||||
best = v;
|
||||
}
|
||||
}
|
||||
let ratio = c0 as f64 / bestc.max(1) as f64;
|
||||
eprintln!("M {b:5} {n:4} {c0:4} {best:#06x}:{bestc:<4} {ratio:.1}x");
|
||||
worst = worst.min(ratio);
|
||||
checked += 1;
|
||||
}
|
||||
assert!(checked >= 5, "only {checked} buckets large enough to judge");
|
||||
// Enriched in every bucket, not just the big ones.
|
||||
assert!(
|
||||
worst > 5.0,
|
||||
"enrichment falls to {worst:.1}x at some set size"
|
||||
);
|
||||
// And specifically in the smallest bucket, where volume cannot explain it.
|
||||
let small_tot = tot.get(&0).copied().unwrap_or(0);
|
||||
let small_c0 = hit.get(&(0usize, base)).copied().unwrap_or(0);
|
||||
assert!(small_tot > 1_000, "small bucket only {small_tot}");
|
||||
assert!(small_c0 > 50, "only {small_c0} small functions contain it");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user