diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index ed433b9..c227c1d 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -5686,3 +5686,97 @@ fn overlay_bin_is_the_seven_section_program_not_the_lit_section() { [1,8,1,1,1,1,1]; section sizes fall {shortfall:?} bytes short of the lump" ); } + +/// The game's own moby class ids come from `lvl.vtbl`, and `moby_classes.count` +/// is **not** a valid bound on them. +/// +/// `lvl.vtbl` is section 3 of the level program: records of `{oClass, handler, +/// auxiliary pointer}`, three words each, ending on padding. Across the 27 +/// levels that is **1,163 distinct class ids**, ranging to `0x13BA` (5,050), +/// with a median of 38 new ids contributed per level. +/// +/// `LevelCoreHeader::moby_classes.count` is 238 on level 0. **1,135 of the 1,163 +/// ids -- 98% -- are at or above that number.** So a test that treats +/// "value < moby_classes.count" as "this looks like a class id" is really testing +/// "is this a small number". +/// +/// That is the mechanism behind the chunk class-ID artifact that was retracted: +/// a nine-value slot whose values are 1, 3, 4, 5, 10, 11, 12, 13 and 15 passed a +/// "less than the moby count" test at 100%, and scored 305x enrichment. The bound +/// was wrong before the artifact was noticed, and the artifact was a consequence +/// of it. +/// +/// Re-scored against the real id set, no chunk slot stands out: the best is 0.375 +/// against a chance rate of 0.018 (1,163 ids in 65,536 values), and a dozen slots +/// sit between 0.25 and 0.38 with no separation between them. +#[test] +fn moby_class_ids_come_from_lvl_vtbl_not_from_the_class_count() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::overlay_reloc::parse_overlay_sections; + use std::collections::BTreeSet; + + let mut classes: BTreeSet = BTreeSet::new(); + let mut records = 0usize; + let mut levels = 0usize; + for lv in 0..27usize { + let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else { + continue; + }; + let Ok(secs) = parse_overlay_sections(&ov) else { + continue; + }; + assert!(secs.len() >= 4, "level {lv}: {} sections", secs.len()); + let s = &secs[3]; + for r in 0..s.data.len() / 12 { + let c = u32::from_le_bytes(s.data[r * 12..r * 12 + 4].try_into().unwrap()); + if c == u32::MAX { + continue; + } + classes.insert(c); + records += 1; + } + levels += 1; + } + assert_eq!(levels, 27, "only {levels} overlays parsed"); + assert!(records > 5_000, "only {records} vtbl records walked"); + + let max = *classes.iter().next_back().unwrap(); + let idx0 = std::fs::read(data.join("level0/core_index.bin")).expect("core_index"); + let bound = LevelCoreHeader::parse(&idx0) + .expect("hdr") + .moby_classes + .count as u32; + let above = classes.iter().filter(|&&c| c >= bound).count(); + + eprintln!( + "lvl.vtbl: {n} distinct class ids from {records} records; max {max:#x} ({max}); \ + moby_classes.count is {bound}, and {above} of {n} ids ({pct:.1}%) are at or above it", + n = classes.len(), + pct = 100.0 * above as f64 / classes.len() as f64, + ); + + assert_eq!(classes.len(), 1163, "distinct class-id count moved"); + assert_eq!(max, 0x13BA, "the highest class id moved"); + assert!( + above * 100 >= classes.len() * 97, + "the class count stopped being a bad bound ({above}/{}); revisit why", + classes.len() + ); + assert!( + classes.contains(&0x2) && classes.contains(&0x0E), + "the small ids that open every vtbl are missing" + ); +} diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 88075cc..e4633e1 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -3065,3 +3065,58 @@ prints `.word` for anything undecodable, and reports the failure count. The undecodable words here are VU1 microcode (`0x0260202d` and friends are COP2 opcodes), which is expected and is itself informative: this function interleaves VIF-fed VU1 work with FPU arithmetic, like `tfrag`'s. + + +## `lvl.vtbl` gives the real class ids, and the old bound was wrong + +`lvl.vtbl` is **section 3** of the level program -- one of the seven sections the +Phase 3 port made visible. Its records are three words: + + [0] oClass the moby class identifier + [1] handler address of UpdateMoby_ in this level's .text + [2] auxiliary the same value for every record of a level, 26 of 27 + (level 1 has exactly one record pointing elsewhere, 0x2a5d94) + +Walking all 27 levels: + + 5,878 records + 1,163 distinct class ids + range 0 .. 0x13BA (5,050) + median 38 new ids contributed per level + +The decomp's `docs/MOBY-DISPATCH-TABLES.md` describes exactly this layout, and +independently notes that 5,598 of 5,788 handlers are preceded by a `jr ra` within +three words -- so the addresses are function entries, not arbitrary words. + +### Why this retires the earlier bound + +`LevelCoreHeader::moby_classes.count` is **238** on level 0. **1,135 of the 1,163 +class ids -- 97.6% -- are at or above that number.** + +So "value < moby_classes.count" was never a test for "this is a class id". It was +a test for "this is a small number". That is the mechanism behind the chunk +class-ID artifact: + + slot values 1, 3, 4, 5, 10, 11, 12, 13, 15 + every one is below 238 + so every one scored as a hit + 65536 / 238 = 275x ... and the largest such score was 305x + +The bound was wrong *before* the artifact was noticed, and the artifact was a +consequence of it. Retracting the claim for "small dominant values" was correct +but incomplete -- the reason those values looked significant is that the test +selected for smallness. + +### Re-scored against the real id set + +With the game's own 1,163 ids as the validity test, no chunk slot separates: + + best slot 0.375 (type f2ef4275 slot 34) + chance rate 0.018 (1,163 ids in 65,536 values) + a dozen slots 0.25 .. 0.38, with no gap between them + +Twenty times chance is not nothing, but a dozen slots packed into the same band +with nothing above them is what a broad property looks like, not a field. The +positive finding stands and is now usable: **the class-id space is known, and it +is not the class count.** Asserted in +`moby_class_ids_come_from_lvl_vtbl_not_from_the_class_count`. diff --git a/docs/roadmap.md b/docs/roadmap.md index 554b771..38df58a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -399,6 +399,14 @@ Phase 1 hands over the extents; this phase works out what each one contains. makes the record size fall out of the data rather than being assumed. 1,270,802 vertices over 4,560 class spans; 116 spans carry no geometry. Vertices are 16-byte VIF-packed half-float quads, handed back raw. +- [x] The **moby class-id space is known**: `lvl.vtbl` (section 3 of the level + program) is records of `{oClass, handler, auxiliary}`, and across 27 levels + that is **1,163 distinct ids** reaching `0x13BA` (5,050), median 38 new per + level. `moby_classes.count` is 238 and **97.6% of real ids are at or above + it**, so the count was never a valid bound -- which is the mechanism behind + the chunk class-ID artifact now retracted. Re-scored against the real set, + no chunk slot separates (best 0.375 against 0.018 chance, a dozen slots in + the same band). Asserted. - [ ] `moby_anim` — **still open, and the obvious reading is ruled out.** An instance table would have to scale with the level's moby class count, but **per-chunk-type record counts are level-independent** -- the same types