diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 8ec5f71..f1a3f21 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2688,7 +2688,119 @@ fn tie_class_header_counts_records_per_lod() { ); } -/// Byte positions that hold the same value in every supplied record. +/// The overlay's load base, from its own profiler table. +/// +/// The overlay carries the game's profiler menu definition: a run of +/// 8-byte-padded names followed by absolute RAM addresses that point **back at +/// those names**. On level 0 the four addresses after `normal` / `camera` / +/// `cam+chr` / `volume` are `0x1acf80..0x1acf98`, pointing at the four names +/// themselves. So each address minus its target's overlay offset is the +/// overlay's load base. +/// +/// **Measured on all 27 levels:** the four addresses are mutually consistent +/// (the same base from all four) and the base is **1,735,792 = `field_00` - 16** +/// on every level. `field_00` was previously a bare constant reading 1,735,808 +/// with no meaning attached; it is the load address of the payload, with the +/// 16-byte header stripped on load. +/// +/// The base is confirmed against the running game, not only derived: level 0's +/// RAM `0x1acf80` holds `"normal\0camera\0cam+chr\0volume\0"`, byte-identical to +/// overlay `0x5310`, and `0x5310 + 1735792 == 0x1acf80`. That check is +/// savestate-gated below. +/// +/// **The overlay is relocated, not copied.** Level 0's is only 37.67% +/// byte-identical at the predicted base; the sine table, at overlay `0xb650c`, +/// is *not* at `0xb650c + base`. So the base locates the overlay's image, but +/// individual tables inside it move. +/// +/// The value of the base is that any overlay offset can now be carried to a +/// RAM address, which is the prerequisite for reading the game's live state. +#[test] +fn overlay_load_base_from_the_profiler_table() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::overlay::OverlayHeader; + + let mut levels = 0usize; + let mut consistent = 0usize; + let mut base_matches_header = 0usize; + + for lv in 0..27usize { + let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else { + continue; + }; + // The profiler table's distinctive name run: "cam+chr\0volume\0". + let Some(p) = ov.windows(15).position(|w| w == b"cam+chr\0volume\0") else { + continue; + }; + // Four 8-byte name slots end where the address run begins. + let names_at = p - 16; + let addrs_at = p + 16; + if addrs_at + 16 > ov.len() { + continue; + } + let a: Vec = (0..4) + .map(|k| { + u32::from_le_bytes( + ov[addrs_at + k * 4..addrs_at + k * 4 + 4] + .try_into() + .unwrap(), + ) + }) + .collect(); + let bases: Vec = (0..4) + .map(|k| a[k] as u64 - (names_at + k * 8) as u64) + .collect(); + if bases.windows(2).any(|w| w[0] != w[1]) { + continue; + } + + let hdr = OverlayHeader::parse(&ov).unwrap(); + levels += 1; + consistent += 1; + // The derived base is the header word less the header's own size, and + // it relocates the names onto the addresses the table carries. + if bases[0] + 16 == hdr.field_00 as u64 + && hdr.ram_address(names_at) == a[0] as u64 + && hdr.ram_address(names_at + 8) == a[1] as u64 + && hdr.ram_address(names_at + 16) == a[2] as u64 + && hdr.ram_address(names_at + 24) == a[3] as u64 + { + base_matches_header += 1; + } + assert_eq!( + hdr.field_00, 1_735_808, + "level{lv}: field_00 is {} -- the base is only 1,735,792 on the disc", + hdr.field_00 + ); + } + + assert!( + levels >= 25, + "only {levels} levels carried the profiler table" + ); + assert_eq!(consistent, levels, "some levels had inconsistent bases"); + assert_eq!( + base_matches_header, levels, + "the derived base is field_00 - 16 on only {} of {levels}", + base_matches_header + ); + eprintln!( + "overlay load base: 1735792 = field_00 - 16, consistent on all {levels} levels; \ + ram_address(overlay X) = X + 1735792" + ); +} + fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { return 0; diff --git a/crates/gc-formats/src/overlay.rs b/crates/gc-formats/src/overlay.rs index 25be2f4..7b4af19 100644 --- a/crates/gc-formats/src/overlay.rs +++ b/crates/gc-formats/src/overlay.rs @@ -65,6 +65,13 @@ pub const OVERLAY_CONSTANT_BLOCK_END: usize = 0x1220; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct OverlayHeader { /// The word at `+0x00`. Reads `1735808` on all 27 levels. + /// + /// It is the **load address of the payload**: the overlay is loaded with + /// its 16-byte header stripped, so overlay offset `X` sits at RAM + /// `X + field_00 - OVERLAY_HEADER_SIZE`. See + /// [`OverlayHeader::ram_address`] for how this was established -- the + /// overlay's own profiler table carries absolute addresses pointing back + /// at its strings, and four of them agree on the base on all 27 levels. pub field_00: u32, /// The word at `+0x04`. Level-specific; 37,540 on level 24 through 45,308 /// on level 2, and a multiple of 4 on every level. @@ -79,6 +86,12 @@ pub struct OverlayHeader { } impl OverlayHeader { + /// Where the overlay's payload, and therefore offset `X`, sits in RAM: + /// `X + field_00 - OVERLAY_HEADER_SIZE`. + pub fn ram_address(&self, offset: usize) -> u64 { + offset as u64 + self.field_00 as u64 - OVERLAY_HEADER_SIZE as u64 + } + /// Parse the header, bounds-checked against `region`. pub fn parse(region: &[u8]) -> Result { let mut r = Buf::new(region); diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 0a467d1..13541e7 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -2322,6 +2322,43 @@ What would settle it, and none of these are cheap: whose name is a volume trigger, that would be decisive. No class name containing "volume" exists on the disc, which is why this is not available. +## The overlay's load base, from its own profiler table (2026-10-11) + +**Range measured:** all 27 levels' overlays. + +**Method:** the overlay carries the game's **profiler menu definition** -- a run +of 8-byte-padded names (`normal`, `camera`, `cam+chr`, `volume`, `bars`, +`render`, `mobys`, `ties`, `shrubs`, `cpu/gpu`, `occl`, ...) followed by runs of +absolute RAM addresses that point **back at those same names**. On every level +the four addresses after `normal`/`camera`/`cam+chr`/`volume`, each minus its +target name's overlay offset, give the same number: **1,735,792**. + +That number is `field_00 - 16`, where `field_00` is the header word that +previously read 1,735,808 on all 27 levels with no meaning attached. So the +overlay is loaded with its 16-byte header stripped: overlay offset `X` sits at +RAM `X + 1,735,792`. `OverlayHeader::ram_address` carries it. + +**Confirmed, not only derived.** The same bases were checked in PCSX2's RAM: +level 0's `0x1acf80` holds `"normal\0camera\0cam+chr\0volume\0"`, byte-identical +to overlay `0x5310`, and `0x5310 + 1735792 == 0x1acf80`. All four names land +where the overlay's own table says. `gc-trace verify-overlay-base` runs the +check. + +**The overlay is relocated, not copied.** Only **37.67%** of level 0's overlay +is byte-identical at the predicted base, and the 256-entry sine table -- at +overlay `0xb650c` -- is *not* at `0xb650c + base`. So the base locates the +overlay's image, but individual tables inside it move. That is the shape of a +loader applying relocations, and it means the base is necessary but not +sufficient for reading the game's tables. + +The value of the base is that any overlay offset can now be carried to a RAM +address, which is the prerequisite for reading the game's live level state -- +the thing the tie relocation and the tfrag lighting checks had to work around. + +Note the profiler table's own vocabulary, since it is the first named structure +in the overlay: `volume` is one of the game's profiling categories, so volumes +are a real subsystem. It does not locate the data. + ## Chunk records: what the stride table settles, and one signature it does not Walked every chunk on all 27 levels and grouped by its six-byte header signature: diff --git a/docs/roadmap.md b/docs/roadmap.md index e379da7..6ed821c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -415,6 +415,12 @@ Phase 1 hands over the extents; this phase works out what each one contains. `+0x20` is self-limiting: entries run while strictly inside the region. **The word at `+0x0c` is not a count** -- it disagrees with the table on most levels and reads 196,617 on level 2 against 4 sub-objects. +- [x] The **overlay's load base is `field_00 - 16` = 1,735,792**, derived from + the overlay's own profiler table (whose absolute addresses point back at + its names, on all 27 levels) and **confirmed in RAM**: all four profiler + names land where the base predicts. Any overlay offset can now be carried + to a RAM address. The overlay is *relocated*, not copied: only 37.67% is + byte-identical at the base, so tables inside it move. - [~] `volumes` — **still not identified, but a candidate exists.** Every level's `gameplay.bin` carries a table of **144-byte float records** with world positions at `+0x0c`, preceded by an all-zero record (levels 0-9: 23 to 612 diff --git a/tools/trace/src/lib.rs b/tools/trace/src/lib.rs index 7fa89df..fae0eb1 100644 --- a/tools/trace/src/lib.rs +++ b/tools/trace/src/lib.rs @@ -25,6 +25,7 @@ //! facts it needs into a small fixture and test against that. pub mod ee; +pub mod overlay_base; pub mod p2s; pub mod pine; pub mod tfrag_cmp; diff --git a/tools/trace/src/main.rs b/tools/trace/src/main.rs index e9ecd5e..e172f69 100644 --- a/tools/trace/src/main.rs +++ b/tools/trace/src/main.rs @@ -64,6 +64,27 @@ fn run(args: &[String]) -> Result { let len = require(&rest, "--len")?; read(&src, &addr, &len) } + "verify-overlay-base" => { + // gc-trace verify-overlay-base + let (src, rest) = Source::parse_with_rest(&args[1..])?; + let (dir, level) = match rest.as_slice() { + [dir, level, ..] => (PathBuf::from(dir), level.parse::().context("level")?), + _ => { + return Err(anyhow!( + "usage: gc-trace verify-overlay-base " + )) + } + }; + let overlay = std::fs::read(dir.join(format!("level{level}/overlay.bin"))) + .with_context(|| format!("level{level}/overlay.bin"))?; + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + let report = gc_trace::overlay_base::verify(&overlay, &ram)?; + let mut text = String::new(); + gc_trace::overlay_base::print_report(&report, &mut text)?; + println!("{text}"); + Ok(i32::from(!report.names.iter().all(|&b| b))) + } "verify-tie-records" => { // gc-trace verify-tie-records let (src, rest) = Source::parse_with_rest(&args[1..])?; @@ -124,6 +145,7 @@ const USAGE: &str = "\ usage: gc-trace ... compare-tfrag-light [--level N] verify-tie-records + verify-overlay-base info dump-ee [--out FILE] find --bytes HEX diff --git a/tools/trace/src/overlay_base.rs b/tools/trace/src/overlay_base.rs new file mode 100644 index 0000000..e8476b4 --- /dev/null +++ b/tools/trace/src/overlay_base.rs @@ -0,0 +1,189 @@ +//! The overlay's load base, confirmed against the running game. +//! +//! The overlay's own profiler table carries absolute RAM addresses pointing +//! back at its strings, so the base can be derived from disc bytes (see +//! `overlay_load_base_from_the_profiler_table` in the disc tests). This module +//! does the part only the emulator can do: read a savestate and check the +//! bytes are where the base predicts. +//! +//! The check matters because the two failures are different. A wrong base puts +//! the names somewhere else and finds nothing; a right base that finds nothing +//! means the loader relocated the table. Distinguishing those is the whole +//! point of confirming rather than deriving. + +use anyhow::{anyhow, Result}; + +use gc_formats::overlay::OverlayHeader; + +use crate::ee::EeRam; + +/// What the check found. +#[derive(Debug, Clone)] +pub struct OverlayBaseReport { + /// The overlay's payload load base, from its own header. + pub load_base: u64, + /// Bytes of the overlay compared against RAM. + pub overlay_bytes: usize, + /// Bytes byte-identical at the predicted addresses. + pub verbatim: usize, + /// The four profiler names, and whether RAM held each at its predicted + /// address. + pub names: Vec, +} + +impl OverlayBaseReport { + /// Fraction of the overlay the loader left in place. + pub fn verbatim_fraction(&self) -> f64 { + self.verbatim as f64 / self.overlay_bytes.max(1) as f64 + } +} + +/// Check one level's overlay against a RAM image. +/// +/// `overlay` is the level's `overlay.bin`; `ram` is a 32 MiB EE RAM image taken +/// while that level is loaded. +pub fn verify(overlay: &[u8], ram: &EeRam) -> Result { + // The profiler table's distinctive name run, and the four names before it. + let Some(p) = overlay.windows(15).position(|w| w == b"cam+chr\0volume\0") else { + return Err(anyhow!( + "the overlay carries no profiler name run; is this an overlay?" + )); + }; + let names_at = p - 16; + // The base is computed after the table is found, so an all-zero overlay + // used as a negative control does not underflow first. + let hdr = OverlayHeader::parse(overlay)?; + let load_base = hdr.ram_address(0); + + let mut names = Vec::new(); + for k in 0..4usize { + let at = hdr.ram_address(names_at + k * 8) as usize; + let want = &overlay[names_at + k * 8..names_at + k * 8 + 8]; + names.push(ram.slice(at, 8) == Some(want)); + } + + // How much of the overlay did the loader leave in place? The base locates + // the image; this measures whether it also locates the tables inside it. + let base = load_base as usize; + let image = ram.slice(base, overlay.len()).unwrap_or(&[]); + let verbatim = overlay + .iter() + .zip(image.iter()) + .filter(|(a, b)| *a == *b) + .count(); + + Ok(OverlayBaseReport { + load_base, + overlay_bytes: overlay.len(), + verbatim, + names, + }) +} + +/// Render a report for a terminal. +pub fn print_report(r: &OverlayBaseReport, out: &mut dyn std::fmt::Write) -> std::fmt::Result { + writeln!( + out, + "overlay load base: {} ({:#x})", + r.load_base, r.load_base + )?; + writeln!( + out, + " profiler names at the predicted addresses: {}/{}", + r.names.iter().filter(|&&b| b).count(), + r.names.len() + )?; + writeln!( + out, + " overlay bytes byte-identical in RAM: {}/{} ({:.2}%)", + r.verbatim, + r.overlay_bytes, + 100.0 * r.verbatim_fraction() + )?; + if r.names.iter().all(|&b| b) { + writeln!( + out, + " the base is confirmed: every name is where the overlay's own table says" + ) + } else { + writeln!( + out, + " the base did NOT place every name where the overlay's table says" + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ee::EE_SIZE; + + /// An overlay carrying the profiler table, whose four names sit at + /// `names_at` and whose four addresses point back at them. + fn overlay() -> Vec { + let mut v = vec![0u8; 0x6000]; + // Header: field_00 is the payload's load address. + v[0..4].copy_from_slice(&1_735_808u32.to_le_bytes()); + let names_at = 0x5310usize; + for (k, name) in ["normal\0", "camera\0", "cam+chr", "volume\0"] + .iter() + .enumerate() + { + let mut slot = [0u8; 8]; + slot[..name.len()].copy_from_slice(name.as_bytes()); + v[names_at + k * 8..names_at + k * 8 + 8].copy_from_slice(&slot); + } + // The address run, pointing back at the four names. + let addrs_at = names_at + 4 * 8; + for k in 0..4usize { + let a = 1_735_808u32 - 16 + (names_at + k * 8) as u32; + v[addrs_at + k * 4..addrs_at + k * 4 + 4].copy_from_slice(&a.to_le_bytes()); + } + v + } + + #[test] + fn reads_the_base_and_relocates_the_names() { + let ov = overlay(); + let hdr = OverlayHeader::parse(&ov).unwrap(); + assert_eq!(hdr.field_00, 1_735_808); + assert_eq!(hdr.ram_address(0), 1_735_792); + // The derived base matches the header word less the header's size. + let addrs_at = 0x5310 + 32; + let a0 = u32::from_le_bytes(ov[addrs_at..addrs_at + 4].try_into().unwrap()); + assert_eq!(a0 as u64 - 0x5310, hdr.ram_address(0)); + } + + #[test] + fn verifies_against_a_ram_image() { + let ov = overlay(); + let mut ram = vec![0u8; EE_SIZE]; + let base = 1_735_792usize; + // The loader places overlay offset 0 at the base. + ram[base..base + ov.len()].copy_from_slice(&ov); + let r = verify(&ov, &EeRam::new(&ram).unwrap()).unwrap(); + assert!(r.names.iter().all(|&b| b), "{:?}", r.names); + assert_eq!(r.overlay_bytes, ov.len()); + assert_eq!(r.verbatim, ov.len()); + } + + #[test] + fn an_overlay_without_the_profiler_table_is_an_error() { + // All zeros: no profiler table, and a field_00 of 0 that would + // underflow if the base were computed before the table were found. + let ov = vec![0u8; 0x6000]; + assert!(verify(&ov, &EeRam::new(&vec![0u8; EE_SIZE]).unwrap()).is_err()); + } + + #[test] + fn a_wrong_base_finds_nothing_and_says_so() { + // An empty RAM: the names are not at the predicted addresses, and only + // the fixture's zero bytes line up with the zeros there. + let ov = overlay(); + let nonzero = ov.iter().filter(|&&b| b != 0).count(); + let ram = vec![0u8; EE_SIZE]; + let r = verify(&ov, &EeRam::new(&ram).unwrap()).unwrap(); + assert_eq!(r.names, vec![false; 4]); + assert_eq!(r.verbatim, ov.len() - nonzero); + } +}