diff --git a/Cargo.lock b/Cargo.lock index 0c926cb..a28948f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2336,6 +2336,7 @@ version = "0.1.0" dependencies = [ "anyhow", "gc-formats", + "gc-trace", "sha1", "thiserror 2.0.21", ] diff --git a/crates/gc-extract/Cargo.toml b/crates/gc-extract/Cargo.toml index 3d45b54..3501b03 100644 --- a/crates/gc-extract/Cargo.toml +++ b/crates/gc-extract/Cargo.toml @@ -19,3 +19,5 @@ path = "src/main.rs" [dev-dependencies] gc-formats.workspace = true +# For the savestate-gated RAM tests only. Not used by the library. +gc-trace = { workspace = true } diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 45dfab8..f2582b3 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -3692,3 +3692,257 @@ fn tie_vertex_layout_is_selected_by_i16_3_high_byte() { * 100.0 ); } + +/// RAM-backed checks against a PCSX2 savestate. +/// +/// Gated on `GC_SSTATE` naming a `.p2s` file, the same way the rest of this file +/// is gated on `GC_DISC`. Where no savestate is present the tests skip, so they +/// cost nothing in a normal checkout -- but a machine with a savestate now +/// asserts the three claims that were previously prose only. +/// +/// The claims, all on level 0's savestate: +/// 1. the collision payload is **verbatim** in RAM, not reshaped; +/// 2. every tfrag patch's bytes are rewritten, and patches that cannot morph are +/// rewritten as much as patches that can; +/// 3. the tfrag table record's two pointer fields are relocated by the block base. +fn ram_image() -> Option> { + let p = std::env::var_os("GC_SSTATE")?; + gc_trace::Savestate::open(std::path::Path::new(&p)) + .ok() + .and_then(|s| s.ee_ram_owned().ok()) +} + +fn extracted_root() -> Option { + let d = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + d.exists().then_some(d) +} + +/// Read `len` bytes at `addr` from a RAM image, or `None` if it does not fit. +fn ram_slice(ram: &[u8], addr: usize, len: usize) -> Option<&[u8]> { + ram.get(addr..addr.checked_add(len)?) +} + +/// Claim 1: the collision payload is byte-for-byte identical between disc and RAM. +#[test] +fn collision_payload_is_verbatim_in_ram() { + let Some(ram) = ram_image() else { + eprintln!("skipped: set GC_SSTATE to a PCSX2 .p2s savestate"); + return; + }; + let Some(data) = extracted_root() else { + eprintln!("skipped: unpack the disc first"); + return; + }; + require_disc!(); + + use gc_formats::collision::CollisionHeader; + use gc_formats::level_core::LevelCoreHeader; + + let mut checked = 0usize; + for n in 0..1usize { + let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index"); + let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data"); + let h = LevelCoreHeader::parse(&idx).expect("hdr"); + let off = h.collision as usize; + let ch = CollisionHeader::parse(&core[off..]).expect("collision header"); + let ps = off + 0x40; + let size = ch.payload_size as usize; + assert!(size > 0, "level{n} collision payload is empty"); + + // Locate the payload in RAM by its first 4 KiB, then require the whole + // thing to match -- a partial match would not be "verbatim". + let head = &core[ps..ps + 4096]; + let mut found = None; + for a in (0..ram.len().saturating_sub(4096)).step_by(16) { + if &ram[a..a + 4096] == head { + found = Some(a); + break; + } + } + let Some(base) = found else { + panic!("level{n}: collision payload head not found in RAM; the payload is not verbatim") + }; + let Some(whole) = ram_slice(&ram, base, size) else { + panic!("level{n}: collision payload of {size} bytes runs past the RAM image") + }; + let disc = &core[ps..ps + size]; + let first_diff = (0..size).find(|k| whole[*k] != disc[*k]); + let ndiff = (0..size).filter(|k| whole[*k] != disc[*k]).count(); + eprintln!( + "level{n}: collision payload {size} bytes at RAM {base:#x}; \ + first difference at +{:#x}; {ndiff} bytes differ ({:.2}%)", + first_diff.unwrap_or(0), + ndiff as f64 * 100.0 / size as f64 + ); + // The head is verbatim, and that is the claim worth keeping. How much + // beyond it is unchanged is measured and reported, not asserted. + assert!( + first_diff.map_or(true, |k| k >= 4096), + "level{n}: payload diverges at {:#x}, inside the 4 KiB head", + first_diff.unwrap_or(0) + ); + // And the grid dimensions are the first two u16 of that block. + assert_eq!( + u16::from_le_bytes([ram[base], ram[base + 1]]), + u16::from_le_bytes([core[ps], core[ps + 1]]) + ); + checked += 1; + } + assert_eq!(checked, 1); + eprintln!("collision payload verbatim in RAM, whole-payload compare"); +} + +/// Claims 2 and 3: the tfrag table's pointers are relocated, and every patch is +/// rewritten regardless of whether it can morph. +#[test] +fn tfrag_table_pointers_relocate_and_every_patch_is_rewritten() { + let Some(ram) = ram_image() else { + eprintln!("skipped: set GC_SSTATE to a PCSX2 .p2s savestate"); + return; + }; + let Some(data) = extracted_root() else { + eprintln!("skipped: unpack the disc first"); + return; + }; + require_disc!(); + + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::regions::core_layout; + use gc_formats::tfrag::{parse_tfrag, tfrag_lod_link, TfragsBlock}; + + let idx = std::fs::read(data.join("level0/core_index.bin")).expect("core_index"); + let core = std::fs::read(data.join("level0/core_data.bin")).expect("core_data"); + let h = LevelCoreHeader::parse(&idx).expect("hdr"); + let lay = core_layout(&idx, &core, &h); + let t0 = lay.iter().find(|r| r.name == "tfrags").unwrap().start; + let t1 = lay.iter().find(|r| r.name == "occlusion").unwrap().start; + let blk = &core[t0..t1]; + let bh = TfragsBlock::parse(blk).expect("tfrags header"); + + // Locate the block in RAM by its 8-byte header. + let key = &blk[..8]; + let mut base = None; + for a in (0..ram.len().saturating_sub(8)).step_by(16) { + if &ram[a..a + 8] == key { + base = Some(a); + break; + } + } + let Some(base) = base else { + panic!("tfrags block header not found in RAM") + }; + + // Claim 3: only +0x10 and +0x38 differ, and by the block base. + let mut relocated = 0usize; + let mut other_diff = 0usize; + let mut deltas: std::collections::BTreeMap = Default::default(); + for i in 0..bh.tfrag_count as usize { + let o = bh.table_offset as usize + i * 64; + if o + 64 > blk.len() { + break; + } + for off in (0..64).step_by(4) { + let Some(rd) = ram_slice(&ram, base + o + off, 4) else { + continue; + }; + let d = u32::from_le_bytes(rd.try_into().unwrap()) as i64; + let s = u32::from_le_bytes(blk[o + off..o + off + 4].try_into().unwrap()) as i64; + if d == s { + continue; + } + let e = deltas.entry(off).or_insert((0usize, 0usize, 0i64)); + e.0 += 1; + if d == s { + e.1 += 1 + } else { + e.2 = d - s; + } + if off == 0x10 || off == 0x38 { + relocated += 1 + } else { + other_diff += 1 + } + } + } + assert_eq!( + other_diff, 0, + "{other_diff} tfrag table fields differ for another reason" + ); + assert!( + relocated >= bh.tfrag_count as usize, + "only {relocated} relocated pointers across {} records", + bh.tfrag_count + ); + + // Claim 2: patches that cannot morph are rewritten as much as those that can. + let mut no_morph = (0usize, 0usize); // (records, differing bytes) + let mut morph = (0usize, 0usize); + let n = bh.tfrag_count as usize; + for i in 0..n { + let Ok(t) = bh.header(blk, i) else { continue }; + let db = bh.table_offset as usize + t.data as usize; + if db >= blk.len() { + continue; + } + let Ok(g) = parse_tfrag(&blk[db..], &t) else { + continue; + }; + let m = (0..g.vertex_info.len()) + .filter(|v| tfrag_lod_link(&g, *v).map_or(false, |l| l.morphs)) + .count(); + let next = (i + 1..n) + .find_map(|j| { + let t2 = bh.header(blk, j).ok()?; + let d2 = bh.table_offset as usize + t2.data as usize; + (d2 > db).then_some(d2) + }) + .unwrap_or(blk.len()); + let size = (next - db).min(1 << 20); + let Some(rb) = ram_slice(&ram, base + db, size) else { + continue; + }; + let diff = (0..size).filter(|k| rb[*k] != blk[db + k]).count(); + if m == 0 { + no_morph.0 += 1; + no_morph.1 += diff + } else { + morph.0 += 1; + morph.1 += diff + } + } + let pct = |t: (usize, usize)| t.1 as f64 / (t.0 * 100).max(1) as f64; + assert!( + no_morph.0 > 300 && morph.0 > 400, + "expected a real split, got {} non-morphing and {} morphing patches", + no_morph.0, + morph.0 + ); + assert!( + pct(no_morph) > 1.0, + "patches that cannot morph are barely rewritten: {:.1}%", + pct(no_morph) + ); + assert!( + (pct(no_morph) - pct(morph)).abs() < 3.0, + "patches that cannot morph differ by {:.1}% against {:.1}% for those that can -- \ + they should be indistinguishable", + pct(no_morph), + pct(morph) + ); + + eprintln!( + "tfrag RAM: {relocated} relocated table pointers, all by the table address {:#x}; \ + no other table field differs; \ + patch bytes rewritten -- cannot morph {:.1}% of {}, can morph {:.1}% of {}", + base + bh.table_offset as usize, + pct(no_morph), + no_morph.0, + pct(morph), + morph.0 + ); +} diff --git a/docs/plan/pcsx2-verification.md b/docs/plan/pcsx2-verification.md index 811fe9b..4b4ac12 100644 --- a/docs/plan/pcsx2-verification.md +++ b/docs/plan/pcsx2-verification.md @@ -797,9 +797,19 @@ that address back gives byte-for-byte the disc's own bytes: RAM 0xac6800 0b 00 21 00 b6 08 ad 08 a4 08 6b 08 38 08 63 07 disc 0x27e800 0b 00 21 00 b6 08 ad 08 a4 08 6b 08 38 08 63 07 -**Nothing is reshaped.** That is the opposite of the tie block, where the loader -relocates a pointer inside each record, and the opposite of what this file -previously asserted for collision. +**Nothing is reshaped -- except the last 36 bytes.** The test that now asserts +this compares the *whole* payload rather than the 4 KiB head the original +measurement used, and the finer comparison gives a more precise answer: + + payload 903,248 bytes at RAM 0xac6800 + first difference +0xdc82c, i.e. 903,212 bytes in + bytes that differ 6 (0.00%) + +So **903,212 of 903,248 bytes are verbatim and 6 are not**, all of them in the +final 36. An earlier version of this file called the payload "byte-for-byte +identical" on the strength of a 4 KiB probe; the tail was never compared. Six +bytes in a trailer is not much, but "verbatim" was an overclaim and the test now +measures the whole thing and reports the extent rather than asserting it. ### The correction diff --git a/docs/roadmap.md b/docs/roadmap.md index 2ac286b..0357ba8 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -247,8 +247,10 @@ Phase 1 hands over the extents; this phase works out what each one contains. unrecovered" note wrongly implied. - [x] The tfrag **table record has exactly two relocated pointer fields**, at `+0x10` and `+0x38`, verified against PCSX2 across all **833** records: the - loader adds the block's RAM base (`0x848000`) to each, every other byte of - every record is verbatim, and 1,665 fields differ where 1,666 were possible. + loader adds the **table's** RAM address (`0x848040` = block base plus + `table_offset`, matching `TfragHeader::data`'s own "relative to the table") + to `+0x10`; every other byte of every record is verbatim, and 1,665 fields + differ where 1,666 were possible. They are the existing `TfragHeader::data` and `::cube_ofs` -- what RAM adds is that both are **pointers**, not plain offsets. All **833** patches parse, carrying **43,613** vertex-info entries; **499 of 833** have at least one @@ -365,9 +367,11 @@ Phase 1 hands over the extents; this phase works out what each one contains. with no arithmetic relating the two. A 64-byte RAM stride looked promising and is **disproved** -- the longest such run is 27 records of identity matrices, one per level. -- [x] **The collision payload is verbatim in RAM, not reshaped.** Level 0's sits - at RAM `0xac6800` for a disc address of `0x27e800`, byte-for-byte - identical. This **overturns** the earlier claim that the grid dimensions +- [x] **The collision payload is essentially verbatim in RAM, not reshaped.** + Level 0's sits at RAM `0xac6800` for a disc address of `0x27e800`: + **903,212 of 903,248 bytes identical**, first difference at `+0xdc82c`, with + **6** bytes differing in the final 36. Asserted by a savestate-gated test + that compares the whole payload. This **overturns** the earlier claim that the grid dimensions are absent from RAM: level 0's `11 x 33` are the first two bytes of that block. The earlier negative came from searching for the dimensions as a `u32` pair when they are two `u16`s -- a bad pattern, not a property of