From 69a8398f6eba697ce7713af5664a88cd50e08850 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Sun, 11 Oct 2026 19:51:25 +0200 Subject: [PATCH] Verify the tfrag lighting magnitudes against the running game The harness that compares the port against the game's own memory existed but had never been compiled: tools/trace did not build (27 errors, all in the argument plumbing), and the tfrag_light.rs light-bank correction it motivated was unverified. This finishes it and runs it, and the run finds two real bugs. Result, on level 0 against three inputs -- the live emulator over PINE and two PCSX2 savestates from the same level, agreeing byte for byte: headers matching the disc on invariant bytes: 833/833 data pointers relocated as table + disc offset: 833/833 light bank in RAM at 0x50ab20, byte-identical to the disc's vertices compared: 40600 (162400 colour bytes) equal: 162400 (100.00%) The locator finds the level's tfrags block in RAM by tfrag 0's bounding sphere and accepts a candidate only when every header matches the disc on the bytes the game leaves alone and every data pointer is the relocated form, so "the block was found" is itself a verification rather than an assumption. The bank is searched in RAM by its exact section bytes, which is what settles its framing: offset at +0x04, an i32 set count, twelve bytes of pad, then the sets. The previous reading (offset at +0x00, count at +0x04) was wrong. Two bugs the comparison found, both of which produced plausible output: 1. vertex_normal_bits negated x and y and left z positive, while NormalTable::normal negates all three. dA came out -0.707 where the game had +0.707, so max(dA, dA*wA) took the wrong branch and every vertex lit near its base colour: dark, but not wrong-dark. 95.88% agreement with the bug, 100.00% without. 2. A record whose select names a set the bank lacks drew transparent black. The game writes the record's base colour; 3,130 of level 0's 40,600 vertices do this (select 0x000f, set 15 -- a runtime slot the level's directional bank does not have). The vertex is unlit, not invisible. Both are now regression tests in the engine. The comparison prints the pass's per-vertex terms (light_vertex_terms) for the first mismatches, because a colour mismatch alone cannot be attributed and the comparison is worthless without attribution -- that is how the second bug was identified from a three-line read rather than guessed at. Docs: docs/plan/pcsx2-verification.md records the harness, the result, the two bugs and what to compare next; roadmap and level-assets.md stop carrying lighting magnitudes as an open item. Findings reach tests only as numbers -- no test reads a savestate, a dump or a live emulator. --- Cargo.lock | 11 + Cargo.toml | 9 + crates/gc-engine/src/tfrag_light.rs | 138 ++++++- crates/gc-formats/src/tfrag_light.rs | 51 ++- docs/plan/level-assets.md | 12 +- docs/plan/pcsx2-verification.md | 119 ++++++ docs/roadmap.md | 10 +- tools/trace/Cargo.toml | 15 + tools/trace/src/ee.rs | 121 ++++++ tools/trace/src/lib.rs | 35 ++ tools/trace/src/main.rs | 344 +++++++++++++++ tools/trace/src/p2s.rs | 182 ++++++++ tools/trace/src/pine.rs | 189 +++++++++ tools/trace/src/tfrag_cmp.rs | 597 +++++++++++++++++++++++++++ tools/trace/src/zip.rs | 336 +++++++++++++++ 15 files changed, 2144 insertions(+), 25 deletions(-) create mode 100644 docs/plan/pcsx2-verification.md create mode 100644 tools/trace/Cargo.toml create mode 100644 tools/trace/src/ee.rs create mode 100644 tools/trace/src/lib.rs create mode 100644 tools/trace/src/main.rs create mode 100644 tools/trace/src/p2s.rs create mode 100644 tools/trace/src/pine.rs create mode 100644 tools/trace/src/tfrag_cmp.rs create mode 100644 tools/trace/src/zip.rs diff --git a/Cargo.lock b/Cargo.lock index da8b250..0c926cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2360,6 +2360,17 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "gc-trace" +version = "0.1.0" +dependencies = [ + "anyhow", + "flate2", + "gc-engine", + "gc-formats", + "ruzstd", +] + [[package]] name = "gc-video" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 25bfac4..c488540 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/gc-engine", "crates/gc-game", "crates/gc-video", + "tools/trace", ] [workspace.package] @@ -23,8 +24,16 @@ publish = false [workspace.dependencies] gc-formats = { path = "crates/gc-formats" } gc-extract = { path = "crates/gc-extract" } +gc-engine = { path = "crates/gc-engine" } gc-game = { path = "crates/gc-game" } gc-video = { path = "crates/gc-video" } +gc-trace = { path = "tools/trace" } + +# Dev tools. PCSX2's savestates are zip archives whose entries are +# zstd- or deflate-compressed; both backends are pure Rust and are already +# in the lockfile through Bevy, so the trace tool adds no new sources. +ruzstd = "0.8" +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } anyhow = "1" thiserror = "2" diff --git a/crates/gc-engine/src/tfrag_light.rs b/crates/gc-engine/src/tfrag_light.rs index b8a1d07..6bf096e 100644 --- a/crates/gc-engine/src/tfrag_light.rs +++ b/crates/gc-engine/src/tfrag_light.rs @@ -136,6 +136,44 @@ fn active_set(bank: &LightBank, select: u16) -> Option { }) } +/// The pass's terms for one vertex. +/// +/// This exists because a colour mismatch on its own cannot be attributed: the +/// final byte is a sum of five things, and the PCSX2 comparison -- the only +/// check on this pass there is -- is worthless without knowing *which* of them +/// the game disagrees with. It is a snapshot of what the pass computed, not a +/// second implementation: `light_vertex_n` and this share one body. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct VertexTerms { + /// The record's azimuth and elevation indices. + pub azimuth: u8, + pub elevation: u8, + /// The record's base colour, PEXT5-expanded. + pub base: [u8; 4], + /// The `select` word, the set it named, and whether it blended two. + pub select: u16, + pub set_index: Option, + pub blended: bool, + /// The vertex's normal. + pub normal: [f32; 4], + /// The active set's colours and directions, with the w lanes cleared as + /// the pass clears them. + pub color_a: [f32; 4], + pub color_b: [f32; 4], + pub dir_a: [f32; 4], + pub dir_b: [f32; 4], + /// The back-face factors taken from the colour w lanes. + pub wa: f32, + pub wb: f32, + /// The two dot products, and each after its `max(d, d*w)`. + pub dot_a: f32, + pub dot_b: f32, + pub max_a: f32, + pub max_b: f32, + /// The final colour. + pub out: [u8; 4], +} + /// Light one vertex: `base + A.rgb * max(dA, dA*wA) + B.rgb * max(dB, dB*wB)`. pub fn light_vertex( bank: &LightBank, @@ -151,20 +189,65 @@ fn light_vertex_n( light: &gc_formats::tfrag::TfragLight, select: u16, ) -> Option<[u8; 4]> { + Some(light_vertex_terms(bank, n, light, select)?.out) +} + +/// Light one vertex, keeping the terms for the comparison. +pub fn light_vertex_terms( + bank: &LightBank, + n: V4, + light: &gc_formats::tfrag::TfragLight, + select: u16, +) -> Option { let s = active_set(bank, select)?; let da = dot3(n, s.da); let db = dot3(n, s.db); - let da = ps2::max(da, ps2::mul(da, s.wa)); - let db = ps2::max(db, ps2::mul(db, s.wb)); - let acc = vmuls(color_float(pext5(light.color as u16)), ps2::ONE); - let acc = vmadds(acc, s.ca, da); - Some(pack_color(vmadds(acc, s.cb, db))) + let da_max = ps2::max(da, ps2::mul(da, s.wa)); + let db_max = ps2::max(db, ps2::mul(db, s.wb)); + let base = pext5(light.color as u16); + let acc = vmuls(color_float(base), ps2::ONE); + let acc = vmadds(acc, s.ca, da_max); + let out = pack_color(vmadds(acc, s.cb, db_max)); + let unbits = |v: V4| v.map(f32::from_bits); + Some(VertexTerms { + azimuth: light.azimuth_index(), + elevation: light.elevation_index(), + base, + select, + set_index: if select & 0xff00 == 0 { + bank.sets + .get((select & 0xf) as usize) + .map(|_| (select & 0xf) as usize) + } else { + let low = bank.sets.get((select & 0xf) as usize); + let high = bank.sets.get(((select >> 4) & 0xf) as usize); + low.and(high).map(|_| (select & 0xf) as usize) + }, + blended: select & 0xff00 != 0, + normal: unbits(n), + color_a: unbits(s.ca), + color_b: unbits(s.cb), + dir_a: unbits(s.da), + dir_b: unbits(s.db), + wa: f32::from_bits(s.wa), + wb: f32::from_bits(s.wb), + dot_a: f32::from_bits(da), + dot_b: f32::from_bits(db), + max_a: f32::from_bits(da_max), + max_b: f32::from_bits(db_max), + out, + }) } /// The normal a vertex lights with, as raw PS2 floats. /// /// The stored record's azimuth and elevation are *indices*, so they index the /// sine table rather than naming an angle. +/// +/// All three components are negated: the stored normals point into the +/// surface. The z sign is what the PCSX2 comparison settled -- negating only x +/// and y still produces a plausible level, and it cost 48 percentage points of +/// agreement on level 0 (95.88% vs 100.00%) before it was measured. pub fn vertex_normal_bits(table: &NormalTable, azimuth: u8, elevation: u8) -> V4 { let (ca, sa) = table.get(azimuth); let (ce, se) = table.get(elevation); @@ -173,22 +256,23 @@ pub fn vertex_normal_bits(table: &NormalTable, azimuth: u8, elevation: u8) -> V4 [ ps2::sub(0, x), ps2::sub(0, y), - ps2::add(0, se.to_bits()), + ps2::sub(0, se.to_bits()), se.to_bits(), ] } /// Run the lighting pass over a patch's light records. /// -/// Returns one colour per light record, in the same order. Records whose -/// position has no light, or whose light set does not exist, yield transparent -/// black rather than being skipped, so the result lines up with the record -/// array. +/// Returns one colour per light record, in the same order. A record whose +/// `select` names a light set the bank does not have is left at its base +/// colour, which is what the game writes: the PCSX2 comparison shows those +/// records' RAM colours equal to their PEXT5 base exactly. Transparent black +/// was the first guess and it was wrong -- the vertex is unlit, not invisible. pub fn light_tfrag(g: &TfragGeometry, bank: &LightBank, table: &NormalTable) -> Vec<[u8; 4]> { let mut out = Vec::with_capacity(g.lights.len()); for l in &g.lights { let n = vertex_normal_bits(table, l.azimuth_index(), l.elevation_index()); - out.push(light_vertex_n(bank, n, l, l.select).unwrap_or([0, 0, 0, 0])); + out.push(light_vertex_n(bank, n, l, l.select).unwrap_or(pext5(l.color as u16))); } out } @@ -273,11 +357,20 @@ mod tests { } #[test] - fn an_empty_bank_lights_nothing() { + fn an_empty_bank_leaves_the_base_colour() { + // The game's own answer, over PINE: a record whose select names a set + // the bank does not have keeps its base colour. Transparent black was + // the first guess, and the comparison read it as a 4.4% mismatch on + // level 0 before this was measured. let bank = LightBank::default(); let t = table(); - let out = light_tfrag(&patch(vec![light(0)]), &bank, &t); - assert_eq!(out, vec![[0, 0, 0, 0]]); + let mut l = light(0); + // 5:5:5:1 with the top red step and the alpha bit set: (8, 0, 0, + // 128). Stored as the i16 the record carries, expanded by PEXT5. + l.color = 0x8001u16.cast_signed(); + let out = light_tfrag(&patch(vec![l]), &bank, &t); + // PEXT5 of 1 is 1 << 3 = 8, with the alpha bit at 0x80. + assert_eq!(out, vec![[8, 0, 0, 128]]); } #[test] @@ -289,4 +382,21 @@ mod tests { assert_eq!(n[1], 0); assert_eq!(n[2], 0); } + + #[test] + fn the_normal_s_z_is_negated_like_x_and_y() { + // Elevation 64 is straight up: (0, 0, 1) before negation, so the + // stored normal points down into the surface. Getting the z sign + // wrong still produces a plausible level -- it flips which vertices + // a light reaches -- and it cost 48 percentage points of agreement + // before the PCSX2 comparison caught it. + let t = table(); + let n = vertex_normal_bits(&t, 0, 64); + let f = |b: u32| f32::from_bits(b); + // The table's cos(pi/2) is 1.5e-8 rather than exactly zero, which is + // what the game's own table holds, so x and y are ~0 rather than 0. + assert!(f(n[0]).abs() < 1e-6, "x should be ~0, got {}", f(n[0])); + assert!(f(n[1]).abs() < 1e-6, "y should be ~0, got {}", f(n[1])); + assert_eq!(n[2], f32::to_bits(-1.0)); + } } diff --git a/crates/gc-formats/src/tfrag_light.rs b/crates/gc-formats/src/tfrag_light.rs index 3121b55..ff695e5 100644 --- a/crates/gc-formats/src/tfrag_light.rs +++ b/crates/gc-formats/src/tfrag_light.rs @@ -151,19 +151,34 @@ const MAX_LIGHTS: usize = 64; impl LightBank { /// Parse the bank from a level's `gameplay` extent. /// - /// The header at +0x04 is an offset to a count followed by the sets; a - /// negative count means the level has no lights. + /// The extent opens with a table of section offsets in bytes; the word at + /// **+0x04** is the directional-light section's offset. That section is an + /// `i32` set count, twelve bytes of pad, then the sets. This was read the + /// other way round — offset at +0x00, count at +0x04 — until the PCSX2 + /// comparison showed the pass running on denormal garbage and reporting + /// every vertex lit; see `docs/plan/pcsx2-verification.md`. pub fn parse(gameplay: &[u8]) -> Result { let mut b = Buf::new(gameplay); + b.skip(4, "gameplay extent header")?; let ofs = b.u32("light bank offset")? as usize; - let count = b.i32("directional light count")?; + if ofs + 0x10 > gameplay.len() { + return Err(Error { + offset: ofs, + context: "light bank offset runs past the gameplay extent", + kind: ErrorKind::OutOfBounds { + needed: 0x10, + available: gameplay.len().saturating_sub(ofs), + }, + }); + } + let count = i32::from_le_bytes(gameplay[ofs..ofs + 4].try_into().unwrap()); if count < 0 { return Ok(LightBank::default()); } let n = (count as usize).min(MAX_LIGHTS); let mut sets = Vec::with_capacity(n); - for _ in 0..n { - sets.push(read_set(gameplay, ofs + 0x10 + sets.len() * 0x40)?); + for i in 0..n { + sets.push(read_set(gameplay, ofs + 0x10 + i * 0x40)?); } Ok(LightBank { sets, @@ -264,8 +279,32 @@ mod tests { #[test] fn a_level_with_no_lights_parses_as_empty() { let mut g = vec![0u8; 32]; - g[0x04..0x08].copy_from_slice(&(-1i32).to_le_bytes()); + g[0x04..0x08].copy_from_slice(&8u32.to_le_bytes()); + g[0x08..0x0c].copy_from_slice(&(-1i32).to_le_bytes()); let b = LightBank::parse(&g).expect("should parse"); assert!(b.sets.is_empty()); } + + #[test] + fn the_bank_offset_is_at_plus_four_not_zero() { + // A synthetic extent laid out the way the game writes it: an offset at + // +0x04, a count at that offset, then the sets at +0x10. + let mut g = vec![0u8; 0x100]; + let at = 0x40usize; + g[0x04..0x08].copy_from_slice(&(at as u32).to_le_bytes()); + g[at..at + 4].copy_from_slice(&1i32.to_le_bytes()); + let set = |g: &mut [u8], o: usize, v: [f32; 4]| { + for (i, f) in v.iter().enumerate() { + g[o + i * 4..o + i * 4 + 4].copy_from_slice(&f.to_bits().to_le_bytes()); + } + }; + set(&mut g, at + 0x10, [1.0, 2.0, 3.0, 4.0]); + set(&mut g, at + 0x20, [0.0, 0.0, -1.0, 0.0]); + let b = LightBank::parse(&g).expect("should parse"); + assert_eq!(b.sets.len(), 1); + assert_eq!(b.sets[0].color_a, [1.0, 2.0, 3.0, 4.0]); + assert_eq!(b.sets[0].dir_a, [0.0, 0.0, -1.0, 0.0]); + // The old reading — offset at +0, count at +4 — must not be what this + // does: at +0 the extent is 0, so it would find an empty bank. + } } diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index 9447c88..02555f7 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -714,9 +714,15 @@ the fidelity rule. resolving texture addresses on the host. - **The guard-band clip test** that sends patches crossing the guard band to the clipping program. -- **Lighting magnitudes.** The pass runs and the output is plausible; whether - it matches the game needs comparison against reference frames, which needs - the PCSX2 loop that is still to come. Nothing in this file establishes it. +- ~~**Lighting magnitudes.**~~ **Settled 2026-10-11** by the PCSX2 + comparison loop (`docs/plan/pcsx2-verification.md`): level 0's 162,400 + colour bytes match the game's EE RAM exactly. Two bugs the comparison found + and fixed, both of which produced output that looked plausible: + `vertex_normal_bits` negated x and y but not z (so `max(dA, dA*wA)` took the + wrong branch and every vertex lit near its base colour -- 95.88% agreement + instead of 100%), and a record whose `select` names a set the bank lacks + drew transparent black where the game writes the record's base colour (3,130 + of level 0's 40,600 vertices, `select 0x000f`). - **tie, moby and shrub geometry.** The class tables parse and the container they point into is identified (below), but none of the three is decoded yet. Moby is the largest of the three and is skeletal, with an animation evaluator diff --git a/docs/plan/pcsx2-verification.md b/docs/plan/pcsx2-verification.md new file mode 100644 index 0000000..cbed57e --- /dev/null +++ b/docs/plan/pcsx2-verification.md @@ -0,0 +1,119 @@ +# PCSX2 verification (`gc-trace`) + +Status as of **2026-10-11**. The tfrag lighting magnitudes are verified; this +is the file that records how, so the next comparison starts from the +harness instead of from scratch. + +Tool: `tools/trace` (`gc-trace`). It never ships, no shipped crate depends on +it, and it reads the game's own memory out of a running emulator, a +savestate, or a raw dump. + +## What exists now + +```text +gc-trace pine-info # protocol handshake +gc-trace dump-ee [--out FILE] # 32 MiB EE RAM image +gc-trace find --bytes HEX # content search (the locators use it too) +gc-trace read --addr HEX --len N +gc-trace info # savestate metadata +gc-trace compare-tfrag-light [--level N] +``` + +`` is `--pine [slot]`, `--state FILE` / `--state latest`, or `--ee FILE`. +It may sit anywhere on the command line. + +The comparison: + +1. **Locates the tfrags block in RAM by content** -- the key is tfrag 0's + bounding sphere, sixteen bytes unique in 32 MiB. A candidate is accepted + only when the block header matches, **every** header matches the disc on + the invariant bytes (everything but the relocated `data` pointer and the + runtime occlusion index), and every `data` pointer is the relocated form + `table + disc offset`. Anything less is a plausible wrong read. +2. **Locates the light bank the same way**, by its exact section bytes. +3. **Runs the pass** over the same disc data with + `gc_engine::tfrag_light::light_tfrag`. +4. **Diffs per channel, per vertex**, bucketed by vertex class, with the first + few mismatching vertices' *terms* (`gc_engine::tfrag_light:: + light_vertex_terms`) printed so a mismatch can be attributed. + +## Result: level 0 (Veldin), verified 2026-10-11 + +Inputs: the live emulator over PINE, and two PCSX2 savestates from the same +level. All three agree byte for byte. + +```text +level0: tfrags block at 0x848000, table at 0x848040, 833 tfrags + headers matching the disc on invariant bytes: 833/833 + data pointers relocated as table + disc offset: 833/833 + light bank in RAM at 0x50ab20, byte-identical to the disc's + +vertices compared: 40600 (162400 colour bytes) + equal: 162400 (100.00%) + runtime-set: 3130 vertices, 3130 fully equal + single-set: 37470 vertices, 37470 fully equal +``` + +What this establishes: + +- **The tfrag locator and the pointer model are right.** 833 of 833 headers + match on invariant bytes and 833 of 833 `data` pointers are relocated + exactly `table + disc offset`, so the block found in RAM is the level's and + the geometry read out of it is the geometry the game drew. +- **The light bank framing is right.** The section found in RAM is + byte-identical to the disc's: offset at `+0x04`, an `i32` set count, twelve + bytes of pad, then 0x40-byte sets. The previous reading -- offset at + `+0x00`, count at `+0x04` -- was wrong, and this is what settled it. +- **The lighting magnitudes match exactly**: every one of the 162,400 colour + bytes the game wrote equals ours. This was the open item in + `docs/roadmap.md` and `docs/plan/level-assets.md` ("Lighting magnitudes are + unverified: they need comparison against reference frames"). + +## Two bugs the comparison found + +Both produced plausible output and neither was visible in a screenshot. + +**1. The normal's z was not negated.** `vertex_normal_bits` negated x and y +and left z positive, while the renderer's `NormalTable::normal` negates all +three. The result: `dA = -0.707` where the game had `+0.707`, so +`max(dA, dA·wA)` picked the wrong branch and every vertex lit near its base +colour -- dark, but not *wrong* dark. Level 0 agreed on 95.88% of colour +bytes with the bug and 100.00% without it. + +**2. A record with no light set drew transparent black.** `select 0x000f` +names set 15; the level's directional bank has 5 sets, so the pass had no set +to light with and yielded `[0,0,0,0]`. The game writes the record's **base +colour**: 3,130 of level 0's 40,600 vertices do this. The vertex is unlit, not +invisible. + +## Finding the next thing + +- **Compare another level.** All three inputs available are level 0; another + level needs a savestate taken in it (F1 in-game, or `gc-trace + pine-savestate SLOT`). The harness takes `--level N` already. +- **Point lights.** The runtime-set cluster is set 15 and above; those slots + are filled by gameplay, so they cannot be verified from a level-load image. + Comparing them needs a frame with point lights in it. +- **Blended sets.** Level 0's vertices are all single-set (`select & 0xff00 == + 0`), so the blend path -- the only path `rsqrt` touches -- is untested + against the game. +- **Distil, then test.** A comparison is not a regression. When a fact from + here becomes load-bearing, distil it into a numbers-only fixture under + `tools/trace/tests/` and test against that; no test may read a savestate, a + dump or a live emulator. + +## Running it + +```sh +cargo run -q -p gc-trace -- pine-info +cargo run -q -p gc-trace -- compare-tfrag-light ~/Documents/gc-extracted --level 0 --pine +cargo run -q -p gc-trace -- compare-tfrag-light ~/Documents/gc-extracted --level 0 \ + --state ~/.config/PCSX2/sstates/.p2s +``` + +Exit status is 0 when every byte matches, 2 on mismatches, 1 on error, so it +can be run as a check. PINE must be enabled in the emulator (Settings → +Advanced); the socket is `$XDG_RUNTIME_DIR/pcsx2.sock` on Linux. + +**Nothing here redistributes game data.** The disc, the BIOS and the +savestates are the user's own; the repository holds the tool and the numbers. diff --git a/docs/roadmap.md b/docs/roadmap.md index 87b8119..00eb5ed 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -65,8 +65,14 @@ early, while they are still cheap to retire. are now bounded and are whole 8-byte records, so shrub measurements are trustworthy for the first time -- the "3.5M records" figure from the unbounded final span is replaced by a measured 9,600. -- [ ] Lighting *magnitudes* are unverified: they need comparison against - reference frames, which needs the PCSX2 comparison loop. +- [x] **Lighting *magnitudes* are verified** by the PCSX2 comparison loop. + `gc-trace compare-tfrag-light` runs the pass over level 0's disc data and + diffs it against the game's own lit colours in EE RAM (live over PINE, or + from a savestate): **all 162,400 colour bytes match**, across 833/833 + headers relocated and the light bank byte-identical. Two bugs it found: + the normal's z was not negated (dark but plausible, 95.88% agreement), + and records with no light set drew transparent black instead of the base + colour. See `docs/plan/pcsx2-verification.md`. - [ ] Then the GS register set and the guard-band clip test. The disc being the exact build the reference projects pin is the single diff --git a/tools/trace/Cargo.toml b/tools/trace/Cargo.toml new file mode 100644 index 0000000..2330e2b --- /dev/null +++ b/tools/trace/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "gc-trace" +description = "PCSX2 ground-truth comparison harness: reads the game's own memory and diffs it against the port" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow = "1" +gc-formats = { workspace = true } +gc-engine = { workspace = true } +ruzstd = { workspace = true } +flate2 = { workspace = true } diff --git a/tools/trace/src/ee.rs b/tools/trace/src/ee.rs new file mode 100644 index 0000000..9bf934f --- /dev/null +++ b/tools/trace/src/ee.rs @@ -0,0 +1,121 @@ +//! EE RAM: a flat 32 MiB window, plus the content search the locators use. +//! +//! The harness never hardcodes an address. Every structure the comparison +//! needs is found by content, because the addresses differ per level and per +//! run, and a pointer read from the wrong base is indistinguishable from a +//! right read from the right one. `EeRam` is deliberately dumb: byte-range +//! reads and a substring search. + +/// The PlayStation 2's main RAM: 32 MiB at physical 0 (no extended RAM). +pub const EE_SIZE: usize = 32 * 1024 * 1024; + +/// A borrowed EE RAM image. +#[derive(Debug, Clone, Copy)] +pub struct EeRam<'a>(&'a [u8]); + +impl<'a> EeRam<'a> { + /// Wrap a RAM image. The length must be exactly 32 MiB, so a short dump is + /// an error rather than a search that silently stops early. + pub fn new(bytes: &'a [u8]) -> anyhow::Result { + if bytes.len() != EE_SIZE { + anyhow::bail!( + "EE RAM image is {} bytes, expected exactly {EE_SIZE}", + bytes.len() + ); + } + Ok(Self(bytes)) + } + + pub fn bytes(&self) -> &'a [u8] { + self.0 + } + + /// Every offset at which `needle` occurs, scanning 32 MiB. + /// + /// A naive first-byte scan keeps this correct for any needle, including + /// one inside a buffer that also contains zeros — which is most of RAM. + pub fn find_all(&self, needle: &[u8]) -> Vec { + if needle.is_empty() || needle.len() > self.0.len() { + return Vec::new(); + } + let first = needle[0]; + let mut out = Vec::new(); + let mut i = 0usize; + while i + needle.len() <= self.0.len() { + if self.0[i] == first && &self.0[i..i + needle.len()] == needle { + out.push(i); + } + i += 1; + } + out + } + + /// The first offset at which `needle` occurs. + pub fn find(&self, needle: &[u8]) -> Option { + self.find_all(needle).into_iter().next() + } + + pub fn slice(&self, at: usize, len: usize) -> Option<&'a [u8]> { + self.0.get(at..at.checked_add(len)?) + } + + /// A little-endian `i32` at `at`. + pub fn i32(&self, at: usize) -> Option { + self.slice(at, 4) + .map(|b| i32::from_le_bytes(b.try_into().unwrap())) + } + + /// A little-endian `u32` at `at`. + pub fn u32(&self, at: usize) -> Option { + self.slice(at, 4) + .map(|b| u32::from_le_bytes(b.try_into().unwrap())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Fill with a non-zero byte: a zero-filled image makes any needle ending + /// in `0x00` match by accident, which is not what this is testing. + fn ram_with(needle: &[u8], at: usize) -> Vec { + let mut v = vec![0xaau8; EE_SIZE]; + v[at..at + needle.len()].copy_from_slice(needle); + v + } + + #[test] + fn only_a_32mib_image_is_accepted() { + EeRam::new(&[0u8; EE_SIZE]).expect("32 MiB is exactly right"); + assert!(EeRam::new(&[0u8; EE_SIZE - 1]).is_err()); + assert!(EeRam::new(&[]).is_err()); + } + + #[test] + fn find_reports_every_occurrence() { + let v = ram_with(&[0xde, 0xad, 0xbe, 0xef], 0x1000); + let ram = EeRam::new(&v).unwrap(); + assert_eq!(ram.find(&[0xde, 0xad, 0xbe, 0xef]), Some(0x1000)); + assert!(ram.find(&[0xde, 0xad, 0xbe, 0xef, 0x00]).is_none()); + } + + #[test] + fn an_empty_or_oversized_needle_finds_nothing() { + let v = vec![7u8; EE_SIZE]; + let ram = EeRam::new(&v).unwrap(); + assert!(ram.find(&[]).is_none()); + // Longer than the image cannot occur inside it, however uniform the + // image is. + let too_long = vec![7u8; EE_SIZE + 1]; + assert!(ram.find(&too_long).is_none()); + } + + #[test] + fn a_needle_at_the_very_end_is_found() { + let mut v = vec![0u8; EE_SIZE]; + let n = [1u8, 2, 3, 4]; + v[EE_SIZE - 4..].copy_from_slice(&n); + let ram = EeRam::new(&v).unwrap(); + assert_eq!(ram.find(&n), Some(EE_SIZE - 4)); + } +} diff --git a/tools/trace/src/lib.rs b/tools/trace/src/lib.rs new file mode 100644 index 0000000..93f1a0f --- /dev/null +++ b/tools/trace/src/lib.rs @@ -0,0 +1,35 @@ +//! `gc-trace` — the PCSX2 ground-truth harness. +//! +//! This crate is a **dev tool**. It never ships and no shipped crate depends +//! on it; the dependency direction is the other way round (`tools/` may read +//! what the engine computes). Its job is the one the rest of the suite cannot +//! do: compare what the real game does against what we compute, from the same +//! disc data. +//! +//! Inputs, in order of convenience: +//! +//! * **PINE** — the live emulator, over its unix socket. Needs PCSX2 running +//! with PINE enabled. No savestate, no files. +//! * **a PCSX2 savestate** (`.p2s`) — a zip with zstd/deflate entries holding +//! `eeMemory.bin` and friends. Repeatable input: the file can be kept, +//! but it is personal data and is never committed. +//! * **a raw EE RAM dump** — 32 MiB, whatever produced it. +//! +//! The one comparison that exists so far is +//! [`compare-tfrag-light`](tfrag_cmp::compare): the game's lit per-vertex +//! tfrag colours in EE RAM against [`gc_engine::tfrag_light::light_tfrag`] over +//! the same disc data. See `docs/plan/pcsx2-verification.md`. +//! +//! **Findings reach tests only as numbers.** A test never reads a savestate, a +//! dump or a live emulator; when a comparison becomes a regression, distil the +//! facts it needs into a small fixture and test against that. + +pub mod ee; +pub mod p2s; +pub mod pine; +pub mod tfrag_cmp; +pub mod zip; + +pub use ee::EeRam; +pub use p2s::Savestate; +pub use tfrag_cmp::{compare, print_report, LevelInputs, Report}; diff --git a/tools/trace/src/main.rs b/tools/trace/src/main.rs new file mode 100644 index 0000000..265e489 --- /dev/null +++ b/tools/trace/src/main.rs @@ -0,0 +1,344 @@ +//! The `gc-trace` command line. +//! +//! ```text +//! gc-trace compare-tfrag-light --level N +//! gc-trace info +//! gc-trace dump-ee [--out FILE] +//! gc-trace find --bytes HEX +//! gc-trace read --addr HEX --len N +//! gc-trace pine-info +//! gc-trace pine-savestate SLOT +//! ``` +//! +//! `` is where the memory comes from, and it is one of: +//! +//! | Flag | Reads | +//! |---|---| +//! | `--state FILE` | a PCSX2 `.p2s` savestate | +//! | `--state latest` | the newest state in PCSX2's folder | +//! | `--ee FILE` | a raw 32 MiB EE RAM dump | +//! | `--pine [slot]` | the running emulator over PINE | + +use std::path::PathBuf; + +use anyhow::{anyhow, Context, Result}; + +use gc_trace::ee::EeRam; +use gc_trace::{compare, LevelInputs}; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + match run(&args) { + Ok(code) => std::process::exit(code), + Err(e) => { + eprintln!("error: {e:#}"); + std::process::exit(1); + } + } +} + +/// Returns the process exit code. +fn run(args: &[String]) -> Result { + let Some(cmd) = args.first() else { + print!("{USAGE}"); + return Ok(0); + }; + match cmd.as_str() { + "info" => { + let src = Source::parse(&args[1..])?; + info(&src) + } + "dump-ee" => { + let (src, rest) = Source::parse_with_rest(&args[1..])?; + let out = flag(&rest, "--out").unwrap_or_else(|| "ee.bin".into()); + dump_ee(&src, &out) + } + "find" => { + let (src, rest) = Source::parse_with_rest(&args[1..])?; + let hex = require(&rest, "--bytes")?; + find(&src, &hex) + } + "read" => { + let (src, rest) = Source::parse_with_rest(&args[1..])?; + let addr = require(&rest, "--addr")?; + let len = require(&rest, "--len")?; + read(&src, &addr, &len) + } + "compare-tfrag-light" => { + let (src, rest) = Source::parse_with_rest(&args[1..])?; + let dir = rest + .first() + .map(PathBuf::from) + .ok_or_else(|| anyhow!("which unpacked level directory?"))?; + let level: usize = flag(&rest, "--level") + .unwrap_or_else(|| "0".into()) + .parse() + .context("--level")?; + compare_tfrag_light(&dir, level, &src) + } + "pine-info" => pine_info(), + "pine-savestate" => { + let slot: u8 = args + .get(1) + .ok_or_else(|| anyhow!("which slot?"))? + .parse() + .context("slot")?; + pine_savestate(slot) + } + _ => { + print!("{USAGE}"); + Ok(0) + } + } +} + +const USAGE: &str = "\ +usage: gc-trace ... + compare-tfrag-light [--level N] + info + dump-ee [--out FILE] + find --bytes HEX + read --addr HEX --len N + pine-info + pine-savestate SLOT + + is one of: + --state FILE a PCSX2 .p2s savestate + --state latest the newest state in PCSX2's folder + --ee FILE a raw 32 MiB EE RAM dump + --pine [slot] the running emulator over PINE +"; + +/// Where a memory image comes from. +enum Source { + State(PathBuf), + Latest, + Raw(PathBuf), + Pine(u32), +} + +impl Source { + /// The first three arguments after the command. + fn parse(args: &[String]) -> Result { + Ok(Self::parse_with_rest(args)?.0) + } + + /// The source flag, and everything else. + /// + /// The flag may sit anywhere: the usage line puts the level directory + /// first, which is the order people type, so a first-argument-only reader + /// would reject its own documented invocation. Whatever is left after the + /// flag and its value is passed back for the caller's own flags. + fn parse_with_rest(args: &[String]) -> Result<(Source, Vec)> { + let Some(at) = args + .iter() + .position(|a| matches!(a.as_str(), "--state" | "--ee" | "--pine")) + else { + return Err(anyhow!("no source; pass --state FILE, --ee FILE or --pine")); + }; + let (flag, tail) = (args[at].as_str(), &args[at + 1..]); + let (src, used) = match flag { + "--state" => { + let v = tail + .first() + .ok_or_else(|| anyhow!("--state needs a path or `latest`"))?; + let src = if v == "latest" { + Source::Latest + } else { + Source::State(PathBuf::from(v)) + }; + (src, 1) + } + "--ee" => { + let v = tail.first().ok_or_else(|| anyhow!("--ee needs a path"))?; + (Source::Raw(PathBuf::from(v)), 1) + } + // `--pine [slot]`: a following slot number is consumed, and + // anything else -- a directory, another flag -- is not. + _ => { + let slot = match tail.first() { + Some(s) if s.parse::().is_ok() => s.parse()?, + _ => gc_trace::pine::DEFAULT_SLOT, + }; + let used = usize::from(tail.first().is_some_and(|s| s.parse::().is_ok())); + (Source::Pine(slot), used) + } + }; + let mut rest: Vec = args[..at].to_vec(); + rest.extend_from_slice(&tail[used..]); + Ok((src, rest)) + } + + /// Load the RAM image this source names. + fn load(&self) -> Result> { + match self { + Source::State(p) => gc_trace::Savestate::open(p) + .and_then(|s| s.ee_ram_owned()) + .with_context(|| format!("savestate {}", p.display())), + Source::Latest => { + let p = latest_state()?; + let bytes = gc_trace::Savestate::open(&p).and_then(|s| s.ee_ram_owned()); + bytes.with_context(|| format!("savestate {} (latest)", p.display())) + } + Source::Raw(p) => { + let b = std::fs::read(p).with_context(|| format!("read {}", p.display()))?; + Ok(b) + } + Source::Pine(slot) => gc_trace::pine::Pine::connect_slot(*slot) + .and_then(|p| p.dump_ee()) + .context("PINE dump"), + } + } +} + +fn latest_state() -> Result { + let dir = pcsx2_state_dir(); + let mut newest: Option<(std::time::SystemTime, PathBuf)> = None; + for entry in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + let p = entry.path(); + if p.extension().map(|e| e == "p2s").unwrap_or(false) { + if let Ok(meta) = entry.metadata() { + if let Ok(t) = meta.modified() { + if newest.as_ref().is_none_or(|(nt, _)| t > *nt) { + newest = Some((t, p)); + } + } + } + } + } + newest + .map(|(_, p)| p) + .ok_or_else(|| anyhow!("no .p2s savestates in {}", dir.display())) +} + +/// PCSX2's savestate folder, per platform. +fn pcsx2_state_dir() -> PathBuf { + if cfg!(target_os = "macos") { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Library/Application Support/PCSX2/sstates") + } else { + let base = std::env::var_os("XDG_CONFIG_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join(".config") + }); + base.join("PCSX2/sstates") + } +} + +fn info(src: &Source) -> Result { + match src { + Source::State(p) => info_state(p)?, + Source::Latest => info_state(&latest_state()?)?, + Source::Raw(_) => { + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + println!("raw EE RAM image, {} bytes", ram.bytes().len()); + } + Source::Pine(slot) => { + let pine = gc_trace::pine::Pine::connect_slot(*slot)?; + println!("PINE: version {}", pine.version()?); + println!("PINE: title {}", pine.title()?); + } + } + Ok(0) +} + +fn info_state(p: &std::path::Path) -> Result<()> { + let state = gc_trace::Savestate::open(p)?; + println!( + "{}: v{:#010x} build {}", + p.display(), + state.version.unwrap_or(0), + state.build + ); + for e in state.entries() { + println!(" {e}"); + } + Ok(()) +} + +fn dump_ee(src: &Source, out: &str) -> Result { + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + std::fs::write(out, ram.bytes()).with_context(|| format!("write {out}"))?; + println!("wrote {} bytes to {out}", ram.bytes().len()); + Ok(0) +} + +fn find(src: &Source, hex: &str) -> Result { + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + let needle = parse_hex(hex)?; + for at in ram.find_all(&needle) { + println!("{at:#x}"); + } + Ok(0) +} + +fn read(src: &Source, addr_hex: &str, len_hex: &str) -> Result { + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + let addr = usize::from_str_radix(addr_hex.trim_start_matches("0x"), 16)?; + let len = usize::from_str_radix(len_hex.trim_start_matches("0x"), 16)?; + let slice = ram + .slice(addr, len) + .ok_or_else(|| anyhow!("{addr:#x}+{len:#x} runs past the RAM image"))?; + for (i, row) in slice.chunks(16).enumerate() { + let hex: Vec = row.iter().map(|b| format!("{b:02x}")).collect(); + println!("{:08x} {}", addr + i * 16, hex.join(" ")); + } + Ok(0) +} + +fn compare_tfrag_light(dir: &std::path::Path, level: usize, src: &Source) -> Result { + let inputs = LevelInputs::load(dir, level)?; + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + let report = compare(&inputs, level, &ram)?; + let mut text = String::new(); + gc_trace::print_report(&report, &mut text)?; + println!("{text}"); + Ok(report.exit_code()) +} + +fn pine_info() -> Result { + let pine = gc_trace::pine::Pine::connect()?; + println!("version: {}", pine.version()?); + println!("title: {}", pine.title()?); + println!( + "PINE is up on {}", + gc_trace::pine::socket_path(gc_trace::pine::DEFAULT_SLOT).display() + ); + Ok(0) +} + +fn pine_savestate(slot: u8) -> Result { + let pine = gc_trace::pine::Pine::connect()?; + pine.save_state(slot)?; + println!("savestate requested for slot {slot}"); + Ok(0) +} + +/// Parse `hex` bytes, ignoring whitespace. +fn parse_hex(s: &str) -> Result> { + let compact: String = s.chars().filter(|c| !c.is_whitespace()).collect(); + if compact.len() % 2 != 0 { + return Err(anyhow!("hex string has an odd length")); + } + (0..compact.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&compact[i..i + 2], 16).map_err(Into::into)) + .collect() +} + +/// Look up `--flag VALUE` in the rest. +fn flag(args: &[String], name: &str) -> Option { + let at = args.iter().position(|a| a == name)?; + args.get(at + 1).cloned() +} + +fn require(args: &[String], name: &str) -> Result { + flag(args, name).ok_or_else(|| anyhow!("missing {name}")) +} diff --git a/tools/trace/src/p2s.rs b/tools/trace/src/p2s.rs new file mode 100644 index 0000000..bef8b04 --- /dev/null +++ b/tools/trace/src/p2s.rs @@ -0,0 +1,182 @@ +//! A PCSX2 savestate (`.p2s`), read for its EE RAM. +//! +//! The format is a zip archive written by libzip, whose entries are the +//! emulator's state: `eeMemory.bin` is main RAM, and there is a version id +//! entry that says which PCSX2 wrote it. All of this is read from the file, +//! never assumed: the version check and the size check are what stop a +//! plausible-looking wrong read — a partial file, a state from a much older +//! PCSX2 — from being compared against as though it were the game's RAM. +//! +//! Compression is whatever PCSX2 chose (zstd by default, deflate or stored if +//! the user configured otherwise), which is [`crate::zip`]'s job. + +use std::path::Path; + +use anyhow::{anyhow, Context, Result}; + +use crate::zip::Zip; + +/// The savestate version PCSX2 v2.x writes; the high 16 bits are the load +/// gate, the low 16 the patch level. +const SAVE_VERSION: u32 = 0x9A59_0000; + +const VERSION_ID: &str = "PCSX2 Savestate Version.id"; +const EE_MEMORY: &str = "eeMemory.bin"; + +/// A loaded savestate. +pub struct Savestate { + bytes: Vec, + zip: Zip, + /// The version word from the version-id entry, if present. + pub version: Option, + /// The version string that follows it. + pub build: String, +} + +impl Savestate { + /// Open and fully validate a savestate file. + pub fn open(path: &Path) -> Result { + let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?; + let state = Self::from_bytes(bytes)?; + state + .validate() + .with_context(|| format!("{}: not a supported savestate", path.display()))?; + Ok(state) + } + + /// The version gate. A state whose version entry is missing or from a + /// different major line may have a different entry layout, so stop here + /// rather than guess which entry is the RAM. + /// + /// Exposed so a fixture is checked the same way a real file is; `open` + /// calls exactly this. + fn validate(&self) -> Result<()> { + let v = self + .version + .ok_or_else(|| anyhow!("no {VERSION_ID} entry; not a PCSX2 v2.x savestate"))?; + if v & 0xFFFF_0000 != SAVE_VERSION { + return Err(anyhow!( + "savestate version {v:#010x}, this reader handles {SAVE_VERSION:#010x} (PCSX2 v2.8.x)" + )); + } + Ok(()) + } + + /// Build from an in-memory image. + pub fn from_bytes(bytes: Vec) -> Result { + let zip = Zip::parse(&bytes)?; + let (version, build) = match zip.find(VERSION_ID) { + Some(_) => { + let id = zip.read(&bytes, VERSION_ID)?; + if id.len() < 4 { + return Err(anyhow!("version id entry is {} bytes", id.len())); + } + // The build string is read out of the first 32 bytes after the + // version word. A longer entry carries more after it, which is + // not read: only the version is load-bearing here. + ( + Some(u32::from_le_bytes(id[..4].try_into().unwrap())), + String::from_utf8_lossy(&id[4..36.min(id.len())]) + .trim_end_matches('\0') + .to_string(), + ) + } + None => (None, String::new()), + }; + Ok(Savestate { + bytes, + zip, + version, + build, + }) + } + + /// The entry names, which is also a cheap sanity check on what is here. + pub fn entries(&self) -> Vec<&str> { + self.zip.entries().iter().map(|e| e.name.as_str()).collect() + } + + /// One decompressed entry. + pub fn entry(&self, name: &str) -> Result> { + self.zip.read(&self.bytes, name) + } + + /// The decompressed `eeMemory.bin` as an owned buffer. + /// + /// Owned rather than borrowed: the caller wants a stable `EeRam` over the + /// bytes, and the zip entry's payload cannot outlive this call anyway. + pub fn ee_ram_owned(&self) -> Result> { + let ram = self.entry(EE_MEMORY)?; + Ok(ram) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::zip::LOCAL_HEADER; + + fn build_zip(entries: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + for (name, payload) in entries { + out.extend_from_slice(&LOCAL_HEADER.to_le_bytes()); + out.extend_from_slice(&20u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // stored + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&crate::zip::crc32(payload).to_le_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(payload); + } + out + } + + fn version_id() -> Vec { + let mut v = SAVE_VERSION.to_le_bytes().to_vec(); + v.extend_from_slice(b"v2.8.2"); + v.resize(36, 0); + v + } + + #[test] + fn a_state_round_trips_its_ram() { + let ram = vec![0x5au8; 16]; + let z = build_zip(&[(VERSION_ID, &version_id()), (EE_MEMORY, &ram)]); + let state = Savestate::from_bytes(z).expect("should read"); + assert_eq!(state.version, Some(SAVE_VERSION)); + assert!(state.build.starts_with("v2.8")); + assert_eq!(state.entry(EE_MEMORY).unwrap(), ram); + } + + #[test] + fn a_foreign_savestate_version_is_an_error() { + let bad = 0x9A3B_0000u32; // a future major line + let mut v = bad.to_le_bytes().to_vec(); + v.resize(36, 0); + let z = build_zip(&[(VERSION_ID, &v), (EE_MEMORY, &[0u8; 8])]); + let state = Savestate::from_bytes(z).expect("reads the archive"); + assert_eq!(state.version, Some(bad)); + let err = state.validate().unwrap_err().to_string(); + assert!(err.contains("version"), "got {err}"); + // The gate reads the high half only: a newer patch level of the same + // major line is a state this reader handles. + let patched = 0x9A59_0001u32; + let mut v = patched.to_le_bytes().to_vec(); + v.resize(36, 0); + let z = build_zip(&[(VERSION_ID, &v), (EE_MEMORY, &[0u8; 8])]); + assert!(Savestate::from_bytes(z).unwrap().validate().is_ok()); + } + + #[test] + fn a_missing_version_entry_is_reported_not_guessed() { + let z = build_zip(&[(EE_MEMORY, &[0u8; 8])]); + let state = Savestate::from_bytes(z).unwrap(); + assert_eq!(state.version, None); + assert!(state.validate().is_err()); + } +} diff --git a/tools/trace/src/pine.rs b/tools/trace/src/pine.rs new file mode 100644 index 0000000..99c0de7 --- /dev/null +++ b/tools/trace/src/pine.rs @@ -0,0 +1,189 @@ +//! PINE: PCSX2's live debug protocol. +//! +//! PCSX2 exposes a unix socket (`$XDG_RUNTIME_DIR/pcsx2.sock` on Linux, +//! `$TMPDIR` on macOS, `/tmp/pcsx2.sock` otherwise, `.N` appended for a +//! non-default slot) that answers framed messages with the emulator's own +//! memory. The framing is simple and versions of it appear in several +//! clients, so it is spelled out from the v2.8.2 sources: +//! +//! ```text +//! request: u32 length | command bytes, command after command +//! reply: u32 length | u8 status | one reply per command, in order +//! ``` +//! +//! A read is `u8 MsgRead64` followed by a `u32` address and answers with the +//! eight bytes at that address; commands batch, so a 32 MiB dump is 84 +//! round-trips of 50,000 reads each, which is the reply-size ceiling. +//! +//! Live reads mean the input is whatever the emulator is doing *right now*. +//! That makes this mode excellent for exploring and poor for a repeatable +//! test, which is why the harness also reads savestates (`crate::p2s`). + +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; + +use anyhow::{anyhow, Context, Result}; + +/// PCSX2's default slot. +pub const DEFAULT_SLOT: u32 = 28011; + +const MSG_READ64: u8 = 3; +const MSG_VERSION: u8 = 8; +const MSG_SAVE_STATE: u8 = 9; +const MSG_TITLE: u8 = 0xB; +const MSG_STATUS: u8 = 0xF; + +/// Reads per batched message. The reply budget is 450,000 bytes, so 50,000 +/// eight-byte reads (400,000 bytes of payload plus framing) is the largest +/// batch that cannot overflow it. +const BATCH: usize = 50_000; + +/// The emulator's state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Running, + Paused, + Shutdown, +} + +/// A connected PINE session. +pub struct Pine { + stream: UnixStream, +} + +impl Pine { + /// Connect to the socket for `slot`. + pub fn connect_slot(slot: u32) -> Result { + let path = socket_path(slot); + let stream = UnixStream::connect(&path).with_context(|| { + format!( + "cannot connect to PINE at {}\n\ + is PCSX2 running with PINE enabled (Settings > Advanced)?", + path.display() + ) + })?; + Ok(Pine { stream }) + } + + /// Connect at the default slot. + pub fn connect() -> Result { + Self::connect_slot(DEFAULT_SLOT) + } + + /// One framed round-trip; `payload` is the concatenated commands. + fn roundtrip(&self, payload: &[u8]) -> Result> { + let mut msg = Vec::with_capacity(4 + payload.len()); + msg.extend_from_slice(&((4 + payload.len()) as u32).to_le_bytes()); + msg.extend_from_slice(payload); + let mut stream = &self.stream; + stream.write_all(&msg).context("PINE write")?; + let mut header = [0u8; 5]; + stream + .read_exact(&mut header) + .context("PINE reply header")?; + let size = u32::from_le_bytes(header[..4].try_into().unwrap()) as usize; + let status = header[4]; + if status != 0 { + return Err(anyhow!("PINE refused the message (status {status:#x})")); + } + let mut body = vec![0u8; size - 5]; + stream.read_exact(&mut body).context("PINE reply body")?; + Ok(body) + } + + /// The emulator's build string, which is how the protocol is discovered. + pub fn version(&self) -> Result { + let body = self.roundtrip(&[MSG_VERSION])?; + Ok(String::from_utf8_lossy(&body) + .trim_end_matches('\0') + .to_string()) + } + + pub fn title(&self) -> Result { + let body = self.roundtrip(&[MSG_TITLE])?; + Ok(String::from_utf8_lossy(&body) + .trim_end_matches('\0') + .to_string()) + } + + /// Read `count` u64s starting at `addr`. + /// + /// Addresses are physical: EE RAM is 0..32 MiB, so a physical address is + /// what this takes and what the game stores once its pointers are + /// relocated. + pub fn read64(&self, addr: u32, count: usize) -> Result> { + let mut out = Vec::with_capacity(count * 8); + let mut done = 0usize; + while done < count { + let n = (count - done).min(BATCH); + let mut payload = Vec::with_capacity(n * 5); + for i in 0..n { + payload.push(MSG_READ64); + payload.extend_from_slice(&(addr + (done + i) as u32 * 8).to_le_bytes()); + } + out.extend_from_slice(&self.roundtrip(&payload)?); + done += n; + } + Ok(out) + } + + /// All of EE RAM, which is what the comparisons want. + pub fn dump_ee(&self) -> Result> { + self.read64(0, crate::ee::EE_SIZE / 8) + } + + pub fn status(&self) -> Result { + let body = self.roundtrip(&[MSG_STATUS])?; + match u32::from_le_bytes(body[..4].try_into().unwrap()) { + 0 => Ok(Status::Running), + 1 => Ok(Status::Paused), + _ => Ok(Status::Shutdown), + } + } + + /// Ask PCSX2 to write a savestate to `slot`. Saving happens on the + /// emulator's own thread, so the state lands shortly after this returns. + pub fn save_state(&self, slot: u8) -> Result<()> { + self.roundtrip(&[MSG_SAVE_STATE, slot])?; + Ok(()) + } +} + +/// The socket path for a slot, following PCSX2's own resolution order. +pub fn socket_path(slot: u32) -> PathBuf { + let base = std::env::var_os("XDG_RUNTIME_DIR") + .or_else(|| std::env::var_os("TMPDIR")) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("/tmp")); + let mut path = base.join("pcsx2.sock"); + if slot != DEFAULT_SLOT { + path = PathBuf::from(format!("{}.{slot}", path.display())); + } + path +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_socket_path_has_the_slot_suffix_only_when_needed() { + let path = socket_path(DEFAULT_SLOT); + assert!(path.ends_with("pcsx2.sock")); + let other = socket_path(28012); + assert!(other.ends_with("pcsx2.sock.28012")); + } + + #[test] + fn a_reply_is_framed_length_then_status_then_body() { + // A failed command is `05 000000 FF` on the wire; the parser reads + // exactly that shape, so check it by constructing the reply the same + // way the reader does. + let mut reply: Vec = 0x05u32.to_le_bytes().to_vec(); + reply.push(0xFF); + assert_eq!(reply.len(), 5); + assert_eq!(u32::from_le_bytes(reply[..4].try_into().unwrap()), 5); + assert_eq!(reply[4], 0xFF); + } +} diff --git a/tools/trace/src/tfrag_cmp.rs b/tools/trace/src/tfrag_cmp.rs new file mode 100644 index 0000000..bf31d4b --- /dev/null +++ b/tools/trace/src/tfrag_cmp.rs @@ -0,0 +1,597 @@ +//! The tfrag lighting comparison: the game's own bytes against ours. +//! +//! `LightTfrags` overwrites every tfrag's RGBA block in EE RAM with its lit +//! per-vertex colours, so after a level loads the RAM holds the answer. The +//! comparison is: +//! +//! 1. **Locate** the tfrags block in RAM by content, never by a fixed +//! address — the load address differs per level and per run. The tfrag-0 +//! bounding sphere is the search key; a candidate is only accepted when +//! *every* header matches the disc on the bytes the game leaves alone +//! (all but the relocated `data` pointer and the runtime occlusion +//! index) and the `data` pointers are relocated exactly as +//! `table + disc_data`. +//! 2. **Locate the light bank** the same way: the bytes of the level's +//! gameplay light section, searched in RAM. It is compared against the +//! disc's so a parse error and a maths error are distinguishable. +//! 3. **Run the pass** over the same disc data with +//! [`gc_engine::tfrag_light::light_tfrag`]. +//! 4. **Diff** per channel, per tfrag, bucketed by vertex class so a +//! mismatch in the blend path (which is the only path `rsqrt` touches) +//! separates from one in the plain path. +//! +//! The pass-ran check is the number of vertices whose RAM colour still equals +//! the disc's unlit placeholder. If that is near the whole level, the pass +//! has not run yet and the comparison means nothing. + +use std::collections::BTreeMap; +use std::path::Path; + +use anyhow::{anyhow, Context, Result}; + +use gc_engine::tfrag_light::{light_tfrag, light_vertex_terms, vertex_normal_bits, VertexTerms}; +use gc_formats::level_core::LevelCoreHeader; +use gc_formats::tfrag::{parse_tfrag, TfragHeader, TfragsBlock, TFRAG_HEADER_SIZE}; +use gc_formats::tfrag_light::{LightBank, NormalTable}; + +use crate::ee::EeRam; + +/// How many mismatching vertices keep their terms in the report. Enough to +/// read a pattern by hand, few enough that the report stays readable. +const BREAKDOWNS: usize = 6; + +/// Result of one comparison, everything a report needs. +#[derive(Debug, Clone)] +pub struct Report { + pub level: usize, + /// RAM address of the tfrags block, its header table, and the tfrag count. + pub block_base: usize, + pub table: usize, + pub tfrag_count: usize, + pub headers_matched: usize, + pub pointers_relocated: usize, + /// RAM address of the directional light bank, when found. + pub bank_ram: Option, + /// Whether the bank was found in RAM **byte-identical to the disc's**. It + /// is a content hit, so a found bank is by construction the disc's; the + /// flag exists to keep the not-found case out of that claim, because then + /// the comparison below runs on the disc's bank with nothing from the + /// game to confirm it. + pub bank_is_disc: bool, + pub vertices_compared: usize, + pub bytes_compared: usize, + pub bytes_equal: usize, + /// Vertices whose RAM colour still equals the disc's unlit placeholder. + pub placeholder_equal: usize, + /// Per-vertex-class tallies: `single-set`, `blended`, and `no-set` for + /// vertices whose `select` names a light set the disc's bank does not have + /// -- runtime lights, which a level-load comparison cannot see. + pub classes: BTreeMap<&'static str, ClassStat>, + /// `ours - ram` per channel, in order. + pub histogram: BTreeMap, + /// The first few differing vertices, for reading by hand. + pub mismatches: Vec, + /// The pass's terms for the first few mismatching vertices. A colour + /// mismatch alone cannot be attributed, and the whole point of the + /// comparison is to say *which* term the game disagrees with. + pub breakdowns: Vec, +} + +/// One mismatching vertex with its terms, so the difference can be read as +/// `ours` versus `ram` per term rather than per byte. +#[derive(Debug, Clone)] +pub struct Breakdown { + pub tfrag: usize, + pub entry: usize, + /// `None` when the pass found no light set for the vertex's `select` -- + /// which is itself the finding, so it is recorded rather than skipped. + pub terms: Option, + /// The record's `select`, always available even when the terms are not. + pub select: u16, + /// The record's base colour, PEXT5-expanded: what the game would write if + /// the light contributed nothing. + pub base: [u8; 4], + /// The disc's stored RGBA for the entry, which is the unlit placeholder. + pub disc: [u8; 4], + pub ram: [u8; 4], +} + +/// Totals for one vertex class. +#[derive(Debug, Clone, Copy, Default)] +pub struct ClassStat { + pub vertices: usize, + pub equal: usize, +} + +/// One differing vertex byte. +#[derive(Debug, Clone, Copy)] +pub struct Mismatch { + pub tfrag: usize, + pub entry: usize, + pub channel: usize, + pub ours: i32, + pub ram: i32, +} + +impl Report { + /// 0 = every byte equal, 2 = mismatches. + pub fn exit_code(&self) -> i32 { + if self.bytes_equal == self.bytes_compared { + 0 + } else { + 2 + } + } +} + +/// Everything the comparison needs from one unpacked level. +pub struct LevelInputs { + pub core_data: Vec, + pub core_index: Vec, + pub gameplay: Vec, + pub overlay: Vec, +} + +impl LevelInputs { + pub fn load(dir: &Path, level: usize) -> Result { + let read = |name: &str| { + std::fs::read(dir.join(format!("level{level}")).join(name)) + .with_context(|| format!("level{level}/{name}")) + }; + Ok(LevelInputs { + core_data: read("core_data.bin")?, + core_index: read("core_index.bin")?, + gameplay: read("gameplay.bin")?, + overlay: read("overlay.bin")?, + }) + } +} + +/// Compare the tfrag lighting of one level against a RAM image. +pub fn compare(inputs: &LevelInputs, level: usize, ram: &EeRam) -> Result { + // Parsed for its validation, not for a field: a core index that does not + // parse cannot name the tfrags region the rest of this reads. + LevelCoreHeader::parse(&inputs.core_index)?; + let tfrags_ofs = i32_at(&inputs.core_index, 0x08) + .ok_or_else(|| anyhow!("core index is too short for the tfrags offset"))?; + let region_end = [0x0c_usize, 0x10, 0x14] + .iter() + .filter_map(|&o| i32_at(&inputs.core_index, o)) + .filter(|&v| v > tfrags_ofs) + .min() + .unwrap_or(inputs.core_data.len() as i32); + let region = &inputs.core_data[tfrags_ofs as usize..region_end as usize]; + let block = TfragsBlock::parse(region)?; + let table_off = block.table_offset as usize; + let count = block.tfrag_count as usize; + if count == 0 { + anyhow::bail!("level{level} has no tfrags"); + } + + let region_len = region.len(); + let block_base = locate_block(region, ram, table_off, count)?; + + // The RAM table, then every header against the disc on the invariant mask. + let table = block_base + table_off; + let mut headers_matched = 0usize; + let mut pointers_relocated = 0usize; + for i in 0..count { + let at = i * TFRAG_HEADER_SIZE; + let disc_h = ®ion[table_off + at..table_off + at + TFRAG_HEADER_SIZE]; + let ram_h = ram + .slice(table + at, TFRAG_HEADER_SIZE) + .ok_or_else(|| anyhow!("tfrag header {i} runs past the RAM image"))?; + if invariant_equal(disc_h, ram_h) { + headers_matched += 1; + } + // data is relocated to an absolute pointer into the RAM copy. + let disc_data = i32_at(disc_h, 0x10).unwrap_or(0); + let ram_data = ram.i32(table + at + 0x10).unwrap_or(0); + if ram_data as usize == block_base + table_off + disc_data as usize { + pointers_relocated += 1; + } + } + + // The bank: search RAM for the gameplay's light section, byte for byte. + // What is searched for is the section's own bytes, so a hit *is* the + // comparison -- a bank that is in RAM by content is the disc's bank, and + // the only other outcome is that it is not there, which is reported + // rather than papered over. + let disc_bank = LightBank::parse(&inputs.gameplay)?; + let section = bank_section(&inputs.gameplay)?; + let bank_ram = if section.is_empty() { + None + } else { + ram.find(section) + }; + + let table_normals = + NormalTable::from_overlay(&inputs.overlay).context("the level overlay's normal table")?; + + let mut report = Report { + level, + block_base, + table, + tfrag_count: count, + headers_matched, + pointers_relocated, + bank_ram, + bank_is_disc: bank_ram.is_some(), + vertices_compared: 0, + bytes_compared: 0, + bytes_equal: 0, + placeholder_equal: 0, + classes: BTreeMap::new(), + histogram: BTreeMap::new(), + mismatches: Vec::new(), + breakdowns: Vec::new(), + }; + + for i in 0..count { + let at = i * TFRAG_HEADER_SIZE; + let disc_h = + TfragHeader::parse(®ion[table_off + at..table_off + at + TFRAG_HEADER_SIZE])?; + let ram_h = TfragHeader::parse(ram.slice(table + at, TFRAG_HEADER_SIZE).unwrap())?; + let vert = disc_h.vert_count as usize; + if vert == 0 { + continue; + } + // The tfrag's own data block in RAM, bounded by the region. `ram_h.data` + // is the already-relocated absolute pointer, and the region ends at + // `block_base + region_len`, so the block is what is between the two. + let ram_data = ram_h.data as usize; + let start = ram_data + .checked_add(0) + .filter(|&s| s < ram.bytes().len()) + .ok_or_else(|| anyhow!("tfrag {i} data pointer {ram_data:#x} is outside RAM"))?; + let region_end_ram = block_base + region_len; + if start >= region_end_ram { + anyhow::bail!("tfrag {i} data pointer {ram_data:#x} is past the tfrags region"); + } + let len = region_end_ram - start; + let block_ram = &ram.bytes()[start..start + len]; + let geom = parse_tfrag(block_ram, &ram_h).context("the game's copy of tfrag {i}")?; + + // The lit colours, from our pass over this same data. + let ours = light_tfrag(&geom, &disc_bank, &table_normals); + + // And the RAM's, plus the disc's unlit placeholder for the same + // entries. Only 0..vert_count mean anything: past them the block is + // scratchpad leftovers that no vertex references. + let rgba_ofs = ram_h.rgba_ofs as usize; + let entries = vert.min(ours.len()); + let disc_data = disc_h.data as usize; + for (e, ours_byte) in ours.iter().take(entries).enumerate() { + let l = &geom.lights[e]; + let select = gem_select(&geom, e); + let class = if select & 0xff00 == 0 { + if disc_bank.sets.len() > (select & 0xf) as usize { + "single-set" + } else { + "runtime-set" + } + } else if disc_bank.sets.len() > (select & 0xf) as usize { + "blended" + } else { + "runtime-blend" + }; + let stat = report.classes.entry(class).or_default(); + stat.vertices += 1; + report.vertices_compared += 1; + let ram_rgba = match ram.slice(start + rgba_ofs + e * 4, 4) { + Some(b) => b, + None => break, + }; + let disc_rgba = region + .get(table_off + disc_data + disc_h.rgba_ofs as usize + e * 4..) + .and_then(|b| b.get(..4)) + .unwrap_or(&[0, 0, 0, 0]); + let mut all_equal = true; + for (ch, ours_byte) in ours_byte.iter().enumerate() { + let o = *ours_byte as i32; + let r = ram_rgba[ch] as i32; + report.bytes_compared += 1; + if o == r { + report.bytes_equal += 1; + } else { + all_equal = false; + *report.histogram.entry(o - r).or_default() += 1; + if report.mismatches.len() < 32 { + report.mismatches.push(Mismatch { + tfrag: i, + entry: e, + channel: ch, + ours: o, + ram: r, + }); + } + } + } + if all_equal { + stat.equal += 1; + } else if report.breakdowns.len() < BREAKDOWNS { + // The terms for this vertex, as the pass computed them, so the + // difference can be attributed to a term. A missing light set + // is kept too: that is a finding, not a gap. + let normal = + vertex_normal_bits(&table_normals, l.azimuth_index(), l.elevation_index()); + let terms = light_vertex_terms(&disc_bank, normal, &geom.lights[e], l.select); + report.breakdowns.push(Breakdown { + tfrag: i, + entry: e, + terms, + select: l.select, + base: gc_engine::tfrag_light::pext5(l.color as u16), + disc: [disc_rgba[0], disc_rgba[1], disc_rgba[2], disc_rgba[3]], + ram: [ram_rgba[0], ram_rgba[1], ram_rgba[2], ram_rgba[3]], + }); + } + if ram_rgba == disc_rgba { + report.placeholder_equal += 1; + } + } + } + + Ok(report) +} + +/// The `select` word for entry `e`, from the parsed light records. +fn gem_select(g: &gc_formats::tfrag::TfragGeometry, e: usize) -> u16 { + g.lights.get(e).map(|l| l.select).unwrap_or(0) +} + +/// The light bank's section bytes: the count word, the pad, then the sets. +/// +/// This is what `LightBank::parse` reads, so searching RAM for it is checking +/// the parse against the game: a hit means the game holds the same bytes the +/// disc holds, and a miss means either the level rewrites the section or the +/// parse does not describe what the game loaded. Both are worth knowing +/// before any colour is compared. +fn bank_section(gameplay: &[u8]) -> Result<&[u8]> { + let ofs = u32_at(gameplay, 0x04) + .ok_or_else(|| anyhow!("gameplay extent is too short for the bank offset"))? + as usize; + let count = i32_at(gameplay, ofs) + .ok_or_else(|| anyhow!("the light bank offset is past the gameplay extent"))?; + if count < 0 { + // No lights on this level, which is not an error. + return Ok(&[]); + } + let end = ofs + 0x10 + 0x40 * count as usize; + gameplay.get(ofs..end).ok_or_else(|| { + anyhow!( + "the light section runs {} bytes past the gameplay extent", + end - gameplay.len() + ) + }) +} + +/// Find the level's tfrags block in RAM. +/// +/// The key is tfrag 0's bounding sphere: sixteen bytes of float that are +/// unique in a 32 MiB image, but the candidate still has to prove itself — +/// the block header must match, *every* header must match the disc on the +/// bytes the game does not rewrite, and every `data` pointer must be the +/// relocated form of the disc's. Anything less and the answer would be a +/// plausible but wrong read. +fn locate_block(region: &[u8], ram: &EeRam, table_off: usize, count: usize) -> Result { + let key = ®ion[table_off..table_off + 16]; + let candidates = ram.find_all(key); + let mut best: Option<(usize, usize)> = None; + for &cand in &candidates { + if cand < table_off { + continue; + } + let base = cand - table_off; + let Some(ram_block_header) = ram.slice(base, 0x10) else { + continue; + }; + if ram_block_header != ®ion[..0x10.min(region.len())] { + continue; + } + let mut matched = 0usize; + for i in 0..count.min(64) { + let at = i * TFRAG_HEADER_SIZE; + let disc_h = match region.get(table_off + at..table_off + at + TFRAG_HEADER_SIZE) { + Some(h) => h, + None => break, + }; + match ram.slice(cand + at, TFRAG_HEADER_SIZE) { + Some(h) if invariant_equal(disc_h, h) => matched += 1, + _ => {} + } + } + if best.is_none_or(|(_, m)| matched > m) { + best = Some((base, matched)); + } + } + match best { + Some((base, matched)) if matched == count.min(64) => Ok(base), + Some((base, matched)) => Err(anyhow!( + "tfrags block at {base:#x} matches {matched} of {count} headers on the invariant bytes; \ + refusing a partial match" + )), + None => Err(anyhow!( + "no tfrags block in the RAM image matches this level ({} candidates for the tfrag-0 \ + bounding sphere)", + candidates.len() + )), + } +} + +/// Bytes a tfrag header must keep between the disc and RAM: everything except +/// the relocated `data` pointer (+0x10..0x14) and the occlusion index +/// (+0x3a..0x3c), which the loader renumbers per tfrag. +fn invariant_equal(disc: &[u8], ram: &[u8]) -> bool { + let keep = |j: usize| !(0x10..0x14).contains(&j) && !(0x3a..0x3c).contains(&j); + disc.iter() + .zip(ram.iter()) + .enumerate() + .filter(|&(j, _)| keep(j)) + .all(|(_, (a, b))| a == b) +} + +fn i32_at(bytes: &[u8], at: usize) -> Option { + bytes + .get(at..at + 4) + .map(|b| i32::from_le_bytes(b.try_into().unwrap())) +} + +fn u32_at(bytes: &[u8], at: usize) -> Option { + bytes + .get(at..at + 4) + .map(|b| u32::from_le_bytes(b.try_into().unwrap())) +} + +/// Render a report for a terminal. +pub fn print_report(r: &Report, out: &mut dyn std::fmt::Write) -> std::fmt::Result { + let _ = writeln!( + out, + "level{}: tfrags block at {:#x}, table at {:#x}, {} tfrags", + r.level, r.block_base, r.table, r.tfrag_count + ); + let _ = writeln!( + out, + " headers matching the disc on invariant bytes: {}/{}", + r.headers_matched, r.tfrag_count + ); + let _ = writeln!( + out, + " data pointers relocated as table + disc offset: {}/{}", + r.pointers_relocated, r.tfrag_count + ); + match r.bank_ram { + Some(at) => writeln!( + out, + " light bank in RAM at {at:#x}, byte-identical to the disc's" + )?, + None => writeln!( + out, + " the disc's light bank is not in RAM by content; the colours below are\\ + \n compared against the disc's bank with nothing from the game to confirm it" + )?, + } + let _ = writeln!( + out, + "\nvertices compared: {} ({} colour bytes)", + r.vertices_compared, r.bytes_compared + ); + let _ = writeln!( + out, + " equal: {} ({:.2}%)", + r.bytes_equal, + 100.0 * r.bytes_equal as f64 / r.bytes_compared.max(1) as f64 + ); + let _ = writeln!( + out, + " unlit placeholders: {} of {} vertices ({:.2}%)", + r.placeholder_equal, + r.vertices_compared, + 100.0 * r.placeholder_equal as f64 / r.vertices_compared.max(1) as f64 + ); + for (class, stat) in &r.classes { + let _ = writeln!( + out, + " {class}: {} vertices, {} fully equal", + stat.vertices, stat.equal + ); + } + let _ = writeln!(out, "\ndifference (ours - ram) per channel:"); + for d in -8..=8 { + let n = r.histogram.get(&d).copied().unwrap_or(0); + if n > 0 { + let bar = "#".repeat((n as f64).log10().ceil() as usize); + let _ = writeln!(out, " {d:+3}: {n:8} {bar}"); + } + } + // The tail in magnitude buckets: a single far value hiding a count of + // differing bytes is the number this report most needs to be legible. + for (lo, hi) in [(9, 33), (33, 65), (65, 129), (129, 256), (256, i32::MAX)] { + let n: usize = r + .histogram + .iter() + .filter(|(&d, _)| !(-lo..=hi).contains(&d)) + .map(|(_, n)| n) + .sum(); + if n > 0 { + let _ = writeln!(out, " |d| in {lo}..{hi}: {n:8}"); + } + } + if !r.mismatches.is_empty() { + let _ = writeln!(out, "\nfirst mismatches:"); + for m in r.mismatches.iter().take(16) { + let _ = writeln!( + out, + " tfrag {} entry {} channel {}: ours {} ram {}", + m.tfrag, m.entry, m.channel, m.ours, m.ram + ); + } + } + for b in &r.breakdowns { + let Some(t) = &b.terms else { + let _ = writeln!( + out, + "\ntfrag {} entry {}: no light set for select {:#06x}; the pass yields transparent\ + \n black, the game wrote {:?}\n (record base {:?}, disc placeholder {:?})", + b.tfrag, b.entry, b.select, b.ram, b.base, b.disc + ); + continue; + }; + let _ = writeln!( + out, + "\ntfrag {} entry {}: ours {:?} ram {:?} (select {:#06x}, set {:?}, blended {})", + b.tfrag, b.entry, t.out, b.ram, t.select, t.set_index, t.blended + ); + let _ = writeln!( + out, + " normal (az {}, el {}) = {:.3} {:.3} {:.3}", + t.azimuth, t.elevation, t.normal[0], t.normal[1], t.normal[2] + ); + let _ = writeln!( + out, + " base {:?} A colour {:.3} {:.3} {:.3} (w {:.3}) B colour {:.3} {:.3} {:.3} (w {:.3})", + t.base, + t.color_a[0], t.color_a[1], t.color_a[2], t.wa, + t.color_b[0], t.color_b[1], t.color_b[2], t.wb + ); + let _ = writeln!( + out, + " dA {:.3} -> max {:.3} dB {:.3} -> max {:.3}", + t.dot_a, t.max_a, t.dot_b, t.max_b + ); + let _ = writeln!( + out, + " dir A {:.3} {:.3} {:.3} dir B {:.3} {:.3} {:.3}", + t.dir_a[0], t.dir_a[1], t.dir_a[2], t.dir_b[0], t.dir_b[1], t.dir_b[2] + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_invariant_mask_excludes_the_pointer_and_the_occlusion_index() { + let disc = vec![0u8; 0x40]; + let mut ram = disc.clone(); + // The relocated pointer: any difference is allowed. + ram[0x10..0x14].fill(0xff); + // The occlusion index, which the loader renumbers. + ram[0x3a] = 1; + ram[0x3b] = 2; + assert!(invariant_equal(&disc, &ram)); + // But the bounding sphere is load-invariant and must match. + let mut bad = ram.clone(); + bad[1] = 0x99; + assert!(!invariant_equal(&disc, &bad)); + } + + #[test] + fn a_ram_image_must_be_exactly_32mib() { + let image = vec![0u8; crate::ee::EE_SIZE]; + assert!(EeRam::new(&image).is_ok()); + assert!(EeRam::new(&vec![0u8; 1024]).is_err()); + } +} diff --git a/tools/trace/src/zip.rs b/tools/trace/src/zip.rs new file mode 100644 index 0000000..3abed19 --- /dev/null +++ b/tools/trace/src/zip.rs @@ -0,0 +1,336 @@ +//! A minimal zip reader: enough for a PCSX2 savestate, nothing more. +//! +//! PCSX2 writes savestates with libzip: a sequence of local file headers +//! followed by a central directory. Entries are stored, deflated or +//! zstd-compressed, so this reader takes the entry's compression method from +//! its local header and decodes with it. Only what the format needs is +//! implemented, and what is not needed is rejected rather than guessed at: +//! +//! * **no streamed entries** — libzip writes the compressed size into the +//! local header, and a `data descriptor` (flag bit 3) is an error. Making +//! that an error rather than a guess matters: misparsed sizes produce a +//! plausible but wrong read of the next entry. +//! * **CRC-32 is checked.** A savestate that was truncated or partially +//! overwritten would otherwise read as a plausible wrong buffer, and this +//! harness exists to compare against real bytes. +//! +//! Zip64 is not needed for a 32 MiB entry, and is not supported. + +use std::io::Read; + +use anyhow::{anyhow, Context, Result}; + +/// One entry in a zip archive. +#[derive(Debug, Clone)] +pub struct Entry { + pub name: String, + /// 0 = stored, 8 = deflate, 93 = zstd. + pub method: u16, + /// Byte range of the compressed payload within the archive. + pub data: std::ops::Range, + /// Expected CRC-32 of the uncompressed payload. + pub crc32: u32, + /// Uncompressed size in bytes. + pub uncompressed_size: usize, +} + +/// A parsed zip archive over a borrowed byte range. +#[derive(Debug, Clone)] +pub struct Zip { + entries: Vec, +} + +pub const LOCAL_HEADER: u32 = 0x0403_4b50; +const METHOD_STORED: u16 = 0; +const METHOD_DEFLATE: u16 = 8; +const METHOD_ZSTD: u16 = 93; + +impl Zip { + /// Walk the local file headers from the start of the archive. + pub fn parse(bytes: &[u8]) -> Result { + let mut entries = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= bytes.len() { + let sig = u32::from_le_bytes(bytes[pos..pos + 4].try_into().unwrap()); + if sig != LOCAL_HEADER { + if pos == 0 { + return Err(anyhow!("not a zip archive")); + } + // The central directory (or an end-of-central-directory + // record) ends the local-header run. + break; + } + let h = LocalHeader::parse(bytes, pos)?; + if h.flags & 0x08 != 0 { + return Err(anyhow!( + "entry {:?} uses a data descriptor; this reader needs sizes in the local header", + h.name + )); + } + let data_start = pos + 30 + h.name_len + h.extra_len; + let data_end = data_start.checked_add(h.compressed_size).ok_or_else(|| { + anyhow!("entry {:?} compressed size overflows the archive", h.name) + })?; + if data_end > bytes.len() { + return Err(anyhow!( + "entry {:?} at +{data_start:#x} runs {} bytes past the end of the archive", + h.name, + data_end - bytes.len() + )); + } + entries.push(Entry { + name: h.name, + method: h.method, + data: data_start..data_end, + crc32: h.crc32, + uncompressed_size: h.uncompressed_size, + }); + pos = data_end; + } + if entries.is_empty() { + return Err(anyhow!("zip archive holds no entries")); + } + Ok(Zip { entries }) + } + + pub fn entries(&self) -> &[Entry] { + &self.entries + } + + pub fn find(&self, name: &str) -> Option<&Entry> { + self.entries.iter().find(|e| e.name == name) + } + + /// Read and decompress one entry. + pub fn read(&self, bytes: &[u8], name: &str) -> Result> { + let e = self + .find(name) + .ok_or_else(|| anyhow!("zip archive has no entry {name:?}"))?; + let raw = &bytes[e.data.clone()]; + let out = match e.method { + METHOD_STORED => raw.to_vec(), + METHOD_DEFLATE => { + let mut d = flate2::read::DeflateDecoder::new(raw); + let mut out = Vec::with_capacity(e.uncompressed_size); + d.read_to_end(&mut out).context("deflate entry")?; + out + } + METHOD_ZSTD => { + let d = + ruzstd::decoding::StreamingDecoder::new(raw).context("zstd entry header")?; + let mut out = Vec::with_capacity(e.uncompressed_size); + d.take(e.uncompressed_size.max(1) as u64) + .read_to_end(&mut out) + .context("zstd entry")?; + out + } + other => return Err(anyhow!("entry {name:?} uses unsupported method {other}")), + }; + if out.len() != e.uncompressed_size { + return Err(anyhow!( + "entry {name:?} decompressed to {} bytes, header says {}", + out.len(), + e.uncompressed_size + )); + } + if crc32(&out) != e.crc32 { + return Err(anyhow!("entry {name:?} failed its CRC-32 check")); + } + Ok(out) + } +} + +struct LocalHeader { + name: String, + method: u16, + flags: u16, + crc32: u32, + compressed_size: usize, + uncompressed_size: usize, + name_len: usize, + extra_len: usize, +} + +impl LocalHeader { + fn parse(bytes: &[u8], pos: usize) -> Result { + let fixed = bytes + .get(pos..pos + 30) + .ok_or_else(|| anyhow!("local header at +{pos:#x} is truncated"))?; + let flags = le16(fixed, 6); + let method = le16(fixed, 8); + let crc32 = le32(fixed, 14); + let compressed_size = le32(fixed, 18) as usize; + let uncompressed_size = le32(fixed, 22) as usize; + let name_len = le16(fixed, 26) as usize; + let extra_len = le16(fixed, 28) as usize; + let name_start = pos + 30; + let name_end = name_start + name_len; + let name = bytes + .get(name_start..name_end) + .ok_or_else(|| anyhow!("entry name at +{pos:#x} runs past the archive"))?; + let name = String::from_utf8(name.to_vec()) + .map_err(|_| anyhow!("entry name at +{pos:#x} is not UTF-8"))?; + Ok(LocalHeader { + name, + method, + flags, + crc32, + compressed_size, + uncompressed_size, + name_len, + extra_len, + }) + } +} + +fn le16(b: &[u8], at: usize) -> u16 { + u16::from_le_bytes(b[at..at + 2].try_into().unwrap()) +} + +fn le32(b: &[u8], at: usize) -> u32 { + u32::from_le_bytes(b[at..at + 4].try_into().unwrap()) +} + +/// CRC-32 (IEEE 802.3), the one zip uses. Written out rather than pulled from +/// a crate because it is fifteen lines and keeps the lockfile unchanged. +pub fn crc32(data: &[u8]) -> u32 { + let mut table = [0u32; 256]; + for (i, slot) in table.iter_mut().enumerate() { + let mut c = i as u32; + for _ in 0..8 { + c = if c & 1 != 0 { + 0xEDB8_8320 ^ (c >> 1) + } else { + c >> 1 + }; + } + *slot = c; + } + let mut c = 0xFFFF_FFFFu32; + for &b in data { + c = table[((c ^ b as u32) & 0xff) as usize] ^ (c >> 8); + } + c ^ 0xFFFF_FFFF +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stored entry, written by hand so the test does not need an encoder. + fn stored_zip(name: &str, payload: &[u8]) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&LOCAL_HEADER.to_le_bytes()); + out.extend_from_slice(&20u16.to_le_bytes()); // version needed + out.extend_from_slice(&0u16.to_le_bytes()); // flags + out.extend_from_slice(&METHOD_STORED.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // time + out.extend_from_slice(&0u16.to_le_bytes()); // date + out.extend_from_slice(&crc32(payload).to_le_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // extra len + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(payload); + // A minimal end-of-central-directory record; the reader stops at it. + out.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); + out.extend_from_slice(&[0u8; 18]); + out + } + + #[test] + fn reads_a_stored_entry_and_checks_its_crc() { + let z = stored_zip("eeMemory.bin", b"0123456789"); + let zip = Zip::parse(&z).expect("should parse"); + assert_eq!(zip.entries().len(), 1); + assert_eq!(zip.entries()[0].name, "eeMemory.bin"); + assert_eq!(zip.read(&z, "eeMemory.bin").unwrap(), b"0123456789"); + } + + #[test] + fn a_corrupted_entry_is_an_error_not_a_wrong_read() { + let mut z = stored_zip("v.bin", b"0123456789"); + let last = z.len() - 19 - 5; + z[last] ^= 0xff; + let zip = Zip::parse(&z).expect("header parsing survives"); + let err = zip.read(&z, "v.bin").unwrap_err().to_string(); + assert!(err.contains("CRC-32"), "got {err}"); + } + + #[test] + fn a_data_descriptor_entry_is_rejected() { + let mut z = stored_zip("v.bin", b"payload"); + // Rewrite the flags field (offset 6) to set bit 3. + z[6] = 0x08; + let err = Zip::parse(&z).unwrap_err().to_string(); + assert!(err.contains("data descriptor"), "got {err}"); + } + + #[test] + fn a_truncated_entry_is_rejected() { + let mut z = stored_zip("v.bin", b"0123456789"); + // The header declares ten payload bytes; leave one of them missing. + // Truncating the end instead would only chew into the central + // directory, which this reader stops at before it gets there. + let declared = 30 + "v.bin".len() + 10; + z.truncate(declared - 1); + let err = Zip::parse(&z).unwrap_err().to_string(); + assert!(err.contains("past the end"), "got {err}"); + } + + #[test] + fn crc32_matches_the_known_value() { + // The standard check value for CRC-32/ISO-HDLC. + assert_eq!(crc32(b"123456789"), 0xCBF4_3926); + } + + /// A deflate entry. The stream is the deflate encoding of `b"AB"` produced + /// by an independent encoder (Python's zlib, raw wbits), so this checks + /// against something this code did not make. + #[test] + fn reads_a_deflate_entry() { + let deflated: &[u8] = &[115, 116, 2, 0]; + let mut out = Vec::new(); + out.extend_from_slice(&LOCAL_HEADER.to_le_bytes()); + out.extend_from_slice(&20u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&METHOD_DEFLATE.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&crc32(b"AB").to_le_bytes()); + out.extend_from_slice(&(deflated.len() as u32).to_le_bytes()); + out.extend_from_slice(&2u32.to_le_bytes()); + out.extend_from_slice(&("v.bin".len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(b"v.bin"); + out.extend_from_slice(deflated); + let zip = Zip::parse(&out).expect("should parse"); + assert_eq!(zip.read(&out, "v.bin").unwrap(), b"AB"); + } + + /// A zstd entry, the way PCSX2 writes them. The frame is the zstd encoding + /// of `b"AB"` from an independent encoder (Python 3.14's zipfile), and it + /// uses raw blocks the way a seven-byte payload needs. + #[test] + fn reads_a_zstd_entry() { + let zstd_frame: &[u8] = &[40, 181, 47, 253, 0, 88, 17, 0, 0, 65, 66]; + let mut out = Vec::new(); + out.extend_from_slice(&LOCAL_HEADER.to_le_bytes()); + out.extend_from_slice(&20u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&93u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&crc32(b"AB").to_le_bytes()); + out.extend_from_slice(&(zstd_frame.len() as u32).to_le_bytes()); + out.extend_from_slice(&2u32.to_le_bytes()); + out.extend_from_slice(&("v.bin".len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(b"v.bin"); + out.extend_from_slice(zstd_frame); + let zip = Zip::parse(&out).expect("should parse"); + assert_eq!(zip.entries()[0].method, 93); + assert_eq!(zip.read(&out, "v.bin").unwrap(), b"AB"); + } +}