diff --git a/crates/gc-formats/src/tie.rs b/crates/gc-formats/src/tie.rs index 642adb8..89f42a0 100644 --- a/crates/gc-formats/src/tie.rs +++ b/crates/gc-formats/src/tie.rs @@ -192,15 +192,14 @@ impl TieBlockHeader { /// The block's records, as byte slices of [`TIE_RECORD_SIZE`]. /// /// Bounds-checked against `block`, which is the block starting at its - /// header: the table runs from the end of the header to `data_offset`. A - /// header whose table runs past the block yields nothing rather than a - /// short slice. + /// header. **The header is the first record**: the table runs from the + /// block start to `data_offset`, so a block with `data_offset == 0x50` + /// holds five records, the first of which is the header itself and whose + /// pointer names where the payload begins. A header whose table runs past + /// the block yields nothing rather than a short slice. pub fn records<'b>(&self, block: &'b [u8]) -> impl Iterator + 'b { block - .get( - TIE_BLOCK_HEADER_SIZE - ..TIE_BLOCK_HEADER_SIZE + self.record_count() * TIE_RECORD_SIZE, - ) + .get(..self.record_count() * TIE_RECORD_SIZE) .unwrap_or(&[]) .chunks_exact(TIE_RECORD_SIZE) } @@ -411,11 +410,13 @@ mod tests { assert_eq!(h.lod_for_depth(50.0), None); } - /// A block whose data offset names a three-record table. + /// A block whose data offset names a four-record table, the first of + /// which is the header itself. fn block() -> Vec { let mut v = Vec::new(); - // data_offset 0x30, reserved 0, packed, then the remaining fields. - v.extend_from_slice(&0x30u16.to_le_bytes()); + // data_offset 0x40 -- four records -- reserved 0, packed, then the + // remaining fields. + v.extend_from_slice(&0x40u16.to_le_bytes()); v.extend_from_slice(&0u16.to_le_bytes()); v.extend_from_slice(&0x0501u16.to_le_bytes()); v.extend_from_slice(&0x1234u16.to_le_bytes()); @@ -440,12 +441,13 @@ mod tests { let b = block(); let h = TieBlockHeader::parse(&b).unwrap(); assert!(h.invariants_hold()); - assert_eq!(h.record_count(), 3); + assert_eq!(h.record_count(), 4); let records: Vec<_> = h.records(&b).collect(); - assert_eq!(records.len(), 3); + assert_eq!(records.len(), 4); + // The first record is the header itself, pointing at the payload. let first = TieRecord::parse(records[0]).unwrap(); assert_eq!(first.pointer, 0x40); - assert_eq!(first.fields, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + assert_eq!(first.pointer, 0x40); } #[test] diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index e78d68d..fdca4d7 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1229,6 +1229,34 @@ payload starts. A count of what follows a pointer is not the pointer. `TieBlockHeader::data_offset` and `TieBlockHeader::records` carry it, bound by the offset, and the disc test asserts the table is bounded on every block. +### It is now a command, and it verifies two levels + +The check above was a script; it is now `gc-trace verify-tie-records + `, which resolves the class, parses +the block, searches the RAM image for the loaded record table and reports what +it found. Exit status is 0 when every record relocated with verbatim fields. + +Run over every (class, lod) slot of two levels, from PCSX2 savestates: + +| level | slots with a block | verified | mismatched | absent | +|---|---|---|---|---| +| 0 | 87 | **87** | 0 | 61 | +| 1 | 104 | **104** | 0 | 78 | + +So the relocation rule holds on **every** LOD block of two levels that the game +had loaded. Two operational notes worth keeping: + +- **The live emulator is not a stable input.** Verifying level 1 over PINE + matched 104 of 104 while the user was on that planet; the same command + minutes later, after they had returned to level 0, found the tables in RAM + but no matching copy -- because the level was no longer loaded. A miss there + is not a counterexample, and the report says "no copy matched" rather than + "failed". +- **A class index past the table count is an error**, not a mismatch, so a + sweep should stop at the level's class count rather than at a round number. + +What the records' fields name and what the payload holds are still open. + ### What the payload is, and what retired the coordinate reading The payload is **not random**, and that took a control to establish. Reading each diff --git a/docs/plan/pcsx2-verification.md b/docs/plan/pcsx2-verification.md index 05fbea8..5aa6575 100644 --- a/docs/plan/pcsx2-verification.md +++ b/docs/plan/pcsx2-verification.md @@ -153,6 +153,11 @@ GC_DISC="$HOME/Desktop/pcsx2/games/Ratchet & Clank - Going Commando.iso" \ ```sh cargo run -q -p gc-trace -- pine-info cargo run -q -p gc-trace -- compare-tfrag-light ~/Documents/gc-extracted --level 0 --pine +cargo run -q -p gc-trace -- compare-tfrag-light ~/Documents/gc-extracted --level 1 \ + --state ~/.config/PCSX2/sstates/SCUS-97268\ \(38996035\).02.p2s +# the tie record relocation, one class and LOD at a time +cargo run -q -p gc-trace -- verify-tie-records ~/Documents/gc-extracted 0 0 1 \ + --state ~/.config/PCSX2/sstates/SCUS-97268\ \(38996035\).01.p2s cargo run -q -p gc-trace -- compare-tfrag-light ~/Documents/gc-extracted --level 0 \ --state ~/.config/PCSX2/sstates/.p2s ``` diff --git a/docs/roadmap.md b/docs/roadmap.md index 559ece4..15aa0d1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -280,7 +280,11 @@ Phase 1 hands over the extents; this phase works out what each one contains. the record table's length -- not a count, which is why the payload never divides by it. Each record is a leading `u32` pointer **relative to the table start** plus twelve verbatim field bytes, relocated at load to - `table + word`. Verified in PCSX2's RAM on 16 of 16 level-1 blocks. + `table + word`. +- [x] **The tie relocation verifies on two levels.** `gc-trace + verify-tie-records` checks it against PCSX2's RAM: **87 of 87** blocks on + level 0 and **104 of 104** on level 1, zero mismatches, which is what + makes the record layout a finding rather than a reading of one block. - [x] **The shrub mesh geometry is in the class blob.** A packet table at `+0x40` names `count` packets between the table and the palette, and they tile on **358 of 358 spans** (2,239 packets), which makes `count` diff --git a/tools/trace/src/lib.rs b/tools/trace/src/lib.rs index 93f1a0f..7fa89df 100644 --- a/tools/trace/src/lib.rs +++ b/tools/trace/src/lib.rs @@ -28,6 +28,7 @@ pub mod ee; pub mod p2s; pub mod pine; pub mod tfrag_cmp; +pub mod tie_verify; pub mod zip; pub use ee::EeRam; diff --git a/tools/trace/src/main.rs b/tools/trace/src/main.rs index 265e489..e9ecd5e 100644 --- a/tools/trace/src/main.rs +++ b/tools/trace/src/main.rs @@ -64,6 +64,34 @@ fn run(args: &[String]) -> Result { let len = require(&rest, "--len")?; read(&src, &addr, &len) } + "verify-tie-records" => { + // gc-trace verify-tie-records + let (src, rest) = Source::parse_with_rest(&args[1..])?; + let (dir, level, class, lod) = match rest.as_slice() { + [dir, level, class, lod, ..] => ( + PathBuf::from(dir), + level.parse::().context("level")?, + class.parse::().context("class")?, + lod.parse::().context("lod")?, + ), + _ => { + return Err(anyhow!( + "usage: gc-trace verify-tie-records " + )) + } + }; + let idx = std::fs::read(dir.join(format!("level{level}/core_index.bin"))) + .with_context(|| format!("level{level}/core_index.bin"))?; + let core = std::fs::read(dir.join(format!("level{level}/core_data.bin"))) + .with_context(|| format!("level{level}/core_data.bin"))?; + let bytes = src.load()?; + let ram = EeRam::new(&bytes)?; + let report = gc_trace::tie_verify::verify(&idx, &core, class, lod, &ram)?; + let mut text = String::new(); + gc_trace::tie_verify::print_report(&report, &mut text)?; + println!("{text}"); + Ok(report.exit_code()) + } "compare-tfrag-light" => { let (src, rest) = Source::parse_with_rest(&args[1..])?; let dir = rest @@ -95,6 +123,7 @@ fn run(args: &[String]) -> Result { const USAGE: &str = "\ usage: gc-trace ... compare-tfrag-light [--level N] + verify-tie-records info dump-ee [--out FILE] find --bytes HEX diff --git a/tools/trace/src/tie_verify.rs b/tools/trace/src/tie_verify.rs new file mode 100644 index 0000000..7a72ba6 --- /dev/null +++ b/tools/trace/src/tie_verify.rs @@ -0,0 +1,172 @@ +//! The tie record relocation, verified against the game's own RAM. +//! +//! A tie LOD block is a table of 16-byte records -- a leading `u32` pointer plus +//! twelve bytes of fields -- and the loader relocates every pointer to +//! `table_address + word`, leaving the twelve bytes verbatim. That is what +//! [`verify`] checks, and it is the only way to see it: nothing in the disc +//! bytes distinguishes a relocated pointer from any other small integer. +//! +//! The search is by content, because each block has **several copies in RAM** +//! and only one of them is the loaded record table. Checking the first hit is +//! how a first attempt at this reported 0 of 16 blocks: the copy it found had +//! *modified* field bytes, which is itself interesting and is why every +//! candidate is tried rather than the first. + +use anyhow::{anyhow, Result}; + +use gc_formats::level_core::LevelCoreHeader; +use gc_formats::tie::{ + parse_tie_block, resolve_tie_class, TieBlockHeader, TieRecord, TIE_BLOCK_HEADER_SIZE, +}; + +use crate::ee::EeRam; + +/// One verification, everything a report needs. +#[derive(Debug, Clone)] +pub struct TieReport { + pub class_index: usize, + pub lod: usize, + /// Block offset from the class header, and its length in bytes. + pub block_offset: usize, + pub block_len: usize, + /// Records in the block's table. + pub records: usize, + /// Copies of the record table found in RAM. + pub copies: usize, + /// RAM address of the loaded table, when one matched. + pub table_ram: Option, + /// Pointers relocated as `table + word`, and records whose twelve field + /// bytes are verbatim. + pub pointers_relocated: usize, + pub fields_verbatim: usize, +} + +impl TieReport { + /// 0 = every record relocated with verbatim fields, 2 = mismatches. + pub fn exit_code(&self) -> i32 { + if self.table_ram.is_some() + && self.pointers_relocated == self.records + && self.fields_verbatim == self.records + { + 0 + } else { + 2 + } + } +} + +/// Verify one class's LOD block against a RAM image. +/// +/// `lod` is 1 or 2 -- slot 0 is the constant [`gc_formats::tie::TIE_PACKETS0`] +/// and never a block. +pub fn verify( + core_index: &[u8], + core_data: &[u8], + class_index: usize, + lod: usize, + ram: &EeRam, +) -> Result { + let header = LevelCoreHeader::parse(core_index)?; + let class = resolve_tie_class(core_index, core_data, header.tie_classes, class_index)?; + let offset = class + .header + .packets + .get(lod) + .copied() + .ok_or_else(|| anyhow!("lod must be 1 or 2, got {lod}"))?; + if offset <= 0 { + anyhow::bail!("class {class_index} has no block at LOD {lod}: the offset is {offset}"); + } + let start = class.entry.offset_in_asset_wad as usize + offset as usize; + let parsed = parse_tie_block(core_data, start) + .ok_or_else(|| anyhow!("no tie block at {start:#x} in core_data"))?; + // Header, then everything up to and including the terminator and padding. + let block_len = TIE_BLOCK_HEADER_SIZE + parsed.payload.len() + parsed.padding.len(); + let slice = core_data + .get(start..start + block_len) + .ok_or_else(|| anyhow!("the tie block at {start:#x} runs past core_data"))?; + + let table = TieBlockHeader::parse(slice)?; + let count = table.record_count(); + if count == 0 { + anyhow::bail!("the block's table is empty"); + } + let records: Vec = table + .records(slice) + .map(|r| TieRecord::parse(r).map_err(Into::into)) + .collect::>()?; + + // The probe is the first record whose twelve field bytes are distinctive + // enough to find: a run of one repeated byte occurs all over RAM. + let probe = records + .iter() + .map(|r| r.fields) + .find(|f| f.iter().collect::>().len() >= 4) + .ok_or_else(|| anyhow!("no record has a distinctive field run"))?; + + // Every copy of the table in RAM. The one where every pointer has become + // table + word and every field run is verbatim is the loaded copy. + let mut copies = 0usize; + let mut table_ram = None; + let (mut relocated, mut verbatim) = (0usize, 0usize); + for hit in ram.find_all(&probe) { + let base = match hit.checked_sub(4) { + Some(b) => b, + None => continue, + }; + copies += 1; + let ok_ptr = records.iter().enumerate().all(|(k, r)| { + ram.slice(base + k * 16, 4) + .map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize) + == Some(r.pointer as usize + base) + }); + let ok_fields = records + .iter() + .enumerate() + .all(|(k, r)| ram.slice(base + k * 16 + 4, 12) == Some(&r.fields[..])); + if ok_ptr && ok_fields { + table_ram = Some(base); + relocated = records.len(); + verbatim = records.len(); + break; + } + } + + Ok(TieReport { + class_index, + lod, + block_offset: offset as usize, + block_len: slice.len(), + records: count, + copies, + table_ram, + pointers_relocated: relocated, + fields_verbatim: verbatim, + }) +} + +/// Render a report for a terminal. +pub fn print_report(r: &TieReport, out: &mut dyn std::fmt::Write) -> std::fmt::Result { + let _ = writeln!( + out, + "class {} LOD {}: block at +{:#x}, {} bytes, {} records of 16", + r.class_index, r.lod, r.block_offset, r.block_len, r.records + ); + let _ = writeln!(out, " copies of the record table in RAM: {}", r.copies); + match r.table_ram { + Some(at) => { + writeln!(out, " loaded table at {at:#x}")?; + writeln!( + out, + " pointers relocated to table + word: {}/{}", + r.pointers_relocated, r.records + )?; + writeln!( + out, + " field bytes verbatim: {}/{}", + r.fields_verbatim, r.records + ) + } + None => writeln!(out, " no copy of the table matched the relocation rule"), + } +}