diff --git a/Cargo.lock b/Cargo.lock index a28948f..3c2d920 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2361,6 +2361,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "gc-overlay-anchors" +version = "0.1.0" +dependencies = [ + "anyhow", +] + [[package]] name = "gc-trace" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c488540..34fce20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/gc-game", "crates/gc-video", "tools/trace", + "tools/overlay-anchors", ] [workspace.package] diff --git a/PROVENANCE.md b/PROVENANCE.md index d41779d..6125b0c 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -7,6 +7,7 @@ code, and those notices travel with anything derived from it. | Project | Role | Licence | Code copied into this repo? | |---|---|---|---| +| [OpenRAC/rac2-gc-decomp](https://github.com/OpenRAC/rac2-gc-decomp) | Byte-exact reconstruction research for the same disc (SHA-256 verified identical) -- supplied `$gp`, the `.lit` load base and the seven-section program layout | GPL-3.0-or-later | Not yet; vendored read-only at `vendor/rac2-gc-decomp` | | [ReRAC](https://github.com/re-rac/rerac) | Native Rust/Bevy port of Ratchet & Clank (2002) — the architecture this follows | ISC | Not yet; derived from, attribute when used | | [OpenRAC / rac2-gc-decomp](https://github.com/OpenRAC/rac2-gc-decomp) | Matching decompilation research for this game | MIT | No — consulted for documentation | | [rac2-gc-decomp community reference](https://github.com/OpenRAC/rac2-gc-decomp/blob/RAC2/docs/COMMUNITY-ENGINE-REFERENCE.md) | Engine architecture notes, moby struct layout | MIT | Documentation only | diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 27083b9..994b3e3 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1835,3 +1835,97 @@ disc, and anything that needs a running PCSX2 is documented in the plan file wit its method. What is *not* acceptable is a disc-readable claim living only in prose, which is how the tie `i16[3]` finding spent two commits before it had a test. + +## An external decompilation project is the same disc, and names the overlay + +`OpenRAC/rac2-gc-decomp` ("Going Commando USA v1.01: byte-exact reconstruction +tools and native-port research. No game assets.") is byte-for-byte the disc this +parser was built against: + + ours 3,828,350,976 bytes + 9db2e33e276133cc283647fa3279b37911955e123d6199d10065547eaa9b1ce5 + theirs 3,828,350,976 bytes + 9db2e33e276133cc283647fa3279b37911955e123d6199d10065547eaa9b1ce5 + +So its findings are usable directly rather than as an analogy. Two of them name +things this file had measured but not explained: + +| what | value | source | status here | +| --- | --- | --- | --- | +| `.lit` load base | `0x1A7C80`, same on all 27 levels | `docs/LEVELS.md` | **newly named** | +| `$gp` | `0x1AEFF0`, same on all 27 levels | `docs/LEVELS.md` | newly available | + +The measured constant `overlay.bin +0x00 = 1,735,808` **is** `0x1A7C80`. The +reason it is identical across all 27 levels is now known rather than merely +observed: every level's literal section loads at the same address, because +loading a level copies its program over the boot's memory from there. So the +first header word is not a size and not a checksum — it is the **load base of +the section this file is**. + +That also settles what the file is. `overlay.bin` is the level's `.lit` section, +which is why it holds both code and the debug string pool. + +The program around it has seven sections, per `docs/LEVELS.md`: + + .lit .bss .data vtbl camvtbl sndvtbl .text + +which is exactly the "seven-section parser and relocation" already listed as a +Phase 3 item for this format. It is now a known target rather than a guess. + +### `+0x0c` is not the `.lit` size + +The obvious next guess -- now that `+0x00` is known to be the `.lit` load base, +`+0x0c` might be the `.lit` size -- is wrong, and measurably so: + +| level | file size | `+0x0c` | excess | +| --- | ---: | ---: | ---: | +| level0 | 2,751,268 | 2,767,752 | +16,484 | +| level11 | 3,151,128 | 2,870,792 | **-280,336** | +| level2 | 2,928,424 | 2,780,808 | **-147,616** | +| level24 | 2,615,776 | 2,763,912 | +148,136 | + +`+0x0c` is **smaller than the file** on 16 of 27 levels, by up to 8.9%. It +cannot be a size of the section it sits at the front of. It is still unnamed. The point here is only that the size hypothesis is excluded +by sign, not by a near miss. + +### Collision: the code that asserts is now locatable + +The collision blocker was "needs the game's own code". The decomp names the +functions (`Camera_CollPrimTest(void *prim, void *grid)`, `MB_CheckCollPill`) +but does **not** decompile them, so the cell layout is still not handed over. +What it does supply is `$gp`, and that is enough to start. + +`Camera_CollPrimTest WARNING! - grid out of bounds! (RAR)` sits at +`overlay.bin+0xd18`, the only copy in the file, so: + + string overlay+0xd18 -> RAM 0x1a8998 + $gp 0x1aeff0 + delta -0x6658 ($lo half 0x99a8) + +MIPS materialises a GP-relative address as a `%hi`/`%lo` pair, and the `$lo` half +is a 16-bit immediate sitting in bytes 1..3 of a 4-byte instruction. Searching +the whole overlay for that half, instruction-aligned, gives: + + 0x354cc8 (overlay+0x1ad048) -- exactly ONE occurrence + +One occurrence in 2.75 MB of code is an unambiguous anchor: `0x354cc8` is the +instruction that materialises the assert string, and the function containing it +is `Camera_CollPrimTest`. That is the route in, and it needs no symbols. + +`tools/overlay-anchors` does this step and is dependency-free; it prints strings, +GP-relative deltas and occurrence counts. Asserting the uniqueness: + + cargo run -q -p gc-overlay-anchors -- \ + /home/reaper/Documents/gc-extracted/level0/overlay.bin \ + "Camera_CollPrimTest WARNING!" + +Note the decomp's `docs/assert-message-names.tsv` holds **code** addresses, not +string addresses (level 0's `0x002db350` disassembles to instructions). They are +function-level anchors and are a second, independent way in. + +### What this project did *not* give up + +Surveyed against the open items: no volume structures, no `TieTex`/tie vertex +layouts, no `SBlk`/`DAW` record fields, no wavelet tables. Collision has the +API but not the data. It is a guide to the code, not a replacement for reading +it. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0357ba8..43557bc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -401,7 +401,9 @@ Phase 1 hands over the extents; this phase works out what each one contains. bounds! (RAR)`, independent support for reading the collision payload's two leading `u16`s as grid dimensions. **The header words are not named and the level-specific tail is not structured yet**; the Phase 3 - seven-section parser and relocation are untouched. + seven-section parser and relocation are untouched. The seven sections are + now named (`.lit .bss .data vtbl camvtbl sndvtbl .text`) and `overlay.bin` + is the `.lit` section, whose `+0x00` load base is `0x1A7C80`. - [x] **The post-shrub chunk container.** A 16-byte header, a 10-byte zero gap, then offsets tagged `0xF0000000 | low24` resolving to `chunk_start + low`; header byte 14 is the record count, equal to `offsets.len()` on 1,123 of @@ -462,7 +464,8 @@ reference function's copy on another overlay. - [ ] Port the masking and code-identity logic - [ ] `Relocation::func` and `Relocation::data` -- [ ] `$gp` for this game (`0x001AEFF0`, against R&C1's `0x166c00`) +- [x] `$gp` for this game (`0x001AEFF0`, against R&C1's `0x166c00`) -- confirmed + by `OpenRAC/rac2-gc-decomp` `docs/LEVELS.md` for all 27 levels - [ ] Extend the level table from 19 to 27 **Gate:** every level-01 function's relocated copy is found on all 27 overlays; diff --git a/tools/overlay-anchors/Cargo.toml b/tools/overlay-anchors/Cargo.toml new file mode 100644 index 0000000..f3c0b56 --- /dev/null +++ b/tools/overlay-anchors/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "gc-overlay-anchors" +description = "Locates MIPS code referencing a string in overlay.bin, via the GP-relative offset" +version = "0.1.0" +edition = "2021" +license = "GPL-3.0-or-later" +publish = false + +[dependencies] +anyhow = "1" diff --git a/tools/overlay-anchors/src/main.rs b/tools/overlay-anchors/src/main.rs new file mode 100644 index 0000000..29e12ab --- /dev/null +++ b/tools/overlay-anchors/src/main.rs @@ -0,0 +1,104 @@ +//! Find the MIPS code that references a string in `overlay.bin`. +//! +//! The overlay has no symbol table, but it has a fixed load base and a fixed +//! `$gp`, so every string has a GP-relative offset. MIPS materialises that as a +//! `%hi`/`%lo` pair, and the `%lo` half is a 16-bit immediate that occurs in the +//! instruction stream. Searching for it turns a string into a code address. +//! +//! This is the step that precedes disassembly, and it needs no disassembler. +use anyhow::{anyhow, Context, Result}; +use std::collections::BTreeSet; + +/// Where a level program is loaded, and its global pointer. Both are stated in +/// OpenRAC/rac2-gc-decomp docs/LEVELS.md and hold for all 27 levels. +const LIT_BASE: u64 = 0x1A7C80; +const GP: u64 = 0x1AEFF0; + +fn main() -> Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let path = args + .first() + .context("usage: gc-overlay-anchors [needle] [--level N]")?; + let overlay = std::fs::read(path).with_context(|| format!("read {path}"))?; + + match args.get(1) { + None => { + list_strings(&overlay); + Ok(()) + } + Some(needle) => { + let needle = needle.as_bytes(); + let hits = find_all(&overlay, needle); + if hits.is_empty() { + return Err(anyhow!( + "{:?} not found in {path}", + String::from_utf8_lossy(needle) + )); + } + for off in &hits { + report(&overlay, needle, *off); + } + Ok(()) + } + } +} + +fn report(overlay: &[u8], needle: &[u8], off: usize) { + let ram = LIT_BASE + off as u64; + let delta = ram as i64 - GP as i64; + let lo = (delta as u16).to_be_bytes(); + eprintln!("{:?}", String::from_utf8_lossy(needle)); + eprintln!(" overlay+{off:#x} -> RAM {ram:#x}"); + eprintln!( + " $gp {GP:#x} -> delta {delta:#x} ({delta}), $lo half {:#06x}", + u16::from_be_bytes(lo) + ); + + // Count instruction-aligned occurrences of the $lo immediate. MIPS puts it in + // bytes 1..3 of a 4-byte instruction, big-endian regardless of file order. + let mut hits = BTreeSet::new(); + for i in (0..overlay.len().saturating_sub(3)).step_by(4) { + if overlay[i + 1] == lo[0] && overlay[i + 2] == lo[1] { + hits.insert(i); + } + } + if hits.is_empty() { + eprintln!(" no instruction-aligned $lo occurrence -- not referenced through $gp\n"); + } else { + eprintln!(" {} instruction-aligned occurrence(s):", hits.len()); + for i in &hits { + eprintln!(" {:#x} (overlay+{:#x})", LIT_BASE + *i as u64, i); + } + eprintln!(); + } +} + +fn find_all(h: &[u8], needle: &[u8]) -> Vec { + if needle.is_empty() || needle.len() > h.len() { + return Vec::new(); + } + (0..=h.len() - needle.len()) + .filter(|&i| &h[i..i + needle.len()] == needle) + .collect() +} + +fn list_strings(d: &[u8]) { + let mut i = 0usize; + while i < d.len() { + if d[i].is_ascii_graphic() || d[i] == b' ' { + let s = i; + while i < d.len() && (d[i].is_ascii_graphic() || d[i] == b' ') { + i += 1 + } + if i - s >= 8 { + println!( + "{:#010x} {}", + LIT_BASE + s as u64, + String::from_utf8_lossy(&d[s..i]) + ); + } + } else { + i += 1; + } + } +}