From 1f903efe23c63db1dbcdc1ed8f0667c77f649b8d Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Tue, 13 Oct 2026 07:07:21 +0200 Subject: [PATCH] An external decomp turns two overlay constants from measured into named OpenRAC/rac2-gc-decomp is the same disc as ours, verified rather than assumed: ours 3,828,350,976 9db2e33e...b1ce5 theirs 3,828,350,976 9db2e33e...b1ce5 Vendored read-only under vendor/ (already gitignored) and added to PROVENANCE. It names two constants this parser had measured but never explained: * overlay.bin +0x00 = 1,735,808 = 0x1A7C80, the .lit load base. It is constant across all 27 levels because every level's literal section loads at the same address. So the first header word is the load base of the section the file IS, which also establishes that overlay.bin is that section -- it holds both code and the debug string pool. $gp = 0x1AEFF0 comes with it, and that closes the roadmap's open "$gp for this game" item. The program around it is seven sections -- .lit .bss .data vtbl camvtbl sndvtbl .text -- which is precisely the "seven-section parser and relocation" Phase 3 item for this format. Known target now rather than a guess. I then tested the obvious follow-up and it is wrong: if +0x00 is the .lit load base, +0x0c ought to be the .lit size. It is SMALLER than the file on 16 of 27 levels, by up to 8.9% (level11: file 3,151,128 vs 2,870,792). Excluded by sign, not by a near miss. Still unnamed. Collision: still blocked, but no longer blocked on tooling. The decomp names Camera_CollPrimTest(void *prim, void *grid) without decompiling it, so the cell layout is not handed over. What it does supply is $gp, and that is enough to start. "Camera_CollPrimTest WARNING! - grid out of bounds! (RAR)" is at overlay+0xd18, the only copy in the file, so: string overlay+0xd18 -> RAM 0x1a8998; delta to $gp -0x6658; $lo 0x99a8 MIPS puts the $lo half in bytes 1..3 of a 4-byte instruction. Searching the whole overlay instruction-aligned gives exactly ONE hit: 0x354cc8 (overlay+0x1ad048) One occurrence in 2.75 MB of code is unambiguous: that is the instruction materialising the assert string, and its enclosing function is Camera_CollPrimTest. That is the route in, and it needs no symbols. tools/overlay-anchors does this step, dependency-free. I started this with capstone for the disassembly proper and dropped it: capstone 0.14 does not re-export Mode/Endian from a public path, and an excluded-from-the-workspace native dependency is a poor trade for a step that does not need one yet. The anchor search is pure byte work, so it is pure Rust. No native dependency is added anywhere, and the tool is an ordinary workspace member. Surveyed the rest of that project against our open items: no volume structures, no tie vertex layouts, no SBlk/DAW fields, no wavelet tables. It is a guide to the code, not a replacement for reading it. --- Cargo.lock | 7 ++ Cargo.toml | 1 + PROVENANCE.md | 1 + docs/formats/core-data-regions.md | 94 +++++++++++++++++++++++++++ docs/roadmap.md | 7 +- tools/overlay-anchors/Cargo.toml | 10 +++ tools/overlay-anchors/src/main.rs | 104 ++++++++++++++++++++++++++++++ 7 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 tools/overlay-anchors/Cargo.toml create mode 100644 tools/overlay-anchors/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index a28948f..3c2d920 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2361,6 +2361,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "gc-overlay-anchors" +version = "0.1.0" +dependencies = [ + "anyhow", +] + [[package]] name = "gc-trace" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c488540..34fce20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/gc-game", "crates/gc-video", "tools/trace", + "tools/overlay-anchors", ] [workspace.package] diff --git a/PROVENANCE.md b/PROVENANCE.md index d41779d..6125b0c 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -7,6 +7,7 @@ code, and those notices travel with anything derived from it. | Project | Role | Licence | Code copied into this repo? | |---|---|---|---| +| [OpenRAC/rac2-gc-decomp](https://github.com/OpenRAC/rac2-gc-decomp) | Byte-exact reconstruction research for the same disc (SHA-256 verified identical) -- supplied `$gp`, the `.lit` load base and the seven-section program layout | GPL-3.0-or-later | Not yet; vendored read-only at `vendor/rac2-gc-decomp` | | [ReRAC](https://github.com/re-rac/rerac) | Native Rust/Bevy port of Ratchet & Clank (2002) — the architecture this follows | ISC | Not yet; derived from, attribute when used | | [OpenRAC / rac2-gc-decomp](https://github.com/OpenRAC/rac2-gc-decomp) | Matching decompilation research for this game | MIT | No — consulted for documentation | | [rac2-gc-decomp community reference](https://github.com/OpenRAC/rac2-gc-decomp/blob/RAC2/docs/COMMUNITY-ENGINE-REFERENCE.md) | Engine architecture notes, moby struct layout | MIT | Documentation only | diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 27083b9..994b3e3 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1835,3 +1835,97 @@ disc, and anything that needs a running PCSX2 is documented in the plan file wit its method. What is *not* acceptable is a disc-readable claim living only in prose, which is how the tie `i16[3]` finding spent two commits before it had a test. + +## An external decompilation project is the same disc, and names the overlay + +`OpenRAC/rac2-gc-decomp` ("Going Commando USA v1.01: byte-exact reconstruction +tools and native-port research. No game assets.") is byte-for-byte the disc this +parser was built against: + + ours 3,828,350,976 bytes + 9db2e33e276133cc283647fa3279b37911955e123d6199d10065547eaa9b1ce5 + theirs 3,828,350,976 bytes + 9db2e33e276133cc283647fa3279b37911955e123d6199d10065547eaa9b1ce5 + +So its findings are usable directly rather than as an analogy. Two of them name +things this file had measured but not explained: + +| what | value | source | status here | +| --- | --- | --- | --- | +| `.lit` load base | `0x1A7C80`, same on all 27 levels | `docs/LEVELS.md` | **newly named** | +| `$gp` | `0x1AEFF0`, same on all 27 levels | `docs/LEVELS.md` | newly available | + +The measured constant `overlay.bin +0x00 = 1,735,808` **is** `0x1A7C80`. The +reason it is identical across all 27 levels is now known rather than merely +observed: every level's literal section loads at the same address, because +loading a level copies its program over the boot's memory from there. So the +first header word is not a size and not a checksum — it is the **load base of +the section this file is**. + +That also settles what the file is. `overlay.bin` is the level's `.lit` section, +which is why it holds both code and the debug string pool. + +The program around it has seven sections, per `docs/LEVELS.md`: + + .lit .bss .data vtbl camvtbl sndvtbl .text + +which is exactly the "seven-section parser and relocation" already listed as a +Phase 3 item for this format. It is now a known target rather than a guess. + +### `+0x0c` is not the `.lit` size + +The obvious next guess -- now that `+0x00` is known to be the `.lit` load base, +`+0x0c` might be the `.lit` size -- is wrong, and measurably so: + +| level | file size | `+0x0c` | excess | +| --- | ---: | ---: | ---: | +| level0 | 2,751,268 | 2,767,752 | +16,484 | +| level11 | 3,151,128 | 2,870,792 | **-280,336** | +| level2 | 2,928,424 | 2,780,808 | **-147,616** | +| level24 | 2,615,776 | 2,763,912 | +148,136 | + +`+0x0c` is **smaller than the file** on 16 of 27 levels, by up to 8.9%. It +cannot be a size of the section it sits at the front of. It is still unnamed. The point here is only that the size hypothesis is excluded +by sign, not by a near miss. + +### Collision: the code that asserts is now locatable + +The collision blocker was "needs the game's own code". The decomp names the +functions (`Camera_CollPrimTest(void *prim, void *grid)`, `MB_CheckCollPill`) +but does **not** decompile them, so the cell layout is still not handed over. +What it does supply is `$gp`, and that is enough to start. + +`Camera_CollPrimTest WARNING! - grid out of bounds! (RAR)` sits at +`overlay.bin+0xd18`, the only copy in the file, so: + + string overlay+0xd18 -> RAM 0x1a8998 + $gp 0x1aeff0 + delta -0x6658 ($lo half 0x99a8) + +MIPS materialises a GP-relative address as a `%hi`/`%lo` pair, and the `$lo` half +is a 16-bit immediate sitting in bytes 1..3 of a 4-byte instruction. Searching +the whole overlay for that half, instruction-aligned, gives: + + 0x354cc8 (overlay+0x1ad048) -- exactly ONE occurrence + +One occurrence in 2.75 MB of code is an unambiguous anchor: `0x354cc8` is the +instruction that materialises the assert string, and the function containing it +is `Camera_CollPrimTest`. That is the route in, and it needs no symbols. + +`tools/overlay-anchors` does this step and is dependency-free; it prints strings, +GP-relative deltas and occurrence counts. Asserting the uniqueness: + + cargo run -q -p gc-overlay-anchors -- \ + /home/reaper/Documents/gc-extracted/level0/overlay.bin \ + "Camera_CollPrimTest WARNING!" + +Note the decomp's `docs/assert-message-names.tsv` holds **code** addresses, not +string addresses (level 0's `0x002db350` disassembles to instructions). They are +function-level anchors and are a second, independent way in. + +### What this project did *not* give up + +Surveyed against the open items: no volume structures, no `TieTex`/tie vertex +layouts, no `SBlk`/`DAW` record fields, no wavelet tables. Collision has the +API but not the data. It is a guide to the code, not a replacement for reading +it. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0357ba8..43557bc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -401,7 +401,9 @@ Phase 1 hands over the extents; this phase works out what each one contains. bounds! (RAR)`, independent support for reading the collision payload's two leading `u16`s as grid dimensions. **The header words are not named and the level-specific tail is not structured yet**; the Phase 3 - seven-section parser and relocation are untouched. + seven-section parser and relocation are untouched. The seven sections are + now named (`.lit .bss .data vtbl camvtbl sndvtbl .text`) and `overlay.bin` + is the `.lit` section, whose `+0x00` load base is `0x1A7C80`. - [x] **The post-shrub chunk container.** A 16-byte header, a 10-byte zero gap, then offsets tagged `0xF0000000 | low24` resolving to `chunk_start + low`; header byte 14 is the record count, equal to `offsets.len()` on 1,123 of @@ -462,7 +464,8 @@ reference function's copy on another overlay. - [ ] Port the masking and code-identity logic - [ ] `Relocation::func` and `Relocation::data` -- [ ] `$gp` for this game (`0x001AEFF0`, against R&C1's `0x166c00`) +- [x] `$gp` for this game (`0x001AEFF0`, against R&C1's `0x166c00`) -- confirmed + by `OpenRAC/rac2-gc-decomp` `docs/LEVELS.md` for all 27 levels - [ ] Extend the level table from 19 to 27 **Gate:** every level-01 function's relocated copy is found on all 27 overlays; diff --git a/tools/overlay-anchors/Cargo.toml b/tools/overlay-anchors/Cargo.toml new file mode 100644 index 0000000..f3c0b56 --- /dev/null +++ b/tools/overlay-anchors/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "gc-overlay-anchors" +description = "Locates MIPS code referencing a string in overlay.bin, via the GP-relative offset" +version = "0.1.0" +edition = "2021" +license = "GPL-3.0-or-later" +publish = false + +[dependencies] +anyhow = "1" diff --git a/tools/overlay-anchors/src/main.rs b/tools/overlay-anchors/src/main.rs new file mode 100644 index 0000000..29e12ab --- /dev/null +++ b/tools/overlay-anchors/src/main.rs @@ -0,0 +1,104 @@ +//! Find the MIPS code that references a string in `overlay.bin`. +//! +//! The overlay has no symbol table, but it has a fixed load base and a fixed +//! `$gp`, so every string has a GP-relative offset. MIPS materialises that as a +//! `%hi`/`%lo` pair, and the `%lo` half is a 16-bit immediate that occurs in the +//! instruction stream. Searching for it turns a string into a code address. +//! +//! This is the step that precedes disassembly, and it needs no disassembler. +use anyhow::{anyhow, Context, Result}; +use std::collections::BTreeSet; + +/// Where a level program is loaded, and its global pointer. Both are stated in +/// OpenRAC/rac2-gc-decomp docs/LEVELS.md and hold for all 27 levels. +const LIT_BASE: u64 = 0x1A7C80; +const GP: u64 = 0x1AEFF0; + +fn main() -> Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let path = args + .first() + .context("usage: gc-overlay-anchors [needle] [--level N]")?; + let overlay = std::fs::read(path).with_context(|| format!("read {path}"))?; + + match args.get(1) { + None => { + list_strings(&overlay); + Ok(()) + } + Some(needle) => { + let needle = needle.as_bytes(); + let hits = find_all(&overlay, needle); + if hits.is_empty() { + return Err(anyhow!( + "{:?} not found in {path}", + String::from_utf8_lossy(needle) + )); + } + for off in &hits { + report(&overlay, needle, *off); + } + Ok(()) + } + } +} + +fn report(overlay: &[u8], needle: &[u8], off: usize) { + let ram = LIT_BASE + off as u64; + let delta = ram as i64 - GP as i64; + let lo = (delta as u16).to_be_bytes(); + eprintln!("{:?}", String::from_utf8_lossy(needle)); + eprintln!(" overlay+{off:#x} -> RAM {ram:#x}"); + eprintln!( + " $gp {GP:#x} -> delta {delta:#x} ({delta}), $lo half {:#06x}", + u16::from_be_bytes(lo) + ); + + // Count instruction-aligned occurrences of the $lo immediate. MIPS puts it in + // bytes 1..3 of a 4-byte instruction, big-endian regardless of file order. + let mut hits = BTreeSet::new(); + for i in (0..overlay.len().saturating_sub(3)).step_by(4) { + if overlay[i + 1] == lo[0] && overlay[i + 2] == lo[1] { + hits.insert(i); + } + } + if hits.is_empty() { + eprintln!(" no instruction-aligned $lo occurrence -- not referenced through $gp\n"); + } else { + eprintln!(" {} instruction-aligned occurrence(s):", hits.len()); + for i in &hits { + eprintln!(" {:#x} (overlay+{:#x})", LIT_BASE + *i as u64, i); + } + eprintln!(); + } +} + +fn find_all(h: &[u8], needle: &[u8]) -> Vec { + if needle.is_empty() || needle.len() > h.len() { + return Vec::new(); + } + (0..=h.len() - needle.len()) + .filter(|&i| &h[i..i + needle.len()] == needle) + .collect() +} + +fn list_strings(d: &[u8]) { + let mut i = 0usize; + while i < d.len() { + if d[i].is_ascii_graphic() || d[i] == b' ' { + let s = i; + while i < d.len() && (d[i].is_ascii_graphic() || d[i] == b' ') { + i += 1 + } + if i - s >= 8 { + println!( + "{:#010x} {}", + LIT_BASE + s as u64, + String::from_utf8_lossy(&d[s..i]) + ); + } + } else { + i += 1; + } + } +}