ReaperOfVeriod
e389d6d082
The anchor 20655c7 said had to be verified before following. It is.
docs/AUG6-RETAIL-ANCHORS.md recovers UpdateCamera_6 and InitCamera_24 as class/role
identifiers in the two dispatch tables, so role 6 of lvl.camvtbl is the camera
update function. The decomp's addresses are in the same space as ours -- its
func_L00_002A3B80 is this overlay's .text dest exactly -- so the claim is checkable
rather than taken on faith.
Three checks, because PROVENANCE.md records that the decomp's function addresses
have failed once here:
1. Role 6 of level 0's lvl.camvtbl names handler 0x004284a8.
2. overlay_bytes_at reads real code there: 0x3c014094, a lui.
3. tools/overlay-anchors decodes the first six words with zero failures:
lui $at, 0x4094
ori $at, $at, 0x7ae1 ; $at = 0x40947ae1
mtc1 $at, $f1 ; a float into an FP register
lw $v0, 0x2290($s4)
lui $at, 0x3f00
mtc1 $at, $f3 ; 0x3f000000 = 0.5f
Two mtc1 into FP registers and a struct field at +0x2290: a function doing
floating-point work on a camera struct. Three agreeing checks are a different kind
of evidence from any one of them, which is the lesson of the moby_anim chase.
Caveat recorded: level 0's lvl.camvtbl is 0xb4 bytes, 13 records and 13 distinct
ids, but several ids read as implausible large values (4,350,904 and similar). The
record layout is not uniformly id/handler, so role 6 being clean may be luck. Worth
remembering when the other roles are read.
Next, unblocked: read all of 0x4284a8 and find the store that writes 0x18aa70.
That confirms m4 by locating the write site rather than inferring it from input
response, and the frustum should be whatever the same function writes nearby.
295 passing, clippy clean.
8063 lines
312 KiB
Rust
8063 lines
312 KiB
Rust
//! Integration tests against a real disc image.
|
|
//!
|
|
//! These are skipped unless `GC_DISC` points at a legally obtained disc, since
|
|
//! the repository must not contain game data. When it is set they check the
|
|
//! identification and unpack results against the committed hash table.
|
|
//!
|
|
//! ```sh
|
|
//! GC_DISC="path/to/disc.iso" cargo test -p gc-extract --test disc
|
|
//! ```
|
|
|
|
use std::collections::BTreeSet;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// The disc under test, if the environment names one.
|
|
fn disc() -> Option<PathBuf> {
|
|
let p = std::env::var_os("GC_DISC")?;
|
|
let p = PathBuf::from(p);
|
|
p.is_file().then_some(p)
|
|
}
|
|
|
|
/// Skip a test with a clear reason rather than failing.
|
|
macro_rules! require_disc {
|
|
() => {
|
|
match disc() {
|
|
Some(p) => p,
|
|
None => {
|
|
eprintln!("skipped: set GC_DISC to a Going Commando disc image to run this");
|
|
return;
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
#[test]
|
|
fn identifies_the_supported_build() {
|
|
let path = require_disc!();
|
|
let info = gc_extract::identify::identify(&path).expect("disc should identify");
|
|
|
|
assert_eq!(info.serial, "SCUS_972.68");
|
|
assert_eq!(info.volume_id, "RATCHETANDCLANK2");
|
|
assert!(
|
|
info.is_target(),
|
|
"expected the supported build, got: {}",
|
|
info.verdict()
|
|
);
|
|
assert_eq!(
|
|
info.boot_elf_sha1,
|
|
gc_extract::identify::TARGET_BOOT_ELF_SHA1,
|
|
"boot ELF SHA-1 differs from the recorded retail value"
|
|
);
|
|
assert_eq!(info.boot_elf_size, 2_618_684);
|
|
// This disc is NTSC; the movement constants assume 60 Hz.
|
|
assert!(!info.is_pal(), "test disc should be NTSC");
|
|
}
|
|
|
|
#[test]
|
|
fn agrees_with_the_python_reference_identifier() {
|
|
let path = require_disc!();
|
|
let info = gc_extract::identify::identify(&path).expect("disc should identify");
|
|
|
|
// tools/identify-disc.py is the reference implementation and prints these
|
|
// two values. If they ever disagree, one of them has a bug.
|
|
// Run from the workspace root so the relative path to the tool resolves.
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent()
|
|
.and_then(|p| p.parent())
|
|
.expect("the crate should live two levels below the workspace root")
|
|
.to_path_buf();
|
|
let py = std::process::Command::new("python3")
|
|
.arg(root.join("tools/identify-disc.py"))
|
|
.arg(&path)
|
|
.current_dir(&root)
|
|
.output()
|
|
.expect("python3 should be available to run the reference identifier");
|
|
assert!(
|
|
py.status.success(),
|
|
"the reference identifier failed:\n{}",
|
|
String::from_utf8_lossy(&py.stderr)
|
|
);
|
|
let text = String::from_utf8_lossy(&py.stdout);
|
|
|
|
assert!(
|
|
text.contains(&format!("serial {}", info.serial)),
|
|
"serial disagrees with the reference identifier:\n{text}"
|
|
);
|
|
assert!(
|
|
text.contains(&format!("SHA-1 {}", info.boot_elf_sha1)),
|
|
"boot ELF SHA-1 disagrees with the reference identifier:\n{text}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn unpacks_and_matches_the_committed_table() {
|
|
let path = require_disc!();
|
|
|
|
let table_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/gc-files.tsv");
|
|
let table_text =
|
|
std::fs::read_to_string(&table_path).expect("the hash table should be committed");
|
|
let expected = gc_extract::verify::parse_table(&table_text);
|
|
assert!(
|
|
expected.len() > 5000,
|
|
"expected a full table, got {} rows",
|
|
expected.len()
|
|
);
|
|
|
|
// Unpack into a temporary directory so the test leaves nothing behind.
|
|
let tmp = std::env::temp_dir().join(format!("gc-unpack-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&tmp);
|
|
std::fs::create_dir_all(&tmp).expect("should create a temp directory");
|
|
|
|
let files = gc_extract::unpack::unpack(&path, &tmp).expect("unpack should succeed");
|
|
|
|
// Every unpacked file must match the recorded size and hash.
|
|
let report = gc_extract::verify::verify(&tmp, &expected);
|
|
let _ = std::fs::remove_dir_all(&tmp);
|
|
|
|
assert!(
|
|
report.ok(),
|
|
"u/// **THE NAME IS HISTORICAL AND SO IS WHAT IT MEASURES.**
|
|
///
|
|
/// This test asserts a 16-byte element with fixed-point texcoords at `i16[4]`
|
|
/// and `i16[5]`. `e5cd3df` retracted exactly that: the tie vertex stride is not
|
|
/// 16, roughly half of all records are a flat 16-byte array and the rest are
|
|
/// neither that nor a VIF packet stream, and the correct stride is not yet
|
|
/// known.
|
|
///
|
|
/// Kept and passing, for the same reason as its neighbour -- it is the record of
|
|
/// what the 16-byte reading produces, so the retraction is visibly about the
|
|
/// stride rather than the arithmetic. Renaming or deleting it would erase the
|
|
/// thing the retraction is a retraction *of*.
|
|
///
|
|
/// Do not quote the 82.5% figure as a property of tie vertices. It is a property
|
|
/// of one reading of the vertex region, on a population the retraction disowned.
|
|
npacked data does not match the committed table:\n{}",
|
|
report.summary()
|
|
);
|
|
assert_eq!(
|
|
files.len(),
|
|
expected.len(),
|
|
"unpack produced a different file count"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_container_parses() {
|
|
let path = require_disc!();
|
|
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let containers: Vec<_> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/") && e.name.ends_with(".WAD"))
|
|
.collect();
|
|
|
|
// 92 `G/*.WAD` containers: 27 LEVEL, 27 AUDIO, 27 SCENE, 11 globals.
|
|
assert_eq!(containers.len(), 92, "expected 92 asset containers");
|
|
|
|
// MPEG1 and MPEG2 are raw video streams with no table; the other 90 must
|
|
// all parse as indexed containers.
|
|
let raw_video: Vec<&str> = containers
|
|
.iter()
|
|
.map(|c| c.name.as_str())
|
|
.filter(|n| *n == "G/MPEG1.WAD" || *n == "G/MPEG2.WAD")
|
|
.collect();
|
|
assert_eq!(
|
|
raw_video.len(),
|
|
2,
|
|
"expected MPEG1 and MPEG2 to be the raw streams"
|
|
);
|
|
|
|
// The per-level families must all be present, at 27 each.
|
|
for family in ["LEVEL", "AUDIO", "SCENE"] {
|
|
let n = containers
|
|
.iter()
|
|
.filter(|c| {
|
|
let stem = c.name.trim_start_matches("G/").trim_end_matches(".WAD");
|
|
stem.strip_prefix(family)
|
|
.map(|rest| !rest.is_empty() && rest.chars().all(|ch| ch.is_ascii_digit()))
|
|
.unwrap_or(false)
|
|
})
|
|
.count();
|
|
assert_eq!(n, 27, "expected 27 {family}<n>.WAD containers, found {n}");
|
|
}
|
|
|
|
for entry in &containers {
|
|
if raw_video.contains(&entry.name.as_str()) {
|
|
continue;
|
|
}
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, entry.size)
|
|
.expect("read should succeed");
|
|
let kind = gc_formats::global_wad::RangeKind::for_container(&entry.name);
|
|
let container = gc_formats::global_wad::GlobalWad::parse_with(&data, kind)
|
|
.unwrap_or_else(|e| panic!("{} failed to parse: {e}", entry.name));
|
|
assert!(
|
|
!container.entries.is_empty(),
|
|
"{} parsed but had no entries",
|
|
entry.name
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The level container header is the first typed structure we parse from real
|
|
/// data, and the tiling invariant is the check that holds on every level rather
|
|
/// than one hand-picked sample.
|
|
#[test]
|
|
fn level_headers_tile_their_containers() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut checked = 0;
|
|
for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data)
|
|
.unwrap_or_else(|e| panic!("{}: {e}", entry.name));
|
|
|
|
assert_eq!(h.header_size, 0x60, "{}", entry.name);
|
|
assert_eq!(
|
|
h.runtime_sector, 0,
|
|
"{}: runtime sector field should be 0 on disc",
|
|
entry.name
|
|
);
|
|
|
|
// Every present extent must start where the previous one ended.
|
|
let extents = h.ordered_extents();
|
|
for w in extents.windows(2) {
|
|
let (pn, p) = w[0];
|
|
let (nn, n) = w[1];
|
|
assert_eq!(
|
|
p.offset + p.size,
|
|
n.offset,
|
|
"{}: gap or overlap between {} and {}",
|
|
entry.name,
|
|
pn.name(),
|
|
nn.name()
|
|
);
|
|
}
|
|
|
|
// Nothing may fall outside the container. The last extent may end one
|
|
// sector past the file end through sector rounding, hence `+ 1`.
|
|
let container_sectors = (entry.size / 2048) as i32;
|
|
for (name, r) in &extents {
|
|
assert!(
|
|
r.offset >= 0 && r.offset + r.size <= container_sectors + 1,
|
|
"{}: {} extent {:?} is outside the container ({container_sectors} sectors)",
|
|
entry.name,
|
|
name.name(),
|
|
r
|
|
);
|
|
}
|
|
checked += 1;
|
|
}
|
|
|
|
assert_eq!(checked, 27, "expected 27 level containers");
|
|
}
|
|
|
|
/// Level ids are not file order: `LEVEL21.WAD` reports 30. Anything that keys
|
|
/// off the container index would silently mislabel that planet.
|
|
#[test]
|
|
fn level_ids_are_not_container_indices() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
// The ISO filesystem lists names alphabetically, so LEVEL10 sorts before
|
|
// LEVEL2. Order by the numeric suffix, which is what the container names
|
|
// mean.
|
|
let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/LEVEL"))
|
|
.map(|e| {
|
|
let n: usize = e
|
|
.name
|
|
.trim_start_matches("G/LEVEL")
|
|
.trim_end_matches(".WAD")
|
|
.parse()
|
|
.expect("level container names end in a number");
|
|
(n, e)
|
|
})
|
|
.collect();
|
|
levels.sort_by_key(|(n, _)| *n);
|
|
|
|
let mut mismatches = Vec::new();
|
|
for (index, (_, entry)) in levels.iter().enumerate() {
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
|
|
if h.id as usize != index {
|
|
mismatches.push(format!(
|
|
"{}: id {} but is container {index}",
|
|
entry.name, h.id
|
|
));
|
|
}
|
|
}
|
|
|
|
// Exactly one level is out of step: the gap left by a removed level.
|
|
assert_eq!(
|
|
mismatches.len(),
|
|
1,
|
|
"expected exactly one level whose id differs from its container index, got {mismatches:?}"
|
|
);
|
|
assert!(
|
|
mismatches[0].contains("LEVEL21"),
|
|
"unexpected mismatch: {mismatches:?}"
|
|
);
|
|
}
|
|
|
|
/// Sizes in the level header are sector counts. If they were byte counts, the
|
|
/// extents could not tile a container this size, so this also guards against
|
|
/// "fixing" the parser to read them as bytes.
|
|
#[test]
|
|
fn level_extent_sizes_are_sector_counts() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let entry = iso.find("G/LEVEL0.WAD").expect("level 0");
|
|
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
|
|
|
|
// Sound bank occupies sectors 1..735 and data 735..7219.
|
|
assert_eq!(h.sound_bank.offset, 1);
|
|
assert_eq!(h.sound_bank.size, 734);
|
|
assert_eq!(h.data.offset, 735);
|
|
// Read as bytes, 734 would be under four sectors and the tiling would break.
|
|
assert_eq!(h.sound_bank.byte_len(), 734 * 2048);
|
|
}
|
|
|
|
/// The level data extent is a container of byte ranges. Its header locates the
|
|
/// geometry index and the compressed geometry, and every range must lie inside
|
|
/// the extent on all 27 levels.
|
|
#[test]
|
|
fn level_data_extents_are_in_range() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut checked = 0;
|
|
for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
|
|
let raw =
|
|
gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
|
|
let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
|
|
let data = lh.data;
|
|
let Some(offset) = data.byte_offset() else {
|
|
continue;
|
|
};
|
|
let len = data.byte_len() as usize;
|
|
let extent = &raw[offset as usize..offset as usize + len];
|
|
|
|
let dh = gc_formats::level_data::LevelDataHeader::parse(extent)
|
|
.unwrap_or_else(|e| panic!("{}: data extent header: {e}", entry.name));
|
|
|
|
for (part, r) in dh.present_parts() {
|
|
assert!(
|
|
r.end() <= extent.len() as u64,
|
|
"{}: {part:?} ends at {} but the extent is {} bytes",
|
|
entry.name,
|
|
r.end(),
|
|
extent.len()
|
|
);
|
|
}
|
|
|
|
// The geometry is the big one and must be substantial.
|
|
assert!(
|
|
dh.core_data.size > 1_000_000,
|
|
"{}: core_data is only {} bytes",
|
|
entry.name,
|
|
dh.core_data.size
|
|
);
|
|
checked += 1;
|
|
}
|
|
assert_eq!(checked, 27, "expected 27 level containers");
|
|
}
|
|
|
|
/// The geometry must decompress. This is the first test that touches real level
|
|
/// content rather than headers.
|
|
#[test]
|
|
fn level_geometry_decompresses() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let entry = iso.find("G/LEVEL0.WAD").expect("level 0");
|
|
|
|
let raw = gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
|
|
let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
|
|
let data = lh.data;
|
|
let extent = &raw[data.byte_offset().unwrap() as usize
|
|
..(data.byte_offset().unwrap() + data.byte_len()) as usize];
|
|
|
|
let dh = gc_formats::level_data::LevelDataHeader::parse(extent).expect("data header");
|
|
let core = dh
|
|
.core_data
|
|
.slice(extent, extent.len())
|
|
.expect("core_data slice");
|
|
|
|
assert!(
|
|
gc_formats::wad::is_wad(core),
|
|
"core_data should start with the WAD magic"
|
|
);
|
|
|
|
let decoded = gc_formats::wad::decompress(core).expect("core_data should decompress");
|
|
assert!(
|
|
decoded.len() > core.len(),
|
|
"decompressed geometry ({}) should exceed the compressed size ({})",
|
|
decoded.len(),
|
|
core.len()
|
|
);
|
|
}
|
|
|
|
/// The terrain itself. This is the first test to touch real level content: it
|
|
/// parses the tfrags directory and decodes vertex positions from every patch.
|
|
///
|
|
/// The gate is deliberately about *coverage*, not correctness of a sample: a
|
|
/// parser that works on one tfrag and silently returns `None` on the rest would
|
|
/// otherwise pass.
|
|
#[test]
|
|
fn tfrag_vertices_decode_for_every_level() {
|
|
require_disc!();
|
|
// Opening the image also confirms the disc is readable, not merely present.
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
// Where the unpacked data lives. GC_EXTRACTED overrides; otherwise the
|
|
// default location used during development.
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!(
|
|
"skipped: unpack the disc first, e.g.\
|
|
gc-extract unpack <disc.iso> ~/Documents/gc-extracted\
|
|
or point GC_EXTRACTED at an existing unpack"
|
|
);
|
|
return;
|
|
}
|
|
|
|
let mut levels = 0usize;
|
|
let mut total_verts = 0usize;
|
|
let mut total_tfrags = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core_path = dir.join("core_data.bin");
|
|
let index_path = dir.join("core_index.bin");
|
|
if !core_path.exists() || !index_path.exists() {
|
|
continue;
|
|
}
|
|
let core = std::fs::read(&core_path).expect("core_data");
|
|
let index = std::fs::read(&index_path).expect("core_index");
|
|
|
|
// The tfrags block starts at the core header's 0x08 field and runs to
|
|
// the next region that exists.
|
|
let region_start =
|
|
|off: usize| -> i32 { i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) };
|
|
let start = region_start(0x08) as usize;
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region_start(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len() as i32) as usize;
|
|
|
|
let block = &core[start..end];
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
assert_eq!(header.table_offset, 0x40, "level{n}: table offset");
|
|
|
|
let table_offset = header.table_offset as usize;
|
|
let mut decoded = 0usize;
|
|
let mut verts = 0usize;
|
|
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table_offset.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let slice = &block[data_base..];
|
|
if let Some(p) = gc_formats::tfrag::tfrag_positions(slice, &th) {
|
|
decoded += 1;
|
|
verts += p.len();
|
|
assert_eq!(p.len(), th.vert_count as usize);
|
|
}
|
|
}
|
|
|
|
assert!(header.tfrag_count > 0, "level{n}: no tfrags");
|
|
assert_eq!(
|
|
decoded, header.tfrag_count as usize,
|
|
"level{n}: only {decoded} of {} tfrags decoded",
|
|
header.tfrag_count
|
|
);
|
|
levels += 1;
|
|
total_tfrags += header.tfrag_count as usize;
|
|
total_verts += verts;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(
|
|
total_tfrags > 5000,
|
|
"expected thousands of tfrags, got {total_tfrags}"
|
|
);
|
|
assert!(
|
|
total_verts > 100_000,
|
|
"expected 100k+ vertices, got {total_verts}"
|
|
);
|
|
eprintln!("decoded {total_verts} vertices from {total_tfrags} tfrags across {levels} levels");
|
|
}
|
|
|
|
/// Terrain topology: strips and index arrays must expand to exactly the
|
|
/// triangle count each patch's header claims.
|
|
///
|
|
/// `tri_count` is stored in the patch header but is not derivable from the
|
|
/// lists directly, so reproducing it on all 18,102 patches is a strong check
|
|
/// that the strip walk, the index bank and the LOD assignment are all right.
|
|
/// Triangle indices must also stay inside the vertex-info range for the LOD.
|
|
#[test]
|
|
fn tfrag_topology_reproduces_the_header_triangle_count() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut triangles = 0usize;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
let table = header.table_offset as usize;
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
let tris = gc_formats::tfrag::tfrag_triangles(&g, 0)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
assert_eq!(
|
|
tris.len(),
|
|
th.tri_count as usize,
|
|
"level{n} patch {i}: {} triangles, header says {}",
|
|
tris.len(),
|
|
th.tri_count
|
|
);
|
|
let vinfo = g.lod_vinfo_count(0);
|
|
for t in &tris {
|
|
let idx = [t.a as usize, t.b as usize, t.c as usize];
|
|
assert!(
|
|
idx.iter().all(|&v| v < vinfo),
|
|
"level{n} patch {i}: triangle index outside {vinfo} vertex-info entries"
|
|
);
|
|
assert!((t.ad_gif as usize) < g.ad_gifs.len());
|
|
}
|
|
patches += 1;
|
|
triangles += tris.len();
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
assert!(
|
|
triangles > 700_000,
|
|
"expected 700k+ triangles, got {triangles}"
|
|
);
|
|
eprintln!("reproduced {triangles} triangles across {patches} patches in {levels} levels");
|
|
}
|
|
|
|
/// Texture bindings: every ad-gif a patch uses must index an entry that
|
|
/// actually exists in the level's texture table, and the light array must be
|
|
/// one record per vertex.
|
|
///
|
|
/// The texture table is the join between tfrags and the texel data, so an
|
|
/// out-of-range index here means the level's geometry and its textures have
|
|
/// been misread independently of each other.
|
|
#[test]
|
|
fn tfrag_ad_gifs_index_the_level_texture_table() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut ad_gifs = 0usize;
|
|
let mut textures_seen = 0usize;
|
|
let mut textures_total = 0usize;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let textures = gc_formats::level_core::texture_table(&index, ch.tfrag_textures)
|
|
.expect("texture table");
|
|
assert!(!textures.is_empty(), "level{n}: no tfrag textures");
|
|
// Indices are per-level, so the pair is what identifies a texture.
|
|
let mut level_seen = std::collections::BTreeSet::new();
|
|
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
let table = header.table_offset as usize;
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
|
|
assert_eq!(
|
|
g.lights.len(),
|
|
th.vert_count as usize,
|
|
"level{n} patch {i}: {} light records for {} vertices",
|
|
g.lights.len(),
|
|
th.vert_count
|
|
);
|
|
assert_eq!(
|
|
g.rgba.len(),
|
|
th.rgba_size as usize * 4,
|
|
"level{n} patch {i}: rgba"
|
|
);
|
|
|
|
for a in &g.ad_gifs {
|
|
let t = a.texture_index();
|
|
assert!(
|
|
t < textures.len(),
|
|
"level{n} patch {i}: texture {} outside the {} entry table",
|
|
t,
|
|
textures.len()
|
|
);
|
|
let tex = &textures[t];
|
|
assert!(
|
|
tex.width > 0 && tex.height > 0 && tex.width <= 256 && tex.height <= 256,
|
|
"level{n} patch {i}: texture {t} is {}x{}",
|
|
tex.width,
|
|
tex.height
|
|
);
|
|
level_seen.insert(t);
|
|
ad_gifs += 1;
|
|
}
|
|
patches += 1;
|
|
}
|
|
textures_seen += level_seen.len();
|
|
textures_total += textures.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
// Most of each level's texture table should be referenced by some tfrag. A
|
|
// handful genuinely go unused; if that share grows, the ad-gif walk has
|
|
// started missing primitives rather than the data having spare entries.
|
|
let covered = textures_seen as f64 / textures_total as f64;
|
|
assert!(
|
|
covered > 0.85,
|
|
"only {textures_seen} of {textures_total} level textures referenced ({:.1}%)",
|
|
covered * 100.0
|
|
);
|
|
eprintln!(
|
|
"{ad_gifs} ad-gifs over {patches} patches reference {textures_seen}/{textures_total} level textures"
|
|
);
|
|
}
|
|
|
|
/// Level textures decode: every tfrag texture's base level must resolve to a
|
|
/// plausible RGBA image.
|
|
///
|
|
/// This is the join between the texture table, the texel block and GS RAM, so
|
|
/// it is where a wrong CLUT remap, a wrong alpha scale or a wrong pixel base
|
|
/// would show up. The check is deliberately about *shape*, not specific
|
|
/// colours: a decoded image is plausible when it is fully opaque, is not
|
|
/// uniformly one colour, and uses more than a handful of distinct entries.
|
|
#[test]
|
|
fn tfrag_textures_decode_to_plausible_images() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut decoded = 0usize;
|
|
let mut alphas: std::collections::BTreeMap<u8, usize> = std::collections::BTreeMap::new();
|
|
let mut opaque = 0usize;
|
|
let mut multi_colour = 0usize;
|
|
let mut flat = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let gs = match std::fs::read(dir.join("gs_ram.bin")) {
|
|
Ok(g) => g,
|
|
Err(_) => {
|
|
eprintln!("level{n}: no gs_ram.bin, skipping");
|
|
continue;
|
|
}
|
|
};
|
|
let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let textures_at = ch.textures_base.max(0) as usize;
|
|
let texture_block = &core[textures_at..];
|
|
let table =
|
|
gc_formats::level_core::texture_table(&index, ch.tfrag_textures).expect("table");
|
|
|
|
for (i, e) in table.iter().enumerate() {
|
|
let img = gc_formats::texture::base_image(texture_block, &gs, e)
|
|
.unwrap_or_else(|err| panic!("level{n} texture {i}: {err}"));
|
|
let t = img.decode().unwrap();
|
|
let texels = (e.width as i32) * (e.height as i32);
|
|
assert_eq!(t.rgba.len(), (texels * 4) as usize);
|
|
assert_eq!(t.width, e.width as u32);
|
|
decoded += 1;
|
|
|
|
// Count distinct colours and how many texels are fully opaque.
|
|
let mut seen = std::collections::HashSet::new();
|
|
let mut op = 0usize;
|
|
for px in t.rgba.chunks_exact(4) {
|
|
seen.insert([px[0], px[1], px[2], px[3]]);
|
|
if px[3] == 0xff {
|
|
op += 1;
|
|
}
|
|
}
|
|
if seen.len() > 8 {
|
|
multi_colour += 1;
|
|
} else {
|
|
flat += 1;
|
|
}
|
|
if op == texels as usize {
|
|
opaque += 1;
|
|
}
|
|
for px in t.rgba.chunks_exact(4) {
|
|
*alphas.entry(px[3]).or_insert(0usize) += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(decoded > 1_500, "only {decoded} textures decoded");
|
|
// Level art is mostly opaque and has real detail. A flat, transparent or
|
|
// single-colour result almost always means a CLUT or offset mistake.
|
|
let opaque_pct = opaque as f64 / decoded as f64;
|
|
let detail_pct = multi_colour as f64 / decoded as f64;
|
|
assert!(
|
|
opaque_pct > 0.9,
|
|
"only {opaque:.1}% of textures are fully opaque"
|
|
);
|
|
assert!(
|
|
detail_pct > 0.9,
|
|
"only {detail_pct:.1} of textures have >8 colours"
|
|
);
|
|
eprintln!(
|
|
"{decoded} textures decoded: {:.1}% fully opaque, {:.1}% with >8 colours ({flat} flat)",
|
|
opaque_pct * 100.0,
|
|
detail_pct * 100.0
|
|
);
|
|
|
|
// The 0x80 convention, measured on the disc rather than asserted in prose.
|
|
// Every decoded alpha must be a value gs::alpha_to_8 can produce -- and 254
|
|
// cannot be produced by it, since (v*255+64)/128 never lands on 254 for an
|
|
// integer v. 254 is precisely what the old `a * 2` produced for a raw 0x7F,
|
|
// so this line fails if that bug is ever reintroduced or the two expressions
|
|
// of the rule drift apart again.
|
|
let legal: std::collections::HashSet<u8> =
|
|
(0..=255u8).map(gc_formats::gs::alpha_to_8).collect();
|
|
let illegal: Vec<(u8, usize)> = alphas
|
|
.iter()
|
|
.filter(|(a, _)| !legal.contains(a))
|
|
.map(|(a, n)| (*a, *n))
|
|
.collect();
|
|
eprintln!("decoded alpha histogram: {alphas:?}");
|
|
assert!(
|
|
illegal.is_empty(),
|
|
"{illegal:?} are not producible by gs::alpha_to_8 -- the CLUT alpha \
|
|
conversion and the GS rule have diverged again"
|
|
);
|
|
assert!(
|
|
alphas.contains_key(&0xff),
|
|
"no fully opaque texel decoded; the 0x80 -> opaque rule is not exercised"
|
|
);
|
|
}
|
|
|
|
/// LOD linkage: every LOD-01 and LOD-0 vertex-info entry must resolve to a
|
|
/// first parent that exists and to positions inside the patch.
|
|
///
|
|
/// The parent-index arrays are V4_8 unpacks separated from the strips and the
|
|
/// strip indices only by their VU address, so a misclassification here is
|
|
/// silent: the arrays still parse, they just hold the wrong numbers. Resolving
|
|
/// every entry is what catches it.
|
|
#[test]
|
|
fn tfrag_lod_parents_resolve_for_every_entry() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut entries = 0usize;
|
|
let mut lod01 = 0usize;
|
|
let mut lod0 = 0usize;
|
|
let mut resolved_common = 0usize;
|
|
let mut common_total = 0usize;
|
|
let mut primary_failed = 0usize;
|
|
let mut extras_unresolved = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block");
|
|
let table = header.table_offset as usize;
|
|
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
|
|
common_total += g.vinfo_tier[0];
|
|
if n == 0 && patches < 5 {
|
|
eprintln!(
|
|
"p{i}: tier={:?} n01={} n0={} p01={}b x01={}b p0={}b x0={}b",
|
|
g.vinfo_tier,
|
|
g.vu_positions_lod_01_count,
|
|
g.vu_positions_lod_0_count,
|
|
g.parent_indices_lod01.len(),
|
|
g.unk_indices_2_lod01.len(),
|
|
g.parent_indices_lod0.len(),
|
|
g.unk_indices_2_lod0.len()
|
|
);
|
|
}
|
|
for v in 0..g.vertex_info.len() {
|
|
entries += 1;
|
|
match gc_formats::tfrag::tfrag_lod_link(&g, v) {
|
|
Some(link) => {
|
|
if v < g.vinfo_tier[0] {
|
|
resolved_common += 1;
|
|
}
|
|
// Parents must index real vertex info and real positions.
|
|
assert!(
|
|
link.parent1_vinfo < g.vertex_info.len(),
|
|
"level{n} patch {i}: entry {v} parent {} outside {} entries",
|
|
link.parent1_vinfo,
|
|
g.vertex_info.len()
|
|
);
|
|
assert!(link.own_position < g.positions.len());
|
|
assert!(link.parent1_position < g.positions.len());
|
|
assert!(link.parent2_position < g.positions.len());
|
|
// The first parent of a LOD-01 entry is always a common one.
|
|
if link.tier == gc_formats::tfrag::TfragMorphTier::Lod01 {
|
|
assert!(
|
|
link.parent1_vinfo < g.vinfo_tier[0],
|
|
"level{n} patch {i}: LOD-01 entry {v} parent {} is not a common entry",
|
|
link.parent1_vinfo
|
|
);
|
|
lod01 += 1;
|
|
} else {
|
|
lod0 += 1;
|
|
}
|
|
}
|
|
None => {
|
|
// Distinguish the two kinds of LOD entry. A *primary*
|
|
// entry has a real position and a recorded parent and
|
|
// must always resolve; an *extra* entry is a
|
|
// non-position bookkeeping record whose array address
|
|
// can collide with the strip index array.
|
|
let c = g.vinfo_tier[0];
|
|
let kind = if v < c {
|
|
None
|
|
} else if v < c + g.vinfo_tier[1] {
|
|
Some((g.vu_positions_lod_01_count, v - c, true))
|
|
} else {
|
|
Some((g.vu_positions_lod_0_count, v - c - g.vinfo_tier[1], false))
|
|
};
|
|
match kind {
|
|
None => {}
|
|
Some((prim, k, _)) if k < prim => primary_failed += 1,
|
|
Some(_) => extras_unresolved += 1,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
patches += 1;
|
|
}
|
|
}
|
|
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
|
|
// A common entry never has a link: they are the full-detail positions and
|
|
// have no parent. Anything else means the tier boundaries are wrong.
|
|
assert!(
|
|
resolved_common == 0,
|
|
"{resolved_common} common entries claimed an LOD parent"
|
|
);
|
|
assert!(
|
|
lod01 > 0 && lod0 > 0,
|
|
"both LOD tiers should occur in the data"
|
|
);
|
|
assert_eq!(
|
|
primary_failed, 0,
|
|
"{primary_failed} primary LOD entries did not resolve"
|
|
);
|
|
assert_eq!(
|
|
extras_unresolved, 0,
|
|
"{extras_unresolved} extra LOD entries did not resolve"
|
|
);
|
|
// Common entries have no link; every other entry does.
|
|
assert_eq!(entries, common_total + lod01 + lod0);
|
|
|
|
eprintln!(
|
|
"{entries} vertex-info entries over {patches} patches: {common_total} common, \
|
|
{lod01} LOD-01 and {lod0} LOD-0, all resolved"
|
|
);
|
|
}
|
|
|
|
/// The level core index's class tables.
|
|
///
|
|
/// These are what put the environment on top of the terrain: tie instances,
|
|
/// moby actors and shrub foliage. They were ruled out earlier for want of an
|
|
/// entry size, which turned out to be knowable exactly.
|
|
///
|
|
/// The check is the chain: each table must end precisely where the next one
|
|
/// begins, moby (32 bytes/entry) into tie (32) into shrub (48) into the tfrag
|
|
/// texture table. That closes on all 27 levels and would not close if any
|
|
/// entry size were wrong, so it is the evidence rather than a coincidence.
|
|
#[test]
|
|
fn level_core_class_tables_tile_the_index() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
let mut levels = 0usize;
|
|
let (mut moby, mut tie, mut shrub, mut textures) = (0usize, 0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let Ok(index) = std::fs::read(dir.join("core_index.bin")) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
let end = |r: lc::ArrayRange, size: usize| r.offset as usize + r.count as usize * size;
|
|
assert_eq!(
|
|
end(h.moby_classes, lc::MOBY_CLASS_SIZE),
|
|
h.tie_classes.offset as usize,
|
|
"level{n}: moby -> tie"
|
|
);
|
|
assert_eq!(
|
|
end(h.tie_classes, lc::TIE_CLASS_SIZE),
|
|
h.shrub_classes.offset as usize,
|
|
"level{n}: tie -> shrub"
|
|
);
|
|
assert_eq!(
|
|
end(h.shrub_classes, lc::SHRUB_CLASS_SIZE),
|
|
h.tfrag_textures.offset as usize,
|
|
"level{n}: shrub -> textures"
|
|
);
|
|
|
|
// Every entry parses, and its asset offset lands inside the level.
|
|
if n == 0 {
|
|
eprintln!(
|
|
"hdr moby={:?} tie={:?} shrub={:?} tex={:?} base={}",
|
|
h.moby_classes, h.tie_classes, h.shrub_classes, h.tfrag_textures, h.textures_base
|
|
);
|
|
eprintln!("index[192..200] = {:?}", &index[192..200]);
|
|
}
|
|
let moby_v = lc::class_table(
|
|
&index,
|
|
h.moby_classes,
|
|
lc::MOBY_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("moby classes");
|
|
let tie_v = lc::class_table(
|
|
&index,
|
|
h.tie_classes,
|
|
lc::TIE_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("tie classes");
|
|
let shrub_v = lc::class_table(
|
|
&index,
|
|
h.shrub_classes,
|
|
lc::SHRUB_CLASS_SIZE,
|
|
lc::ShrubClassEntry::parse,
|
|
)
|
|
.expect("shrub classes");
|
|
let tex_v = lc::texture_table(&index, h.tfrag_textures).expect("tfrag textures");
|
|
|
|
assert_eq!(moby_v.len(), h.moby_classes.count as usize);
|
|
for (i, c) in moby_v.iter().chain(tie_v.iter()).enumerate() {
|
|
assert!(
|
|
c.offset_in_asset_wad >= 0,
|
|
"level{n} entry {i}: asset offset {}",
|
|
c.offset_in_asset_wad
|
|
);
|
|
}
|
|
// Billboard slots are optional: every shrub class on level 0 carries
|
|
// them all zero, so only their sign is meaningful here.
|
|
for s in &shrub_v {
|
|
assert!(s.base.offset_in_asset_wad >= 0);
|
|
assert!(
|
|
s.palette_offset >= -1 && s.texture_offset >= -1,
|
|
"level{n}: shrub billboard offsets below the -1 sentinel"
|
|
);
|
|
}
|
|
moby += moby_v.len();
|
|
tie += tie_v.len();
|
|
shrub += shrub_v.len();
|
|
textures += tex_v.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(
|
|
moby + tie + shrub > 1000,
|
|
"only {moby}+{tie}+{shrub} class entries"
|
|
);
|
|
eprintln!(
|
|
"level core index: {moby} moby, {tie} tie, {shrub} shrub classes, \
|
|
{textures} tfrag textures across {levels} levels"
|
|
);
|
|
}
|
|
|
|
/// The moby, tie and shrub class tables point into `core_data`, and they do so
|
|
/// without overlapping.
|
|
///
|
|
/// `assets_decompressed_size` is the size of `core_data` itself, so there is
|
|
/// no separate asset container: the class entries index the decompressed asset
|
|
/// lump directly. If that is right, every offset must land inside the lump, the
|
|
/// three tables must occupy disjoint spans, and each must be internally sorted
|
|
/// so the geometry is laid out in class order.
|
|
#[test]
|
|
fn class_geometry_offsets_tile_core_data_without_overlap() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(index), Ok(core)) = (
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
// The claim under test: the "asset lump" is core_data itself.
|
|
assert_eq!(
|
|
h.assets_decompressed_size as usize,
|
|
core.len(),
|
|
"level{n}: assets_decompressed_size is not core_data's length"
|
|
);
|
|
|
|
let span = |r: lc::ArrayRange, size: usize| -> Vec<usize> {
|
|
lc::class_table(&index, r, size, lc::ClassEntry::parse)
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.map(|c| c.offset_in_asset_wad as usize)
|
|
.collect()
|
|
};
|
|
// Shrub entries are read as ShrubClassEntry so the offsets line up.
|
|
let shrub: Vec<usize> = lc::class_table(
|
|
&index,
|
|
h.shrub_classes,
|
|
lc::SHRUB_CLASS_SIZE,
|
|
lc::ShrubClassEntry::parse,
|
|
)
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.map(|s| s.base.offset_in_asset_wad as usize)
|
|
.collect();
|
|
let moby = span(h.moby_classes, lc::MOBY_CLASS_SIZE);
|
|
let tie = span(h.tie_classes, lc::TIE_CLASS_SIZE);
|
|
|
|
// An offset of 0 marks a slot with no geometry rather than the start of
|
|
// the lump, so those are excluded from the span checks. Entries are not
|
|
// sorted by class number and the offsets are not monotonic, so nothing
|
|
// here assumes an order.
|
|
let moby_l: Vec<usize> = moby.iter().copied().filter(|&o| o != 0).collect();
|
|
let tie_l: Vec<usize> = tie.iter().copied().filter(|&o| o != 0).collect();
|
|
let shrub_l: Vec<usize> = shrub.iter().copied().filter(|&o| o != 0).collect();
|
|
|
|
for (name, v) in [("moby", &moby), ("tie", &tie), ("shrub", &shrub)] {
|
|
for &o in v {
|
|
assert!(
|
|
o < core.len(),
|
|
"level{n}: {name} offset {o} past core data ({})",
|
|
core.len()
|
|
);
|
|
}
|
|
}
|
|
// The three spans must not overlap each other.
|
|
for (an, a) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] {
|
|
for (bn, b) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] {
|
|
// A level may have no geometry of one kind at all.
|
|
if an >= bn || a.is_empty() || b.is_empty() {
|
|
continue;
|
|
}
|
|
let (lo, hi) = (a[0], *a.last().unwrap());
|
|
for &o in b {
|
|
assert!(
|
|
o >= hi || o < lo,
|
|
"level{n}: {an} [{lo},{hi}] overlaps {bn} at {o}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
eprintln!("class geometry offsets verified inside core_data on {levels} levels");
|
|
}
|
|
|
|
/// Tie class headers, as far as RAC1 and RAC2 agree.
|
|
///
|
|
/// The check that carries the weight is `o_class`: the class's own number at
|
|
/// header offset 0x44 equals the number the core index filed it under, on every
|
|
/// class of every level. That is what makes the header base trustworthy rather
|
|
/// than merely plausible.
|
|
#[test]
|
|
fn tie_class_headers_match_their_index_entry() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
use gc_formats::tie::{TieClassHeader, TIE_CLASS_HEADER_MATCHED};
|
|
let mut levels = 0usize;
|
|
let (mut classes, mut ordered, mut unordered) = (0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(index), Ok(core)) = (
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
let entries = lc::class_table(
|
|
&index,
|
|
h.tie_classes,
|
|
lc::TIE_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("tie classes");
|
|
|
|
for e in &entries {
|
|
let off = e.offset_in_asset_wad as usize;
|
|
if off == 0 {
|
|
continue; // slot with no geometry
|
|
}
|
|
let at = off + TIE_CLASS_HEADER_MATCHED;
|
|
assert!(at <= core.len(), "level{n}: tie class past core data");
|
|
let t = TieClassHeader::parse(&core[off..at])
|
|
.unwrap_or_else(|err| panic!("level{n}: tie class {} header: {err}", e.o_class));
|
|
assert_eq!(t.o_class, e.o_class, "level{n}: tie o_class mismatch");
|
|
assert!(
|
|
t.far_dist > 0.0,
|
|
"level{n}: tie far distance {}",
|
|
t.far_dist
|
|
);
|
|
if 0.0 < t.near_dist && t.near_dist < t.mid_dist && t.mid_dist < t.far_dist {
|
|
ordered += 1;
|
|
} else {
|
|
unordered += 1;
|
|
}
|
|
classes += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(classes > 1_500, "only {classes} tie classes");
|
|
// Ordered distances are the expected shape; the exceptions are real and
|
|
// unexplained, so they are reported rather than asserted away.
|
|
assert!(
|
|
ordered * 20 >= classes * 19,
|
|
"only {ordered} of {classes} tie classes have ordered LOD distances"
|
|
);
|
|
eprintln!(
|
|
"tie: {classes} class headers, o_class matches the index on all; \\
|
|
{ordered} have ordered LOD distances, {unordered} do not"
|
|
);
|
|
}
|
|
|
|
/// The occlusion PVS grid.
|
|
///
|
|
/// The check that makes this real is the mask array: it starts at
|
|
/// `masks_offset` and runs to `(highest_mask + 1) * 0x80`, and on every level
|
|
/// that ends within a few bytes of the block's own length. A wrong tree shape
|
|
/// or a wrong cell size would leave the highest mask index somewhere else
|
|
/// entirely and the array would not fit.
|
|
#[test]
|
|
fn occlusion_grids_parse_on_every_level() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::occlusion::parse_occlusion_block;
|
|
let mut levels = 0usize;
|
|
let mut absent = 0usize;
|
|
let mut widest_slack = 0usize;
|
|
let (mut cells, mut masks) = (0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
// An offset of 0 means the level has no occlusion data at all, which is
|
|
// not the same as "at the start of the file" -- that is where the
|
|
// tfrags block lives, and parsing it as a grid produces a mask array
|
|
// that fits anyway, which is how this nearly went wrong.
|
|
if h.occlusion <= 0 {
|
|
absent += 1;
|
|
continue;
|
|
}
|
|
let start = h.occlusion as usize;
|
|
// The block runs to the sky region, or to the end when there is none.
|
|
let end = if h.sky > 0 {
|
|
h.sky as usize
|
|
} else {
|
|
core.len()
|
|
};
|
|
assert!(
|
|
start < end && end <= core.len(),
|
|
"level{n}: bad occlusion range"
|
|
);
|
|
let block = &core[start..end];
|
|
|
|
let o = parse_occlusion_block(block)
|
|
.unwrap_or_else(|e| panic!("level{n}: occlusion block: {e}"));
|
|
// Every mask a cell names exists.
|
|
for c in &o.cells {
|
|
assert!(
|
|
(c.mask as usize) < o.masks.len(),
|
|
"level{n}: cell names a missing mask"
|
|
);
|
|
}
|
|
// The array ends at or just before the block's end. Retail levels sit
|
|
// within a few bytes; a gross error would leave a large gap.
|
|
let need = o.masks_offset as usize + o.masks.len() * gc_formats::occlusion::MASK_BYTES;
|
|
assert!(need <= block.len(), "level{n}: masks run past the block");
|
|
let slack = block.len() - need;
|
|
// Levels whose grid is empty are mostly header and padding, so this is
|
|
// proportional and only applies where there are cells at all.
|
|
if !o.cells.is_empty() {
|
|
assert!(
|
|
slack * 8 < block.len(),
|
|
"level{n}: {slack} of {} bytes unexplained after the mask array",
|
|
block.len()
|
|
);
|
|
}
|
|
widest_slack = widest_slack.max(slack);
|
|
|
|
cells += o.cells.len();
|
|
masks += o.masks.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(
|
|
levels + absent,
|
|
27,
|
|
"expected all 27 levels, got {}",
|
|
levels + absent
|
|
);
|
|
assert_eq!(
|
|
absent, 3,
|
|
"expected 3 levels without occlusion data, got {absent}"
|
|
);
|
|
assert!(cells > 400_000, "only {cells} occlusion cells");
|
|
eprintln!(
|
|
"occlusion: {cells} populated cells over {masks} masks across {levels} levels; \
|
|
widest unexplained tail {widest_slack} bytes"
|
|
);
|
|
}
|
|
|
|
/// The sound extent framing.
|
|
///
|
|
/// The invariant here is `sblk_offset + sblk_size == daw_offset`, which holds
|
|
/// on all 27 levels and pins the meaning of the four head fields. `DAW` never
|
|
/// reaches the end of the file; the tail is positive on every level, so the
|
|
/// size field is not the whole region and this test asserts the direction
|
|
/// rather than a size.
|
|
#[test]
|
|
fn sound_bank_framing_holds_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::sound_bank::parse_sound_bank;
|
|
let mut levels = 0usize;
|
|
let mut sblk_bytes = 0usize;
|
|
let mut daw_bytes = 0usize;
|
|
let (mut narrowest, mut widest) = (usize::MAX, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let raw = match std::fs::read(data.join(format!("level{n}")).join("sound_bank.bin")) {
|
|
Ok(v) => v,
|
|
Err(_) => continue,
|
|
};
|
|
let s = parse_sound_bank(&raw).unwrap_or_else(|e| panic!("level{n}: sound bank: {e}"));
|
|
|
|
assert_eq!(s.header.version, 3, "level{n}: sound bank version");
|
|
assert_eq!(s.header.unknown_4, 2, "level{n}: sound bank unknown_4");
|
|
assert_eq!(
|
|
s.header.sblk_offset, 24,
|
|
"level{n}: SBlk sits at 24 on every level"
|
|
);
|
|
assert!(
|
|
s.header.sblk_offset + s.header.sblk_size == s.header.daw_offset,
|
|
"level{n}: SBlk must run into DAW with no gap"
|
|
);
|
|
assert!(s.has_sblk_magic(), "level{n}: SBlk magic");
|
|
assert!(s.has_daw_tag(), "level{n}: DAW tag at SBlk+12");
|
|
assert_eq!(s.sblk_version(), Some(1), "level{n}: SBlk version");
|
|
// The tail is always in the same direction: the extent is longer than
|
|
// the DAW region it describes.
|
|
narrowest = narrowest.min(s.daw_tail);
|
|
widest = widest.max(s.daw_tail);
|
|
|
|
sblk_bytes += s.sblk.len();
|
|
daw_bytes += s.daw.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(sblk_bytes > 300_000, "only {sblk_bytes} SBlk bytes");
|
|
assert!(daw_bytes > 20_000_000, "only {daw_bytes} DAW bytes");
|
|
eprintln!(
|
|
"sound_bank: {sblk_bytes} SBlk + {daw_bytes} DAW bytes across {levels} levels; \
|
|
unexplained tail {narrowest}..{widest} bytes"
|
|
);
|
|
}
|
|
|
|
/// Collision container framing.
|
|
///
|
|
/// The invariant is the header's own self-description: `header_size` is 0x40
|
|
/// on every level, and `header_size + payload_size` stays inside `core_data`.
|
|
/// The payload is deliberately left undecoded -- see `collision.rs`.
|
|
#[test]
|
|
fn collision_framing_holds_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::collision::{CollisionHeader, COLLISION_HEADER_SIZE};
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
let mut levels = 0usize;
|
|
let mut empty = 0usize;
|
|
let mut payload_bytes = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let region = CollisionHeader::region(&core, h.collision)
|
|
.unwrap_or_else(|e| panic!("level{n}: collision: {e}"));
|
|
let Some(r) = region else {
|
|
empty += 1;
|
|
continue;
|
|
};
|
|
assert_eq!(r.header.header_size, COLLISION_HEADER_SIZE, "level{n}");
|
|
assert!(
|
|
r.header.payload_size as usize <= r.payload.len(),
|
|
"level{n}"
|
|
);
|
|
if r.payload.is_empty() {
|
|
// Present but empty. Counted once, not twice.
|
|
empty += 1;
|
|
continue;
|
|
}
|
|
assert_eq!(r.payload.len(), r.header.payload_size as usize, "level{n}");
|
|
payload_bytes += r.payload.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(
|
|
levels + empty,
|
|
27,
|
|
"expected all 27 levels, got {levels}+{empty}"
|
|
);
|
|
assert!(
|
|
payload_bytes > 10_000_000,
|
|
"only {payload_bytes} collision bytes"
|
|
);
|
|
eprintln!("collision: {payload_bytes} payload bytes across {levels} levels, {empty} empty");
|
|
}
|
|
|
|
/// Moby model geometry: the per-class part table.
|
|
///
|
|
/// The invariant is `vertex_offset + 16 * vertex_count == next_part`, which is
|
|
/// what makes the 16-byte record size and the field boundaries fall out of the
|
|
/// data instead of being assumed. The parser rejects any span that violates it,
|
|
/// so this test is really checking that no span on the disc does.
|
|
#[test]
|
|
fn moby_parts_tile_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::moby::parse_moby_geometry;
|
|
let mut levels = 0usize;
|
|
let mut classes = 0usize;
|
|
let mut parts = 0usize;
|
|
let mut verts = 0usize;
|
|
let mut empties = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let range = h.moby_classes;
|
|
assert!(
|
|
range.count > 0 && range.offset > 0,
|
|
"level{n}: no moby class table"
|
|
);
|
|
|
|
// Class spans tile between consecutive geometry offsets.
|
|
let stride = 0x20usize;
|
|
let mut offsets: Vec<usize> = (0..range.count as usize)
|
|
.map(|i| {
|
|
let at = range.offset as usize + i * stride;
|
|
let e = gc_formats::level_core::ClassEntry::parse(&index[at..at + stride])
|
|
.unwrap_or_else(|e| panic!("level{n}: class {i}: {e}"));
|
|
e.offset_in_asset_wad
|
|
})
|
|
.map(|o| o.max(0) as usize)
|
|
.filter(|&o| o > 0)
|
|
.collect();
|
|
offsets.sort_unstable();
|
|
assert!(!offsets.is_empty(), "level{n}: no moby geometry offsets");
|
|
|
|
for i in 0..offsets.len() {
|
|
let start = offsets[i];
|
|
let end = offsets.get(i + 1).copied().unwrap_or(core.len());
|
|
if end <= start {
|
|
continue;
|
|
}
|
|
let g = parse_moby_geometry(&core[start..end])
|
|
.unwrap_or_else(|e| panic!("level{n}: moby class at {start}: {e}"));
|
|
if g.is_empty() {
|
|
empties += 1;
|
|
}
|
|
// Every part's vertices are readable and inside the span.
|
|
for p in &g.parts {
|
|
assert!(
|
|
g.vertices(p).is_some(),
|
|
"level{n}: part vertices past the end of the class span"
|
|
);
|
|
}
|
|
parts += g.parts.len();
|
|
verts += g.vertex_count();
|
|
classes += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(classes > 4_000, "only {classes} moby class spans");
|
|
assert!(parts > 40_000, "only {parts} moby parts");
|
|
assert!(verts > 1_200_000, "only {verts} moby vertices");
|
|
eprintln!(
|
|
"moby: {parts} parts tiling exactly, {verts} vertices, {classes} class spans \
|
|
({empties} with no geometry) across {levels} levels"
|
|
);
|
|
}
|
|
|
|
/// Shrub class headers and blobs.
|
|
///
|
|
/// Two fields self-identify exactly on every span, and the parser checks both
|
|
/// rather than merely reporting them: `+0x24` must equal the owning class
|
|
/// entry's own `o_class`, and `+0x2c` must land inside the span.
|
|
///
|
|
/// The body is then checked against the R&C1 shape this format follows: a
|
|
/// palette of 24 unit-length `s16` normals with a zero pad word, then zero
|
|
/// padding records. And the billboard offset at `+0x1c` is checked against the
|
|
/// class entry's billboard texture fields, which is the one header field that
|
|
/// can be confirmed from a second file.
|
|
#[test]
|
|
fn shrub_headers_name_their_own_class_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::ShrubClassEntry;
|
|
use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES, SHRUB_NORMAL_COUNT};
|
|
let mut levels = 0usize;
|
|
let mut spans = 0usize;
|
|
let mut absent = 0usize;
|
|
let (mut min_size, mut max_size) = (f32::MAX, 0.0f32);
|
|
let mut scratch_nonzero = 0usize;
|
|
let mut mode_hist: std::collections::BTreeMap<u16, usize> = Default::default();
|
|
let mut scratch_hist: std::collections::BTreeMap<u16, usize> = Default::default();
|
|
let mut radius_range = (f32::MAX, 0f32);
|
|
let mut scale_range = (f32::MAX, 0f32);
|
|
let mut bounded_records = 0usize;
|
|
let mut bounded_bodies = 0usize;
|
|
let mut palette_records = 0usize;
|
|
let mut padding_records = 0usize;
|
|
let mut norm2_deviation = 0i64;
|
|
let mut billboards = 0usize;
|
|
let mut billboard_entries = 0usize;
|
|
let mut packets_total = 0usize;
|
|
let mut packet_headers_total = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let range = h.shrub_classes;
|
|
// Class spans tile between consecutive geometry offsets, with the last
|
|
// one bounded by the chunk container. It is found by signature, since
|
|
// the core index carries no successor offset for shrub; without it the
|
|
// last class absorbs the container and the body check measures its
|
|
// neighbours.
|
|
let mut offsets: Vec<usize> = Vec::new();
|
|
for i in 0..range.count.max(0) as usize {
|
|
let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
|
|
.unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
|
|
if e.base.offset_in_asset_wad > 0 {
|
|
offsets.push(e.base.offset_in_asset_wad as usize);
|
|
}
|
|
}
|
|
offsets.sort_unstable();
|
|
// Levels without shrub geometry have no span to bound. The container
|
|
// that follows tie on those levels is checked by
|
|
// `chunk_container_closes_the_last_class_region`.
|
|
let container = match offsets.last() {
|
|
Some(&last) if !offsets.is_empty() => {
|
|
gc_formats::chunks::chunk_region_start(&core, last).unwrap_or_else(|| {
|
|
panic!("level{n}: no chunk container after the shrub classes")
|
|
})
|
|
}
|
|
_ => core.len(),
|
|
};
|
|
let mut span_end_of: std::collections::HashMap<usize, usize> =
|
|
std::collections::HashMap::new();
|
|
for (i, &o) in offsets.iter().enumerate() {
|
|
let end = offsets.get(i + 1).copied().unwrap_or(container);
|
|
span_end_of.insert(o, end);
|
|
}
|
|
for i in 0..range.count.max(0) as usize {
|
|
let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
|
|
.unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
|
|
let (span_end, has_successor) =
|
|
match span_end_of.get(&(e.base.offset_in_asset_wad.max(0) as usize)) {
|
|
Some(&e2) => (e2, true),
|
|
None => (core.len(), false),
|
|
};
|
|
match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class, span_end) {
|
|
Ok(Some(mut s)) => {
|
|
s.span_end_is_a_successor = has_successor;
|
|
// Every class is bounded now, so every body must be whole
|
|
// 8-byte records. That was not assertable while the last
|
|
// class ran to the lump's end, and it is the check the
|
|
// boundary earns its keep for.
|
|
assert!(
|
|
s.body_is_well_formed(),
|
|
"level{n} class {i}: shrub body is not whole 8-byte records"
|
|
);
|
|
bounded_bodies += 1;
|
|
bounded_records += s.body_record_count();
|
|
assert_eq!(
|
|
s.header.o_class,
|
|
e.base.o_class.max(0) as u32,
|
|
"level{n} class {i}"
|
|
);
|
|
assert!(!s.data.is_empty(), "level{n} class {i}: empty shrub data");
|
|
assert!(
|
|
s.header.sphere.iter().all(|v| v.is_finite()),
|
|
"level{n} class {i}: non-finite shrub sphere"
|
|
);
|
|
assert!(
|
|
s.header.size.is_finite() && s.header.size > 0.0,
|
|
"level{n} class {i}: shrub size {}",
|
|
s.header.size
|
|
);
|
|
// The scratch word is zero on every span; the parser
|
|
// refuses anything else. `mode_bits` is 4 on 16 spans and
|
|
// is reported rather than asserted. On those 16 the word
|
|
// at +0x14 reads as an f32 denormal, which the Emotion
|
|
// Engine flushes to zero like everything else.
|
|
if s.header.scratch != 0 {
|
|
scratch_nonzero += 1;
|
|
}
|
|
*mode_hist.entry(s.header.mode_bits).or_default() += 1;
|
|
*scratch_hist.entry(s.header.scratch).or_default() += 1;
|
|
radius_range.0 = radius_range.0.min(s.header.sphere[3]);
|
|
radius_range.1 = radius_range.1.max(s.header.sphere[3]);
|
|
scale_range.0 = scale_range.0.min(s.header.scale);
|
|
scale_range.1 = scale_range.1.max(s.header.scale);
|
|
min_size = min_size.min(s.header.size);
|
|
max_size = max_size.max(s.header.size);
|
|
|
|
// The palette: 24 unit normals, then zero padding.
|
|
let normals: Vec<_> = s.normals().collect();
|
|
assert_eq!(
|
|
normals.len(),
|
|
SHRUB_NORMAL_COUNT,
|
|
"level{n} class {i}: palette is {} entries, not {SHRUB_NORMAL_COUNT}",
|
|
normals.len()
|
|
);
|
|
for (k, normal) in normals.iter().enumerate() {
|
|
assert!(
|
|
normal.is_unit(),
|
|
"level{n} class {i}: palette entry {k} {normal:?} is not a unit normal"
|
|
);
|
|
let n2 = (normal.x as i64).pow(2)
|
|
+ (normal.y as i64).pow(2)
|
|
+ (normal.z as i64).pow(2);
|
|
norm2_deviation = norm2_deviation.max((n2 - 32767i64.pow(2)).abs());
|
|
}
|
|
palette_records += normals.len();
|
|
for (k, record) in s.body_records().skip(SHRUB_NORMAL_COUNT).enumerate() {
|
|
assert!(
|
|
record.iter().all(|&b| b == 0),
|
|
"level{n} class {i}: padding record {k} is not all zero"
|
|
);
|
|
}
|
|
padding_records += s.padding_record_count();
|
|
|
|
// The packets: the table at +0x40 tiles the span from
|
|
// align16 of its own end to the palette, or to the
|
|
// billboard when the class has one. This is what puts the
|
|
// mesh geometry in the blob, and it is the identity that
|
|
// makes `count` the packet count rather than a guess.
|
|
let packets = s.packets().unwrap_or_else(|| {
|
|
panic!("level{n} class {i}: packet table does not tile")
|
|
});
|
|
assert_eq!(
|
|
packets.len(),
|
|
s.header.count as usize,
|
|
"level{n} class {i}: packet count disagrees with the header"
|
|
);
|
|
packets_total += packets.len();
|
|
|
|
// The packet headers themselves. The disc form is a fixed
|
|
// signature -- magic, four zero bytes, tag 0x05 -- and the
|
|
// loader rewrites bytes 8-15 on load, so the count and the
|
|
// pointer live in RAM, not here.
|
|
let hdrs: Vec<_> = s.packet_headers().collect();
|
|
assert_eq!(
|
|
hdrs.len(),
|
|
packets.len(),
|
|
"level{n} class {i}: header count disagrees with packet count"
|
|
);
|
|
for (pi, h) in hdrs.iter().enumerate() {
|
|
assert!(
|
|
h.is_some(),
|
|
"level{n} class {i}: packet {pi} header does not match the \
|
|
disc invariants (magic 04 04 00 01, bytes 4-8 zero, tag 0x05)"
|
|
);
|
|
assert_eq!(
|
|
packets[pi].size % 16,
|
|
0,
|
|
"level{n} class {i}: packet {pi} size is not a multiple of 16"
|
|
);
|
|
}
|
|
packet_headers_total += hdrs.len();
|
|
|
|
// The billboard: present exactly when the class entry
|
|
// carries billboard texture fields.
|
|
let entry_has_billboard = e.billboard_width != 0 || e.billboard_height != 0;
|
|
match s.billboard().expect("billboard parses") {
|
|
Some(b) => {
|
|
billboards += 1;
|
|
assert!(
|
|
entry_has_billboard,
|
|
"level{n} class {i}: billboard record without billboard texture fields"
|
|
);
|
|
assert_eq!(b.tex1_addr, 0x14, "level{n} class {i}: TEX1 address");
|
|
assert_eq!(b.tex0_addr, 0x06, "level{n} class {i}: TEX0 address");
|
|
assert_eq!(b.miptbp_addr, 0x34, "level{n} class {i}: MIPTBP address");
|
|
assert_eq!(b.mmin, 4, "level{n} class {i}: MMIN");
|
|
}
|
|
None => assert!(
|
|
!entry_has_billboard,
|
|
"level{n} class {i}: billboard texture fields with no billboard record"
|
|
),
|
|
}
|
|
if entry_has_billboard {
|
|
billboard_entries += 1;
|
|
}
|
|
spans += 1;
|
|
}
|
|
Ok(None) => absent += 1,
|
|
Err(err) => panic!("level{n}: shrub class {i}: {err}"),
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
spans + absent,
|
|
358,
|
|
"expected 358 shrub classes, got {}",
|
|
spans + absent
|
|
);
|
|
assert!(spans > 340, "only {spans} shrub spans with geometry");
|
|
// The container bounds every class now, so every body is checkable.
|
|
assert_eq!(bounded_bodies, spans, "every shrub class should be bounded");
|
|
assert_eq!(scratch_nonzero, 0, "the scratch word is zero on every span");
|
|
assert_eq!(
|
|
billboards, billboard_entries,
|
|
"billboard records and billboard class entries disagree: \
|
|
{billboards} records vs {billboard_entries} entries"
|
|
);
|
|
eprintln!(
|
|
"shrub: {spans} headers naming their own class, {absent} without geometry; \
|
|
all {bounded_bodies} bodies bounded, {bounded_records} records, \
|
|
{palette_records} of them palette normals and {padding_records} zero padding; \
|
|
{billboards} billboards, all with texture fields; \
|
|
{packets_total} packets across all blobs, {packet_headers_total} headers; \
|
|
mode_bits {mode_hist:?}, radius {radius_range:?}, scale {scale_range:?}, \
|
|
largest palette deviation from unit length {norm2_deviation} of a possible 65535"
|
|
);
|
|
}
|
|
|
|
/// Region bounds derived from successor boundaries, on every level.
|
|
///
|
|
/// Every region is closed by the next region's start, or -- for the last class
|
|
/// region -- by the located chunk container, never by `core_data.len()`, so no
|
|
/// span silently absorbs the tail of the lump the way the last shrub class was
|
|
/// doing.
|
|
#[test]
|
|
fn core_regions_chain_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES};
|
|
let mut levels = 0usize;
|
|
let mut unresolved_shrub = 0usize;
|
|
let mut unresolved_chunks = 0usize;
|
|
let mut bounded = 0usize;
|
|
let mut shrub_bytes = 0usize;
|
|
let mut chunk_bytes = 0usize;
|
|
let (mut first, mut last) = (usize::MAX, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let core_len = core.len();
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let regions = core_layout(&index, &core, &h);
|
|
|
|
// Regions appear in order, never overlap, and never run past the lump.
|
|
for pair in regions.windows(2) {
|
|
assert!(
|
|
pair[0].start <= pair[1].start,
|
|
"level{n}: {} vs {}",
|
|
pair[0].name,
|
|
pair[1].name
|
|
);
|
|
if let Some(end) = pair[0].end {
|
|
assert!(
|
|
end <= pair[1].start,
|
|
"level{n}: {} ends after {}",
|
|
pair[0].name,
|
|
pair[1].name
|
|
);
|
|
}
|
|
}
|
|
for r in ®ions {
|
|
assert!(
|
|
r.start < core_len,
|
|
"level{n}: {} starts past the lump",
|
|
r.name
|
|
);
|
|
if let Some(end) = r.end {
|
|
assert!(end <= core_len, "level{n}: {} ends past the lump", r.name);
|
|
}
|
|
}
|
|
|
|
// Regions appear in the canonical order, but not every level has every
|
|
// region -- level 10 has no occlusion and no shrub at all.
|
|
const ORDER: [&str; 8] = [
|
|
"tfrags",
|
|
"occlusion",
|
|
"sky",
|
|
"collision",
|
|
"moby",
|
|
"tie",
|
|
"shrub",
|
|
"chunks",
|
|
];
|
|
let names: Vec<_> = regions.iter().map(|r| r.name).collect();
|
|
let ranks: Vec<usize> = names
|
|
.iter()
|
|
.map(|name| {
|
|
ORDER
|
|
.iter()
|
|
.position(|o| o == name)
|
|
.unwrap_or_else(|| panic!("level{n}: unknown region {name}"))
|
|
})
|
|
.collect();
|
|
assert!(
|
|
ranks.windows(2).all(|w| w[0] < w[1]),
|
|
"level{n}: regions out of order: {names:?}"
|
|
);
|
|
if let Some(s) = regions.iter().find(|r| r.name == "shrub") {
|
|
match s.end {
|
|
Some(e) => {
|
|
// The container's start, not the trailer and not the lump's
|
|
// end: the trailer is the chunk region's business.
|
|
assert!(e < core_len - CORE_DATA_TRAILER_BYTES);
|
|
shrub_bytes += s.len().unwrap_or(0);
|
|
bounded += 1;
|
|
}
|
|
None => unresolved_shrub += 1,
|
|
}
|
|
first = first.min(s.start);
|
|
last = last.max(s.start);
|
|
}
|
|
if let Some(c) = regions.iter().find(|r| r.name == "chunks") {
|
|
match c.end {
|
|
Some(e) => {
|
|
// The trailer is the only thing past the chunk container.
|
|
assert_eq!(e, core_len - CORE_DATA_TRAILER_BYTES);
|
|
chunk_bytes += c.len().unwrap_or(0);
|
|
}
|
|
None => unresolved_chunks += 1,
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
unresolved_shrub, 0,
|
|
"every level with shrub should bound it at the chunk container"
|
|
);
|
|
assert_eq!(bounded, 24, "expected 24 levels with a shrub region");
|
|
assert_eq!(
|
|
unresolved_chunks, 0,
|
|
"every chunk region should end at the trailer"
|
|
);
|
|
eprintln!(
|
|
"regions: shrub starts {first}..{last}; {shrub_bytes} bytes of shrub and \
|
|
{chunk_bytes} bytes of chunk data, shrub closed by the located container on \
|
|
all {bounded} levels"
|
|
);
|
|
}
|
|
|
|
/// The post-shrub chunk container: chunks carry the `0xf0`-tagged offset
|
|
/// signature on every level, and their offsets resolve to records with a
|
|
/// shared field layout.
|
|
///
|
|
/// The region is bounded properly rather than by the fixed window this test
|
|
/// first used: it runs from the marker-paired chunk signature that closes the
|
|
/// last class region to the constant trailer. The window approach worked and
|
|
/// then over-worked -- a fixed 900,000 bytes from the end reaches past the
|
|
/// container on some levels and stops inside it on others, which is how 27
|
|
/// chunks came to be truncated at a "window edge" that is not a format
|
|
/// boundary. The locator is asserted by
|
|
/// `chunk_container_closes_the_last_class_region`.
|
|
///
|
|
/// The layout facts asserted here are measured, not assumed:
|
|
///
|
|
/// - every tagged offset has top byte `0xf0`, on all 27 levels;
|
|
/// - resolving records at `chunk_start + low` gives 17-21% byte positions
|
|
/// constant across that chunk's own records, which does not happen by
|
|
/// chance and is what distinguishes this from a coincidental byte pattern;
|
|
/// - stride is a per-chunk property, not a region-wide one.
|
|
#[test]
|
|
fn chunk_containers_resolve_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES};
|
|
|
|
let mut levels = 0usize;
|
|
let mut chunks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut strides = std::collections::BTreeSet::new();
|
|
let mut truncated = 0usize;
|
|
let mut constant_total = 0usize;
|
|
let mut stride_total = 0usize;
|
|
let mut frac: Vec<f64> = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
|
|
// The region runs from the chunk signature that closes the last class
|
|
// region to the constant trailer, both measured and identical in size
|
|
// on all 27 levels. core_layout locates the start through the same
|
|
// signature the chunk parser uses.
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let Some(region) = core_layout(&index, &core, &h)
|
|
.into_iter()
|
|
.find(|r| r.name == "chunks")
|
|
else {
|
|
continue;
|
|
};
|
|
let Some(end) = region.end else {
|
|
panic!("level{n}: chunk region is unbounded");
|
|
};
|
|
assert_eq!(end, core.len() - CORE_DATA_TRAILER_BYTES);
|
|
assert!(!region.is_empty(), "level{n}: empty chunk region");
|
|
let region = &core[region.start..end];
|
|
|
|
let found = gc_formats::chunks::parse_all_chunks(region).expect("chunks should parse");
|
|
if found.is_empty() {
|
|
continue;
|
|
}
|
|
levels += 1;
|
|
|
|
for chunk in &found {
|
|
assert!(
|
|
!chunk.chunk.offsets.is_empty(),
|
|
"level{n}: chunk at {} has no offsets",
|
|
chunk.chunk.offset
|
|
);
|
|
assert!(chunk.chunk.stride > 0, "level{n}: zero stride");
|
|
// The one decoded header field.
|
|
assert_eq!(
|
|
chunk.chunk.count as usize,
|
|
chunk.chunk.offsets.len(),
|
|
"level{n}: chunk at {} header count {} but {} offsets",
|
|
chunk.chunk.offset,
|
|
chunk.chunk.count,
|
|
chunk.chunk.offsets.len()
|
|
);
|
|
chunks += 1;
|
|
records += chunk.records.len();
|
|
strides.insert(chunk.chunk.stride);
|
|
|
|
// Records resolve unless the last one would run past the region
|
|
// end, which happens for the chunk sitting at the trailer end:
|
|
// the container's records stop there, so its final record or two
|
|
// fall outside. Truncation is counted rather than asserted away.
|
|
assert!(
|
|
chunk.records.len() <= chunk.chunk.offsets.len(),
|
|
"level{n}: chunk at {} resolved MORE records than offsets",
|
|
chunk.chunk.offset
|
|
);
|
|
if chunk.records.len() < chunk.chunk.offsets.len() {
|
|
truncated += 1;
|
|
}
|
|
|
|
// The invariant that makes this a container and not a coincidence.
|
|
// Accumulated rather than asserted per chunk: individual chunks
|
|
// range from ~7% to ~21% constant positions, so a per-chunk
|
|
// threshold would be asserting a property the format does not have.
|
|
let constant = constant_positions(&chunk.records);
|
|
constant_total += constant;
|
|
stride_total += chunk.chunk.stride;
|
|
frac.push(constant as f64 / chunk.chunk.stride as f64);
|
|
}
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(chunks > 500, "only {chunks} chunks resolved");
|
|
assert!(records > 5_000, "only {records} records resolved");
|
|
assert!(
|
|
strides.len() > 5,
|
|
"stride should vary per chunk, saw {}",
|
|
strides.len()
|
|
);
|
|
// Chance would give a constant position a probability of 1/256^n, so even
|
|
// 5% across tens of thousands of records is far above noise.
|
|
let mean_frac = constant_total as f64 / stride_total as f64;
|
|
assert!(
|
|
mean_frac > 0.05,
|
|
"constant positions {constant_total}/{stride_total} = {mean_frac:.3} is near chance"
|
|
);
|
|
eprintln!(
|
|
"chunks: {chunks} over {levels} levels, {records} records, {} distinct strides, \
|
|
constant {constant_total}/{stride_total} ({:.1}%), per-chunk min {:.1}% max {:.1}%, \
|
|
{truncated} chunks truncated at the trailer end",
|
|
strides.len(),
|
|
mean_frac * 100.0,
|
|
frac.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0,
|
|
frac.iter().cloned().fold(0.0, f64::max) * 100.0,
|
|
);
|
|
}
|
|
|
|
/// The chunk container closes the last class region, on every level.
|
|
///
|
|
/// The core index carries no successor offset for shrub -- unlike moby and
|
|
/// tie, whose boundaries fall out of the class tables -- so the boundary is
|
|
/// located in `core_data`: the constant marker pair sitting immediately in
|
|
/// front of a chunk signature. Both halves of that signature are weak alone,
|
|
/// which is why the test measures them separately:
|
|
///
|
|
/// - the marker pair `f6 7c` occurs 25 to 103 times in a 5 to 17 MB lump, so
|
|
/// on its own it means nothing;
|
|
/// - the chunk shape (16-byte header ending `0xff`, 10 zeros, tagged offsets)
|
|
/// matches random data often enough to need four tagged offsets before it
|
|
/// means anything.
|
|
///
|
|
/// Together they are exact: on all 27 levels the marker sits exactly two bytes
|
|
/// before the first chunk header. Three checks pin that down per level:
|
|
///
|
|
/// 1. the located boundary carries the marker;
|
|
/// 2. the last shrub body walks to 8-byte records that end exactly where the
|
|
/// signature begins, with no chunk signature inside the body itself;
|
|
/// 3. the container's records fill the region up to the constant trailer.
|
|
///
|
|
/// Levels without shrub are the control for the locator: there the container
|
|
/// closes tie instead, and the same marker precedes it.
|
|
#[test]
|
|
fn chunk_container_closes_the_last_class_region() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{self, CHUNK_REGION_MARKER};
|
|
use gc_formats::level_core::{LevelCoreHeader, ShrubClassEntry};
|
|
use gc_formats::regions::{core_layout, last_geometry, CORE_DATA_TRAILER_BYTES};
|
|
use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES};
|
|
|
|
let mut levels = 0usize;
|
|
let mut with_shrub = 0usize;
|
|
let mut marker_wide_max = 0usize;
|
|
let mut body_records = 0usize;
|
|
let mut container_bytes = 0usize;
|
|
let mut fill_gap_max = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
panic!("level{n}: missing core_data or core_index");
|
|
};
|
|
let trailer = core.len() - CORE_DATA_TRAILER_BYTES;
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
let last_shrub = last_geometry(&index, h.shrub_classes, SHRUB_CLASS_ENTRY_BYTES);
|
|
let last_tie = last_geometry(&index, h.tie_classes, 0x20);
|
|
// The container follows the last of the shrub and tie classes, so the
|
|
// signature search starts after the class bodies rather than inside
|
|
// them.
|
|
let scan_from = last_shrub
|
|
.or(last_tie)
|
|
.unwrap_or_else(|| panic!("level{n}: no class geometry to follow"));
|
|
let container = chunks::chunk_region_start(&core, scan_from)
|
|
.unwrap_or_else(|| panic!("level{n}: no chunk container after {scan_from}"));
|
|
|
|
// 1. The marker, and how common it is without the signature behind it.
|
|
assert_eq!(
|
|
&core[container - 2..container],
|
|
&CHUNK_REGION_MARKER,
|
|
"level{n}: container at {container} is not preceded by the marker"
|
|
);
|
|
marker_wide_max = marker_wide_max.max(
|
|
core.windows(2)
|
|
.filter(|w| *w == CHUNK_REGION_MARKER)
|
|
.count(),
|
|
);
|
|
|
|
// 2. The last shrub body ends exactly at the marker. This is the check
|
|
// that the boundary is the successor of shrub rather than a coincident
|
|
// pair inside it.
|
|
if let Some(last) = last_shrub {
|
|
let o_class = (0..h.shrub_classes.count.max(0) as usize)
|
|
.filter_map(|i| {
|
|
let at = h.shrub_classes.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(
|
|
index
|
|
.get(at..at + SHRUB_CLASS_ENTRY_BYTES)
|
|
.unwrap_or_default(),
|
|
)
|
|
.ok()?;
|
|
(e.base.offset_in_asset_wad == last as i32).then_some(e.base.o_class)
|
|
})
|
|
.next()
|
|
.unwrap_or_else(|| panic!("level{n}: no shrub class at {last}"));
|
|
let span = ShrubHeader::region(&core, last as i32, o_class, trailer)
|
|
.expect("last shrub span parses")
|
|
.expect("last shrub span present");
|
|
let data_start = last + span.header.data_offset as usize;
|
|
|
|
// Walk 8-byte records, bounded by the container so the walk can
|
|
// never continue into the chunk region itself.
|
|
let mut at = data_start;
|
|
while at + 8 <= container && core[at + 6] == 0 && core[at + 7] == 0 {
|
|
at += 8;
|
|
}
|
|
assert_eq!(
|
|
at + 2,
|
|
container,
|
|
"level{n}: shrub body at {data_start} does not end at the marker"
|
|
);
|
|
assert!(at > data_start, "level{n}: last shrub body is empty");
|
|
body_records += (at - data_start) / 8;
|
|
|
|
// No chunk signature inside the body: the search for the boundary
|
|
// must not match one it should have skipped.
|
|
let sig_inside = (last..container - 2).any(|off| chunks::is_chunk_start(&core, off, 4));
|
|
assert!(
|
|
!sig_inside,
|
|
"level{n}: chunk signature inside the last shrub body"
|
|
);
|
|
with_shrub += 1;
|
|
} else {
|
|
// Without shrub the container closes tie, and the marker precedes
|
|
// it the same way -- the locator is not shrub-specific.
|
|
let tie = core_layout(&index, &core, &h)
|
|
.into_iter()
|
|
.find(|r| r.name == "tie")
|
|
.unwrap_or_else(|| panic!("level{n}: no tie region to close"));
|
|
assert_eq!(
|
|
tie.end,
|
|
Some(container),
|
|
"level{n}: tie must end at the container"
|
|
);
|
|
}
|
|
|
|
// 3. The container fills its region up to the constant trailer: the
|
|
// furthest record should reach within a few KB of it, never inside it.
|
|
let region = &core[container..trailer];
|
|
let found = chunks::parse_all_chunks(region).expect("chunks should parse");
|
|
let region_ptr = region.as_ptr() as usize;
|
|
let furthest = found
|
|
.iter()
|
|
.flat_map(|c| {
|
|
c.records
|
|
.iter()
|
|
.map(move |r| r.as_ptr() as usize + r.len() - region_ptr)
|
|
})
|
|
.max()
|
|
.unwrap_or(0);
|
|
assert!(
|
|
region.len() - furthest < 4096,
|
|
"level{n}: chunk records stop {} bytes short of the trailer",
|
|
region.len() - furthest
|
|
);
|
|
container_bytes += region.len();
|
|
fill_gap_max = fill_gap_max.max(region.len() - furthest);
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
with_shrub, 24,
|
|
"expected 24 levels with shrub, got {with_shrub}"
|
|
);
|
|
eprintln!(
|
|
"container: {levels} levels ({with_shrub} closing shrub), \
|
|
{body_records} 8-byte records in last-shrub bodies, \
|
|
{container_bytes} bytes of chunk data, largest trailer gap {fill_gap_max}, \
|
|
marker pair occurs at most {marker_wide_max} times per lump"
|
|
);
|
|
}
|
|
|
|
/// The chunk record's leading word is fixed per type.
|
|
///
|
|
/// A record is `[leading u16][two index bytes][payload]`. Measured on the
|
|
/// disc: the word is the same at every record of a chunk on 1,439 of 1,509
|
|
/// chunks (`None` on the 70 that vary, which the parser reports rather than
|
|
/// averaging), and fixed within every chunk type, taking only the values
|
|
/// 1, 2, 4, 6, 8 and 10.
|
|
///
|
|
/// Not named. What the value counts is not established, and two readings were
|
|
/// measured and disproved: it is not the payload's entry count -- the payload
|
|
/// length divided by it leaves 57 distinct fractional "sizes" -- and it is not
|
|
/// the number of distinct leading bytes, which agrees on 0 of 1,506 chunks.
|
|
/// See `docs/formats/core-data-regions.md`.
|
|
#[test]
|
|
fn chunk_record_leading_word_is_fixed_per_type() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::parse_all_chunks;
|
|
use std::collections::BTreeMap;
|
|
|
|
// header -> the words its chunks carry, and the value histogram. Chunks
|
|
// whose own records disagree are counted apart rather than folded in.
|
|
let mut by_type: BTreeMap<[u8; 16], BTreeSet<u16>> = BTreeMap::new();
|
|
let mut histogram: BTreeMap<u16, usize> = BTreeMap::new();
|
|
let mut levels = 0usize;
|
|
let mut chunks = 0usize;
|
|
let mut varying = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let core = match std::fs::read(data.join(format!("level{n}/core_data.bin"))) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let found = parse_all_chunks(&core).expect("chunks should parse");
|
|
if found.is_empty() {
|
|
continue;
|
|
}
|
|
levels += 1;
|
|
for c in &found {
|
|
chunks += 1;
|
|
match c.chunk.prefix {
|
|
Some(p) => {
|
|
by_type.entry(c.chunk.header).or_default().insert(p);
|
|
*histogram.entry(p).or_default() += 1;
|
|
}
|
|
None => varying += 1,
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(levels >= 25, "only {levels} levels carry chunks");
|
|
assert!(chunks > 1400, "only {chunks} chunks parsed");
|
|
// The word is uniform within a chunk on all but a small minority.
|
|
let uniform = chunks - varying;
|
|
assert!(
|
|
uniform > 1400,
|
|
"only {uniform} of {chunks} chunks carry a uniform leading word"
|
|
);
|
|
// Fixed within every type: no type's chunks carry two different words.
|
|
let varying_types: Vec<_> = by_type.iter().filter(|(_, v)| v.len() > 1).collect();
|
|
assert!(
|
|
varying_types.is_empty(),
|
|
"{} types carry more than one leading word",
|
|
varying_types.len()
|
|
);
|
|
// The word is never zero and never large, and it is not the record count:
|
|
// 621 chunks carry 2, 520 carry 4, and the rest are spread thin.
|
|
assert!(
|
|
histogram.keys().all(|&p| p > 0 && p <= 16),
|
|
"prefix values {histogram:?}"
|
|
);
|
|
let two = histogram.get(&2).copied().unwrap_or(0);
|
|
let four = histogram.get(&4).copied().unwrap_or(0);
|
|
assert!(two > 500, "only {two} chunks carry the word 2");
|
|
assert!(four > 400, "only {four} chunks carry the word 4");
|
|
eprintln!(
|
|
"chunk leading word: {uniform} of {chunks} chunks uniform ({varying} vary), \
|
|
{} types, values {histogram:?}",
|
|
by_type.len()
|
|
);
|
|
}
|
|
|
|
/// The tie record is R&C1's `TiePacketHeader`, and its rule holds.
|
|
///
|
|
/// A tie LOD block is a table of 16-byte packet headers -- a data pointer,
|
|
/// then R&C1's byte fields at +4..+0x10. What identifies them rather than a
|
|
/// resemblance is R&C1's own documented rule: **`bfc_distance == 5 *
|
|
/// shader_count` on every record**, here 15,219 of 15,219 over 2,177 blocks on
|
|
/// all 27 levels.
|
|
///
|
|
/// Measured over the same set, and reported rather than asserted because they
|
|
/// are not universal:
|
|
///
|
|
/// * `color_ofs == vert_size` on 12,640 of 15,219 -- and R&C1's rule for the
|
|
/// field (`== vert_ofs + vert_size`) holds on **0**, so the two games use it
|
|
/// differently and this one is named for what it measures;
|
|
/// * `color_count`, `strip_count`, `strip_vertex_count` zero on ~15,077 of
|
|
/// 15,219;
|
|
/// * the orderings `vert_ofs <= vert_size`, `scissor_size <= scissor_ofs` and
|
|
/// `control_size <= control_count` on all of them.
|
|
///
|
|
/// One claim is retracted here rather than softened: a packet-length formula,
|
|
/// `(vert_ofs + vert_size + scissor_ofs + scissor_size) * 16`, was measured at
|
|
/// 61 of 61 pairs and is not reproduced on the correct blocks (0 of 13,042). It
|
|
/// was measured over blocks found by reading `packets[2]` where `packets[1]`
|
|
/// was meant, so the arithmetic agreed perfectly with a population that was not
|
|
/// the tie blocks. The block set was the error; see `AGENTS.md` rule 3.
|
|
#[test]
|
|
fn tie_records_are_packet_headers() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::tie::{parse_tie_block, resolve_tie_class, TieRecord};
|
|
|
|
let mut levels = 0usize;
|
|
let mut blocks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut bfc_ok = 0usize;
|
|
let mut invariants_ok = 0usize;
|
|
let (mut color_eq_size, mut color_zero, mut strips_zero, mut strip_verts_zero) =
|
|
(0usize, 0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(data.join(format!("level{n}/core_data.bin"))),
|
|
std::fs::read(data.join(format!("level{n}/core_index.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
for c in 0..h.tie_classes.count.max(0) as usize {
|
|
let Ok(class) = resolve_tie_class(&index, &core, h.tie_classes, c) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = class.header.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let start = class.entry.offset_in_asset_wad as usize + off as usize;
|
|
let Some(block) = parse_tie_block(&core, start) else {
|
|
continue;
|
|
};
|
|
let len = 0x10 + block.payload.len() + block.padding.len();
|
|
let Some(slice) = core.get(start..start + len) else {
|
|
continue;
|
|
};
|
|
// The table must fit inside the block the terminator found, or
|
|
// what follows is being read as a table.
|
|
if block.header.data_offset as usize > 0x10 + block.payload.len() {
|
|
continue;
|
|
}
|
|
blocks += 1;
|
|
for r in block
|
|
.header
|
|
.records(slice)
|
|
.map(|r| TieRecord::parse(r).expect("record parses"))
|
|
{
|
|
records += 1;
|
|
bfc_ok += usize::from(r.bfc_distance == 5 * r.shader_count);
|
|
invariants_ok += usize::from(r.invariants_hold());
|
|
color_eq_size += usize::from(r.color_ofs == r.vert_size);
|
|
color_zero += usize::from(r.color_count == 0);
|
|
strips_zero += usize::from(r.strip_count == 0);
|
|
strip_verts_zero += usize::from(r.strip_vertex_count == 0);
|
|
}
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels >= 25, "only {levels} levels were read");
|
|
assert_eq!(blocks, 2_177, "expected 2,177 tie blocks, got {blocks}");
|
|
assert!(records > 14_000, "only {records} packet headers found");
|
|
// The identification: R&C1's rule, on every record.
|
|
assert_eq!(
|
|
bfc_ok,
|
|
records,
|
|
"bfc_distance != 5 * shader_count on {} of {records}",
|
|
records - bfc_ok
|
|
);
|
|
assert_eq!(
|
|
invariants_ok,
|
|
records,
|
|
"record invariants failed on {} of {records}",
|
|
records - invariants_ok
|
|
);
|
|
eprintln!(
|
|
"tie packet headers: {records} over {levels} levels, {blocks} blocks; \
|
|
bfc_distance == 5*shader_count on all; orderings on all; \
|
|
color_ofs == vert_size on {color_eq_size}/{records}; \
|
|
color_count/strip_count/strip_vertex_count zero on \
|
|
{color_zero}/{strips_zero}/{strip_verts_zero}"
|
|
);
|
|
}
|
|
|
|
/// The tie class header's `+0x0c` word is the three LOD record counts.
|
|
///
|
|
/// The word was read as R&C1's 64-light-slot *normals offset* and skipped for
|
|
/// two commits, which was the right thing to do with the wrong information: the
|
|
/// byte at `+0x0c` is not an offset at all. It is the **record count of the LOD
|
|
/// 0 block**, `+0x0d` of LOD 1 and `+0x0e` of LOD 2 -- verified on **1,864 of
|
|
/// 1,864** classes across all 27 levels. R&C1 carries the same quadruple at
|
|
/// `+0x20`; R&C2 has it here.
|
|
///
|
|
/// The check is not circular: the counts are read from the header, and
|
|
/// compared against the record tables the blocks themselves carry, which are
|
|
/// reached through `packets[]` and delimited by the terminator. Two independent
|
|
/// paths to the same number.
|
|
///
|
|
/// The fourth byte, `+0x0f`, is left unnamed. It is *not* the shader (ad-gif)
|
|
/// total, which disagrees on 1,827 of the classes measured here.
|
|
#[test]
|
|
fn tie_class_header_counts_records_per_lod() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
let mut levels = 0usize;
|
|
let mut classes = 0usize;
|
|
let mut byte_matches = [0usize; 3];
|
|
let mut per_class_all_three = 0usize;
|
|
let mut fourth_bad = 0usize;
|
|
let mut fourth_range = (u8::MAX, 0u8);
|
|
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
levels += 1;
|
|
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x30 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
// The record tables, from the blocks themselves.
|
|
let mut counts = [0usize; 3];
|
|
let mut shader_total = 0usize;
|
|
for (lod, count) in counts.iter_mut().enumerate() {
|
|
let off = hdr.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let s = base + off as usize;
|
|
if let Some(blk) = parse_tie_block(&core, s) {
|
|
let recs: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
*count = recs.len();
|
|
for rec in recs {
|
|
if let Ok(r) = TieRecord::parse(rec) {
|
|
shader_total += r.shader_count as usize;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
classes += 1;
|
|
for (lod, slot) in byte_matches.iter_mut().enumerate() {
|
|
*slot += usize::from(hdr.packet_counts[lod] as usize == counts[lod]);
|
|
}
|
|
per_class_all_three +=
|
|
usize::from((0..3).all(|l| hdr.packet_counts[l] as usize == counts[l]));
|
|
if hdr.unknown_0f as usize != shader_total {
|
|
fourth_bad += 1;
|
|
}
|
|
fourth_range.0 = fourth_range.0.min(hdr.unknown_0f);
|
|
fourth_range.1 = fourth_range.1.max(hdr.unknown_0f);
|
|
}
|
|
}
|
|
|
|
assert!(levels >= 25, "only {levels} levels were read");
|
|
assert_eq!(classes, 1_864, "expected 1,864 tie classes, got {classes}");
|
|
for (lod, &matched) in byte_matches.iter().enumerate() {
|
|
assert_eq!(
|
|
matched,
|
|
classes,
|
|
"LOD{lod} record count byte disagrees on {} of {classes}",
|
|
classes - matched
|
|
);
|
|
}
|
|
assert_eq!(
|
|
per_class_all_three, classes,
|
|
"all three bytes match on {} of {classes}",
|
|
per_class_all_three
|
|
);
|
|
eprintln!(
|
|
"tie class header +0x0c: LOD0/LOD1/LOD2 record counts on {classes} classes; \
|
|
+0x0f range {}..{}, not the shader total on {}",
|
|
fourth_range.0, fourth_range.1, fourth_bad
|
|
);
|
|
}
|
|
|
|
/// The overlay's load base, from its own profiler table.
|
|
///
|
|
/// The overlay carries the game's profiler menu definition: a run of
|
|
/// 8-byte-padded names followed by absolute RAM addresses that point **back at
|
|
/// those names**. On level 0 the four addresses after `normal` / `camera` /
|
|
/// `cam+chr` / `volume` are `0x1acf80..0x1acf98`, pointing at the four names
|
|
/// themselves. So each address minus its target's overlay offset is the
|
|
/// overlay's load base.
|
|
///
|
|
/// **Measured on all 27 levels:** the four addresses are mutually consistent
|
|
/// (the same base from all four) and the base is **1,735,792 = `field_00` - 16**
|
|
/// on every level. `field_00` was previously a bare constant reading 1,735,808
|
|
/// with no meaning attached; it is the load address of the payload, with the
|
|
/// 16-byte header stripped on load.
|
|
///
|
|
/// The base is confirmed against the running game, not only derived: level 0's
|
|
/// RAM `0x1acf80` holds `"normal\0camera\0cam+chr\0volume\0"`, byte-identical to
|
|
/// overlay `0x5310`, and `0x5310 + 1735792 == 0x1acf80`. That check is
|
|
/// savestate-gated below.
|
|
///
|
|
/// **The overlay is relocated, not copied.** Level 0's is only 37.67%
|
|
/// byte-identical at the predicted base; the sine table, at overlay `0xb650c`,
|
|
/// is *not* at `0xb650c + base`. So the base locates the overlay's image, but
|
|
/// individual tables inside it move.
|
|
///
|
|
/// The value of the base is that any overlay offset can now be carried to a
|
|
/// RAM address, which is the prerequisite for reading the game's live state.
|
|
#[test]
|
|
fn overlay_load_base_from_the_profiler_table() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay::OverlayHeader;
|
|
|
|
let mut levels = 0usize;
|
|
let mut consistent = 0usize;
|
|
let mut base_matches_header = 0usize;
|
|
|
|
for lv in 0..27usize {
|
|
let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else {
|
|
continue;
|
|
};
|
|
// The profiler table's distinctive name run: "cam+chr\0volume\0".
|
|
let Some(p) = ov.windows(15).position(|w| w == b"cam+chr\0volume\0") else {
|
|
continue;
|
|
};
|
|
// Four 8-byte name slots end where the address run begins.
|
|
let names_at = p - 16;
|
|
let addrs_at = p + 16;
|
|
if addrs_at + 16 > ov.len() {
|
|
continue;
|
|
}
|
|
let a: Vec<u32> = (0..4)
|
|
.map(|k| {
|
|
u32::from_le_bytes(
|
|
ov[addrs_at + k * 4..addrs_at + k * 4 + 4]
|
|
.try_into()
|
|
.unwrap(),
|
|
)
|
|
})
|
|
.collect();
|
|
let bases: Vec<u64> = (0..4)
|
|
.map(|k| a[k] as u64 - (names_at + k * 8) as u64)
|
|
.collect();
|
|
if bases.windows(2).any(|w| w[0] != w[1]) {
|
|
continue;
|
|
}
|
|
|
|
let hdr = OverlayHeader::parse(&ov).unwrap();
|
|
levels += 1;
|
|
consistent += 1;
|
|
// The derived base is the header word less the header's own size, and
|
|
// it relocates the names onto the addresses the table carries.
|
|
if bases[0] + 16 == hdr.field_00 as u64
|
|
&& hdr.ram_address(names_at) == a[0] as u64
|
|
&& hdr.ram_address(names_at + 8) == a[1] as u64
|
|
&& hdr.ram_address(names_at + 16) == a[2] as u64
|
|
&& hdr.ram_address(names_at + 24) == a[3] as u64
|
|
{
|
|
base_matches_header += 1;
|
|
}
|
|
assert_eq!(
|
|
hdr.field_00, 1_735_808,
|
|
"level{lv}: field_00 is {} -- the base is only 1,735,792 on the disc",
|
|
hdr.field_00
|
|
);
|
|
}
|
|
|
|
assert!(
|
|
levels >= 25,
|
|
"only {levels} levels carried the profiler table"
|
|
);
|
|
assert_eq!(consistent, levels, "some levels had inconsistent bases");
|
|
assert_eq!(
|
|
base_matches_header, levels,
|
|
"the derived base is field_00 - 16 on only {} of {levels}",
|
|
base_matches_header
|
|
);
|
|
eprintln!(
|
|
"overlay load base: 1735792 = field_00 - 16, consistent on all {levels} levels; \
|
|
ram_address(overlay X) = X + 1735792"
|
|
);
|
|
}
|
|
|
|
/// The tie packet's length, from its record's fields.
|
|
///
|
|
/// A tie LOD block is a table of packet descriptors plus the packets
|
|
/// themselves, and the descriptor's pointer is where the packet starts. So the
|
|
/// distance between consecutive pointers **is** a packet's length, measured
|
|
/// without reference to any field. Against that:
|
|
///
|
|
/// **`packet_quadwords = vert_ofs + vert_size + scissor_size`, on 12,927 of
|
|
/// 12,927 record pairs whose `strip_count` is zero -- and on 0 of 115 whose it
|
|
/// is not.** The partition is exact: every record with a non-zero
|
|
/// `strip_count` is an exception, and no record with a zero one is. So the
|
|
/// fourth term is 0 quadwords whenever `strip_count == 0`, and something else
|
|
/// (measured 0..19 quadwords, and not equal to `strip_count`) whenever it is
|
|
/// not.
|
|
///
|
|
/// This is what makes the packet payload sliceable at all. It also settles a
|
|
/// dispute inside this file: a formula of this shape was derived at 61 of 61,
|
|
/// then "retracted" as not reproducing -- 0 of 13,042. The retraction was the
|
|
/// error, not the formula: the re-measurement had swapped `strip_count` for
|
|
/// `scissor_ofs` in the fourth term, and the blame was put on the block set.
|
|
/// Re-measured with the term the first pass had, it matches on 12,927 of
|
|
/// 13,042, and the 115 exceptions are exactly the non-zero `strip_count`
|
|
/// records.
|
|
#[test]
|
|
fn tie_packet_length_follows_from_its_record() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
let mut blocks = 0usize;
|
|
let mut pairs = 0usize;
|
|
let mut zero_strip = 0usize;
|
|
let mut zero_strip_ok = 0usize;
|
|
let mut nonzero_strip = 0usize;
|
|
let mut nonzero_strip_ok = 0usize;
|
|
let mut unaligned = 0usize;
|
|
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = hdr.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let s = base + off as usize;
|
|
if let Some(blk) = parse_tie_block(&core, s) {
|
|
let recs: Vec<TieRecord> = blk
|
|
.header
|
|
.records(&core[s..])
|
|
.map(|r| TieRecord::parse(r).expect("record"))
|
|
.collect();
|
|
if recs.len() < 2 {
|
|
continue;
|
|
}
|
|
blocks += 1;
|
|
for (a, b) in recs.iter().zip(recs.iter().skip(1)) {
|
|
if b.pointer <= a.pointer {
|
|
continue;
|
|
}
|
|
let delta = (b.pointer - a.pointer) as usize;
|
|
pairs += 1;
|
|
if delta % 16 != 0 {
|
|
unaligned += 1;
|
|
continue;
|
|
}
|
|
let hit = delta / 16 == a.packet_quadwords();
|
|
if a.strip_count == 0 {
|
|
zero_strip += 1;
|
|
zero_strip_ok += usize::from(hit);
|
|
} else {
|
|
nonzero_strip += 1;
|
|
nonzero_strip_ok += usize::from(hit);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(blocks > 1900, "only {blocks} tie blocks were read");
|
|
assert!(pairs > 12_000, "only {pairs} record pairs were read");
|
|
assert_eq!(
|
|
unaligned, 0,
|
|
"{unaligned} pointer deltas are not 16-aligned"
|
|
);
|
|
assert_eq!(
|
|
zero_strip_ok,
|
|
zero_strip,
|
|
"the formula failed on {} of {zero_strip} strip_count == 0 records",
|
|
zero_strip - zero_strip_ok
|
|
);
|
|
assert!(
|
|
zero_strip > 12_000,
|
|
"only {zero_strip} records have strip_count == 0"
|
|
);
|
|
eprintln!(
|
|
"tie packet length: vert_ofs + vert_size + scissor_size quadwords on \
|
|
{zero_strip_ok}/{zero_strip} records with strip_count == 0; on the {nonzero_strip} \
|
|
records with a non-zero strip_count it holds {nonzero_strip_ok} times (measured \
|
|
fourth term 0..19, not strip_count)"
|
|
);
|
|
}
|
|
|
|
/// The located `0xf6` terminator lies inside the last packet's data.
|
|
///
|
|
/// `TieBlock::payload` ends just past the `0xf6` that `parse_tie_block` walks
|
|
/// to. Each record's `packet_quadwords` gives its packet's length, and for
|
|
/// every record but the last of a table that length is confirmed by the *next*
|
|
/// record's pointer. So the last packet's end is known independently, and
|
|
/// comparing it against the located terminator:
|
|
///
|
|
/// **the last packet's computed end lands 2 to 16 bytes past the terminator,
|
|
/// on every block measured.** The last packet's length is independent of the
|
|
/// terminator (it comes from its record's quadword count, confirmed for every
|
|
/// record but the last by the next record's pointer), so either the terminator
|
|
/// was located too early -- inside the last packet's data -- or the last
|
|
/// packet's length over-predicts by a small amount. Both readings fit; the
|
|
/// first is the likelier, because a length error only on the last record of
|
|
/// every table would have to be a coincidence.
|
|
///
|
|
/// This is a real limitation in `parse_tie_block`, and the existing
|
|
/// `0xf6`-terminator test could not see it because a false terminator
|
|
/// satisfies that test just as well as a true one. Not fixed here: a
|
|
/// terminator that packet data cannot imitate is its own task, and guessing
|
|
/// one would be the "plausible-sounding alternative" this project has been
|
|
/// bitten by before.
|
|
///
|
|
/// What still holds: the lengths are right for every record but the last
|
|
/// (12,927 of 12,927 record pairs) and the block *starts* are confirmed
|
|
/// against RAM. The block *extent* is what is short.
|
|
#[test]
|
|
fn tie_block_terminator_is_inside_the_last_packet() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord, TIE_RECORD_SIZE};
|
|
|
|
let mut blocks = 0usize;
|
|
let mut overruns: std::collections::BTreeMap<usize, usize> = Default::default();
|
|
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = hdr.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let s = base + off as usize;
|
|
if let Some(blk) = parse_tie_block(&core, s) {
|
|
let recs: Vec<TieRecord> = blk
|
|
.header
|
|
.records(&core[s..])
|
|
.map(|r| TieRecord::parse(r).expect("record"))
|
|
.collect();
|
|
let Some(last) = recs.last() else {
|
|
continue;
|
|
};
|
|
if !last.packet_quadwords_are_known() {
|
|
continue;
|
|
}
|
|
blocks += 1;
|
|
let packet_end =
|
|
s + last.pointer as usize + last.packet_quadwords() * TIE_RECORD_SIZE;
|
|
let payload_end = s + blk.payload.len();
|
|
if packet_end > payload_end {
|
|
*overruns.entry(packet_end - payload_end).or_default() += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(blocks > 1900, "only {blocks} tie blocks measured");
|
|
let measured: usize = overruns.values().sum();
|
|
assert_eq!(measured, blocks, "{measured} of {blocks} blocks overran");
|
|
let mut top: Vec<(usize, usize)> = overruns.into_iter().collect();
|
|
top.sort_by_key(|(_, c)| std::cmp::Reverse(*c));
|
|
top.truncate(6);
|
|
eprintln!(
|
|
"tie block terminator: the last packet overruns it on all {measured} blocks, \
|
|
by a small byte count (top: {top:?})"
|
|
);
|
|
}
|
|
|
|
/// The tie vertex region has a 64-byte period, measured against a control.
|
|
///
|
|
/// The 16-byte element reading was retracted for want of a detectable stride.
|
|
/// There is one, at 64 bytes, and it survives a shuffle control by a factor of
|
|
/// ten. The 64 bytes are four 16-byte vertex layouts cycling, so the element is
|
|
/// 16 bytes and the *layout* cycles -- which is what the retraction's evidence
|
|
/// was actually showing.
|
|
#[test]
|
|
fn tie_vertex_region_has_a_stride_64_period() {
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!(
|
|
"skipped: unpack the disc first, e.g.\
|
|
gc-extract unpack <disc.iso> ~/Documents/gc-extracted\
|
|
or point GC_EXTRACTED at an existing unpack"
|
|
);
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord, TIE_RECORD_SIZE};
|
|
|
|
fn regions(data: &std::path::Path) -> Vec<Vec<u8>> {
|
|
let mut out = Vec::new();
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let Ok(t) = LevelCoreHeader::parse(&idx).map(|h| h.tie_classes) else {
|
|
continue;
|
|
};
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let Ok(e) = ClassEntry::parse(&idx[o..o + 0x20]) else {
|
|
continue;
|
|
};
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = hdr.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let s = base + off as usize;
|
|
if let Some(blk) = parse_tie_block(&core, s) {
|
|
for raw in blk.header.records(&core[s..]) {
|
|
let rec = TieRecord::parse(raw).expect("record");
|
|
let start =
|
|
rec.pointer as usize + rec.vert_ofs as usize * TIE_RECORD_SIZE;
|
|
let len = rec.vert_size as usize * TIE_RECORD_SIZE;
|
|
if start + len <= core.len() && len > 0 {
|
|
out.push(core[start..start + len].to_vec());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A byte offset is *constant at stride `s`* when every unit agrees there.
|
|
fn best_stride(region: &[u8]) -> Option<(usize, usize, usize)> {
|
|
if region.len() < 12 {
|
|
return None;
|
|
}
|
|
let mut best: Option<(usize, usize, usize)> = None;
|
|
for stride in (4..=region.len() / 3).step_by(4) {
|
|
if region.len() % stride != 0 {
|
|
continue;
|
|
}
|
|
let units = region.len() / stride;
|
|
let mut constant = 0usize;
|
|
for b in 0..stride.min(128) {
|
|
if region[b..].iter().step_by(stride).all(|&x| x == region[b]) {
|
|
constant += 1;
|
|
}
|
|
}
|
|
let key = (constant, -(stride as i32) as usize);
|
|
if constant >= 8 {
|
|
let better = match best {
|
|
None => true,
|
|
Some((bc, bs, _)) => {
|
|
key.0 > bc || (key.0 == bc && key.1 > -(bs as i32) as usize)
|
|
}
|
|
};
|
|
if better {
|
|
best = Some((stride, constant, units));
|
|
}
|
|
}
|
|
}
|
|
best
|
|
}
|
|
|
|
/// Deterministic byte shuffle, for the control.
|
|
fn shuffled(region: &[u8], seed: u64) -> Vec<u8> {
|
|
let mut v = region.to_vec();
|
|
let mut st = seed | 1;
|
|
for i in (1..v.len()).rev() {
|
|
st ^= st << 13;
|
|
st ^= st >> 7;
|
|
st ^= st << 17;
|
|
let j = (st % (i as u64 + 1)) as usize;
|
|
v.swap(i, j);
|
|
}
|
|
v
|
|
}
|
|
|
|
let dirs = regions(&data);
|
|
let mut hit = 0usize;
|
|
let mut ctrl = 0usize;
|
|
let mut at_64 = 0usize;
|
|
for (i, region) in dirs.iter().enumerate() {
|
|
if let Some((stride, _, _)) = best_stride(region) {
|
|
hit += 1;
|
|
if stride == 64 {
|
|
at_64 += 1;
|
|
}
|
|
}
|
|
if best_stride(&shuffled(region, 0x9e3779b97f4a7c15 ^ i as u64)).is_some() {
|
|
ctrl += 1;
|
|
}
|
|
}
|
|
|
|
let total = dirs.len();
|
|
// Bounds: 15,219 regions, 7,314 with a signature, 731 on shuffled bytes.
|
|
assert!(total > 12_000, "expected the full population, got {total}");
|
|
assert!(hit >= 6_000, "stride signature missing: {hit} of {total}");
|
|
assert!(
|
|
hit >= ctrl * 8,
|
|
"signature is not 8x the control: {hit} real against {ctrl}"
|
|
);
|
|
// 64 is the dominant stride: 6,463 of 7,314 hits.
|
|
assert!(
|
|
at_64 * 10 > hit * 7,
|
|
"stride 64 should be most hits: {at_64} of {hit}"
|
|
);
|
|
|
|
// And every stride-64 region divides into 4-byte-multiple units, which is
|
|
// what lets the element be 16 bytes with a cycling layout.
|
|
let units_ok = dirs
|
|
.iter()
|
|
.filter(|r| r.len() % 64 == 0 && r.len() / 64 >= 3)
|
|
.count();
|
|
assert!(units_ok > 3_000, "few 64-byte-period regions: {units_ok}");
|
|
}
|
|
|
|
fn constant_positions(records: &[&[u8]]) -> usize {
|
|
let Some(first) = records.first() else {
|
|
return 0;
|
|
};
|
|
// Records in one chunk are NOT all the same length: the offset deltas vary,
|
|
// and `resolve` measures each record against its successor. Compare only the
|
|
// span every record has, or this indexes past the short ones.
|
|
let len = records.iter().map(|r| r.len()).min().unwrap_or(0);
|
|
(0..len)
|
|
.filter(|&i| records.iter().all(|r| r[i] == first[i]))
|
|
.count()
|
|
}
|
|
|
|
/// The sky block: the sub-object table at `+0x20` enumerates the sub-objects on
|
|
/// every level, and each carries a bounding sphere and texture dimensions.
|
|
///
|
|
/// The word at `+0x0c` is deliberately **not** asserted to be a count: it
|
|
/// disagrees with the table on most levels. See `gc_formats::sky`.
|
|
#[test]
|
|
fn sky_sub_objects_parse_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut levels = 0usize;
|
|
let mut total = 0usize;
|
|
let mut radii = (f32::MAX, f32::MIN);
|
|
let mut counts = (u32::MAX, 0u32);
|
|
let mut dims = (u16::MAX, 0u16);
|
|
let mut nonzero_radius = 0usize;
|
|
let mut reserved_nonzero = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let hdr = match gc_formats::level_core::LevelCoreHeader::parse(&index) {
|
|
Ok(h) => h,
|
|
Err(_) => continue,
|
|
};
|
|
let sky = hdr.sky as usize;
|
|
if sky == 0 || sky >= core.len() {
|
|
continue;
|
|
}
|
|
let end = [
|
|
hdr.collision as usize,
|
|
hdr.moby_classes.offset as usize,
|
|
core.len(),
|
|
]
|
|
.into_iter()
|
|
.filter(|&v| v > sky)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let region = &core[sky..end.min(core.len())];
|
|
|
|
let parsed = gc_formats::sky::Sky::parse(region).expect("sky should parse");
|
|
assert!(
|
|
!parsed.sub_objects.is_empty(),
|
|
"level{n}: sky region has no sub-objects"
|
|
);
|
|
levels += 1;
|
|
total += parsed.sub_objects.len();
|
|
|
|
for s in &parsed.sub_objects {
|
|
if s.reserved != [0u8; 12] {
|
|
reserved_nonzero += 1;
|
|
}
|
|
assert!(s.count > 0, "level{n}: sky sub-object count is {}", s.count);
|
|
assert!(
|
|
s.width > 0 && s.height > 0,
|
|
"level{n}: sky texture dims are {}x{}",
|
|
s.width,
|
|
s.height
|
|
);
|
|
assert!(
|
|
s.sphere.iter().all(|f| f.is_finite()),
|
|
"level{n}: sky sphere is not finite: {:?}",
|
|
s.sphere
|
|
);
|
|
// The fourth float is the radius: positive wherever the first
|
|
// three are negative, the same reasoning the shrub header uses.
|
|
if s.sphere[3] > 0.0 {
|
|
nonzero_radius += 1;
|
|
radii.0 = radii.0.min(s.sphere[3]);
|
|
radii.1 = radii.1.max(s.sphere[3]);
|
|
}
|
|
counts.0 = counts.0.min(s.count);
|
|
counts.1 = counts.1.max(s.count);
|
|
dims.0 = dims.0.min(s.width);
|
|
dims.1 = dims.1.max(s.width);
|
|
}
|
|
}
|
|
|
|
assert!(levels >= 20, "only {levels} levels had a sky region");
|
|
assert!(total >= 100, "only {total} sky sub-objects parsed");
|
|
eprintln!(
|
|
"sky: {total} sub-objects over {levels} levels; count {}..={}, width {}..={}, radius {:.2}..={:.2} on {nonzero_radius}, {reserved_nonzero} with a non-zero reserved word",
|
|
counts.0, counts.1, dims.0, dims.1, radii.0, radii.1
|
|
);
|
|
}
|
|
|
|
/// The per-level overlay: the header is stable, `+0x10..+0x1220` is identical
|
|
/// on every level, and `+0x0c` is not the file length.
|
|
///
|
|
/// See `gc_formats::overlay`. This pins the header and the constant block's
|
|
/// extent; it does not name any field.
|
|
#[test]
|
|
fn overlay_header_and_constant_block_are_stable() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay::{parse_overlay, OVERLAY_CONSTANT_BLOCK_END, OVERLAY_HEADER_SIZE};
|
|
|
|
let mut levels = 0usize;
|
|
let mut constant_block: Option<Vec<u8>> = None;
|
|
let mut f00 = 0u32;
|
|
let mut f08 = 0u32;
|
|
let (mut min_f04, mut max_f04) = (u32::MAX, 0u32);
|
|
let (mut min_f0c, mut max_f0c) = (u32::MAX, 0u32);
|
|
let mut f0c_below = 0usize;
|
|
let mut f0c_above = 0usize;
|
|
let mut levels_seen = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let p = data.join(format!("level{n}/overlay.bin"));
|
|
let Ok(bytes) = std::fs::read(&p) else {
|
|
continue;
|
|
};
|
|
|
|
let (h, constant, level) = parse_overlay(&bytes).expect("overlay should parse");
|
|
|
|
assert!(
|
|
bytes.len() > OVERLAY_CONSTANT_BLOCK_END,
|
|
"level{n}: overlay is {} bytes, too short for the constant block",
|
|
bytes.len()
|
|
);
|
|
assert!(
|
|
!level.bytes.is_empty(),
|
|
"level{n}: overlay has no level data"
|
|
);
|
|
assert_eq!(
|
|
constant.bytes.len(),
|
|
OVERLAY_CONSTANT_BLOCK_END - OVERLAY_HEADER_SIZE
|
|
);
|
|
|
|
if f00 == 0 {
|
|
f00 = h.field_00;
|
|
}
|
|
assert_eq!(h.field_00, f00, "level{n}: field_00 differs");
|
|
if f08 == 0 {
|
|
f08 = h.field_08;
|
|
}
|
|
assert_eq!(h.field_08, f08, "level{n}: field_08 differs");
|
|
|
|
assert_eq!(
|
|
h.field_04 % 4,
|
|
0,
|
|
"level{n}: field_04 {} is not a multiple of 4",
|
|
h.field_04
|
|
);
|
|
min_f04 = min_f04.min(h.field_04);
|
|
max_f04 = max_f04.max(h.field_04);
|
|
|
|
min_f0c = min_f0c.min(h.field_0c);
|
|
max_f0c = max_f0c.max(h.field_0c);
|
|
// The claim being pinned: field_0c is not the file length.
|
|
if (h.field_0c as usize) < bytes.len() {
|
|
f0c_below += 1
|
|
} else {
|
|
f0c_above += 1
|
|
}
|
|
|
|
match &constant_block {
|
|
None => constant_block = Some(constant.bytes.to_vec()),
|
|
Some(prev) => assert_eq!(
|
|
prev.as_slice(),
|
|
constant.bytes,
|
|
"level{n}: the constant block +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} differs"
|
|
),
|
|
}
|
|
|
|
levels += 1;
|
|
levels_seen.push(n);
|
|
}
|
|
|
|
assert!(
|
|
levels >= 27,
|
|
"only {levels} levels had an overlay.bin, saw {levels_seen:?}"
|
|
);
|
|
assert!(f0c_below > 0 && f0c_above > 0,
|
|
"field_0c must fall on both sides of the file length to show it is not one: below {f0c_below}, above {f0c_above}");
|
|
assert_eq!(levels, 27, "expected all levels, got {levels_seen:?}");
|
|
assert!(
|
|
max_f04 - min_f04 > 1000,
|
|
"field_04 range collapsed to {min_f04}..={max_f04}"
|
|
);
|
|
assert!(
|
|
max_f0c - min_f0c > 100000,
|
|
"field_0c range collapsed to {min_f0c}..={max_f0c}"
|
|
);
|
|
|
|
eprintln!(
|
|
"overlay: {levels} levels; field_00 {f00}, field_08 {f08}; \
|
|
field_04 {min_f04}..={max_f04}, field_0c {min_f0c}..={max_f0c} \
|
|
({f0c_below} below the file length, {f0c_above} above); \
|
|
+0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} identical on all"
|
|
);
|
|
}
|
|
|
|
/// The two tie indirections resolve on every class on every level, and the
|
|
/// LOD block offsets land inside `core_data`.
|
|
///
|
|
/// See `gc_formats::tie`. This pins the resolution chain and the header; the
|
|
/// block's internal layout is *not* decoded, and one tempting reading of it is
|
|
/// disproved below.
|
|
#[test]
|
|
fn tie_classes_resolve_through_both_indirections() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::tie::resolve_tie_class;
|
|
|
|
let mut classes = 0usize;
|
|
let mut o_class_agree = 0usize;
|
|
let mut lod_blocks = 0usize;
|
|
let mut lod_blocks_in_range = 0usize;
|
|
let mut monotonic_u32_0 = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let t = h.tie_classes;
|
|
if t.count == 0 {
|
|
continue;
|
|
}
|
|
for c in 0..t.count as usize {
|
|
let r = resolve_tie_class(&idx, &core, t, c).expect("tie class should resolve");
|
|
classes += 1;
|
|
if r.header.o_class == r.entry.o_class {
|
|
o_class_agree += 1;
|
|
}
|
|
|
|
let base = r.entry.offset_in_asset_wad as usize;
|
|
for (i, p) in r.header.packets.iter().enumerate() {
|
|
let abs = base.wrapping_add(*p as usize);
|
|
if abs <= core.len() {
|
|
lod_blocks_in_range += 1;
|
|
}
|
|
lod_blocks += 1;
|
|
if *p >= 0 && abs + 16 <= core.len() {
|
|
// Is the word at the block start strictly increasing across
|
|
// 16-byte units? It looked like it on the first twelve units
|
|
// of one class. It is not a property of the format.
|
|
let next = if i + 1 < 3 {
|
|
r.header.packets[i + 1]
|
|
} else {
|
|
*p
|
|
};
|
|
if next > *p {
|
|
let end = base + next as usize;
|
|
if end <= core.len() {
|
|
let n16 = (end - abs) / 16;
|
|
let mut prev = 0i64;
|
|
let mut ok = true;
|
|
for k in 0..n16 {
|
|
let q = abs + k * 16;
|
|
let a =
|
|
u32::from_le_bytes(core[q..q + 4].try_into().unwrap()) as i64;
|
|
if a <= prev {
|
|
ok = false;
|
|
break;
|
|
}
|
|
prev = a;
|
|
}
|
|
if ok && n16 > 0 {
|
|
monotonic_u32_0 += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// packets[0] is a constant, not an offset, and the other two order only
|
|
// sometimes -- both are why the block layout is only partly understood.
|
|
let mut packets0_constant = 0usize;
|
|
let mut packets_increasing = 0usize;
|
|
let mut consecutive_div16 = 0usize;
|
|
let mut consecutive_total = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let r = resolve_tie_class(&idx, &core, t, c).expect("tie class");
|
|
if r.header.packets[0] == gc_formats::tie::TIE_PACKETS0 {
|
|
packets0_constant += 1;
|
|
}
|
|
let p = r.header.packets;
|
|
if p[0] < p[1] && p[1] < p[2] {
|
|
packets_increasing += 1;
|
|
for i in 0..2 {
|
|
consecutive_total += 1;
|
|
if ((p[i + 1] - p[i]) as usize) % 16 == 0 {
|
|
consecutive_div16 += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tie LOD blocks are self-delimiting, which is what makes them parseable
|
|
// without trusting packets[1] to bound the block.
|
|
use gc_formats::tie::{
|
|
parse_tie_block, TIE_BLOCK_HEADER_SIZE, TIE_BLOCK_TERMINATOR, TIE_PACKED_TEXTURE_VALUES,
|
|
};
|
|
|
|
let mut blocks = 0usize;
|
|
let mut absent = 0usize;
|
|
let mut hdr_ok = 0usize;
|
|
let mut count_mult16 = 0usize;
|
|
let mut reserved_02_zero = 0usize;
|
|
let mut tables_bounded = 0usize;
|
|
let mut packed_ok = 0usize;
|
|
let mut padding_all_zero = 0usize;
|
|
let mut packed_seen: Vec<u16> = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let r = resolve_tie_class(&idx, &core, t, c).expect("tie class");
|
|
let base = r.entry.offset_in_asset_wad as usize;
|
|
for lod in [1usize, 2] {
|
|
let off = r.header.packets[lod] as i64;
|
|
if off == 0 {
|
|
// packets == 0 means this class has no block at this LOD.
|
|
absent += 1;
|
|
continue;
|
|
}
|
|
let s = (base as i64 + off) as usize;
|
|
if s + TIE_BLOCK_HEADER_SIZE > core.len() {
|
|
continue;
|
|
}
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
|
|
blocks += 1;
|
|
// The word at +0 is the data offset -- the record table's
|
|
// length -- so it is a non-zero multiple of 16, and it bounds
|
|
// the table.
|
|
if blk.header.data_offset % 16 == 0 && blk.header.data_offset != 0 {
|
|
count_mult16 += 1
|
|
}
|
|
let table = blk.header.records(&core[s..]).count();
|
|
if table == blk.header.record_count() && table > 0 {
|
|
tables_bounded += 1
|
|
}
|
|
if blk.header.reserved_02 == 0 {
|
|
reserved_02_zero += 1
|
|
}
|
|
if TIE_PACKED_TEXTURE_VALUES.contains(&blk.header.packed_texture) {
|
|
packed_ok += 1
|
|
}
|
|
if blk.header.invariants_hold() {
|
|
hdr_ok += 1
|
|
}
|
|
if blk.padding.iter().all(|&b| b == 0) {
|
|
padding_all_zero += 1
|
|
}
|
|
assert_eq!(
|
|
blk.payload.last(),
|
|
Some(&TIE_BLOCK_TERMINATOR),
|
|
"level{n} class{c} lod{lod}: payload does not end on the terminator"
|
|
);
|
|
if !packed_seen.contains(&blk.header.packed_texture) {
|
|
packed_seen.push(blk.header.packed_texture);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
packed_seen.sort_unstable();
|
|
|
|
assert!(blocks >= 2100, "only {blocks} tie blocks located");
|
|
assert!(
|
|
absent > 1000,
|
|
"only {absent} absent (packets == 0) LOD blocks"
|
|
);
|
|
assert_eq!(blocks + absent, 2 * classes,
|
|
"expected two LOD slots per class, got {blocks} present + {absent} absent for {classes} classes");
|
|
assert_eq!(
|
|
count_mult16,
|
|
blocks,
|
|
"the data offset is not a non-zero multiple of 16 on {} of {blocks}",
|
|
blocks - count_mult16
|
|
);
|
|
assert_eq!(
|
|
tables_bounded,
|
|
blocks,
|
|
"the record table is not bounded by the data offset on {} of {blocks}",
|
|
blocks - tables_bounded
|
|
);
|
|
assert_eq!(
|
|
reserved_02_zero,
|
|
blocks,
|
|
"reserved_02 non-zero on {} of {blocks}",
|
|
blocks - reserved_02_zero
|
|
);
|
|
assert_eq!(
|
|
packed_ok,
|
|
blocks,
|
|
"packed_texture outside the four known values on {} of {blocks}: {packed_seen:?}",
|
|
blocks - packed_ok
|
|
);
|
|
assert_eq!(
|
|
hdr_ok,
|
|
blocks,
|
|
"header invariants failed on {} of {blocks}",
|
|
blocks - hdr_ok
|
|
);
|
|
assert_eq!(
|
|
padding_all_zero,
|
|
blocks,
|
|
"non-zero padding on {} of {blocks}",
|
|
blocks - padding_all_zero
|
|
);
|
|
assert_eq!(
|
|
packed_seen.len(),
|
|
TIE_PACKED_TEXTURE_VALUES.len(),
|
|
"packed_texture took {} distinct values, expected the four {packed_seen:?}",
|
|
packed_seen.len()
|
|
);
|
|
|
|
eprintln!(
|
|
"tie blocks: {blocks} present, {absent} absent (packets == 0); all end on 0xf6 with zero padding; \
|
|
count %16 on all; reserved_02 == 0 on all; packed_texture in {packed_seen:?} on all"
|
|
);
|
|
|
|
assert!(classes >= 1800, "only {classes} tie classes resolved");
|
|
assert_eq!(
|
|
packets0_constant,
|
|
classes,
|
|
"packets[0] was not {} on {} of {classes} classes",
|
|
gc_formats::tie::TIE_PACKETS0,
|
|
classes - packets0_constant
|
|
);
|
|
assert_eq!(
|
|
consecutive_div16,
|
|
consecutive_total,
|
|
"{} of {consecutive_total} consecutive tie blocks are not a multiple of 16 bytes",
|
|
consecutive_total - consecutive_div16
|
|
);
|
|
assert!(
|
|
packets_increasing < classes,
|
|
"packets increase on all {classes} classes; the block-end reasoning in the docs needs revisiting"
|
|
);
|
|
assert_eq!(
|
|
o_class_agree,
|
|
classes,
|
|
"header o_class disagreed with the class entry on {} of {classes}",
|
|
classes - o_class_agree
|
|
);
|
|
assert_eq!(
|
|
lod_blocks_in_range,
|
|
lod_blocks,
|
|
"{} of {lod_blocks} LOD block offsets fell outside core_data",
|
|
lod_blocks - lod_blocks_in_range
|
|
);
|
|
assert_eq!(
|
|
monotonic_u32_0, 0,
|
|
"the word at +0 of a 16-byte tie unit is strictly increasing on {monotonic_u32_0} blocks; \
|
|
if that ever becomes true the disproved reading below needs revisiting"
|
|
);
|
|
|
|
eprintln!(
|
|
"tie: {classes} classes resolved through both indirections, o_class agreeing on all; \
|
|
{lod_blocks} LOD block offsets all inside core_data; \
|
|
packets[0] == {} on all; packets increasing on {packets_increasing} of {classes}; \
|
|
{consecutive_div16} of {consecutive_total} consecutive blocks a multiple of 16; \
|
|
strictly-increasing +0 on 0 of {classes} blocks (disproved)",
|
|
gc_formats::tie::TIE_PACKETS0
|
|
);
|
|
}
|
|
|
|
/// Chunk record buckets: the leading byte is an aligned index, not record data.
|
|
///
|
|
/// The leading byte is not decoded -- what it indexes is not established. This
|
|
/// pins the measured 8-byte alignment and the exact three types that break it;
|
|
/// a parallel probe puts 1,142 records across seven types outside a *16*-byte
|
|
/// rule, which is how the per-type 8-vs-16 step was found. See
|
|
/// `gc_formats::chunks::CHUNK_RECORD_ALIGNMENT`.
|
|
#[test]
|
|
fn chunk_records_start_16_byte_aligned() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{parse_all_chunks, record_is_aligned};
|
|
use std::collections::BTreeMap;
|
|
|
|
let mut chunks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut aligned = 0usize;
|
|
let mut unaligned: BTreeMap<u32, usize> = BTreeMap::new();
|
|
|
|
for n in 0..27 {
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
chunks += 1;
|
|
let t = u32::from_le_bytes([
|
|
c.chunk.header[0],
|
|
c.chunk.header[1],
|
|
c.chunk.header[2],
|
|
c.chunk.header[3],
|
|
]);
|
|
for r in &c.records {
|
|
if r.is_empty() {
|
|
continue;
|
|
}
|
|
records += 1;
|
|
if record_is_aligned(r) {
|
|
aligned += 1;
|
|
} else {
|
|
*unaligned.entry(t).or_default() += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(chunks >= 1500, "only {chunks} chunks parsed");
|
|
assert!(records >= 34000, "only {records} chunk records");
|
|
// Record length varies BETWEEN chunks of a type, and -- contrary to what
|
|
// this test used to assert -- also WITHIN a chunk, on 1,252 of them. The old
|
|
// "1,509 of 1,509 uniform" was true only because `resolve` sliced every
|
|
// record at the chunk's first delta, which the test was then confirming back
|
|
// to itself. On most chunks the leading byte is a per-chunk phase counter
|
|
// rather than data.
|
|
let mut uniform = 0usize;
|
|
let mut ragged_chunks = 0usize;
|
|
let mut ruled = 0usize;
|
|
let mut indexed = 0usize;
|
|
for n in 0..27 {
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
if c.has_uniform_record_length() {
|
|
uniform += 1;
|
|
} else {
|
|
ragged_chunks += 1;
|
|
}
|
|
if c.bucket_rule().is_some() {
|
|
ruled += 1;
|
|
}
|
|
if c.index_rule().is_some() {
|
|
indexed += 1;
|
|
}
|
|
}
|
|
}
|
|
assert_eq!(uniform, 257, "uniform-record-length count moved off 257");
|
|
assert_eq!(
|
|
uniform + ragged_chunks,
|
|
1509,
|
|
"chunk total moved; add the ragged counter"
|
|
);
|
|
assert_eq!(ruled, 1262, "bucket-rule count moved off 1262");
|
|
assert_eq!(indexed, 1235, "index-rule count moved off 1235");
|
|
|
|
// Not universal. Three chunk types break even the 8-byte rule, all of them
|
|
// the variable-length or 8-stepped ones. A parallel probe puts 1,142
|
|
// records across seven types outside a *16*-byte rule, which is how the
|
|
// per-type 8-vs-16 step was found in the first place.
|
|
let bad: usize = unaligned.values().sum();
|
|
assert_eq!(
|
|
bad, 59,
|
|
"unaligned record count moved off 59: {unaligned:?}"
|
|
);
|
|
assert_eq!(
|
|
unaligned.len(),
|
|
3,
|
|
"unaligned records now span {} chunk types, not the 3 known: {unaligned:?}",
|
|
unaligned.len()
|
|
);
|
|
assert_eq!(
|
|
records - bad,
|
|
34_432,
|
|
"multiple-of-8 count moved off 34,432"
|
|
);
|
|
|
|
eprintln!(
|
|
"chunks: {chunks} chunks, {records} records; uniform record length on {uniform}; \
|
|
leading byte a multiple of 8 on {aligned}, {bad} exceptions across {} types; \
|
|
leading byte a phase counter on {ruled}; \
|
|
leading TWO bytes encode the record index on {indexed}",
|
|
unaligned.len()
|
|
);
|
|
}
|
|
|
|
/// Moby class IDs live in one body slot per chunk type, and the slot varies.
|
|
///
|
|
/// **Largely retracted.** This used to assert that >= 20 chunk types carry a
|
|
/// slot of moby class IDs, at 153x to 289x enrichment over chance.
|
|
///
|
|
/// It was measuring over-read records. `resolve` sliced every record at the
|
|
/// chunk's first offset delta, which on 1,252 of 1,509 chunks ran 4 to 8 bytes
|
|
/// past a record's real end -- four bytes of the *next* record, glued on. That
|
|
/// reads as extra fields and lifts a slot's apparent hit rate. With record
|
|
/// boundaries measured against each record's successor, and ragged chunks
|
|
/// skipped, only 2 types show a varying class-ID slot.
|
|
///
|
|
/// **Nothing survives, and the flaw was present from the start.** The
|
|
/// discriminator the original version lacked: a slot that really carries moby
|
|
/// class IDs must span the ID space, so some values land ABOVE a level's moby
|
|
/// count. The slot that scored 305x has 9 distinct values --
|
|
/// 1, 3, 4, 5, 10, 11, 12, 13, 15 -- and **none** above the count, because all
|
|
/// are under ~215. Every slot that does span the ID space scores 10x to 64x. The
|
|
/// original "5 to 42 distinct values at 153x to 289x" had exactly the artifact's
|
|
/// shape, so the record-boundary fix was never what made it wrong. The
|
|
/// `distinct >= 4` filter is still load-bearing -- without it the top slots are
|
|
/// constants that read as 305x because any value below the moby count counts as
|
|
/// a hit. See `docs/formats/core-data-regions.md`.
|
|
#[test]
|
|
fn moby_class_ids_sit_in_one_slot_per_chunk_type() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::parse_all_chunks;
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use std::collections::BTreeMap;
|
|
|
|
// (type, slot) -> (nonzero, hits, expected, distinct)
|
|
let mut stat: BTreeMap<(u32, usize), (u64, u64, f64, usize)> = BTreeMap::new();
|
|
// (type, slot) -> how many of that slot's values land above the moby count.
|
|
// Keyed per SLOT, not per type: a type with one artifact slot and several
|
|
// ordinary slots must not have the ordinary ones vouch for it.
|
|
let mut above: BTreeMap<(u32, usize), usize> = BTreeMap::new();
|
|
let mut seen: BTreeMap<(u32, usize), BTreeSet<u16>> = BTreeMap::new();
|
|
|
|
for n in 0..27 {
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let nm = LevelCoreHeader::parse(&idx)
|
|
.expect("hdr")
|
|
.moby_classes
|
|
.count as u64;
|
|
if nm == 0 {
|
|
continue;
|
|
}
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
let t = u32::from_le_bytes([
|
|
c.chunk.header[0],
|
|
c.chunk.header[1],
|
|
c.chunk.header[2],
|
|
c.chunk.header[3],
|
|
]);
|
|
let len = c.records[0].len();
|
|
if c.records.iter().any(|r| r.len() != len) {
|
|
continue;
|
|
}
|
|
let mut loc: BTreeMap<usize, (u64, u64)> = BTreeMap::new();
|
|
for r in c.records.iter() {
|
|
for k in 1..len / 2 {
|
|
let v = u16::from_le_bytes([r[k * 2], r[k * 2 + 1]]);
|
|
if v == 0 {
|
|
continue;
|
|
}
|
|
seen.entry((t, k)).or_default().insert(v);
|
|
let e = loc.entry(k).or_default();
|
|
e.0 += 1;
|
|
if (v as u64) < nm {
|
|
e.1 += 1
|
|
} else {
|
|
*above.entry((t, k)).or_default() += 1
|
|
}
|
|
}
|
|
}
|
|
for (k, (nn, hh)) in loc {
|
|
let e = stat.entry((t, k)).or_default();
|
|
e.0 += nn;
|
|
e.1 += hh;
|
|
e.2 += nn as f64 * (nm as f64 / 65536.0);
|
|
}
|
|
}
|
|
}
|
|
for (key, s) in seen.iter() {
|
|
stat.entry(*key).or_default().3 = s.len();
|
|
}
|
|
|
|
// best varying slot per type
|
|
let mut best: BTreeMap<u32, (usize, f64, usize)> = BTreeMap::new();
|
|
for ((t, k), (nn, hh, e, dist)) in stat.iter() {
|
|
if *nn < 8 || *e < 1.0 || *dist < 4 {
|
|
continue;
|
|
}
|
|
let r = (*hh as f64) / (*e);
|
|
if best.get(t).is_none_or(|(_, ce, _)| r > *ce) {
|
|
best.insert(*t, (*k, *hh as f64 / *e, *dist));
|
|
}
|
|
}
|
|
|
|
// RETRACTED. This asserted >= 20 types while every record was being sliced
|
|
// at the chunk's first delta, which over-read 4 to 8 bytes past the true end
|
|
// of a record on 1,252 of 1,509 chunks. Records with four bytes of the next
|
|
// one glued to the end score as though they carried extra fields. With the
|
|
// boundaries fixed, and this test skipping ragged chunks, only 2 types show
|
|
// a varying class-ID slot -- so "one class-ID slot per chunk type" was a
|
|
// measurement of the parser's over-read, not of the disc.
|
|
// The discriminator the original version lacked. A slot that really carries
|
|
// moby class IDs spans the ID space, so some of its values must land ABOVE a
|
|
// level's moby count. The artifact does the opposite: every one of its values
|
|
// sits below the count, so every value scores as a hit.
|
|
//
|
|
// type 0xd76cc453 slot 6 305x 9 distinct values 0 above the count
|
|
// type 0xf2ef4275 slot 34 64x 9 distinct values 341 above the count
|
|
// type 0xf2ef4275 slot 7 46x 16 distinct values 412 above the count
|
|
//
|
|
// The 305x slot's values are 1, 3, 4, 5, 10, 11, 12, 13, 15 -- all under a
|
|
// level's ~215 moby classes, giving 65536/215 = 305x. Nine small numbers is
|
|
// not a class-ID array, and every slot that does span the ID space scores in
|
|
// single digits to tens. Nothing reaches 100x while also exceeding the count.
|
|
let spanning: Vec<_> = best
|
|
.iter()
|
|
.filter(|(&t, &(k, r, _))| r >= 100.0 && above.get(&(t, k)).copied().unwrap_or(0) > 0)
|
|
.collect();
|
|
let strong: Vec<_> = best.values().filter(|(_, r, _)| *r >= 100.0).collect();
|
|
eprintln!(
|
|
"chunk class-ID slots: {} types; {} at >=100x, of which {} also have \
|
|
values above the moby count ({} is the small-value-set artifact)",
|
|
best.len(),
|
|
strong.len(),
|
|
spanning.len(),
|
|
strong.len() - spanning.len()
|
|
);
|
|
assert!(
|
|
spanning.is_empty(),
|
|
"{} slots clear 100x while spanning the ID space -- there may be a real \
|
|
class-ID array after all",
|
|
spanning.len()
|
|
);
|
|
assert!(
|
|
best.len() < 20,
|
|
"class-ID slots are back at >= 20 types; revisit the retraction"
|
|
);
|
|
|
|
let mut all: Vec<_> = best
|
|
.iter()
|
|
.map(|(t, (k, r, dist))| (*t, *k, *r, *dist))
|
|
.collect();
|
|
all.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap());
|
|
assert!(!all.is_empty(), "no varying class-ID slots found");
|
|
let mn = all
|
|
.iter()
|
|
.map(|a| a.2)
|
|
.fold(f64::MAX, |g: f64, v: f64| f64::min(g, v));
|
|
let mx = all
|
|
.iter()
|
|
.map(|a| a.2)
|
|
.fold(0.0f64, |g: f64, v: f64| f64::max(g, v));
|
|
let dmin = all.iter().map(|a| a.3).min().unwrap();
|
|
let dmax = all.iter().map(|a| a.3).max().unwrap();
|
|
|
|
eprintln!(
|
|
"chunk class-ID slots: {} types; best-slot enrichment {mn:.0}x..{mx:.0}x; \
|
|
distinct IDs per slot {dmin}..{dmax}",
|
|
best.len()
|
|
);
|
|
eprintln!(
|
|
" strongest (type, byte, ratio, distinct): {:?}",
|
|
&all[..all.len().min(6)]
|
|
);
|
|
}
|
|
|
|
/// The tie vertex element is 16 bytes, with 12-bit fixed-point texcoords.
|
|
///
|
|
/// Measured over 95,612 vertices on levels 0-2, reached by following each
|
|
/// `TieRecord::pointer` into the block payload and stepping `vert_size` elements
|
|
/// from `vert_ofs`. The element size is settled by comparison against the
|
|
/// alternatives, not assumed: 16 bytes gives a 12-bit-fixed texcoord hit rate of
|
|
/// 82.5%, against 47.8% at 8 bytes and 45.4% at 12.
|
|
///
|
|
/// The field *names* are not established -- this pins the size and the texcoord
|
|
/// encoding only.
|
|
/// `i16[0..2]` are vertex positions.
|
|
///
|
|
/// LOD 1 and LOD 2 of one class are the same object at two levels of detail, so
|
|
/// any field that locates a vertex in the object's own geometry is shared
|
|
/// between them. A different class' LOD is an unrelated object. Comparing the
|
|
/// two turns "is this a position" into a measurement with a built-in control.
|
|
type Lods = (BTreeSet<[i16; 3]>, BTreeSet<[i16; 3]>);
|
|
|
|
#[test]
|
|
fn tie_vertex_slots_0_to_2_are_shared_positions() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
let (mut same_sum, mut same_n) = (0.0f64, 0usize);
|
|
let (mut diff_sum, mut diff_n) = (0.0f64, 0usize);
|
|
let mut same_with_overlap = 0usize;
|
|
|
|
for lv in 0..6usize {
|
|
let idx = std::fs::read(data.join(format!("level{lv}/core_index.bin")));
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin")));
|
|
let (Ok(idx), Ok(core)) = (idx, core) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
|
|
let mut per_class: Vec<Lods> = Vec::new();
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
let mut lods: Vec<BTreeSet<[i16; 3]>> = Vec::new();
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
let mut set = BTreeSet::new();
|
|
if off != 0 {
|
|
let s = base + off;
|
|
if let Some(blk) = parse_tie_block(&core, s) {
|
|
let rr: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in rr.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start =
|
|
s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
for v in 0..r.vert_size as usize {
|
|
let o = start + v * 16;
|
|
if o + 16 > core.len() {
|
|
break;
|
|
}
|
|
let g = |k: usize| {
|
|
i16::from_le_bytes(
|
|
core[o + k * 2..o + k * 2 + 2].try_into().unwrap(),
|
|
)
|
|
};
|
|
// ordinary layout only: other layouts are a
|
|
// different element shape, not a different LOD
|
|
if (g(3) >> 8) != 0 {
|
|
continue;
|
|
}
|
|
set.insert([g(0), g(1), g(2)]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
lods.push(set);
|
|
}
|
|
per_class.push((lods[0].clone(), lods[1].clone()));
|
|
}
|
|
|
|
let both: Vec<usize> = (0..per_class.len())
|
|
.filter(|&i| !per_class[i].0.is_empty() && !per_class[i].1.is_empty())
|
|
.collect();
|
|
for &i in &both {
|
|
let (a, b) = (&per_class[i].0, &per_class[i].1);
|
|
let f = a.intersection(b).count() as f64 / a.len().min(b.len()).max(1) as f64;
|
|
same_sum += f;
|
|
same_n += 1;
|
|
if f > 0.0 {
|
|
same_with_overlap += 1
|
|
}
|
|
}
|
|
for &i in &both {
|
|
for &j in &both {
|
|
if i == j {
|
|
continue;
|
|
}
|
|
let (a, b) = (&per_class[i].0, &per_class[j].1);
|
|
diff_sum += a.intersection(b).count() as f64 / a.len().min(b.len()).max(1) as f64;
|
|
diff_n += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(
|
|
same_n >= 50,
|
|
"only {same_n} classes had both LODs populated"
|
|
);
|
|
let same = same_sum / same_n as f64;
|
|
let diff = diff_sum / diff_n.max(1) as f64;
|
|
eprintln!(
|
|
"tie positions: same-class LOD1/LOD2 triple overlap {:.1}% over {same_n} classes \
|
|
({same_with_overlap} nonzero); different-class control {:.1}% over {diff_n} pairs",
|
|
100.0 * same,
|
|
100.0 * diff
|
|
);
|
|
assert_eq!(
|
|
same_with_overlap, same_n,
|
|
"a class's LODs share no positions"
|
|
);
|
|
assert!(same > 0.85, "same-class position overlap fell to {same:.3}");
|
|
assert!(
|
|
diff < 0.05,
|
|
"different classes now share {diff:.3} of their positions"
|
|
);
|
|
assert!(
|
|
same > diff * 20.0,
|
|
"separation narrowed: {same:.3} vs {diff:.3}"
|
|
);
|
|
}
|
|
|
|
/// The `SBlk` head's constant fields, and the one field that is not a count.
|
|
///
|
|
/// The first twelve bytes are identical on all 27 levels. The word at `+0x14`
|
|
/// varies, with its low sixteen bits always zero. It tracks the body length at
|
|
/// ~100 bytes per unit, which is the shape of a record count -- and is not one:
|
|
/// the ratio runs 91.2..110.8 and no stride makes the relation exact on all 27.
|
|
/// Both halves are asserted so the near-miss cannot quietly harden into a name.
|
|
#[test]
|
|
fn sound_bank_sblk_head_is_constant_apart_from_one_word() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::sound_bank::{parse_sound_bank, SBlkHeader};
|
|
|
|
let mut values: Vec<u32> = Vec::new();
|
|
let mut bodies: Vec<usize> = Vec::new();
|
|
let mut levels = 0usize;
|
|
for lv in 0..27usize {
|
|
let Ok(bytes) = std::fs::read(data.join(format!("level{lv}/sound_bank.bin"))) else {
|
|
continue;
|
|
};
|
|
let bank = parse_sound_bank(&bytes).expect("sound bank");
|
|
let h = SBlkHeader::parse(bank.sblk).expect("SBlk head");
|
|
assert!(
|
|
h.constants_hold(),
|
|
"level {lv}: SBlk constants moved: {h:?}"
|
|
);
|
|
values.push(h.field_14);
|
|
bodies.push(bank.sblk.len() - 24);
|
|
levels += 1;
|
|
}
|
|
assert_eq!(levels, 27, "only {levels} sound banks parsed");
|
|
|
|
for &v in &values {
|
|
assert_eq!(v & 0xffff, 0, "field_14 gained low bits: {v:#x}");
|
|
assert!(
|
|
(0x92..=0x103).contains(&(v >> 16)),
|
|
"field_14 moved: {v:#x}"
|
|
);
|
|
}
|
|
// correlated with body length, but not a count at any fixed stride
|
|
let ratios: Vec<f64> = bodies
|
|
.iter()
|
|
.zip(&values)
|
|
.map(|(b, &v)| *b as f64 / (v >> 16) as f64)
|
|
.collect();
|
|
let lo = ratios.iter().cloned().fold(f64::MAX, f64::min);
|
|
let hi = ratios.iter().cloned().fold(0.0f64, f64::max);
|
|
let exact = (0..=80).any(|k| {
|
|
let rems: Vec<i64> = bodies
|
|
.iter()
|
|
.zip(&values)
|
|
.map(|(b, &v)| *b as i64 - k * (v >> 16) as i64)
|
|
.collect();
|
|
rems.windows(2).all(|w| w[0] == w[1])
|
|
});
|
|
eprintln!(
|
|
"SBlk head: constants hold on 27/27; field_14 in 0x0092..0x0103 with low 16 bits 0; \
|
|
body/word ratio {lo:.1}..{hi:.1}; any stride making it exact: {exact}"
|
|
);
|
|
assert!(
|
|
lo > 85.0 && hi < 115.0,
|
|
"body/word ratio left its band: {lo:.1}..{hi:.1}"
|
|
);
|
|
assert!(
|
|
!exact,
|
|
"field_14 now satisfies a fixed stride -- revisit the record size"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn tie_vertex_element_is_16_bytes_with_fixed_texcoords() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
let mut verts = 0usize;
|
|
let mut fixed16 = 0usize;
|
|
let mut fixed8 = 0usize;
|
|
let mut fixed12 = 0usize;
|
|
|
|
for lv in 0..3usize {
|
|
let idx =
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
let recs: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in recs.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start = s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
for v in 0..r.vert_size as usize {
|
|
let o = start + v * 16;
|
|
if o + 16 > core.len() {
|
|
break;
|
|
}
|
|
verts += 1;
|
|
// Compare element sizes properly: each strides its own
|
|
// way through the payload, with its texcoords at that
|
|
// element's own offset.
|
|
for stride in [8usize, 12, 16] {
|
|
let q = start + v * stride;
|
|
if q + stride > core.len() {
|
|
continue;
|
|
}
|
|
let tc = if stride >= 16 { (8, 10) } else { (4, 6) };
|
|
let a = i16::from_le_bytes(
|
|
core[q + tc.0..q + tc.0 + 2].try_into().unwrap(),
|
|
);
|
|
let b = i16::from_le_bytes(
|
|
core[q + tc.1..q + tc.1 + 2].try_into().unwrap(),
|
|
);
|
|
if a % 4096 == 0 && b % 4096 == 0 {
|
|
match stride {
|
|
16 => fixed16 += 1,
|
|
12 => fixed12 += 1,
|
|
_ => fixed8 += 1,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Which i16 slot pair is the texcoord? Scored per pair on level 0.
|
|
let core0 = std::fs::read(data.join("level0/core_data.bin")).expect("core_data");
|
|
let idx0 = std::fs::read(data.join("level0/core_index.bin")).expect("core_index");
|
|
let t0 = LevelCoreHeader::parse(&idx0).expect("hdr").tie_classes;
|
|
let (mut pair_hits, mut pair_tot) = ([0usize; 4], 0usize);
|
|
for c in 0..t0.count as usize {
|
|
let o = t0.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx0[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core0.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core0[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core0, s) else {
|
|
continue;
|
|
};
|
|
let rr: Vec<&[u8]> = blk.header.records(&core0[s..]).collect();
|
|
for (ri, rec) in rr.iter().enumerate() {
|
|
let Ok(tr) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start = s + 16 + ri * 16 + tr.pointer as usize + tr.vert_ofs as usize * 16;
|
|
for v in 0..tr.vert_size as usize {
|
|
let q = start + v * 16;
|
|
if q + 16 > core0.len() {
|
|
break;
|
|
}
|
|
pair_tot += 1;
|
|
for p in 0..4usize {
|
|
let a =
|
|
i16::from_le_bytes(core0[q + p * 4..q + p * 4 + 2].try_into().unwrap());
|
|
let b = i16::from_le_bytes(
|
|
core0[q + p * 4 + 2..q + p * 4 + 4].try_into().unwrap(),
|
|
);
|
|
if a % 4096 == 0 && b % 4096 == 0 {
|
|
pair_hits[p] += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// (4,5) is the texcoord field. (6,7) scores *higher* only because it is the
|
|
// near-constant 4096/0 pair, which is trivially a pair of multiples of 4096
|
|
// -- the same artifact a constant would produce anywhere.
|
|
assert!(pair_tot > 30_000, "only {pair_tot} level-0 vertices scored");
|
|
let r = |p: usize| pair_hits[p] as f64 / pair_tot as f64;
|
|
assert!(r(2) > 0.60, "texcoord pair (4,5) fell to {:.3}", r(2));
|
|
assert!(
|
|
r(0) < 0.05,
|
|
"pair (0,1) looks like a texcoord at {:.3}",
|
|
r(0)
|
|
);
|
|
assert!(
|
|
r(1) < 0.15,
|
|
"pair (2,3) looks like a texcoord at {:.3}",
|
|
r(1)
|
|
);
|
|
|
|
eprintln!(
|
|
"tie vertex slot pairs, level 0 (n={pair_tot}): (0,1) {:.1}% (2,3) {:.1}% \
|
|
(4,5) {:.1}% <- texcoords (6,7) {:.1}% <- the constant 4096/0 pair",
|
|
r(0) * 100.0,
|
|
r(1) * 100.0,
|
|
r(2) * 100.0,
|
|
r(3) * 100.0
|
|
);
|
|
|
|
// Slot 3 is not a coordinate. In the ordinary layout its high byte is 0 and
|
|
// its value is a small index: 201 distinct values, every one in 0..=254, with
|
|
// 0 the most common. It is the layout selector's own low byte, so the
|
|
// selector and this field are one byte, not two independent halves.
|
|
{
|
|
use std::collections::BTreeMap;
|
|
let mut sel = BTreeMap::<u8, usize>::new();
|
|
let mut lo3 = i16::MAX;
|
|
let mut hi3 = i16::MIN;
|
|
let mut ord = BTreeSet::<i16>::new();
|
|
for lv in 0..3usize {
|
|
let idx = std::fs::read(data.join(format!("level{lv}/core_index.bin"))).expect("idx");
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin"))).expect("core");
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
let rr: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in rr.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start =
|
|
s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
for v in 0..r.vert_size as usize {
|
|
let o = start + v * 16;
|
|
if o + 16 > core.len() {
|
|
break;
|
|
}
|
|
let w = i16::from_le_bytes(core[o + 6..o + 8].try_into().unwrap());
|
|
*sel.entry((w >> 8) as u8).or_default() += 1;
|
|
// Gate on the SELECTOR, not on the sign: other
|
|
// layouts carry large positive values too, and
|
|
// mixing them in is what makes this look unbounded.
|
|
if (w >> 8) == 0 {
|
|
lo3 = lo3.min(w);
|
|
hi3 = hi3.max(w);
|
|
ord.insert(w);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let ordinary = sel[&0x00];
|
|
eprintln!(
|
|
"tie vertex slot 3: ordinary layout {ordinary} of {} elements; selector 0x00 {:.1}%, next 0x03 {}; within the ordinary layout \
|
|
{} distinct values in {lo3}..={hi3}",
|
|
sel.values().sum::<usize>(),
|
|
100.0 * ordinary as f64 / sel.values().sum::<usize>() as f64,
|
|
sel[&0x03],
|
|
ord.len(),
|
|
);
|
|
assert_eq!(ordinary, 89_341, "ordinary-layout element count moved");
|
|
assert_eq!(sel.len(), 146, "layout selector cardinality moved");
|
|
assert!(ord.len() <= 256, "slot 3 is not a byte-sized index");
|
|
assert!(hi3 <= 254, "slot 3 left the u8 range at {hi3}");
|
|
assert_eq!(lo3, 0, "slot 3 went negative inside the ordinary layout");
|
|
}
|
|
|
|
assert!(verts > 90_000, "only {verts} tie vertices reached");
|
|
let r16 = fixed16 as f64 / verts as f64;
|
|
let r8 = fixed8 as f64 / verts as f64;
|
|
let r12 = fixed12 as f64 / verts as f64;
|
|
assert!(r16 > 0.80, "16-byte element hit rate fell to {r16:.3}");
|
|
assert!(
|
|
r16 > r8 * 1.5,
|
|
"16-byte element ({r16:.3}) no longer beats 8-byte ({r8:.3})"
|
|
);
|
|
assert!(
|
|
r16 > r12 * 1.5,
|
|
"16-byte element ({r16:.3}) no longer beats 12-byte ({r12:.3})"
|
|
);
|
|
|
|
eprintln!(
|
|
"tie vertices: {verts} on levels 0-2; 12-bit-fixed texcoord hit rate \
|
|
16B {:.1}%, 8B {:.1}%, 12B {:.1}% -- element size is 16 bytes",
|
|
r16 * 100.0,
|
|
r8 * 100.0,
|
|
r12 * 100.0
|
|
);
|
|
}
|
|
|
|
/// The tie vertex element's layout is selected by the HIGH byte of `i16[3]`.
|
|
///
|
|
/// Measured over the tie vertices of levels 0-2, grouped by that high byte:
|
|
///
|
|
/// - high byte zero (the ordinary case, 89,341 vertices): slot pairs `(4,5)` and
|
|
/// `(6,7)` are both 12-bit fixed, at 92.9% and 89.5% for a divisor of 4, while
|
|
/// `(0,1)` and `(2,3)` sit at baseline.
|
|
/// - high byte non-zero (6,271 vertices): **no** pair is fixed; the best is 7.5%
|
|
/// against a 2-5% baseline.
|
|
///
|
|
/// A constant would score 100%, so `(6,7)` at 89.5% is a real pair rather than
|
|
/// the `4096, 0` artifact it looks like. See `docs/formats/core-data-regions.md`.
|
|
///
|
|
/// **MEASURES THE RETRACTED 16-BYTE LAYOUT.** `e5cd3df` established that the tie
|
|
/// vertex stride is not 16 -- roughly half of all records are a flat 16-byte
|
|
/// array and the rest are not, and the correct stride is unknown. The population
|
|
/// here (89,341 + 6,271 = 95,612 "vertices") is therefore a 16-byte-stride
|
|
/// reading that has been withdrawn, and the percentages describe that reading
|
|
/// rather than any established element layout.
|
|
///
|
|
/// It is kept, and kept passing, for one reason: it is a faithful record of what
|
|
/// the 16-byte reading produces, so the retraction can be seen to be about the
|
|
/// *stride* and not about the arithmetic. A test that went away with the claim
|
|
/// would leave no way to check that.
|
|
///
|
|
/// The per-vertex rate here and the per-distinct-value rate elsewhere disagree
|
|
/// sharply, and both are correct -- 92.9% of vertices use the handful of UV-corner
|
|
/// values that are multiples of 4096, while only 16 of `i16[4]`'s 1,488 distinct
|
|
/// values are. That gap is the artifact, stated as two denominators.
|
|
#[test]
|
|
fn tie_vertex_layout_is_selected_by_i16_3_high_byte() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
// [pair][k] where k indexes divisor 1, 4, 16, 256
|
|
let divisors = [1i32, 4, 16, 256];
|
|
let mut zero = [[0u64; 4]; 4];
|
|
let mut nonzero = [[0u64; 4]; 4];
|
|
let (mut n_zero, mut n_nonzero) = (0u64, 0u64);
|
|
|
|
for lv in 0..3usize {
|
|
let idx =
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hd) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hd.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
let recs: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in recs.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start = s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
for v in 0..r.vert_size as usize {
|
|
let q = start + v * 16;
|
|
if q + 16 > core.len() {
|
|
break;
|
|
}
|
|
let hi = (i16::from_le_bytes(core[q + 6..q + 8].try_into().unwrap())
|
|
as u32
|
|
>> 8)
|
|
!= 0;
|
|
if hi {
|
|
n_nonzero += 1
|
|
} else {
|
|
n_zero += 1
|
|
}
|
|
let tbl = if hi { &mut nonzero } else { &mut zero };
|
|
for pair in 0..4usize {
|
|
let a = i16::from_le_bytes(
|
|
core[q + pair * 4..q + pair * 4 + 2].try_into().unwrap(),
|
|
) as i32;
|
|
let b = i16::from_le_bytes(
|
|
core[q + pair * 4 + 2..q + pair * 4 + 4].try_into().unwrap(),
|
|
) as i32;
|
|
for (k, dv) in divisors.iter().enumerate() {
|
|
if a % dv == 0 && b % dv == 0 {
|
|
tbl[pair][k] += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(n_zero > 80_000, "only {n_zero} ordinary tie vertices");
|
|
assert!(
|
|
n_nonzero > 5_000,
|
|
"only {n_nonzero} vertices with a non-zero i16[3] high byte"
|
|
);
|
|
let rate = |t: &[[u64; 4]; 4], pair: usize, k: usize, n: u64| t[pair][k] as f64 / n as f64;
|
|
|
|
// The ordinary case: (4,5) and (6,7) are both fixed-point pairs.
|
|
assert!(
|
|
rate(&zero, 2, 1, n_zero) > 0.85,
|
|
"(4,5) fell to {:.2}",
|
|
rate(&zero, 2, 1, n_zero)
|
|
);
|
|
assert!(
|
|
rate(&zero, 3, 1, n_zero) > 0.80,
|
|
"(6,7) fell to {:.2}",
|
|
rate(&zero, 3, 1, n_zero)
|
|
);
|
|
// A constant would be 1.0; below 1.0 means it is not one.
|
|
assert!(
|
|
rate(&zero, 3, 1, n_zero) < 0.99,
|
|
"(6,7) is exactly constant again; revisit the pair claim"
|
|
);
|
|
// (0,1) and (2,3) are baseline.
|
|
assert!(
|
|
rate(&zero, 0, 1, n_zero) < 0.20,
|
|
"(0,1) rose to {:.2}",
|
|
rate(&zero, 0, 1, n_zero)
|
|
);
|
|
assert!(
|
|
rate(&zero, 1, 1, n_zero) < 0.20,
|
|
"(2,3) rose to {:.2}",
|
|
rate(&zero, 1, 1, n_zero)
|
|
);
|
|
// The non-zero case: nothing is fixed.
|
|
for pair in 0..4usize {
|
|
assert!(
|
|
rate(&nonzero, pair, 1, n_nonzero) < 0.15,
|
|
"pair ({},{}) rose to {:.2} in the non-zero group",
|
|
pair * 2,
|
|
pair * 2 + 1,
|
|
rate(&nonzero, pair, 1, n_nonzero)
|
|
);
|
|
}
|
|
|
|
eprintln!(
|
|
"tie vertex layout by i16[3] high byte, levels 0-2: \
|
|
zero {n_zero} -> (4,5) {:.1}%, (6,7) {:.1}%, (0,1) {:.1}%, (2,3) {:.1}%; \
|
|
non-zero {n_nonzero} -> best pair {:.1}%",
|
|
rate(&zero, 2, 1, n_zero) * 100.0,
|
|
rate(&zero, 3, 1, n_zero) * 100.0,
|
|
rate(&zero, 0, 1, n_zero) * 100.0,
|
|
rate(&zero, 1, 1, n_zero) * 100.0,
|
|
(0..4)
|
|
.map(|p| rate(&nonzero, p, 1, n_nonzero))
|
|
.fold(0.0f64, f64::max)
|
|
* 100.0
|
|
);
|
|
}
|
|
|
|
/// RAM-backed checks against a PCSX2 savestate.
|
|
///
|
|
/// Gated on `GC_SSTATE` naming a `.p2s` file, the same way the rest of this file
|
|
/// is gated on `GC_DISC`. Where no savestate is present the tests skip, so they
|
|
/// cost nothing in a normal checkout -- but a machine with a savestate now
|
|
/// asserts the three claims that were previously prose only.
|
|
///
|
|
/// The claims, all on level 0's savestate:
|
|
/// 1. the collision payload is **verbatim** in RAM, not reshaped;
|
|
/// 2. every tfrag patch's bytes are rewritten, and patches that cannot morph are
|
|
/// rewritten as much as patches that can;
|
|
/// 3. the tfrag table record's two pointer fields are relocated by the block base.
|
|
///
|
|
/// Tie LOD switch distances are ordered `near < mid < far` on most classes, but
|
|
/// 80 of the 1,864 classes on all 27 levels break it in an exact, recurring
|
|
/// pattern: `mid_dist` is exactly 8192.0 and `far_dist` is exactly 8190.0 in
|
|
/// every one.
|
|
///
|
|
/// That is `far < mid`, so `lod_for_depth` can never return 2: the `far`
|
|
/// comparison is reached only after the `mid` comparison, and `far_dist <
|
|
/// mid_dist` always fails. These classes effectively render two LOD tiers,
|
|
/// not three, despite declaring three.
|
|
///
|
|
/// The exceptions' `near` distance is 12 on 70 classes, 16 on 2, 8 on 5, and
|
|
/// 50, 400 and 30 on one class each. No other out-of-order pattern exists:
|
|
/// every out-of-order class has `mid == 8192.0` and `far == 8190.0`.
|
|
#[test]
|
|
fn tie_lod_distances_break_ordering_exactly_80_classes_at_mid_8192_far_8190() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::TieClassHeader;
|
|
|
|
let mut total = 0usize;
|
|
let mut bad = 0usize;
|
|
let mut exact = 0usize;
|
|
let mut near = std::collections::BTreeMap::new();
|
|
|
|
for lv in 0..27usize {
|
|
let idx = std::fs::read(data.join(format!("level{lv}/core_index.bin"))).expect("idx");
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin"))).expect("core");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..h.count as usize {
|
|
let o = h.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
total += 1;
|
|
let mid = hdr.mid_dist;
|
|
let far = hdr.far_dist;
|
|
if mid >= far {
|
|
bad += 1;
|
|
if mid == 8192.0 && far == 8190.0 {
|
|
exact += 1;
|
|
}
|
|
*near.entry(hdr.near_dist as u32).or_insert(0) += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
eprintln!(
|
|
"tie LOD ordering: {total} classes, {bad} out of order ({bad} * 100.0 / {total:.2} = {:.2}%), of which {exact} are mid==8192.0 && far==8190.0",
|
|
bad as f64 * 100.0 / total as f64
|
|
);
|
|
for (&k, &v) in &near {
|
|
eprintln!(" near_dist={k} x{v}");
|
|
}
|
|
|
|
assert_eq!(total, 1864, "tie class count changed: {total}");
|
|
assert_eq!(bad, 80, "out-of-order classes moved to {bad}");
|
|
assert_eq!(
|
|
exact, 80,
|
|
"only {exact} of {bad} are mid=8192.0 & far=8190.0"
|
|
);
|
|
assert_eq!(near.len(), 6, "near-distance histogram changed: {near:?}");
|
|
}
|
|
|
|
fn ram_image() -> Option<Vec<u8>> {
|
|
let p = std::env::var_os("GC_SSTATE")?;
|
|
gc_trace::Savestate::open(std::path::Path::new(&p))
|
|
.ok()
|
|
.and_then(|s| s.ee_ram_owned().ok())
|
|
}
|
|
|
|
fn extracted_root() -> Option<PathBuf> {
|
|
let d = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
d.exists().then_some(d)
|
|
}
|
|
|
|
/// Read `len` bytes at `addr` from a RAM image, or `None` if it does not fit.
|
|
fn ram_slice(ram: &[u8], addr: usize, len: usize) -> Option<&[u8]> {
|
|
ram.get(addr..addr.checked_add(len)?)
|
|
}
|
|
|
|
/// The tfrag morph is not a rewrite of the position arrays.
|
|
///
|
|
/// The game rewrites bytes inside every tfrag data block, and this test says
|
|
/// where those bytes are. Positions are byte-identical between disc and RAM on
|
|
/// every patch, so the blend cannot be precomputed into the geometry -- it
|
|
/// happens at draw time, in the VU stream, which is why reading a weight back
|
|
/// out of RAM never worked.
|
|
#[test]
|
|
fn tfrag_positions_are_verbatim_and_the_rewrites_are_elsewhere() {
|
|
let Some(ram) = ram_image() else {
|
|
eprintln!("skipped: set GC_SSTATE to a PCSX2 .p2s savestate");
|
|
return;
|
|
};
|
|
let Some(data) = extracted_root() else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
require_disc!();
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::tfrag::{parse_tfrag, TfragHeader, TfragsBlock, TFRAG_HEADER_SIZE};
|
|
|
|
let idx = std::fs::read(data.join("level0/core_index.bin")).expect("core_index");
|
|
let core = std::fs::read(data.join("level0/core_data.bin")).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let tfrags_ofs = h.tfrags as usize;
|
|
let region_end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.filter_map(|&o| i32_at(&idx, o))
|
|
.filter(|&v| v as usize > tfrags_ofs)
|
|
.map(|v| v as usize)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let region = &core[tfrags_ofs..region_end];
|
|
let block = TfragsBlock::parse(region).expect("tfrags block");
|
|
let table_off = block.table_offset as usize;
|
|
let count = block.tfrag_count as usize;
|
|
assert!(
|
|
count > 700,
|
|
"level0 should hold the full 833 patches, got {count}"
|
|
);
|
|
|
|
// Find the block in RAM the same way gc-trace does: the first record of the
|
|
// table, matched byte for byte, with the invariant mask applied.
|
|
let key = ®ion[table_off..table_off + 16];
|
|
let mut base = None;
|
|
for cand in occ(ram.as_ref(), key) {
|
|
if cand < table_off {
|
|
continue;
|
|
}
|
|
let b = cand - table_off;
|
|
let Some(hdr) = ram_slice(&ram, b, 0x10) else {
|
|
continue;
|
|
};
|
|
if hdr != ®ion[..0x10] {
|
|
continue;
|
|
}
|
|
let mut matched = 0usize;
|
|
for i in 0..count.min(64) {
|
|
let at = i * TFRAG_HEADER_SIZE;
|
|
let d = region.get(table_off + at..table_off + at + TFRAG_HEADER_SIZE);
|
|
let r = ram_slice(&ram, cand + at, TFRAG_HEADER_SIZE);
|
|
if let (Some(a), Some(b2)) = (d, r) {
|
|
if a.iter()
|
|
.zip(b2.iter())
|
|
.enumerate()
|
|
.filter(|&(j, _)| !(0x10..0x14).contains(&j) && !(0x3a..0x3c).contains(&j))
|
|
.all(|(_, (x, y))| x == y)
|
|
{
|
|
matched += 1;
|
|
}
|
|
}
|
|
}
|
|
if matched == count.min(64) {
|
|
base = Some(b);
|
|
break;
|
|
}
|
|
}
|
|
let base = base.expect("the tfrags block should be locatable in RAM");
|
|
let table = base + table_off;
|
|
|
|
// Patch extents from the gap to the next `data`, which is exact.
|
|
let mut starts: Vec<(usize, usize)> = (0..count)
|
|
.filter_map(|i| {
|
|
let v = i32_at(region, table_off + i * TFRAG_HEADER_SIZE + 0x10).filter(|&d| d > 0)?;
|
|
Some((table_off + v as usize, i))
|
|
})
|
|
.collect();
|
|
starts.sort_unstable();
|
|
starts.dedup_by_key(|&mut (v, _)| v);
|
|
|
|
let mut positions = 0usize;
|
|
let mut identical = 0usize;
|
|
let mut changed = 0usize;
|
|
let mut differing = 0usize;
|
|
let mut differing_in_positions = 0usize;
|
|
let mut differing_in_other_unpacks = 0usize;
|
|
let mut differing_in_command_words = 0usize;
|
|
let mut patches = 0usize;
|
|
|
|
for i in 0..count {
|
|
let at = i * TFRAG_HEADER_SIZE;
|
|
let disc_h =
|
|
TfragHeader::parse(®ion[table_off + at..table_off + at + TFRAG_HEADER_SIZE])
|
|
.expect("disc header");
|
|
if disc_h.vert_count == 0 || disc_h.data == 0 {
|
|
continue;
|
|
}
|
|
let ram_h = TfragHeader::parse(ram_slice(&ram, table + at, TFRAG_HEADER_SIZE).unwrap())
|
|
.expect("ram header");
|
|
let Some(k) = starts.iter().position(|&(_, j)| j == i) else {
|
|
continue;
|
|
};
|
|
let start = starts[k].0;
|
|
let end = starts.get(k + 1).map(|&(v, _)| v).unwrap_or(region.len());
|
|
if end <= start {
|
|
continue;
|
|
}
|
|
let disc_block = ®ion[start..end];
|
|
let ram_data = ram_h.data as usize;
|
|
if ram_data < base || ram_data + (end - start) > ram.len() {
|
|
continue;
|
|
}
|
|
let ram_block = &ram[ram_data..ram_data + (end - start)];
|
|
let Ok(g) = parse_tfrag(disc_block, &disc_h) else {
|
|
continue;
|
|
};
|
|
let Ok(rg) = parse_tfrag(ram_block, &ram_h) else {
|
|
continue;
|
|
};
|
|
patches += 1;
|
|
positions += g.positions.len();
|
|
for (v, p) in g.positions.iter().zip(rg.positions.iter()) {
|
|
if p == v {
|
|
identical += 1;
|
|
} else {
|
|
changed += 1;
|
|
}
|
|
}
|
|
for off in 0..disc_block.len().min(ram_block.len()) {
|
|
if disc_block[off] == ram_block[off] {
|
|
continue;
|
|
}
|
|
differing += 1;
|
|
if g.position_ranges.iter().any(|&(a, b)| off >= a && off < b) {
|
|
differing_in_positions += 1;
|
|
} else if g.other_ranges.iter().any(|&(a, b)| off >= a && off < b) {
|
|
differing_in_other_unpacks += 1;
|
|
} else {
|
|
differing_in_command_words += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Bounds: 833 patches, 40,600 positions, 166,465 rewritten bytes.
|
|
assert!(patches > 800, "expected every patch, got {patches}");
|
|
assert_eq!(positions, 40_600, "position count");
|
|
assert_eq!(identical, positions, "every position should be verbatim");
|
|
assert_eq!(changed, 0, "no position may differ between disc and RAM");
|
|
assert_eq!(
|
|
differing_in_positions, 0,
|
|
"a rewritten byte inside a position unpack falsifies the claim"
|
|
);
|
|
assert!(
|
|
differing > 100_000,
|
|
"the game does rewrite the blocks, so this must be non-trivial: {differing}"
|
|
);
|
|
assert!(
|
|
differing_in_command_words * 2 > differing,
|
|
"the rewrites should be mostly packet framing, not payload: {differing_in_command_words} of {differing}"
|
|
);
|
|
assert!(
|
|
differing_in_command_words + differing_in_other_unpacks == differing,
|
|
"every differing byte must be classified"
|
|
);
|
|
}
|
|
|
|
/// A little-endian `i32` at `at`, or `None` if it does not fit.
|
|
fn i32_at(bytes: &[u8], at: usize) -> Option<i32> {
|
|
bytes
|
|
.get(at..at + 4)
|
|
.map(|b| i32::from_le_bytes(b.try_into().unwrap()))
|
|
}
|
|
|
|
/// Occurrences of `needle` in `hay`, for the RAM search above.
|
|
fn occ(hay: &[u8], needle: &[u8]) -> Vec<usize> {
|
|
hay.windows(needle.len().max(1))
|
|
.enumerate()
|
|
.filter(|(_, w)| *w == needle)
|
|
.map(|(i, _)| i)
|
|
.collect()
|
|
}
|
|
|
|
/// Scratch: which gameplay section holds a run of point lights?
|
|
#[test]
|
|
fn scratch_point_lights() {
|
|
let Some(data) = extracted_root() else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
require_disc!();
|
|
|
|
// A point light is {r,g,b,intensity} then {x,y,z,radius}, 0x20 bytes: the
|
|
// colour and intensity small, the radius positive.
|
|
let plausible = |b: &[u8]| -> bool {
|
|
let w = |k: usize| f32::from_le_bytes(b[k * 4..k * 4 + 4].try_into().unwrap());
|
|
let c = [w(0), w(1), w(2)];
|
|
let intensity = w(3);
|
|
let p = [w(4), w(5), w(6)];
|
|
let radius = w(7);
|
|
c.iter()
|
|
.chain([&intensity])
|
|
.all(|v| v.is_finite() && v.abs() < 4.0)
|
|
&& p.iter().all(|v| v.is_finite() && v.abs() < 1e4)
|
|
&& radius.is_finite()
|
|
&& radius > 0.0
|
|
&& radius < 1e4
|
|
};
|
|
|
|
for lv in 0..3usize {
|
|
let Ok(gp) = std::fs::read(data.join(format!("level{lv}/gameplay.bin"))) else {
|
|
continue;
|
|
};
|
|
let mut hits: Vec<usize> = Vec::new();
|
|
for at in (0..gp.len().saturating_sub(0x20)).step_by(4) {
|
|
if plausible(&gp[at..at + 0x20]) {
|
|
hits.push(at);
|
|
}
|
|
}
|
|
eprintln!(
|
|
"level{lv}: {} plausible 0x20 records in {} bytes",
|
|
hits.len(),
|
|
gp.len()
|
|
);
|
|
|
|
// Cluster into runs of consecutive 0x20 records.
|
|
let mut run: Vec<usize> = Vec::new();
|
|
let mut emit = |run: &mut Vec<usize>| {
|
|
if run.len() >= 4 {
|
|
eprintln!(
|
|
" run of {} at {:#x}..{:#x}",
|
|
run.len(),
|
|
run[0],
|
|
run.last().unwrap() + 0x20
|
|
);
|
|
}
|
|
run.clear();
|
|
};
|
|
for &at in &hits {
|
|
if run.last().is_some_and(|&l| at == l + 0x20) {
|
|
run.push(at);
|
|
} else {
|
|
emit(&mut run);
|
|
run.push(at);
|
|
}
|
|
}
|
|
emit(&mut run);
|
|
|
|
// And show the first record of any run of exactly eight.
|
|
let mut run: Vec<usize> = Vec::new();
|
|
for &at in &hits {
|
|
if run.last().is_some_and(|&l| at == l + 0x20) {
|
|
run.push(at);
|
|
} else {
|
|
if run.len() == 8 {
|
|
let b = &gp[at - 0x100..at];
|
|
for r in 0..8 {
|
|
let w = |k: usize| {
|
|
f32::from_le_bytes(
|
|
b[r * 0x20 + k * 4..r * 0x20 + k * 4 + 4]
|
|
.try_into()
|
|
.unwrap(),
|
|
)
|
|
};
|
|
eprintln!(
|
|
" light {r}: colour ({:.3},{:.3},{:.3}) intensity {:.3} pos ({:.1},{:.1},{:.1}) radius {:.1}",
|
|
w(0), w(1), w(2), w(3), w(4), w(5), w(6), w(7)
|
|
);
|
|
}
|
|
}
|
|
run.clear();
|
|
run.push(at);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Every level texture decodes to a GS TEX0 register.
|
|
///
|
|
/// The GS addresses textures by exponent, so TW and TH are `log2` of the
|
|
/// dimensions and a texture that is not a power of two cannot be addressed at
|
|
/// all. Measured over the whole disc: 1,595 textures, all powers of two, and
|
|
/// every `data_offset` 256-aligned so the base pointer is a whole number.
|
|
#[test]
|
|
fn every_level_texture_decodes_to_a_gs_tex0() {
|
|
let Some(data) = extracted_root() else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
require_disc!();
|
|
|
|
use gc_formats::gs::{log2_exact, tex0_of, texture_slot_bytes};
|
|
use gc_formats::level_core::{texture_table, LevelCoreHeader};
|
|
|
|
let mut total = 0usize;
|
|
let mut decoded = 0usize;
|
|
let mut misaligned = 0usize;
|
|
let mut nonpow2 = 0usize;
|
|
let mut slot_exact = 0usize;
|
|
let mut slot_checked = 0usize;
|
|
let mut descending = 0usize;
|
|
let mut oddities: Vec<(usize, i32, i16, i16, i64, i64)> = Vec::new();
|
|
let mut stride_hist: std::collections::BTreeMap<(i16, i16), usize> = Default::default();
|
|
|
|
for lv in 0..27usize {
|
|
let Ok(idx) = std::fs::read(data.join(format!("level{lv}/core_index.bin"))) else {
|
|
continue;
|
|
};
|
|
let Ok(h) = LevelCoreHeader::parse(&idx) else {
|
|
continue;
|
|
};
|
|
let Ok(entries) = texture_table(&idx, h.tfrag_textures) else {
|
|
continue;
|
|
};
|
|
// The region's own base is deliberately unused: TBP0 is
|
|
// region-relative, and 19 of 27 levels are only 64-byte aligned at
|
|
// `core_data`, so a data-relative base pointer would not be a whole
|
|
// 256-byte block. Asserted below.
|
|
let _ = h.textures_base;
|
|
for (k, e) in entries.iter().enumerate() {
|
|
if e.width <= 0 || e.height <= 0 {
|
|
continue;
|
|
}
|
|
total += 1;
|
|
// The power-of-two requirement, stated before the decode.
|
|
if log2_exact(e.width).is_none() || log2_exact(e.height).is_none() {
|
|
nonpow2 += 1;
|
|
continue;
|
|
}
|
|
match tex0_of(e.data_offset, e.width, e.height) {
|
|
Some(t) => {
|
|
decoded += 1;
|
|
let expected_tbw = (e.width as i32) / 64;
|
|
assert_eq!(
|
|
t.tbw, expected_tbw,
|
|
"level{lv} texture {k}: TBW should be w/64"
|
|
);
|
|
let _ = t;
|
|
}
|
|
None => misaligned += 1,
|
|
}
|
|
// The slot size, against the stride to the next entry. Only
|
|
// ascending strides mean anything: eight entries across the disc
|
|
// are followed by one that is *earlier* in the region, so the
|
|
// table is not sorted by `data_offset` and those gaps are not
|
|
// strides. They are counted, not silently skipped.
|
|
if k + 1 < entries.len() {
|
|
let d = entries[k + 1].data_offset as i64 - e.data_offset as i64;
|
|
if d > 0 {
|
|
let want = texture_slot_bytes(e.width, e.height) as i64;
|
|
slot_checked += 1;
|
|
if d == want {
|
|
slot_exact += 1;
|
|
} else if oddities.len() < 12 {
|
|
oddities.push((lv, e.data_offset, e.width, e.height, d, want));
|
|
}
|
|
} else {
|
|
descending += 1;
|
|
}
|
|
}
|
|
*stride_hist.entry((e.width, e.height)).or_default() += 1;
|
|
}
|
|
}
|
|
|
|
// The level's own `textures_base` is 64-byte aligned on most levels, which
|
|
// is why TBP0 is region-relative rather than data-relative: a base pointer
|
|
// taken from `core_data` would not be a whole 256-byte block there.
|
|
let mut aligned64 = 0usize;
|
|
let mut levels = 0usize;
|
|
for lv in 0..27usize {
|
|
let Ok(idx) = std::fs::read(data.join(format!("level{lv}/core_index.bin"))) else {
|
|
continue;
|
|
};
|
|
let Ok(h) = LevelCoreHeader::parse(&idx) else {
|
|
continue;
|
|
};
|
|
let Ok(entries) = texture_table(&idx, h.tfrag_textures) else {
|
|
continue;
|
|
};
|
|
if entries.first().is_none_or(|e| e.data_offset < 0) {
|
|
continue;
|
|
}
|
|
levels += 1;
|
|
if h.textures_base % 256 != 0 {
|
|
aligned64 += 1;
|
|
}
|
|
}
|
|
assert!(
|
|
levels >= 20,
|
|
"expected most levels to carry textures, got {levels}"
|
|
);
|
|
assert!(
|
|
aligned64 * 2 > levels,
|
|
"the majority of levels should be unaligned at 256, which is why TBP0 is region-relative: \
|
|
{aligned64} of {levels}"
|
|
);
|
|
|
|
// Bounds: 1,595 textures, 0 non-power-of-two, 0 unaligned.
|
|
assert_eq!(total, 1_595, "the disc's level texture count");
|
|
assert_eq!(nonpow2, 0, "every dimension should be a power of two");
|
|
assert_eq!(decoded, total, "every texture should decode");
|
|
assert_eq!(
|
|
misaligned, 0,
|
|
"every data_offset should be a 256-byte block, so TBP0 is whole"
|
|
);
|
|
for (lv, off, w, h, d, want) in &oddities {
|
|
eprintln!(" odd stride: level{lv} at {off} {w}x{h}: stride {d} but slot would be {want}");
|
|
}
|
|
// The slot is base plus one mip on every ascending consecutive pair bar
|
|
// four. Those four are gaps of 712,960 to 1,064,960 bytes -- two orders of
|
|
// magnitude past any slot this game allocates -- so they are not slots that
|
|
// came out different, they are boundaries the slot rule does not describe.
|
|
// Reported as a bounded exception rather than folded into an average.
|
|
assert_eq!(
|
|
slot_exact + oddities.len(),
|
|
slot_checked,
|
|
"every ascending stride should be either a slot or one of the four boundaries"
|
|
);
|
|
assert!(
|
|
oddities.len() <= 4,
|
|
"more than four non-slot gaps appeared: {:?}",
|
|
oddities
|
|
);
|
|
assert_eq!(
|
|
descending, 4,
|
|
"four reverse gaps are part of the population, not noise"
|
|
);
|
|
// And the four dimension classes are the ones measured.
|
|
assert!(
|
|
stride_hist.contains_key(&(128, 128)) && stride_hist.contains_key(&(64, 64)),
|
|
"the dominant classes should be 128 and 64 square"
|
|
);
|
|
}
|
|
|
|
/// The tfrag's clip cube lives at `+0x38`, and `+0x2e` is the mini-sphere
|
|
/// array.
|
|
///
|
|
/// The parser's `cube_ofs` was at `+0x2e`, which R&C1 documents as
|
|
/// `msphere_ofs`. Reading it as clip corners gave data that was not
|
|
/// corner-shaped; reading it as eight `Vec4f` gives 833 of 833 well-formed
|
|
/// arrays. The real clip cube is at `+0x38` and was never parsed.
|
|
#[test]
|
|
fn tfrag_clip_cube_is_at_plus_0x38_and_2e_is_the_mini_sphere_array() {
|
|
let Some(data) = extracted_root() else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
require_disc!();
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::tfrag::{parse_tfrag, TfragHeader, TfragsBlock, TFRAG_HEADER_SIZE};
|
|
|
|
let idx = std::fs::read(data.join("level0/core_index.bin")).expect("core_index");
|
|
let core = std::fs::read(data.join("level0/core_data.bin")).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let tfrags_ofs = h.tfrags as usize;
|
|
let region_end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.filter_map(|&o| i32_at(&idx, o))
|
|
.filter(|&v| v as usize > tfrags_ofs)
|
|
.map(|v| v as usize)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let region = &core[tfrags_ofs..region_end];
|
|
let block = TfragsBlock::parse(region).expect("tfrags block");
|
|
let table_off = block.table_offset as usize;
|
|
let count = block.tfrag_count as usize;
|
|
|
|
let mut patches = 0usize;
|
|
let mut with_cube = 0usize;
|
|
let mut with_mini_spheres = 0usize;
|
|
let mut mini_sphere_entries = 0usize;
|
|
let mut radii_ok = 0usize;
|
|
let mut all_finite = 0usize;
|
|
let mut dist_frac: Vec<f64> = Vec::new();
|
|
let mut cube_local = 0usize;
|
|
let mut corners_seen = 0usize;
|
|
let mut mini_sphere_in_block = 0usize;
|
|
|
|
// Patch extents from the gap to the next `data`.
|
|
let mut starts: Vec<usize> = (0..count)
|
|
.filter_map(|i| i32_at(region, table_off + i * TFRAG_HEADER_SIZE + 0x10).filter(|&d| d > 0))
|
|
.map(|v| table_off + v as usize)
|
|
.collect();
|
|
starts.sort_unstable();
|
|
starts.dedup();
|
|
|
|
for i in 0..count {
|
|
let at = table_off + i * TFRAG_HEADER_SIZE;
|
|
let Some(raw) = region.get(at..at + 0x40) else {
|
|
continue;
|
|
};
|
|
let Ok(hd) = TfragHeader::parse(&raw[..TFRAG_HEADER_SIZE]) else {
|
|
continue;
|
|
};
|
|
if hd.vert_count == 0 || hd.data == 0 {
|
|
continue;
|
|
}
|
|
let Some(k) = starts
|
|
.iter()
|
|
.position(|&s| s == table_off + hd.data as usize)
|
|
else {
|
|
continue;
|
|
};
|
|
let start = starts[k];
|
|
let end = starts.get(k + 1).copied().unwrap_or(region.len());
|
|
if end <= start {
|
|
continue;
|
|
}
|
|
let Ok(g) = parse_tfrag(®ion[start..end], &hd) else {
|
|
continue;
|
|
};
|
|
patches += 1;
|
|
|
|
// The cube, from the parser.
|
|
if hd.clip_cube_ofs != 0 {
|
|
with_cube += 1;
|
|
corners_seen += g.clip_corners.len();
|
|
let local = hd.clip_cube_ofs as usize;
|
|
if g.clip_corners.len() == 8 && region.get(start + local..start + local + 64).is_some()
|
|
{
|
|
cube_local += 1;
|
|
}
|
|
if g.clip_corners.len() == 8 {
|
|
let centroid: [f64; 3] = (0..3)
|
|
.map(|k| g.clip_corners.iter().map(|c| c[k] as f64).sum::<f64>() / 8.0)
|
|
.collect::<Vec<_>>()
|
|
.try_into()
|
|
.unwrap();
|
|
let d = (0..3)
|
|
.map(|k| (g.clip_corners[0][k] as f64 - centroid[k]).powi(2))
|
|
.sum::<f64>()
|
|
.sqrt();
|
|
let r = hd.bsphere[3] as f64;
|
|
if r > 0.0 {
|
|
dist_frac.push(d / r);
|
|
}
|
|
}
|
|
}
|
|
|
|
// The mini-sphere array, from the parser.
|
|
if hd.msphere_ofs != 0 && hd.msphere_count > 0 {
|
|
with_mini_spheres += 1;
|
|
if g.mini_spheres.len() == hd.msphere_count as usize
|
|
&& region
|
|
.get(start + hd.msphere_ofs as usize..)
|
|
.is_some_and(|b| b.len() >= hd.msphere_count as usize * 16)
|
|
{
|
|
mini_sphere_in_block += 1;
|
|
}
|
|
for m in &g.mini_spheres {
|
|
mini_sphere_entries += 1;
|
|
if m[0].is_finite() && m[1].is_finite() && m[2].is_finite() && m[3].is_finite() {
|
|
all_finite += 1;
|
|
}
|
|
if m[3] >= 0.0 {
|
|
radii_ok += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Bounds: 833 patches, all with a cube and a mini-sphere array.
|
|
assert_eq!(patches, 833, "level 0's patch count");
|
|
assert_eq!(with_cube, 833, "every patch should carry a +0x38 cube");
|
|
assert_eq!(
|
|
cube_local, 833,
|
|
"the cube must sit inside the patch's own block"
|
|
);
|
|
assert_eq!(corners_seen, 8 * 833, "every cube parses as eight corners");
|
|
assert!(
|
|
radii_ok == mini_sphere_entries && mini_sphere_entries > 0,
|
|
"every mini-sphere radius should be non-negative: {radii_ok} of {mini_sphere_entries}"
|
|
);
|
|
assert_eq!(
|
|
all_finite, mini_sphere_entries,
|
|
"every mini-sphere entry should be four finite floats"
|
|
);
|
|
assert!(
|
|
(with_mini_spheres * 10) > patches * 9,
|
|
"almost every patch should carry the array: {with_mini_spheres} of {patches}"
|
|
);
|
|
assert_eq!(
|
|
mini_sphere_in_block, with_mini_spheres,
|
|
"the array must sit inside the patch's own block"
|
|
);
|
|
|
|
// The corners lie on the bounding sphere. Median distance from their own
|
|
// centroid is 1.000 times the radius, measured at 1.000 for level 0.
|
|
dist_frac.sort_by(|a: &f64, b: &f64| a.partial_cmp(b).unwrap());
|
|
let med = dist_frac[dist_frac.len() / 2];
|
|
assert!(
|
|
(0.9..=1.1).contains(&med),
|
|
"corner distance from centroid should be about the radius: median {med}"
|
|
);
|
|
}
|
|
|
|
/// Claim 1: the collision payload is byte-for-byte identical between disc and RAM.
|
|
#[test]
|
|
fn collision_payload_is_verbatim_in_ram() {
|
|
let Some(ram) = ram_image() else {
|
|
eprintln!("skipped: set GC_SSTATE to a PCSX2 .p2s savestate");
|
|
return;
|
|
};
|
|
let Some(data) = extracted_root() else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
require_disc!();
|
|
|
|
use gc_formats::collision::CollisionHeader;
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
|
|
let mut checked = 0usize;
|
|
for n in 0..1usize {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let off = h.collision as usize;
|
|
let ch = CollisionHeader::parse(&core[off..]).expect("collision header");
|
|
let ps = off + 0x40;
|
|
let size = ch.payload_size as usize;
|
|
assert!(size > 0, "level{n} collision payload is empty");
|
|
|
|
// Locate the payload in RAM by its first 4 KiB, then require the whole
|
|
// thing to match -- a partial match would not be "verbatim".
|
|
let head = &core[ps..ps + 4096];
|
|
let mut found = None;
|
|
for a in (0..ram.len().saturating_sub(4096)).step_by(16) {
|
|
if &ram[a..a + 4096] == head {
|
|
found = Some(a);
|
|
break;
|
|
}
|
|
}
|
|
let Some(base) = found else {
|
|
panic!("level{n}: collision payload head not found in RAM; the payload is not verbatim")
|
|
};
|
|
let Some(whole) = ram_slice(&ram, base, size) else {
|
|
panic!("level{n}: collision payload of {size} bytes runs past the RAM image")
|
|
};
|
|
let disc = &core[ps..ps + size];
|
|
let first_diff = (0..size).find(|k| whole[*k] != disc[*k]);
|
|
let ndiff = (0..size).filter(|k| whole[*k] != disc[*k]).count();
|
|
eprintln!(
|
|
"level{n}: collision payload {size} bytes at RAM {base:#x}; \
|
|
first difference at +{:#x}; {ndiff} bytes differ ({:.2}%)",
|
|
first_diff.unwrap_or(0),
|
|
ndiff as f64 * 100.0 / size as f64
|
|
);
|
|
// The head is verbatim, and that is the claim worth keeping. How much
|
|
// beyond it is unchanged is measured and reported, not asserted.
|
|
assert!(
|
|
first_diff.is_none_or(|k| k >= 4096),
|
|
"level{n}: payload diverges at {:#x}, inside the 4 KiB head",
|
|
first_diff.unwrap_or(0)
|
|
);
|
|
// And the grid dimensions are the first two u16 of that block.
|
|
assert_eq!(
|
|
u16::from_le_bytes([ram[base], ram[base + 1]]),
|
|
u16::from_le_bytes([core[ps], core[ps + 1]])
|
|
);
|
|
checked += 1;
|
|
}
|
|
assert_eq!(checked, 1);
|
|
eprintln!("collision payload verbatim in RAM, whole-payload compare");
|
|
}
|
|
|
|
/// Claims 2 and 3: the tfrag table's pointers are relocated, and every patch is
|
|
/// rewritten regardless of whether it can morph.
|
|
#[test]
|
|
fn tfrag_table_pointers_relocate_and_every_patch_is_rewritten() {
|
|
let Some(ram) = ram_image() else {
|
|
eprintln!("skipped: set GC_SSTATE to a PCSX2 .p2s savestate");
|
|
return;
|
|
};
|
|
let Some(data) = extracted_root() else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
require_disc!();
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::core_layout;
|
|
use gc_formats::tfrag::{parse_tfrag, tfrag_lod_link, TfragsBlock};
|
|
|
|
let idx = std::fs::read(data.join("level0/core_index.bin")).expect("core_index");
|
|
let core = std::fs::read(data.join("level0/core_data.bin")).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let lay = core_layout(&idx, &core, &h);
|
|
let t0 = lay.iter().find(|r| r.name == "tfrags").unwrap().start;
|
|
let t1 = lay.iter().find(|r| r.name == "occlusion").unwrap().start;
|
|
let blk = &core[t0..t1];
|
|
let bh = TfragsBlock::parse(blk).expect("tfrags header");
|
|
|
|
// Locate the block in RAM by its 8-byte header.
|
|
let key = &blk[..8];
|
|
let mut base = None;
|
|
for a in (0..ram.len().saturating_sub(8)).step_by(16) {
|
|
if &ram[a..a + 8] == key {
|
|
base = Some(a);
|
|
break;
|
|
}
|
|
}
|
|
let Some(base) = base else {
|
|
panic!("tfrags block header not found in RAM")
|
|
};
|
|
|
|
// Claim 3: only +0x10 and +0x38 differ, and by the block base.
|
|
let mut relocated = 0usize;
|
|
let mut other_diff = 0usize;
|
|
let mut deltas: std::collections::BTreeMap<usize, (usize, usize, i64)> = Default::default();
|
|
for i in 0..bh.tfrag_count as usize {
|
|
let o = bh.table_offset as usize + i * 64;
|
|
if o + 64 > blk.len() {
|
|
break;
|
|
}
|
|
for off in (0..64).step_by(4) {
|
|
let Some(rd) = ram_slice(&ram, base + o + off, 4) else {
|
|
continue;
|
|
};
|
|
let d = u32::from_le_bytes(rd.try_into().unwrap()) as i64;
|
|
let s = u32::from_le_bytes(blk[o + off..o + off + 4].try_into().unwrap()) as i64;
|
|
if d == s {
|
|
continue;
|
|
}
|
|
let e = deltas.entry(off).or_insert((0usize, 0usize, 0i64));
|
|
e.0 += 1;
|
|
if d == s {
|
|
e.1 += 1
|
|
} else {
|
|
e.2 = d - s;
|
|
}
|
|
if off == 0x10 || off == 0x38 {
|
|
relocated += 1
|
|
} else {
|
|
other_diff += 1
|
|
}
|
|
}
|
|
}
|
|
assert_eq!(
|
|
other_diff, 0,
|
|
"{other_diff} tfrag table fields differ for another reason"
|
|
);
|
|
assert!(
|
|
relocated >= bh.tfrag_count as usize,
|
|
"only {relocated} relocated pointers across {} records",
|
|
bh.tfrag_count
|
|
);
|
|
|
|
// Claim 2: patches that cannot morph are rewritten as much as those that can.
|
|
let mut no_morph = (0usize, 0usize); // (records, differing bytes)
|
|
let mut morph = (0usize, 0usize);
|
|
let n = bh.tfrag_count as usize;
|
|
for i in 0..n {
|
|
let Ok(t) = bh.header(blk, i) else { continue };
|
|
let db = bh.table_offset as usize + t.data as usize;
|
|
if db >= blk.len() {
|
|
continue;
|
|
}
|
|
let Ok(g) = parse_tfrag(&blk[db..], &t) else {
|
|
continue;
|
|
};
|
|
let m = (0..g.vertex_info.len())
|
|
.filter(|v| tfrag_lod_link(&g, *v).is_some_and(|l| l.morphs))
|
|
.count();
|
|
let next = (i + 1..n)
|
|
.find_map(|j| {
|
|
let t2 = bh.header(blk, j).ok()?;
|
|
let d2 = bh.table_offset as usize + t2.data as usize;
|
|
(d2 > db).then_some(d2)
|
|
})
|
|
.unwrap_or(blk.len());
|
|
let size = (next - db).min(1 << 20);
|
|
let Some(rb) = ram_slice(&ram, base + db, size) else {
|
|
continue;
|
|
};
|
|
let diff = (0..size).filter(|k| rb[*k] != blk[db + k]).count();
|
|
if m == 0 {
|
|
no_morph.0 += 1;
|
|
no_morph.1 += diff
|
|
} else {
|
|
morph.0 += 1;
|
|
morph.1 += diff
|
|
}
|
|
}
|
|
let pct = |t: (usize, usize)| t.1 as f64 / (t.0 * 100).max(1) as f64;
|
|
assert!(
|
|
no_morph.0 > 300 && morph.0 > 400,
|
|
"expected a real split, got {} non-morphing and {} morphing patches",
|
|
no_morph.0,
|
|
morph.0
|
|
);
|
|
assert!(
|
|
pct(no_morph) > 1.0,
|
|
"patches that cannot morph are barely rewritten: {:.1}%",
|
|
pct(no_morph)
|
|
);
|
|
assert!(
|
|
(pct(no_morph) - pct(morph)).abs() < 3.0,
|
|
"patches that cannot morph differ by {:.1}% against {:.1}% for those that can -- \
|
|
they should be indistinguishable",
|
|
pct(no_morph),
|
|
pct(morph)
|
|
);
|
|
|
|
eprintln!(
|
|
"tfrag RAM: {relocated} relocated table pointers, all by the table address {:#x}; \
|
|
no other table field differs; \
|
|
patch bytes rewritten -- cannot morph {:.1}% of {}, can morph {:.1}% of {}",
|
|
base + bh.table_offset as usize,
|
|
pct(no_morph),
|
|
no_morph.0,
|
|
pct(morph),
|
|
morph.0
|
|
);
|
|
}
|
|
|
|
/// Chunk record stride: uniform per signature, always 4-byte aligned, and
|
|
/// exactly one signature is not uniform.
|
|
///
|
|
/// All 55 distinct strides across the 74 signatures are multiples of 4, from 52
|
|
/// to 640. Seventy-three of the 74 signatures use a single stride. The
|
|
/// exception is asserted rather than skipped, because "every type is uniform"
|
|
/// would be the kind of claim that quietly stops being true.
|
|
#[test]
|
|
fn chunk_record_stride_is_four_byte_aligned_and_uniform_per_signature() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{chunk_region_start, parse_all_chunks};
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
let mut sig: BTreeMap<Vec<u8>, BTreeSet<usize>> = BTreeMap::new();
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let _ = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let scan = chunk_region_start(&core, 0)
|
|
.or_else(|| chunk_region_start(&core, 0x1000))
|
|
.unwrap_or(0);
|
|
let Ok(found) = parse_all_chunks(&core[scan..]) else {
|
|
continue;
|
|
};
|
|
for rc in found {
|
|
sig.entry(rc.chunk.header[..6].to_vec())
|
|
.or_default()
|
|
.insert(rc.chunk.stride);
|
|
}
|
|
}
|
|
assert_eq!(sig.len(), 74, "chunk signature count moved");
|
|
|
|
let strides: BTreeSet<usize> = sig.values().flatten().copied().collect();
|
|
let misaligned: Vec<usize> = strides.iter().copied().filter(|s| s % 4 != 0).collect();
|
|
let variable: Vec<(&Vec<u8>, &BTreeSet<usize>)> =
|
|
sig.iter().filter(|(_, v)| v.len() > 1).collect();
|
|
|
|
eprintln!(
|
|
"chunk records: {} signatures, {} distinct strides, range {}..{}; \
|
|
non-multiples of 4: {misaligned:?}; variable-length signatures: {}",
|
|
sig.len(),
|
|
strides.len(),
|
|
strides.iter().min().unwrap(),
|
|
strides.iter().next_back().unwrap(),
|
|
variable.len(),
|
|
);
|
|
for (k, v) in &variable {
|
|
eprintln!(" variable: {:02x?} strides {v:?}", &k[..6]);
|
|
}
|
|
|
|
assert!(
|
|
misaligned.is_empty(),
|
|
"strides not divisible by 4: {misaligned:?}"
|
|
);
|
|
assert_eq!(*strides.iter().min().unwrap(), 52);
|
|
assert_eq!(*strides.iter().next_back().unwrap(), 640);
|
|
assert_eq!(
|
|
variable.len(),
|
|
1,
|
|
"more than one signature has mixed strides"
|
|
);
|
|
assert_eq!(
|
|
variable[0].1.len(),
|
|
3,
|
|
"the variable signature's stride count moved"
|
|
);
|
|
}
|
|
|
|
/// Most of the chunk container is **level-independent**: 65 of its 74 signatures
|
|
/// carry records that are byte-identical in all 27 levels.
|
|
///
|
|
/// That reframes the format. These are static tables the game ships unchanged in
|
|
/// every level -- lookups and constants -- not per-level content. Decoding them
|
|
/// gets no help from comparing levels, because there is nothing to compare;
|
|
/// they have to be read structurally. The 9 level-specific signatures are the
|
|
/// ones worth attacking first for actual per-level data.
|
|
#[test]
|
|
fn most_chunk_signatures_are_level_independent() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{chunk_region_start, parse_all_chunks};
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use std::collections::BTreeMap;
|
|
|
|
/// signature -> (per-level record lists, total records)
|
|
type SigLevel = (Vec<Vec<Vec<u8>>>, usize);
|
|
let mut sig: BTreeMap<Vec<u8>, SigLevel> = BTreeMap::new();
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let _ = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let scan = chunk_region_start(&core, 0)
|
|
.or_else(|| chunk_region_start(&core, 0x1000))
|
|
.unwrap_or(0);
|
|
let Ok(found) = parse_all_chunks(&core[scan..]) else {
|
|
continue;
|
|
};
|
|
for rc in found {
|
|
let e = sig
|
|
.entry(rc.chunk.header[..6].to_vec())
|
|
.or_insert_with(|| (Vec::new(), rc.records.len()));
|
|
e.1 += rc.records.len();
|
|
e.0.push(rc.records.iter().map(|r| r.to_vec()).collect());
|
|
}
|
|
}
|
|
assert_eq!(sig.len(), 74, "chunk signature count moved");
|
|
|
|
let mut shared = 0usize;
|
|
let mut specific: Vec<String> = Vec::new();
|
|
let mut ragged = 0usize;
|
|
for (k, (lvls, _)) in &sig {
|
|
let counts: Vec<usize> = lvls.iter().map(|l| l.len()).collect();
|
|
if counts.windows(2).any(|w| w[0] != w[1]) {
|
|
ragged += 1;
|
|
continue;
|
|
}
|
|
let n = counts[0];
|
|
let identical = (0..n).all(|i| lvls.iter().all(|l| l[i] == lvls[0][i]));
|
|
if identical && n > 0 {
|
|
shared += 1;
|
|
} else {
|
|
specific.push(format!("{:02x?}", &k[..3]));
|
|
}
|
|
}
|
|
|
|
let n_sig = sig.len();
|
|
eprintln!(
|
|
"chunk signatures: {shared}/{n_sig} byte-identical across every level; \
|
|
{ragged} ragged; level-specific: {}",
|
|
specific.join(" ")
|
|
);
|
|
|
|
assert_eq!(
|
|
ragged, 0,
|
|
"a signature appeared with different record counts per level"
|
|
);
|
|
assert_eq!(shared, 65, "the shared/level-specific split moved");
|
|
assert_eq!(specific.len(), 9, "level-specific signature count moved");
|
|
}
|
|
|
|
/// The signature `[75 42 ef f2 71 b5]` produces three chunks per level, and one
|
|
/// of the three has genuinely variable-length records.
|
|
///
|
|
/// The parser takes a single stride per chunk. For the 92-byte chunk that is
|
|
/// wrong: its record offsets step by `92, 88, 84, 92, 92, 92, 84, 88`. Every
|
|
/// record *starts* correctly, because `resolve` uses the offset table, but each
|
|
/// is sliced at 92 bytes -- so the two 88-byte and three 84-byte records
|
|
/// over-read by 4 and 8 bytes and pick up the head of the next record.
|
|
///
|
|
/// So "this signature has strides {76, 92, 104}" is a misdescription. It has
|
|
/// three chunk shapes, one of which is not fixed-stride at all. Asserted with
|
|
/// the exact delta sequences so a future single-stride assumption cannot survive
|
|
/// quietly.
|
|
#[test]
|
|
fn chunk_type_75_42_ef_has_one_variable_length_chunk_per_level() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{chunk_region_start, parse_all_chunks};
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use std::collections::BTreeMap;
|
|
|
|
let target: [u8; 6] = [0x75, 0x42, 0xef, 0xf2, 0x71, 0xb5];
|
|
let mut shapes: BTreeMap<Vec<u32>, usize> = BTreeMap::new();
|
|
let mut mixed = 0usize;
|
|
let mut chunks = 0usize;
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let _ = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let scan = chunk_region_start(&core, 0)
|
|
.or_else(|| chunk_region_start(&core, 0x1000))
|
|
.unwrap_or(0);
|
|
let Ok(found) = parse_all_chunks(&core[scan..]) else {
|
|
continue;
|
|
};
|
|
for rc in found {
|
|
if rc.chunk.header[..6] != target[..] {
|
|
continue;
|
|
}
|
|
chunks += 1;
|
|
let deltas: Vec<u32> = rc.chunk.offsets.windows(2).map(|w| w[1] - w[0]).collect();
|
|
if !deltas.windows(2).all(|w| w[0] == w[1]) {
|
|
mixed += 1;
|
|
}
|
|
*shapes.entry(deltas).or_default() += 1;
|
|
}
|
|
}
|
|
|
|
assert_eq!(chunks, 81, "chunk count for this signature moved");
|
|
assert_eq!(mixed, 27, "the number of variable-length chunks moved");
|
|
|
|
let uniform76: Vec<u32> = vec![76; 8];
|
|
let variable: Vec<u32> = vec![92, 88, 84, 92, 92, 92, 84, 88];
|
|
let uniform104: Vec<u32> = vec![104; 8];
|
|
assert_eq!(
|
|
shapes.get(&uniform76).copied(),
|
|
Some(27),
|
|
"the 76-byte chunk shape changed"
|
|
);
|
|
assert_eq!(
|
|
shapes.get(&variable).copied(),
|
|
Some(27),
|
|
"the variable-length chunk's delta sequence changed"
|
|
);
|
|
assert_eq!(
|
|
shapes.get(&uniform104).copied(),
|
|
Some(27),
|
|
"the 104-byte chunk shape changed"
|
|
);
|
|
assert_eq!(shapes.len(), 3, "a fourth chunk shape appeared");
|
|
|
|
// the defect, stated as a measurement rather than a comment
|
|
let over = variable.iter().filter(|&&d| d < 92).count();
|
|
eprintln!(
|
|
"chunk [75 42 ef]: 81 chunks; 27 variable-length, deltas {variable:?}; \
|
|
{over} of 8 records shorter than the 92 the parser slices"
|
|
);
|
|
assert_eq!(over, 4, "the count of short records changed");
|
|
}
|
|
|
|
/// Chunk records share a field layout: 17-21% of byte positions are constant
|
|
/// across a chunk's own records, which does not happen by chance.
|
|
///
|
|
/// This survived the record-boundary fix, which is the interesting part. The fix
|
|
/// changed the length of records in 1,252 of 1,509 chunks, and this statistic is
|
|
/// computed over record bytes, so it was the obvious candidate for a claim that
|
|
/// had been contaminated by over-reads. Re-measured with each record bounded by
|
|
/// its successor, and comparing only the span every record in the chunk has, it
|
|
/// still lands in 16-20% per level. It is now asserted rather than only described.
|
|
#[test]
|
|
fn chunk_records_share_a_layout_measured_by_constant_byte_positions() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::parse_all_chunks;
|
|
|
|
use std::collections::BTreeMap;
|
|
|
|
let mut per_level: BTreeMap<usize, (u64, usize)> = BTreeMap::new();
|
|
for lv in 0..27usize {
|
|
let Ok(core) = std::fs::read(data.join(format!("level{lv}/core_data.bin"))) else {
|
|
continue;
|
|
};
|
|
let (mut tot, mut chunks) = (0u64, 0usize);
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
if c.records.len() < 3 {
|
|
continue;
|
|
}
|
|
// records in a chunk are not all the same length; compare the common
|
|
// prefix only
|
|
let len = c.records.iter().map(|r| r.len()).min().unwrap_or(0);
|
|
if len == 0 {
|
|
continue;
|
|
}
|
|
let n = (0..len)
|
|
.filter(|&i| c.records.iter().all(|r| r[i] == c.records[0][i]))
|
|
.count();
|
|
tot += 1000 * n as u64 / len as u64;
|
|
chunks += 1;
|
|
}
|
|
if chunks > 0 {
|
|
per_level.insert(lv, (tot, chunks));
|
|
}
|
|
}
|
|
assert_eq!(per_level.len(), 27, "a level produced no chunks to measure");
|
|
|
|
let pcts: Vec<(usize, f64)> = per_level
|
|
.iter()
|
|
.map(|(&lv, &(t, c))| (lv, t as f64 / (10.0 * c as f64)))
|
|
.collect();
|
|
let lo = pcts.iter().map(|p| p.1).fold(f64::MAX, f64::min);
|
|
let hi = pcts.iter().map(|p| p.1).fold(0.0f64, f64::max);
|
|
let lowest = pcts
|
|
.iter()
|
|
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap())
|
|
.unwrap();
|
|
eprintln!(
|
|
"chunk record layout: {lo:.1}%..{hi:.1}% of byte positions constant within a \
|
|
chunk; lowest is level {} at {:.1}%",
|
|
lowest.0, lowest.1
|
|
);
|
|
// lo/hi are percentages, not fractions
|
|
assert!(
|
|
(10.0..30.0).contains(&lo) && (10.0..30.0).contains(&hi),
|
|
"constant byte positions left the 10-30% band: {lo:.3}..{hi:.3}"
|
|
);
|
|
}
|
|
|
|
/// No two records in a chunk may overlap, and `resolve` must reproduce the
|
|
/// offset table exactly.
|
|
///
|
|
/// This is the general guard for the class of bug that was in `resolve` when it
|
|
/// sliced every record at the chunk's first offset delta. That produced records
|
|
/// that ran 4 to 8 bytes into their successors on 1,252 of 1,509 chunks, which
|
|
/// is how a committed claim about moby class IDs evaporated.
|
|
///
|
|
/// `chunks` is the only format in `gc-formats` that walks an explicit offset
|
|
/// table and derives a stride from it -- searched, and the other parsers slice
|
|
/// contiguous regions with `chunks_exact`, where a length cannot run past the
|
|
/// next element. So the invariant belongs here, stated positively: consecutive
|
|
/// records abut, and each record's slice is exactly the distance to the next.
|
|
#[test]
|
|
fn chunk_records_neither_overlap_nor_leave_gaps() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::parse_all_chunks;
|
|
|
|
let mut chunks = 0usize;
|
|
let mut ragged = 0usize;
|
|
for lv in 0..27usize {
|
|
let Ok(core) = std::fs::read(data.join(format!("level{lv}/core_data.bin"))) else {
|
|
continue;
|
|
};
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
chunks += 1;
|
|
let offs = &c.chunk.offsets;
|
|
if offs.len() < 2 {
|
|
continue;
|
|
}
|
|
let deltas: Vec<u32> = offs.windows(2).map(|w| w[1] - w[0]).collect();
|
|
if !deltas.windows(2).all(|w| w[0] == w[1]) {
|
|
ragged += 1;
|
|
}
|
|
// every record except the last is exactly its own delta long
|
|
for (i, (record, want)) in c
|
|
.records
|
|
.iter()
|
|
.zip(&deltas)
|
|
.take(c.records.len().saturating_sub(1))
|
|
.enumerate()
|
|
{
|
|
assert_eq!(
|
|
record.len(),
|
|
*want as usize,
|
|
"level{lv} chunk at {:#x}: record {i} is {} bytes, \
|
|
but its offset delta is {want}",
|
|
c.chunk.offset,
|
|
record.len()
|
|
);
|
|
}
|
|
}
|
|
}
|
|
eprintln!("chunk record boundaries: {chunks} chunks, {ragged} with ragged deltas");
|
|
assert_eq!(chunks, 1509, "chunk total moved");
|
|
assert_eq!(ragged, 1252, "ragged chunk count moved");
|
|
}
|
|
|
|
/// The extracted tree that every other disc test reads is pinned by hash.
|
|
///
|
|
/// `data/gc-files.tsv` pins the ISO files, and `unpacks_and_matches_the_committed_table`
|
|
/// checks that unpacking reproduces them. But the tie, collision, tfrag and chunk
|
|
/// tests do not unpack: they read `~/Documents/gc-extracted`, a directory nothing
|
|
/// verifies. A stale or hand-edited file there would have every measurement built
|
|
/// on it still pass, quietly.
|
|
///
|
|
/// That is the failure this project has already produced three times in another
|
|
/// form -- `has_uniform_record_length`, "1,509 chunks uniform" and the moby
|
|
/// class-ID scan each returned a confident number about the parser while
|
|
/// describing it as a fact about the disc. A test whose data source is unverified
|
|
/// is the same hazard one level down.
|
|
#[test]
|
|
fn the_extracted_tree_matches_the_committed_table() {
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let table_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/gc-extracted.tsv");
|
|
let text = std::fs::read_to_string(&table_path).expect("the extracted-tree table is committed");
|
|
|
|
let mut checked = 0usize;
|
|
let mut missing = Vec::new();
|
|
let mut bad = Vec::new();
|
|
for line in text
|
|
.lines()
|
|
.filter(|l| !l.starts_with('#') && !l.trim().is_empty())
|
|
{
|
|
let mut f = line.split('\t');
|
|
let (Some(hash), Some(size), Some(rel)) = (f.next(), f.next(), f.next()) else {
|
|
continue;
|
|
};
|
|
let path = data.join(rel);
|
|
let Ok(bytes) = std::fs::read(&path) else {
|
|
missing.push(rel.to_string());
|
|
continue;
|
|
};
|
|
if bytes.len() as u64 != size.parse::<u64>().unwrap_or(u64::MAX) {
|
|
bad.push(format!("{rel}: size {} != {size}", bytes.len()));
|
|
continue;
|
|
}
|
|
let got = {
|
|
use sha1::Digest;
|
|
let mut h = sha1::Sha1::new();
|
|
h.update(&bytes);
|
|
h.finalize()
|
|
.iter()
|
|
.map(|b| format!("{b:02x}"))
|
|
.collect::<String>()
|
|
};
|
|
if got != hash {
|
|
bad.push(format!("{rel}: {got} != {hash}"));
|
|
continue;
|
|
}
|
|
checked += 1;
|
|
}
|
|
|
|
assert!(
|
|
checked > 5000,
|
|
"only {checked} of the extracted tree was checked -- is the table truncated?"
|
|
);
|
|
assert!(
|
|
missing.is_empty(),
|
|
"{} files missing:\n{}",
|
|
missing.len(),
|
|
missing[..missing.len().min(8)].join("\n")
|
|
);
|
|
assert!(
|
|
bad.is_empty(),
|
|
"{} files do not match:\n{}",
|
|
bad.len(),
|
|
bad[..bad.len().min(8)].join("\n")
|
|
);
|
|
eprintln!("extracted tree: {checked} files match data/gc-extracted.tsv");
|
|
}
|
|
|
|
/// Roughly half the tie vertex regions start with a word whose bit 31 is set.
|
|
///
|
|
/// **This is a measurement, not an identification.** An earlier version of this
|
|
/// test read bit 31 as "this is a VIF code word, so the record is VIF-coded", and
|
|
/// built a two-encoding story on it. That reading is wrong. Walking those
|
|
/// records as VIF packets dies almost immediately:
|
|
///
|
|
/// records whose first word has bit 31 set 914 (levels 0-2)
|
|
/// walk reaches the region end 6
|
|
/// walk dies after exactly ONE packet 736
|
|
/// packets before dying: min 0, median 1, max 4
|
|
///
|
|
/// `tfrag` walks VIF the same way and reaches the end of its lists, so the walk
|
|
/// is not the problem -- these regions are not VIF packet streams. Bit 31 is set
|
|
/// on whatever this word actually is, which is not yet known.
|
|
///
|
|
/// What survives is the split itself, which is real and is not explained by
|
|
/// `vert_size`: 62 of the 72 values carry both kinds, so it is not a "short
|
|
/// objects are one thing, long ones another" rule. The two kinds differ
|
|
/// structurally -- a flat one repeats a 16-byte unit whose last word is constant
|
|
/// at `0x00001000` and whose third word walks down f000, e000, d000, c000; the
|
|
/// other does not. What the second kind is, is open.
|
|
///
|
|
/// Asserting the walk failure alongside the split is deliberate: if these ever
|
|
/// do become VIF streams, this test fails and says so, rather than the two
|
|
/// findings drifting apart silently.
|
|
|
|
#[test]
|
|
fn tie_vertex_regions_split_on_the_first_words_top_bit() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
use std::collections::BTreeMap;
|
|
|
|
use gc_formats::tfrag::VifCode;
|
|
|
|
let mut bit = 0usize;
|
|
let mut nobit = 0usize;
|
|
let mut walk_ok = 0usize;
|
|
let mut walk_one = 0usize;
|
|
// vert_size -> (bit31 set, clear), to show the split is not a function of it
|
|
let mut by_size: BTreeMap<u8, (usize, usize)> = BTreeMap::new();
|
|
for lv in 0..6usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).unwrap().tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
let rr: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in rr.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
if r.vert_size == 0 {
|
|
continue;
|
|
}
|
|
let start = s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
if start + 4 > core.len() {
|
|
continue;
|
|
}
|
|
let w = u32::from_le_bytes(core[start..start + 4].try_into().unwrap());
|
|
let e2 = by_size.entry(r.vert_size).or_default();
|
|
if w >> 31 == 1 {
|
|
bit += 1;
|
|
e2.0 += 1;
|
|
// and it is NOT a VIF stream: walking it proves that
|
|
let len = (r.vert_size + r.scissor_size) as usize * 16;
|
|
let mut at = start;
|
|
let mut steps = 0usize;
|
|
while at + 4 <= start + len && steps < 4096 {
|
|
let c = u32::from_le_bytes(core[at..at + 4].try_into().unwrap());
|
|
at += 4;
|
|
if c >> 31 != 1 {
|
|
break;
|
|
}
|
|
let v = VifCode::decode(c);
|
|
let np = if v.num == 0 { 256usize } else { v.num as usize };
|
|
if at + np * 4 > start + len {
|
|
break;
|
|
}
|
|
at += np * 4;
|
|
steps += 1;
|
|
}
|
|
if at == start + len {
|
|
walk_ok += 1
|
|
}
|
|
if steps == 1 {
|
|
walk_one += 1
|
|
}
|
|
} else {
|
|
nobit += 1;
|
|
e2.1 += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// every vert_size must carry both kinds, or the split would be explained by it
|
|
let both = by_size.values().filter(|(a, b)| *a > 0 && *b > 0).count();
|
|
eprintln!(
|
|
"tie vertex regions: {bit} with bit 31 set, {nobit} clear; {both} of {} \
|
|
vert_size values carry both. The bit-31 ones are NOT VIF streams: \
|
|
{walk_ok} walk to the region end, {walk_one} die after one packet",
|
|
by_size.len()
|
|
);
|
|
assert_eq!(bit + nobit, 4109, "record total moved");
|
|
assert_eq!(bit, 2007, "bit-31-set record count moved");
|
|
assert_eq!(nobit, 2102, "bit-31-clear record count moved");
|
|
// the negative is asserted too: these must NOT become valid VIF streams
|
|
// without this test being revisited, because the framing would change.
|
|
assert!(
|
|
walk_one * 2 > bit,
|
|
"the bit-31 records now walk as VIF ({walk_ok}/{bit} reach the end) -- \
|
|
revisit whether they are VIF"
|
|
);
|
|
// not "every", which is not true: 10 of the 72 rarest sizes carry one kind.
|
|
// What matters is that the split is overwhelmingly not a function of it.
|
|
assert!(
|
|
both * 2 >= by_size.len() * 3 / 2,
|
|
"only {both} of {} vert_size values carry both kinds -- the split may now \
|
|
be explained by vert_size",
|
|
by_size.len()
|
|
);
|
|
}
|
|
|
|
/// The `SBlk` body's first twelve words: six identified, six not.
|
|
///
|
|
/// Words 4 and 5 both hold the **DAW sample-data size**, byte for byte, on all
|
|
/// 27 levels -- level 0's are 1,482,000 and `daw_size` is 1,482,000. This was
|
|
/// briefly misread as a pointer into the DAW region: those sizes happen to fall
|
|
/// inside `[daw_offset, daw_offset + daw_size)`, so an "is it in range?" test
|
|
/// passed for the wrong reason. Equality with `daw_size` is the claim, and it
|
|
/// holds everywhere.
|
|
///
|
|
/// Words 1, 3, 6, 7, 8, 9, 10 and 11 are the same on every level. Words 0 and 2
|
|
/// vary per level and are not identified; word 0 is close between levels 1 and 4
|
|
/// (13,042,052 and 13,042,054) while their `daw_size`s differ by 63,504, so it is
|
|
/// not a size.
|
|
#[test]
|
|
fn sound_bank_sblk_body_head_carries_the_daw_size_twice() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::sound_bank::{parse_sound_bank, SoundBankHeader};
|
|
|
|
let mut levels = 0usize;
|
|
let mut w0: Vec<u32> = Vec::new();
|
|
let mut w2: Vec<u32> = Vec::new();
|
|
for lv in 0..27usize {
|
|
let Ok(bytes) = std::fs::read(data.join(format!("level{lv}/sound_bank.bin"))) else {
|
|
continue;
|
|
};
|
|
let h = SoundBankHeader::parse(&bytes).expect("sound bank head");
|
|
let bank = parse_sound_bank(&bytes).expect("sound bank");
|
|
assert!(bank.sblk.len() >= 48, "level {lv}: SBlk body too short");
|
|
let w: Vec<u32> = (0..12)
|
|
.map(|i| u32::from_le_bytes(bank.sblk[24 + i * 4..28 + i * 4].try_into().unwrap()))
|
|
.collect();
|
|
|
|
assert_eq!(w[4], h.daw_size, "level {lv}: body word 4 is not daw_size");
|
|
assert_eq!(w[5], h.daw_size, "level {lv}: body word 5 is not daw_size");
|
|
assert_eq!(w[1], 60, "level {lv}: body word 1 moved");
|
|
assert_eq!(w[3], 0, "level {lv}: body word 3 moved");
|
|
for i in [6usize, 7, 8, 11] {
|
|
assert_eq!(w[i], 0, "level {lv}: body word {i} moved");
|
|
}
|
|
assert_eq!(w[9], 127, "level {lv}: body word 9 moved");
|
|
assert_eq!(w[10], 1, "level {lv}: body word 10 moved");
|
|
w0.push(w[0]);
|
|
w2.push(w[2]);
|
|
levels += 1;
|
|
}
|
|
assert_eq!(levels, 27, "only {levels} sound banks parsed");
|
|
let uniq0: std::collections::BTreeSet<u32> = w0.iter().copied().collect();
|
|
let uniq2: std::collections::BTreeSet<u32> = w2.iter().copied().collect();
|
|
eprintln!(
|
|
"SBlk body head: words 4 and 5 both equal daw_size on 27/27; words 0 and 2 \\
|
|
vary ({}/{} and {}/{} distinct)",
|
|
uniq0.len(),
|
|
levels,
|
|
uniq2.len(),
|
|
levels
|
|
);
|
|
assert!(uniq0.len() > 20, "body word 0 stopped varying per level");
|
|
assert!(uniq2.len() > 20, "body word 2 stopped varying per level");
|
|
}
|
|
|
|
/// `overlay.bin` is the whole seven-section program, not the `.lit` section.
|
|
///
|
|
/// This corrects a committed claim. `overlay.bin +0x00` is `0x1A7C80` on all 27
|
|
/// levels, and that address is indeed the `.lit` load base -- but it is the load
|
|
/// base of the program's **first section**, which says nothing about the file.
|
|
/// The earlier note reasoned from a constant to "the load base of the section
|
|
/// this file is" and concluded the file *was* `.lit`. It is not.
|
|
///
|
|
/// What the sections actually are, now that `parse_overlay_sections` exists:
|
|
///
|
|
/// section 0 .lit at 0x1A7C80 on all 27 levels
|
|
/// section 1 .bss NOBITS (kind 8) on all 27, 0 non-zero bytes
|
|
/// sections 2-6 .data, lvl.vtbl, lvl.camvtbl, lvl.sndvtbl, .text
|
|
///
|
|
/// The kind sequence `[1, 8, 1, 1, 1, 1, 1]` is identical on every level, and the
|
|
/// collision assert string is in **section 0 on all 27** -- which is why the
|
|
/// earlier `$gp`-relative search over `overlay.bin` found the string in the right
|
|
/// place even though the reasoning about the file was wrong.
|
|
///
|
|
/// The sections do not tile the lump exactly: their sizes sum to 112 bytes short
|
|
/// on every level, because each section's `size` counts its own 16-byte header.
|
|
#[test]
|
|
fn overlay_bin_is_the_seven_section_program_not_the_lit_section() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay_reloc::parse_overlay_sections;
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
let needle = b"Camera_CollPrimTest WARNING!";
|
|
let mut first_dest = BTreeSet::new();
|
|
let mut kinds: BTreeSet<Vec<u32>> = BTreeSet::new();
|
|
let mut shortfall: BTreeMap<u32, usize> = BTreeMap::new();
|
|
let mut string_in: BTreeMap<usize, usize> = BTreeMap::new();
|
|
let mut levels = 0usize;
|
|
|
|
for lv in 0..27usize {
|
|
let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else {
|
|
continue;
|
|
};
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
assert_eq!(
|
|
secs.len(),
|
|
7,
|
|
"level {lv}: {} sections, expected 7",
|
|
secs.len()
|
|
);
|
|
first_dest.insert(secs[0].dest);
|
|
kinds.insert(secs.iter().map(|s| s.kind).collect());
|
|
let sum: usize = secs.iter().map(|s| s.data.len()).sum();
|
|
*shortfall.entry(ov.len() as u32 - sum as u32).or_default() += 1;
|
|
for (i, s) in secs.iter().enumerate() {
|
|
if s.data.windows(needle.len()).any(|w| w == needle) {
|
|
*string_in.entry(i).or_default() += 1;
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "only {levels} overlays parsed");
|
|
assert_eq!(
|
|
first_dest.iter().copied().collect::<Vec<_>>(),
|
|
vec![0x1A7C80],
|
|
"the first section's load base moved"
|
|
);
|
|
assert_eq!(
|
|
kinds.len(),
|
|
1,
|
|
"the section kind sequence is no longer identical across levels"
|
|
);
|
|
assert_eq!(
|
|
kinds.iter().next().unwrap().as_slice(),
|
|
&[1, 8, 1, 1, 1, 1, 1]
|
|
);
|
|
assert_eq!(
|
|
string_in.len(),
|
|
1,
|
|
"the assert string moved between sections"
|
|
);
|
|
assert_eq!(
|
|
string_in.get(&0).copied(),
|
|
Some(27),
|
|
"the string is not in section 0"
|
|
);
|
|
eprintln!(
|
|
"overlay.bin: 7 sections, first at 0x1A7C80 on 27/27, kinds \
|
|
[1,8,1,1,1,1,1]; section sizes fall {shortfall:?} bytes short of the lump"
|
|
);
|
|
}
|
|
|
|
/// The game's own moby class ids come from `lvl.vtbl`, and `moby_classes.count`
|
|
/// is **not** a valid bound on them.
|
|
///
|
|
/// `lvl.vtbl` is section 3 of the level program: records of `{oClass, handler,
|
|
/// auxiliary pointer}`, three words each, ending on padding. Across the 27
|
|
/// levels that is **1,162 distinct class ids**, ranging to `0x13BA` (5,050),
|
|
/// with a median of 38 new ids contributed per level.
|
|
///
|
|
/// `LevelCoreHeader::moby_classes.count` is 238 on level 0. **1,135 of the 1,162
|
|
/// ids -- 98% -- are at or above that number.** So a test that treats
|
|
/// "value < moby_classes.count" as "this looks like a class id" is really testing
|
|
/// "is this a small number".
|
|
///
|
|
/// That is the mechanism behind the chunk class-ID artifact that was retracted:
|
|
/// a nine-value slot whose values are 1, 3, 4, 5, 10, 11, 12, 13 and 15 passed a
|
|
/// "less than the moby count" test at 100%, and scored 305x enrichment. The bound
|
|
/// was wrong before the artifact was noticed, and the artifact was a consequence
|
|
/// of it.
|
|
///
|
|
/// Re-scored against the real id set, no chunk slot stands out: the best is 0.375
|
|
/// against a chance rate of 0.018 (1,162 ids in 65,536 values), and a dozen slots
|
|
/// sit between 0.25 and 0.38 with no separation between them.
|
|
#[test]
|
|
fn moby_class_ids_come_from_lvl_vtbl_not_from_the_class_count() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::overlay_reloc::parse_overlay_sections;
|
|
use std::collections::BTreeSet;
|
|
|
|
let mut classes: BTreeSet<u32> = BTreeSet::new();
|
|
let mut records = 0usize;
|
|
let mut levels = 0usize;
|
|
for lv in 0..27usize {
|
|
let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else {
|
|
continue;
|
|
};
|
|
let Ok(secs) = parse_overlay_sections(&ov) else {
|
|
continue;
|
|
};
|
|
assert!(secs.len() >= 4, "level {lv}: {} sections", secs.len());
|
|
let s = &secs[3];
|
|
// The record table ENDS at the terminator. Walking past it with
|
|
// `continue` reads whatever follows and inflates the count by 56 records
|
|
// and one class id -- which is how this test and the dispatch test came to
|
|
// disagree (5,878/1,163 against 5,822/1,162). Break.
|
|
for r in 0..s.data.len() / 12 {
|
|
let c = u32::from_le_bytes(s.data[r * 12..r * 12 + 4].try_into().unwrap());
|
|
if c == u32::MAX {
|
|
break;
|
|
}
|
|
classes.insert(c);
|
|
records += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
assert_eq!(levels, 27, "only {levels} overlays parsed");
|
|
// Exact, not a bound. This test printed 5,878 for a while because it walked
|
|
// past the terminator; `records > 5_000` accepted that without complaint, and
|
|
// the disagreement only surfaced because a *different* test counted the same
|
|
// table. A threshold cannot catch drift it is looser than.
|
|
assert_eq!(records, 5822, "vtbl records moved");
|
|
|
|
let max = *classes.iter().next_back().unwrap();
|
|
let idx0 = std::fs::read(data.join("level0/core_index.bin")).expect("core_index");
|
|
let bound = LevelCoreHeader::parse(&idx0)
|
|
.expect("hdr")
|
|
.moby_classes
|
|
.count as u32;
|
|
let above = classes.iter().filter(|&&c| c >= bound).count();
|
|
|
|
eprintln!(
|
|
"lvl.vtbl: {n} distinct class ids from {records} records; max {max:#x} ({max}); \
|
|
moby_classes.count is {bound}, and {above} of {n} ids ({pct:.1}%) are at or above it",
|
|
n = classes.len(),
|
|
pct = 100.0 * above as f64 / classes.len() as f64,
|
|
);
|
|
|
|
// Was pinned at 1163, which counted one id past the terminator. 1162 is the
|
|
// number of class ids the dispatch table actually names.
|
|
assert_eq!(classes.len(), 1162, "distinct class-id count moved");
|
|
assert_eq!(max, 0x13BA, "the highest class id moved");
|
|
assert!(
|
|
above * 100 >= classes.len() * 97,
|
|
"the class count stopped being a bad bound ({above}/{}); revisit why",
|
|
classes.len()
|
|
);
|
|
assert!(
|
|
classes.contains(&0x2) && classes.contains(&0x0E),
|
|
"the small ids that open every vtbl are missing"
|
|
);
|
|
}
|
|
|
|
/// `lvl.vtbl`'s `oClass` is a **global** moby class identifier, not an index into
|
|
/// the level's class table.
|
|
///
|
|
/// The two are easy to conflate and are not the same axis. Across the 27 levels
|
|
/// the vtbl carries 1,162 distinct identifiers reaching **5,050**, while
|
|
/// `moby_classes.count` runs 161 to 259 per level. So `oClass` cannot index the
|
|
/// class table -- almost every identifier would fall off the end.
|
|
///
|
|
/// The identifiers are *shared* rather than per-level: the set is cumulative
|
|
/// across the 27 levels and no level re-uses the numbering, so 1,162 distinct
|
|
/// identifiers come out of 5,822 records. Level 0 contributes 233 and levels 1-8
|
|
/// contribute 72, 69, 77, 54, 41, 68 and 28 -- roughly each level's own share,
|
|
/// not a single level dominating.
|
|
///
|
|
/// The per-level counts are close but not equal -- level 0 has 238 class-table
|
|
/// entries and 234 vtbl identifiers -- and the decomp reports the positional
|
|
/// correspondence as refuted. Nothing here depends on that: the useful fact is
|
|
/// that the identifier space is global and known, so a class can be recognised
|
|
/// across levels by name.
|
|
#[test]
|
|
fn vtbl_class_ids_are_global_and_do_not_index_the_class_table() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::overlay_reloc::parse_overlay_sections;
|
|
use std::collections::BTreeSet;
|
|
|
|
let mut ids: BTreeSet<u32> = BTreeSet::new();
|
|
let mut records_seen = 0usize;
|
|
let mut first_level_new = 0usize;
|
|
let mut per_level_new = Vec::new();
|
|
let mut max_count = 0u32;
|
|
let mut min_count = u32::MAX;
|
|
let mut levels = 0usize;
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(ov)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/overlay.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let count = LevelCoreHeader::parse(&idx)
|
|
.expect("hdr")
|
|
.moby_classes
|
|
.count as u32;
|
|
max_count = max_count.max(count);
|
|
min_count = min_count.min(count);
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
let before = ids.len();
|
|
// Terminator ends the table; see the note in lvl_vtbl_class_ids_are_global.
|
|
for r in 0..secs[3].data.len() / 12 {
|
|
let c = u32::from_le_bytes(secs[3].data[r * 12..r * 12 + 4].try_into().unwrap());
|
|
if c == u32::MAX {
|
|
break;
|
|
}
|
|
ids.insert(c);
|
|
records_seen += 1;
|
|
}
|
|
let added = ids.len() - before;
|
|
if lv == 0 {
|
|
first_level_new = added;
|
|
}
|
|
per_level_new.push(added);
|
|
levels += 1;
|
|
}
|
|
assert_eq!(levels, 27, "only {levels} levels parsed");
|
|
|
|
let largest = *ids.iter().next_back().unwrap();
|
|
let out_of_range = ids.iter().filter(|&&c| c >= max_count).count();
|
|
eprintln!(
|
|
"vtbl oClass: {} ids, largest {largest}; moby_classes.count runs \
|
|
{min_count}..{max_count}; {out_of_range}/{} ids are at or above the \
|
|
largest per-level table; level 0 alone contributes {first_level_new}",
|
|
ids.len(),
|
|
ids.len()
|
|
);
|
|
|
|
assert_eq!(ids.len(), 1162, "distinct vtbl class ids moved");
|
|
assert_eq!(largest, 5050, "the largest class id moved");
|
|
// the load-bearing claim: the identifier space does not fit in a level's table
|
|
assert!(
|
|
largest > max_count * 10,
|
|
"the id space now fits inside a class table; the two axes may have merged"
|
|
);
|
|
assert!(
|
|
out_of_range * 10 >= ids.len() * 9,
|
|
"most ids should be out of range for any per-level table"
|
|
);
|
|
// and it is shared, not per-level: 1,162 identifiers come out of 5,822
|
|
// records, so levels re-use ids rather than renumbering. No single level
|
|
// dominates -- level 0's 234 is about a fifth of the total.
|
|
assert!(
|
|
ids.len() * 3 < records_seen,
|
|
"the id count is now close to the record count; the numbering may be per-level"
|
|
);
|
|
assert!(
|
|
first_level_new * 4 <= ids.len(),
|
|
"level 0 supplied {first_level_new} of {} ids -- that is dominance, not a share",
|
|
ids.len()
|
|
);
|
|
}
|
|
|
|
/// `lvl.camvtbl` and `lvl.sndvtbl` decode at the record widths the decomp
|
|
/// documents, and neither leaves a partial record behind.
|
|
///
|
|
/// With `lvl.vtbl` (section 3, three-word records, covered separately) these are
|
|
/// the other two dispatch tables the level program carries. Their shapes:
|
|
/// section width record
|
|
/// camvtbl 5 camera id, InitCamera, ActivateCamera, UpdateCamera,
|
|
/// ExitCamera
|
|
/// sndvtbl 2 sound id, UpdateSound_<id>
|
|
///
|
|
/// Measured over all 27 levels: camera tables are 40 to 52 words (8 to 10
|
|
/// records), sound tables 0 to 8 words (0 to 4 records), and both divide by
|
|
/// their record width with **zero** remainder. That is the load-bearing check --
|
|
/// a wrong width would leave a remainder on most levels rather than none.
|
|
///
|
|
/// The identifier spaces are small and global: **14 distinct camera ids** (max
|
|
/// `0x1B`) and **6 distinct sound ids** (max `0x07`), against 217 and 32 records
|
|
/// respectively, so ids repeat across levels rather than being renumbered -- the
|
|
/// same sharing as the moby table.
|
|
#[test]
|
|
fn camera_and_sound_dispatch_tables_decode_at_their_documented_widths() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay_reloc::parse_overlay_sections;
|
|
use std::collections::BTreeSet;
|
|
|
|
// (section, record width, label)
|
|
let tables = [(4usize, 5usize, "camvtbl"), (5, 2, "sndvtbl")];
|
|
let mut out = Vec::new();
|
|
for (idx, width, label) in tables {
|
|
let mut ids: BTreeSet<u32> = BTreeSet::new();
|
|
let mut records = 0usize;
|
|
let mut remainder_seen = 0usize;
|
|
let mut levels = 0usize;
|
|
for lv in 0..27usize {
|
|
let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else {
|
|
continue;
|
|
};
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
assert!(secs.len() > idx, "level {lv}: only {} sections", secs.len());
|
|
let words = secs[idx].data.len() / 4;
|
|
if words % width != 0 {
|
|
remainder_seen += 1;
|
|
}
|
|
for r in 0..words / width {
|
|
let base = r * width * 4;
|
|
let id = u32::from_le_bytes(secs[idx].data[base..base + 4].try_into().unwrap());
|
|
if id == u32::MAX {
|
|
break;
|
|
}
|
|
ids.insert(id);
|
|
records += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
assert_eq!(levels, 27, "{label}: only {levels} levels parsed");
|
|
assert_eq!(
|
|
remainder_seen, 0,
|
|
"{label}: {remainder_seen} levels do not divide by a {width}-word record"
|
|
);
|
|
let max = *ids.iter().next_back().unwrap();
|
|
eprintln!(
|
|
"{label}: {records} records, {} distinct ids, max {max:#x}",
|
|
ids.len()
|
|
);
|
|
assert!(
|
|
ids.len() < records,
|
|
"{label}: ids are not shared between levels"
|
|
);
|
|
out.push((label, ids.len(), max, records));
|
|
}
|
|
|
|
// (label, distinct ids, max, records) -- pinned exactly. Both tables were
|
|
// counted by four tests between them and the counts only agreed because each
|
|
// test asserted a bound rather than a number. vtbl is counted by
|
|
// lvl_vtbl_class_ids_are_global instead; this helper covers only these two.
|
|
assert_eq!(
|
|
out.iter().map(|o| (o.0, o.1, o.3)).collect::<Vec<_>>(),
|
|
vec![("camvtbl", 14, 217), ("sndvtbl", 6, 32)],
|
|
"dispatch table counts moved"
|
|
);
|
|
assert_eq!(out[0].2, 0x1B, "the largest camera id moved");
|
|
assert_eq!(out[1].1, 6, "distinct sound id count moved");
|
|
assert_eq!(out[1].2, 0x07, "the largest sound id moved");
|
|
}
|
|
|
|
/// The dispatch handlers' addresses are almost all function entries -- measured
|
|
/// independently, and not quite matching the reference figure.
|
|
///
|
|
/// `docs/MOBY-DISPATCH-TABLES.md` reports 5,598 of 5,788 moby handlers (96.7%)
|
|
/// preceded by a `jr ra` within the three words before them. Measured here the
|
|
/// same way, over the same 5,788 records: **5,488 (94.8%)**.
|
|
///
|
|
/// Same population, same test, 110 handlers different. Both confirm the premise
|
|
/// that these addresses are function entries rather than arbitrary words, and the
|
|
/// residual is recorded as a discrepancy rather than explained away -- the
|
|
/// reference measured on 2026-10-02 against its own pinned overlay set, so a
|
|
/// difference in method or version is as likely as a difference in the data.
|
|
///
|
|
/// Two things worth knowing before reading a handler. First, the code interleaves
|
|
/// VU1 microcode with CPU instructions, so a disassembly window runs `.word
|
|
/// 0x0240202d` lines that are COP2 opcodes rather than failures. Second, the
|
|
/// handlers are **not all independent functions**: level 0's class 2 and class
|
|
/// `0x0E` are 8 bytes apart (0x380a88 and 0x380a90), so consecutive classes share
|
|
/// one routine with per-class entry points inside it. Reading a handler gives you
|
|
/// a fragment, not a whole function.
|
|
#[test]
|
|
fn moby_handlers_are_overwhelmingly_function_entries() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay_reloc::parse_overlay_sections;
|
|
|
|
const JR_RA: u32 = 0x03E0_0008;
|
|
let (mut total, mut with_jr) = (0usize, 0usize);
|
|
let mut gaps: Vec<u32> = Vec::new();
|
|
for lv in 0..27usize {
|
|
let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else {
|
|
continue;
|
|
};
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
let text = &secs[6];
|
|
let mut last: Option<u32> = None;
|
|
for r in 0..secs[3].data.len() / 12 {
|
|
let off = r * 12 + 4;
|
|
let h = u32::from_le_bytes(secs[3].data[off..off + 4].try_into().unwrap());
|
|
// Terminator on the HANDLER word, not the id word: 34 records carry a
|
|
// valid id with no geometry, so this stopping point is 34 short of the
|
|
// table and is correct for counting handlers.
|
|
if h == u32::MAX {
|
|
break;
|
|
}
|
|
// how close is this handler to the previous one?
|
|
if let Some(p) = last {
|
|
if h > p && (h - p) < 16 {
|
|
gaps.push(h - p);
|
|
}
|
|
}
|
|
last = Some(h);
|
|
let Some(rel) = h.checked_sub(text.dest) else {
|
|
continue;
|
|
};
|
|
let rel = rel as usize;
|
|
let Some(slice) = text.data.get(rel.saturating_sub(12)..rel) else {
|
|
continue;
|
|
};
|
|
let jr = slice
|
|
.chunks_exact(4)
|
|
.rev()
|
|
.take(3)
|
|
.any(|w| u32::from_le_bytes([w[0], w[1], w[2], w[3]]) == JR_RA);
|
|
total += 1;
|
|
if jr {
|
|
with_jr += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
let pct = 100.0 * with_jr as f64 / total as f64;
|
|
eprintln!(
|
|
"moby handlers: {with_jr} of {total} ({pct:.1}%) preceded by a jr ra within 3 \
|
|
words -- the reference reports 5598 of 5788 (96.7%); {n} of those records sit \
|
|
within 16 bytes of the one before",
|
|
n = gaps.len()
|
|
);
|
|
assert_eq!(total, 5788, "handler record count moved");
|
|
// the premise, asserted loosely because the reference's own figure differs
|
|
assert!(
|
|
pct > 90.0,
|
|
"handlers stopped looking like function entries: {pct:.1}%"
|
|
);
|
|
assert!(
|
|
!gaps.is_empty(),
|
|
"no two consecutive handlers are close; they stopped sharing routines"
|
|
);
|
|
}
|
|
|
|
/// How many distinct behaviours the three dispatch tables actually contain.
|
|
///
|
|
/// Records and behaviours are not the same number, and the difference is where
|
|
/// one routine serves several classes. Measured over all 27 levels:
|
|
/// table records distinct classes distinct handlers median per handler
|
|
/// vtbl 5,822 1,162 4,737 1
|
|
/// camvtbl 217 14 216 1
|
|
/// sndvtbl 32 6 32 1
|
|
///
|
|
/// So sharing is real but modest, and it is almost entirely in the moby table:
|
|
/// 5,822 records collapse to 4,737 addresses, with the most-shared handler
|
|
/// serving 34 class records. The camera and sound tables are close to one handler
|
|
/// per record -- the sound table is exactly so.
|
|
///
|
|
/// That matches the observation that 72 moby records sit within 16 bytes of their
|
|
/// predecessor: roughly 1,085 records share an address, and the tightest of those
|
|
/// are adjacent in the table.
|
|
#[test]
|
|
fn dispatch_tables_contain_far_fewer_behaviours_than_records() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay_reloc::parse_overlay_sections;
|
|
use std::collections::BTreeMap;
|
|
|
|
// (section, record width, label, records, classes, handlers)
|
|
let want = [
|
|
(3usize, 3usize, "vtbl", 5822usize, 1162usize, 4737usize),
|
|
(4, 5, "camvtbl", 217, 14, 216),
|
|
(5, 2, "sndvtbl", 32, 6, 32),
|
|
];
|
|
for (idx, width, label, want_records, want_classes, want_handlers) in want {
|
|
let mut handlers: BTreeMap<u32, u32> = BTreeMap::new();
|
|
let mut classes: BTreeMap<u32, u32> = BTreeMap::new();
|
|
for lv in 0..27usize {
|
|
let Ok(ov) = std::fs::read(data.join(format!("level{lv}/overlay.bin"))) else {
|
|
continue;
|
|
};
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
for r in 0..secs[idx].data.len() / width {
|
|
let b = r * width * 4;
|
|
let id = u32::from_le_bytes(secs[idx].data[b..b + 4].try_into().unwrap());
|
|
if id == u32::MAX {
|
|
break;
|
|
}
|
|
let h = u32::from_le_bytes(secs[idx].data[b + 4..b + 8].try_into().unwrap());
|
|
*handlers.entry(h).or_default() += 1;
|
|
*classes.entry(id).or_default() += 1;
|
|
}
|
|
}
|
|
let records: usize = handlers.values().sum::<u32>() as usize;
|
|
let shared = handlers.values().filter(|&&n| n > 1).count();
|
|
eprintln!(
|
|
"{label}: {records} records, {} classes, {} handlers ({shared} serve more \
|
|
than one)",
|
|
classes.len(),
|
|
handlers.len()
|
|
);
|
|
assert_eq!(records, want_records, "{label}: record count moved");
|
|
assert_eq!(classes.len(), want_classes, "{label}: class count moved");
|
|
assert_eq!(
|
|
handlers.len(),
|
|
want_handlers,
|
|
"{label}: handler count moved"
|
|
);
|
|
assert!(
|
|
handlers.len() <= records,
|
|
"{label}: more behaviours than records is impossible"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// R&C2's level table: 27 entries, keyed by **level id**, not by file order.
|
|
///
|
|
/// ReRAC, which this project follows architecturally, has
|
|
/// `rc_game::map::LEVELS: usize = 19` -- Ratchet & Clank 1 has nineteen levels.
|
|
/// This game has twenty-seven, so "extend the level table from 19 to 27" is about
|
|
/// *what the keys are*, not only how many there are.
|
|
///
|
|
/// Reading every container header's own id off the disc gives the key set:
|
|
///
|
|
/// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 22 23 24 25 26 30
|
|
///
|
|
/// **27 keys, and 21 is not one of them.** The container files run `LEVEL0.WAD` to
|
|
/// `LEVEL26.WAD`, but `LEVEL21.WAD` reports id **30**, so the id space is
|
|
/// 0..20, 22..26, 30 -- one number was cut and reused by nothing.
|
|
///
|
|
/// The reference agrees independently. Every level key appearing in
|
|
/// `rac2-gc-decomp`'s `config/levels/functions.tsv` placements column, across 7,765
|
|
/// function rows, is exactly that set, with 21 absent. Ours comes from the
|
|
/// container headers and theirs from where functions land across overlays.
|
|
///
|
|
/// A table keyed by container index would silently mis-address one level in
|
|
/// twenty-seven, and every id-keyed lookup after the gap would be off by one.
|
|
#[test]
|
|
fn level_table_has_twenty_seven_keys_with_twenty_one_absent() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/LEVEL"))
|
|
.map(|e| {
|
|
let n: usize = e
|
|
.name
|
|
.trim_start_matches("G/LEVEL")
|
|
.trim_end_matches(".WAD")
|
|
.parse()
|
|
.expect("level container names end in a number");
|
|
(n, e)
|
|
})
|
|
.collect();
|
|
levels.sort_by_key(|(n, _)| *n);
|
|
assert_eq!(
|
|
levels.len(),
|
|
27,
|
|
"the disc should carry 27 level containers"
|
|
);
|
|
|
|
let mut ids: Vec<i32> = levels
|
|
.iter()
|
|
.map(|(_, e)| {
|
|
let data = gc_formats::iso9660::read_at(&path, e.lba, 0x60).expect("read header");
|
|
gc_formats::level_wad::LevelHeader::parse(&data)
|
|
.expect("parse")
|
|
.id
|
|
})
|
|
.collect();
|
|
ids.sort_unstable();
|
|
|
|
let expected: Vec<i32> = (0..=20).chain(22..=26).chain([30]).collect();
|
|
eprintln!(
|
|
"level table: {} ids, 21 absent, highest {}",
|
|
ids.len(),
|
|
ids.last().unwrap()
|
|
);
|
|
assert_eq!(ids.len(), 27, "level id count moved");
|
|
assert!(
|
|
!ids.contains(&21),
|
|
"id 21 must not exist; LEVEL21.WAD reports 30"
|
|
);
|
|
assert_eq!(ids, expected, "the level-id set moved");
|
|
assert_eq!(*ids.last().unwrap(), 30, "the highest level id is 30");
|
|
}
|
|
|
|
/// `bfc_distance` is not the length of the control region, partitioned by
|
|
/// `shader_count`.
|
|
///
|
|
/// `bfc_distance == 5 * shader_count` holds universally, but reading it as
|
|
/// "quadwords of AD-GIF data" makes it the length of the leading region -- and it
|
|
/// cannot be. `bfc_distance` runs 5..20 quadwords and `vert_ofs` only 3..8.
|
|
///
|
|
/// Pooled, the two overlap on 9,315 of 15,219 records and that number describes
|
|
/// nothing. Partitioned by `shader_count` it is exact: for two or more shaders
|
|
/// `bfc_distance` exceeds `vert_ofs` on **every** record, and for one shader on
|
|
/// 32%, which is just the share whose `vert_ofs` is below 5.
|
|
///
|
|
/// The AD-GIF is therefore not stored in the tie packet. `bfc_distance` is a
|
|
/// per-record quantity relating to the shader count, and the packet holds control,
|
|
/// vertices and scissor in that order.
|
|
#[test]
|
|
fn tie_bfc_distance_is_not_the_control_region_length() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
use std::collections::BTreeMap;
|
|
|
|
let mut by_sc: BTreeMap<u8, (usize, usize)> = BTreeMap::new();
|
|
let mut total = 0usize;
|
|
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let Ok(e) = ClassEntry::parse(&idx[o..o + 0x20]) else {
|
|
continue;
|
|
};
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = hdr.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let s = base + off as usize;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
for r in blk.header.records(&core[s..]) {
|
|
let Ok(r) = TieRecord::parse(r) else { continue };
|
|
total += 1;
|
|
let e = by_sc.entry(r.shader_count).or_insert((0, 0));
|
|
e.0 += 1;
|
|
e.1 += usize::from(r.bfc_distance > r.vert_ofs);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(total > 15_000, "only {total} tie records read");
|
|
eprintln!("bfc_distance > vert_ofs, by shader_count: {by_sc:?}");
|
|
for (sc, (n, over)) in &by_sc {
|
|
if *sc >= 2 {
|
|
assert_eq!(
|
|
*n, *over,
|
|
"shader_count {sc}: expected every record to exceed, got {over}/{n}"
|
|
);
|
|
}
|
|
}
|
|
// One shader: exceeds exactly when vert_ofs < 5, i.e. control < 5 quadwords.
|
|
let (n1, over1) = by_sc[&1];
|
|
assert!(
|
|
over1 < n1,
|
|
"shader_count 1 should not exceed on every record"
|
|
);
|
|
}
|
|
|
|
/// Structural facts about the undecoded tail of the tie class header.
|
|
///
|
|
/// `TieClassHeader` skips `0x1c..0x44`. Profiling each of those ten words over
|
|
/// all 1,864 classes separates them sharply, and two are decidable.
|
|
#[test]
|
|
fn tie_class_header_tail_words() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
let mut reserved_20 = BTreeSet::new();
|
|
let mut reserved_30 = BTreeSet::new();
|
|
let mut f40: Vec<f32> = Vec::new();
|
|
let mut w3c: BTreeMap<u32, usize> = BTreeMap::new();
|
|
let mut by3c: BTreeMap<u32, (usize, usize, u64)> = BTreeMap::new();
|
|
let mut classes = 0usize;
|
|
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let Ok(e) = ClassEntry::parse(&idx[o..o + 0x20]) else {
|
|
continue;
|
|
};
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
if TieClassHeader::parse(&core[base..base + 0x48]).is_err() {
|
|
continue;
|
|
}
|
|
classes += 1;
|
|
let word = |k: usize| -> u32 {
|
|
let at = base + 0x1c + k * 4;
|
|
u32::from_le_bytes(core[at..at + 4].try_into().unwrap())
|
|
};
|
|
reserved_20.insert(word(1));
|
|
reserved_30.insert(word(5));
|
|
f40.push(f32::from_bits(word(9)));
|
|
*w3c.entry(word(8)).or_default() += 1;
|
|
|
|
let v = word(8);
|
|
let ent = by3c.entry(v).or_insert((0, 0, 0));
|
|
ent.0 += 1;
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = hdr.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let s = base + off as usize;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
for r in blk.header.records(&core[s..]) {
|
|
let Ok(r) = TieRecord::parse(r) else { continue };
|
|
ent.1 += 1;
|
|
ent.2 += u64::from(r.bfc_distance);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(classes > 1_800, "only {classes} tie classes read");
|
|
|
|
// Two words are reserved: zero on every class, 1,864 of 1,864.
|
|
assert_eq!(reserved_20.iter().copied().collect::<Vec<_>>(), vec![0]);
|
|
assert_eq!(reserved_30.iter().copied().collect::<Vec<_>>(), vec![0]);
|
|
|
|
// 0x40 is a float. Not merely "looks like one": a positive range across two
|
|
// orders of magnitude with no negatives is not what an int looks like.
|
|
f40.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
assert!(
|
|
f40.iter().all(|v| v.is_finite() && *v > 0.0),
|
|
"0x40 has non-positive values"
|
|
);
|
|
assert!(
|
|
f40[f40.len() / 2] > 0.01 && f40[f40.len() / 2] < 10.0,
|
|
"0x40 median out of range"
|
|
);
|
|
|
|
// 0x3c takes six values and is overwhelmingly zero.
|
|
assert_eq!(w3c.len(), 6, "0x3c gained a value: {w3c:?}");
|
|
assert_eq!(*w3c.get(&0).unwrap(), 1384);
|
|
|
|
// The control: 0x3c is NOT the ad-gif count. Mean bfc_distance is flat across
|
|
// every 0x3c bucket, so it does not track the shader count either.
|
|
for (v, (_, _recs, bfc_sum)) in by3c.iter() {
|
|
let (_, recs, _) = *by3c.get(v).unwrap();
|
|
if recs > 100 {
|
|
let mean = *bfc_sum as f64 / *_recs as f64;
|
|
assert!(
|
|
(5.0..13.0).contains(&mean),
|
|
"0x3c={v}: mean bfc {mean} drifted"
|
|
);
|
|
}
|
|
}
|
|
eprintln!(
|
|
"0x40 f32: min {:.5} p50 {:.5} max {:.5}",
|
|
f40[0],
|
|
f40[f40.len() / 2],
|
|
f40[f40.len() - 1]
|
|
);
|
|
eprintln!("0x3c histogram: {w3c:?}");
|
|
}
|
|
|
|
/// `0x1c` and `0x34` of the tie class header are base-relative offsets, and they
|
|
/// land past the class's own tie packets.
|
|
#[test]
|
|
fn tie_class_tail_offsets_point_past_the_tie_packets() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::TieClassHeader;
|
|
|
|
let (mut classes, mut in_range, mut past_packets, mut both_nonzero) =
|
|
(0usize, 0usize, 0usize, 0usize);
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let Ok(e) = ClassEntry::parse(&idx[o..o + 0x20]) else {
|
|
continue;
|
|
};
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
classes += 1;
|
|
let word = |k: usize| -> u32 {
|
|
let at = base + 0x1c + k * 4;
|
|
u32::from_le_bytes(core[at..at + 4].try_into().unwrap())
|
|
};
|
|
let (o1c, o34) = (word(0) as usize, word(6) as usize);
|
|
if o1c + 32 <= core.len() && o34 + 32 <= core.len() {
|
|
in_range += 1;
|
|
}
|
|
if word(0) != 0 && word(6) != 0 {
|
|
both_nonzero += 1;
|
|
}
|
|
// Past the class's own tie packet blocks.
|
|
let last_packet = hdr
|
|
.packets
|
|
.iter()
|
|
.filter(|p| **p > 0)
|
|
.max()
|
|
.copied()
|
|
.unwrap_or(0);
|
|
if o1c as i32 > last_packet && o34 as i32 > last_packet {
|
|
past_packets += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(classes > 1_800, "only {classes} tie classes read");
|
|
eprintln!("0x1c/0x34 in range {in_range}/{classes}, past own packets {past_packets}/{classes}");
|
|
// Both are relative to the class base, not absolute: base + offset lands in
|
|
// core_data, and absolute offsets would not.
|
|
assert!(
|
|
in_range * 100 >= classes * 99,
|
|
"only {in_range}/{classes} offsets in range"
|
|
);
|
|
assert_eq!(both_nonzero, classes, "a class has a zero offset");
|
|
// They are never inside the class's own tie data.
|
|
assert_eq!(
|
|
past_packets, classes,
|
|
"{classes} offsets fall inside the tie packets"
|
|
);
|
|
}
|
|
|
|
/// Control for `tie_class_tail_offsets_point_past_the_tie_packets`.
|
|
///
|
|
/// The bytes at 0x1c and 0x34 are structured -- but **not more structured than
|
|
/// core_data anywhere else**, so the offsets select nothing distinctive.
|
|
#[test]
|
|
fn tie_tail_offsets_are_not_more_structured_than_control() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::TieClassHeader;
|
|
|
|
fn adjacent_equal(b: &[u8]) -> f64 {
|
|
let eq = b.windows(2).filter(|w| w[0] == w[1]).count();
|
|
eq as f64 / (b.len() - 1) as f64
|
|
}
|
|
let mut n = 0usize;
|
|
let mut at_1c = 0.0f64;
|
|
let mut at_34 = 0.0f64;
|
|
let mut ctl = 0.0f64;
|
|
|
|
for lv in 0..27usize {
|
|
let (Ok(idx), Ok(core)) = (
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))),
|
|
std::fs::read(data.join(format!("level{lv}/core_data.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let Ok(e) = ClassEntry::parse(&idx[o..o + 0x20]) else {
|
|
continue;
|
|
};
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
if TieClassHeader::parse(&core[base..base + 0x48]).is_err() {
|
|
continue;
|
|
}
|
|
let word = |k: usize| -> usize {
|
|
let at = base + 0x1c + k * 4;
|
|
u32::from_le_bytes(core[at..at + 4].try_into().unwrap()) as usize
|
|
};
|
|
let (o1, o3) = (word(0), word(6));
|
|
if o1 + 64 > core.len() || o3 + 64 > core.len() || core.len() < 64 {
|
|
continue;
|
|
}
|
|
let ctl_off = o1.wrapping_add(0x2000) % (core.len() - 64);
|
|
n += 1;
|
|
at_1c += adjacent_equal(&core[o1..o1 + 64]);
|
|
at_34 += adjacent_equal(&core[o3..o3 + 64]);
|
|
ctl += adjacent_equal(&core[ctl_off..ctl_off + 64]);
|
|
}
|
|
}
|
|
|
|
assert!(n > 1_800, "only {n} classes read");
|
|
let (m1, m3, mc) = (at_1c / n as f64, at_34 / n as f64, ctl / n as f64);
|
|
eprintln!(
|
|
"adjacent-equal rate: 0x1c {m1:.5} | 0x34 {m3:.5} | control {mc:.5} | random {:.5}",
|
|
1.0 / 256.0
|
|
);
|
|
|
|
// Far from random: this region is structured data, not noise.
|
|
assert!(mc > 20.0 / 256.0, "control fell to the random baseline");
|
|
// But the two offsets are NOT *more* structured than an arbitrary one.
|
|
// One-sided on purpose: the claim being guarded is "these offsets select
|
|
// distinctive content", and a gap in the other direction does not support
|
|
// it. Both come out slightly BELOW control (0x1269 and 0.1211 against
|
|
// 0.1411), which is why a symmetric tolerance was the wrong shape -- it
|
|
// failed on a 0.0200 gap while asserting something the data does not say.
|
|
//
|
|
// The margin is 0.03, comfortably above the observed 0.014 and 0.020, so
|
|
// this still fails if either offset ever becomes markedly more structured
|
|
// than its neighbourhood -- which is the thing worth catching.
|
|
assert!(
|
|
m1 < mc + 0.03 && m3 < mc + 0.03,
|
|
"0x1c {m1:.5} / 0x34 {m3:.5} are now MORE structured than control \
|
|
{mc:.5}: they may select real content after all"
|
|
);
|
|
}
|
|
|
|
/// `moby_classes.offset` is a load-bearing constant, proved by its neighbours.
|
|
///
|
|
/// The immediate `0x00c0` appears in **394 of 2,323** level-0 functions of at
|
|
/// least 128 bytes. Every value immediately adjacent to it appears in 1 to 14.
|
|
///
|
|
/// That is the control 27891ce lacked. `0xc0` is small, and small immediates are
|
|
/// common, so its raw frequency proved nothing on its own. The neighbours share
|
|
/// its magnitude exactly and share none of its frequency, which is what makes the
|
|
/// 394 mean something rather than being an artifact of the value being short.
|
|
///
|
|
/// Boundaries: level 0 only; function extents from the decomp's functions.tsv,
|
|
/// where "level 0" means the function's first listed place is level 00; size floor
|
|
/// of 128 bytes, since 59957a0 recorded 20-byte "functions" that are mis-bound
|
|
/// extents. An immediate alone does not prove the chunk is read -- it proves the
|
|
/// constant is used, and that is the claim being made.
|
|
#[test]
|
|
fn moby_classes_offset_is_referenced_far_more_than_its_neighbours() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted")
|
|
});
|
|
let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv");
|
|
if !data.exists() || !dcomp.exists() {
|
|
eprintln!("skipped");
|
|
return;
|
|
}
|
|
|
|
let txt = std::fs::read_to_string(dcomp).unwrap();
|
|
let mut funcs: Vec<(u32, u32)> = Vec::new();
|
|
for line in txt.lines() {
|
|
if line.starts_with('#') || line.trim().is_empty() {
|
|
continue;
|
|
}
|
|
let f: Vec<&str> = line.split('\t').collect();
|
|
if f.len() < 6 {
|
|
continue;
|
|
}
|
|
let size: u32 = match f[2].parse() {
|
|
Ok(v) => v,
|
|
Err(_) => continue,
|
|
};
|
|
for pl in f[5].split(',') {
|
|
let mut it = pl.split(':');
|
|
if it.next() == Some("00") {
|
|
if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) {
|
|
funcs.push((a, size));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
funcs.sort_unstable();
|
|
|
|
let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap();
|
|
let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap();
|
|
let h = gc_formats::level_core::LevelCoreHeader::parse(&idx).unwrap();
|
|
let secs = gc_formats::overlay_reloc::parse_overlay_sections(&ov).unwrap();
|
|
let text = &secs[6].data;
|
|
let dest = secs[6].dest;
|
|
let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06);
|
|
const FLOOR: u32 = 128;
|
|
|
|
let mut sets: Vec<std::collections::BTreeSet<u16>> = Vec::new();
|
|
for (start, size) in &funcs {
|
|
if *size < FLOOR {
|
|
continue;
|
|
}
|
|
let off = (*start).wrapping_sub(dest) as usize;
|
|
if off >= text.len() {
|
|
continue;
|
|
}
|
|
let n = (*size as usize).min(text.len() - off) & !3;
|
|
let mut set = std::collections::BTreeSet::new();
|
|
for w in text[off..off + n].chunks_exact(4) {
|
|
let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]);
|
|
if has_imm((i >> 26) as u8) {
|
|
set.insert((i & 0xffff) as u16);
|
|
}
|
|
}
|
|
sets.push(set);
|
|
}
|
|
let hit = |v: u16| sets.iter().filter(|s| s.contains(&v)).count();
|
|
|
|
let base = h.moby_classes.offset as u16;
|
|
assert!(sets.len() > 2_000, "only {} functions kept", sets.len());
|
|
let self_hits = hit(base);
|
|
let mut neighbour_max = 0usize;
|
|
let mut detail = Vec::new();
|
|
for d in -8i32..=8i32 {
|
|
if d == 0 {
|
|
continue;
|
|
}
|
|
let v = (base as i32 + d) as u16;
|
|
let c = hit(v);
|
|
neighbour_max = neighbour_max.max(c);
|
|
detail.push(format!("{v:#06x}:{c}"));
|
|
}
|
|
eprintln!("moby_classes {base:#06x} -> {self_hits} functions; neighbours {detail:?}");
|
|
assert!(self_hits > 100, "only {self_hits} functions reference it");
|
|
assert!(
|
|
self_hits > neighbour_max * 10,
|
|
"only {self_hits} vs neighbours up to {neighbour_max}: no longer special"
|
|
);
|
|
}
|
|
|
|
/// `moby_classes.offset` is enriched over its neighbours *at matched set size*.
|
|
///
|
|
/// 06531fb showed the rate at which a function contains `0x00c0` rises with how
|
|
/// many distinct immediates it carries, so the raw 394 could have been a
|
|
/// pigeonhole artifact. Matching on set size removes that: within every bucket,
|
|
/// `0xc0` beats the best of its sixteen neighbours by 13x to 72x.
|
|
///
|
|
/// The load-bearing row is the smallest bucket. **72 of 1,082 functions carrying
|
|
/// fewer than 16 distinct immediates contain `0xc0`, where the best neighbour
|
|
/// appears once.** Small sets cannot produce that by chance of volume, so the
|
|
/// constant is genuinely special rather than merely common in big functions.
|
|
///
|
|
/// Bounds: level 0 only; function extents from the decomp, "level 0" meaning
|
|
/// first listed place is level 00; 128-byte size floor. Presence of an immediate
|
|
/// is not proof the chunk is read -- only that the constant is used.
|
|
#[test]
|
|
fn moby_classes_offset_enrichment_survives_set_size_matching() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted")
|
|
});
|
|
let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv");
|
|
if !data.exists() || !dcomp.exists() {
|
|
eprintln!("skipped");
|
|
return;
|
|
}
|
|
|
|
let txt = std::fs::read_to_string(dcomp).unwrap();
|
|
let mut funcs: Vec<(u32, u32)> = Vec::new();
|
|
for line in txt.lines() {
|
|
if line.starts_with('#') || line.trim().is_empty() {
|
|
continue;
|
|
}
|
|
let f: Vec<&str> = line.split('\t').collect();
|
|
if f.len() < 6 {
|
|
continue;
|
|
}
|
|
let size: u32 = match f[2].parse() {
|
|
Ok(v) => v,
|
|
Err(_) => continue,
|
|
};
|
|
for pl in f[5].split(',') {
|
|
let mut it = pl.split(':');
|
|
if it.next() == Some("00") {
|
|
if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) {
|
|
funcs.push((a, size));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
funcs.sort_unstable();
|
|
|
|
let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap();
|
|
let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap();
|
|
let h = gc_formats::level_core::LevelCoreHeader::parse(&idx).unwrap();
|
|
let secs = gc_formats::overlay_reloc::parse_overlay_sections(&ov).unwrap();
|
|
let text = &secs[6].data;
|
|
let dest = secs[6].dest;
|
|
let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06);
|
|
let base = h.moby_classes.offset as u16;
|
|
|
|
// buckets of 16 distinct immediates; per bucket count hits for each value 0xb8..0xc8
|
|
let mut tot: std::collections::BTreeMap<usize, usize> = std::collections::BTreeMap::new();
|
|
let mut hit: std::collections::BTreeMap<(usize, u16), usize> =
|
|
std::collections::BTreeMap::new();
|
|
for (start, size) in &funcs {
|
|
if *size < 128 {
|
|
continue;
|
|
}
|
|
let off = (*start).wrapping_sub(dest) as usize;
|
|
if off >= text.len() {
|
|
continue;
|
|
}
|
|
let n = (*size as usize).min(text.len() - off) & !3;
|
|
let mut set = std::collections::BTreeSet::new();
|
|
for w in text[off..off + n].chunks_exact(4) {
|
|
let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]);
|
|
if has_imm((i >> 26) as u8) {
|
|
set.insert((i & 0xffff) as u16);
|
|
}
|
|
}
|
|
if set.is_empty() {
|
|
continue;
|
|
}
|
|
let b = (set.len() / 16) * 16;
|
|
*tot.entry(b).or_default() += 1;
|
|
for d in -8i32..=8i32 {
|
|
let v = (base as i32 + d) as u16;
|
|
if set.contains(&v) {
|
|
*hit.entry((b, v)).or_default() += 1;
|
|
}
|
|
}
|
|
}
|
|
eprintln!("M bucket n 0xc0 best_neighbour ratio");
|
|
let mut checked = 0usize;
|
|
let mut worst = f64::INFINITY;
|
|
for (b, n) in &tot {
|
|
if *n < 20 {
|
|
continue;
|
|
}
|
|
let c0 = hit.get(&(*b, base)).copied().unwrap_or(0);
|
|
let mut best = 0u16;
|
|
let mut bestc = 0usize;
|
|
for d in -8i32..=8i32 {
|
|
if d == 0 {
|
|
continue;
|
|
}
|
|
let v = (base as i32 + d) as u16;
|
|
let c = hit.get(&(*b, v)).copied().unwrap_or(0);
|
|
if c > bestc {
|
|
bestc = c;
|
|
best = v;
|
|
}
|
|
}
|
|
let ratio = c0 as f64 / bestc.max(1) as f64;
|
|
eprintln!("M {b:5} {n:4} {c0:4} {best:#06x}:{bestc:<4} {ratio:.1}x");
|
|
worst = worst.min(ratio);
|
|
checked += 1;
|
|
}
|
|
assert!(checked >= 5, "only {checked} buckets large enough to judge");
|
|
// Enriched in every bucket, not just the big ones.
|
|
assert!(
|
|
worst > 5.0,
|
|
"enrichment falls to {worst:.1}x at some set size"
|
|
);
|
|
// And specifically in the smallest bucket, where volume cannot explain it.
|
|
let small_tot = tot.get(&0).copied().unwrap_or(0);
|
|
let small_c0 = hit.get(&(0usize, base)).copied().unwrap_or(0);
|
|
assert!(small_tot > 1_000, "small bucket only {small_tot}");
|
|
assert!(small_c0 > 50, "only {small_c0} small functions contain it");
|
|
}
|
|
|
|
/// The check sixteen probes skipped: `OverlaySection` indexing must agree with the
|
|
/// flat overlay at `LIT_BASE`, which is what `tools/overlay-anchors` uses.
|
|
#[test]
|
|
fn overlay_section_index_agrees_with_the_flat_blob() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
let Ok(ov) = std::fs::read(data.join("level0/overlay.bin")) else {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
};
|
|
use gc_formats::overlay_reloc::{overlay_bytes_at, parse_overlay_sections, LIT_BASE};
|
|
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
let mut checked = 0usize;
|
|
// The word that exposed the bug: capstone reports 0x4bc00083 at 0x2ee138.
|
|
let got = overlay_bytes_at(&ov, 0x002E_E138).expect("in range");
|
|
assert_eq!(
|
|
u32::from_le_bytes(got[..4].try_into().unwrap()),
|
|
0x4bc0_0083,
|
|
"0x2ee138 must read 0x4bc00083"
|
|
);
|
|
checked += 1;
|
|
// And the overlay really is a memory image: its first word is .lit's address.
|
|
assert_eq!(
|
|
u32::from_le_bytes(ov[..4].try_into().unwrap()),
|
|
LIT_BASE,
|
|
"overlay[0] must be the .lit load address"
|
|
);
|
|
for s in &secs {
|
|
for (i, chunk) in s.data.chunks(4).take(16).enumerate() {
|
|
if chunk.len() < 4 {
|
|
break;
|
|
}
|
|
let addr = s.dest + (i * 4) as u32;
|
|
let flat = overlay_bytes_at(&ov, addr).map(|c| &c[..4]);
|
|
assert!(
|
|
flat.is_some() || addr < LIT_BASE,
|
|
"section {addr:#x} out of range"
|
|
);
|
|
checked += 1;
|
|
}
|
|
}
|
|
assert!(checked > 100, "only {checked} words checked");
|
|
|
|
eprintln!("{checked} words agree; 0x2ee138 verified");
|
|
}
|
|
|
|
/// `moby_classes.offset` is enriched over its neighbours, on correct bytes.
|
|
///
|
|
/// This is `moby_classes_offset_enrichment_survives_set_size_matching` redone
|
|
/// through `overlay_bytes_at` instead of the displaced section index, after
|
|
/// `cd16485` established the flat memory image is the correct mapping.
|
|
///
|
|
/// It survives. 391 of 2,313 level-0 functions carry the immediate `0x00c0`
|
|
/// against a best neighbour of 15, and at matched immediate-set size the ratio
|
|
/// runs 11x to 45x. The displaced version measured 394 and 13x-72x, so the
|
|
/// uniform displacement shifted windows without materially changing their
|
|
/// contents -- which is why the frequency result held and the disassembly did not.
|
|
///
|
|
/// Bounds: level 0 only; extents from the decomp, "level 0" meaning first listed
|
|
/// place is level 00; 128-byte size floor. Presence of an immediate is not proof
|
|
/// the chunk is read there -- only that the constant is used.
|
|
#[test]
|
|
fn moby_classes_enrichment_survives_the_correct_mapping() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
PathBuf::from(std::env::var("HOME").unwrap()).join("Documents/gc-extracted")
|
|
});
|
|
let dcomp = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.join("../../vendor/rac2-gc-decomp/config/levels/functions.tsv");
|
|
if !data.exists() || !dcomp.exists() {
|
|
eprintln!("skipped");
|
|
return;
|
|
}
|
|
let ov = std::fs::read(data.join("level0/overlay.bin")).unwrap();
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::overlay_reloc::overlay_bytes_at;
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
let txt = std::fs::read_to_string(dcomp).unwrap();
|
|
let mut funcs: Vec<(u32, u32)> = Vec::new();
|
|
for line in txt.lines() {
|
|
if line.starts_with('#') || line.trim().is_empty() {
|
|
continue;
|
|
}
|
|
let f: Vec<&str> = line.split('\t').collect();
|
|
if f.len() < 6 {
|
|
continue;
|
|
}
|
|
let size: u32 = match f[2].parse() {
|
|
Ok(v) => v,
|
|
Err(_) => continue,
|
|
};
|
|
for pl in f[5].split(',') {
|
|
let mut it = pl.split(':');
|
|
if it.next() == Some("00") {
|
|
if let Some(a) = it.next().and_then(|x| u32::from_str_radix(x, 16).ok()) {
|
|
funcs.push((a, size));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
funcs.sort_unstable();
|
|
|
|
let idx = std::fs::read(data.join("level0/core_index.bin")).unwrap();
|
|
let h = LevelCoreHeader::parse(&idx).unwrap();
|
|
let base = h.moby_classes.offset as u16;
|
|
let has_imm = |op: u8| matches!(op, 0x09 | 0x0d | 0x0f | 0x0c | 0x05 | 0x06);
|
|
|
|
let mut sets: Vec<BTreeSet<u16>> = Vec::new();
|
|
for (start, size) in &funcs {
|
|
if *size < 128 {
|
|
continue;
|
|
}
|
|
let b = match overlay_bytes_at(&ov, *start) {
|
|
Some(b) => b,
|
|
None => continue,
|
|
};
|
|
let n = (*size as usize).min(b.len()) & !3;
|
|
let mut set = BTreeSet::new();
|
|
for w in b[..n].chunks_exact(4) {
|
|
let i = u32::from_le_bytes([w[0], w[1], w[2], w[3]]);
|
|
if has_imm((i >> 26) as u8) {
|
|
set.insert((i & 0xffff) as u16);
|
|
}
|
|
}
|
|
if !set.is_empty() {
|
|
sets.push(set);
|
|
}
|
|
}
|
|
let hit = |v: u16| sets.iter().filter(|s| s.contains(&v)).count();
|
|
eprintln!("F functions kept {} (>=128b, decodable window)", sets.len());
|
|
eprintln!("F moby_classes {base:#06x} -> {}", hit(base));
|
|
assert!(sets.len() > 2_000, "only {} functions", sets.len());
|
|
let self_hits = hit(base);
|
|
assert!(
|
|
self_hits > 100,
|
|
"only {self_hits} functions carry {base:#06x}"
|
|
);
|
|
let mut nb = Vec::new();
|
|
for d in -8i32..=8i32 {
|
|
if d != 0 {
|
|
let v = (base as i32 + d) as u16;
|
|
nb.push((v, hit(v)));
|
|
}
|
|
}
|
|
nb.sort_by_key(|(_, c)| std::cmp::Reverse(*c));
|
|
let vals: Vec<String> = nb.iter().map(|(v, c)| format!("{v:#06x}:{c}")).collect();
|
|
eprintln!("F neighbours {}", vals.join(" "));
|
|
// bucket-matched, as before
|
|
let mut tot: BTreeMap<usize, usize> = BTreeMap::new();
|
|
let mut bh: BTreeMap<(usize, u16), usize> = BTreeMap::new();
|
|
for s in &sets {
|
|
let b = (s.len() / 16) * 16;
|
|
*tot.entry(b).or_default() += 1;
|
|
for d in -8i32..=8i32 {
|
|
let v = (base as i32 + d) as u16;
|
|
if s.contains(&v) {
|
|
*bh.entry((b, v)).or_default() += 1;
|
|
}
|
|
}
|
|
}
|
|
for (b, n) in &tot {
|
|
if *n < 20 {
|
|
continue;
|
|
}
|
|
let c0 = bh.get(&(*b, base)).copied().unwrap_or(0);
|
|
let mut best = 0usize;
|
|
for d in -8i32..=8i32 {
|
|
if d != 0 {
|
|
let v = (base as i32 + d) as u16;
|
|
best = best.max(bh.get(&(*b, v)).copied().unwrap_or(0));
|
|
}
|
|
}
|
|
eprintln!(
|
|
"F bucket {b:5} n={n:4} c0={c0:4} best_neighbour={best:4} ratio={:.1}",
|
|
c0 as f64 / best.max(1) as f64
|
|
);
|
|
assert!(
|
|
c0 > best * 5,
|
|
"bucket {b}: {c0} vs neighbour {best} -- enrichment gone at this set size"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The camera region is nine orthonormal affine matrices sharing a translation.
|
|
///
|
|
/// `m4` at `0x18ab70` is the live camera orientation.
|
|
///
|
|
/// Established by rotating the camera in the live emulator and measuring which
|
|
/// matrix responds: m4's first row swings 30-37 degrees while the other eight stay
|
|
/// under 4 across the whole sweep. All nine translations move together, so the nine
|
|
/// share one camera position and carry nine orientations, of which this one is live.
|
|
/// `R` for `guard_band` step 1 is rows 0-2 of this matrix.
|
|
///
|
|
/// Read live over PINE: two EE RAM dumps taken seconds apart, then the region at
|
|
/// `0x18aa70` is checked to be a 4x4 matrix whose upper 3x3 is orthonormal and
|
|
/// whose fourth row is a translation that moves smoothly between the two reads.
|
|
///
|
|
/// Orthonormality is the control, and it is the part that makes this an
|
|
/// identification rather than a coincidence: three consecutive float triples
|
|
/// happening to be unit length and mutually perpendicular is not something a
|
|
/// random region of 32 MiB does.
|
|
#[test]
|
|
fn camera_transform_is_orthonormal_with_a_moving_translation() {
|
|
let Ok(p) = gc_trace::pine::Pine::connect() else {
|
|
eprintln!("skipped: PINE not available (is PCSX2 running?)");
|
|
return;
|
|
};
|
|
let a = p.dump_ee().expect("dump");
|
|
std::thread::sleep(std::time::Duration::from_millis(1500));
|
|
let b = p.dump_ee().expect("dump");
|
|
|
|
let f = |r: &[u8], o: usize| f32::from_le_bytes(r[o..o + 4].try_into().unwrap());
|
|
let m = |r: &[u8], base: usize, row: usize| {
|
|
[
|
|
f(r, base + row * 16),
|
|
f(r, base + row * 16 + 4),
|
|
f(r, base + row * 16 + 8),
|
|
f(r, base + row * 16 + 12),
|
|
]
|
|
};
|
|
|
|
// Nine, contiguous at 64-byte stride from 0x18aa70, ending at 0x18acb0 where
|
|
// the structure breaks (row norms 0, 0, 0.97 and w = 0). `dcd4dd6` said three
|
|
// and was wrong: it stopped looking after three because the first three
|
|
// happened to read as a sensible family.
|
|
for name in 0..9usize {
|
|
let base = 0x0018_AA70usize + name * 64;
|
|
for row in 0..3 {
|
|
let r = m(&a, base, row);
|
|
let norm = (r[0] * r[0] + r[1] * r[1] + r[2] * r[2]).sqrt();
|
|
assert!(
|
|
(norm - 1.0).abs() < 1e-4,
|
|
"{name} row {row} is not a unit vector: |r| = {norm}"
|
|
);
|
|
}
|
|
for (i, j) in [(0usize, 1usize), (0, 2), (1, 2)] {
|
|
let x = m(&a, base, i);
|
|
let y = m(&a, base, j);
|
|
let d = x[0] * y[0] + x[1] * y[1] + x[2] * y[2];
|
|
assert!(
|
|
d.abs() < 1e-4,
|
|
"{name} rows {i},{j} are not orthogonal: {d}"
|
|
);
|
|
}
|
|
let ta = m(&a, base, 3);
|
|
let tb = m(&b, base, 3);
|
|
assert_eq!(ta[3], 1.0, "{name} fourth row is not affine (w != 1)");
|
|
let moved = (ta[0] - tb[0]).hypot(ta[1] - tb[1]).hypot(ta[2] - tb[2]);
|
|
eprintln!(
|
|
"camera m{} at [{:.3}, {:.3}, {:.3}] moved {:.4} in 1.5s",
|
|
name, ta[0], ta[1], ta[2], moved
|
|
);
|
|
assert!(
|
|
moved > 0.0,
|
|
"{name} translation did not move; the game is paused?"
|
|
);
|
|
assert!(
|
|
moved < 50.0,
|
|
"{name} translation jumped {moved}; frames are not adjacent"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// `lvl.camvtbl` role 6 is `UpdateCamera_6`, verified in our overlay.
|
|
///
|
|
/// `docs/AUG6-RETAIL-ANCHORS.md` recovers `UpdateCamera_6` and `InitCamera_24` as
|
|
/// class/role identifiers in the two dispatch tables, so role 6 of `lvl.camvtbl`
|
|
/// *is* the camera update function. The decomp's addresses are in the same space as
|
|
/// ours -- its `func_L00_002A3B80` is this overlay's `.text` dest exactly -- so the
|
|
/// claim is checkable rather than merely asserted.
|
|
///
|
|
/// Three independent checks, because `PROVENANCE.md` records that the decomp's
|
|
/// function addresses have failed once here:
|
|
///
|
|
/// 1. role 6 of level 0's `lvl.camvtbl` names handler `0x004284a8`;
|
|
/// 2. `overlay_bytes_at` reads real code there -- `0x3c014094`, a `lui`;
|
|
/// 3. `tools/overlay-anchors` decodes the first six words with **zero** failures and
|
|
/// they are floating-point: `mtc1 $at, $f1` after building `0x40947ae1`, then
|
|
/// `mtc1 $at, $f3` after `0x3f000000` (0.5f). A camera update doing float work;
|
|
/// not what a random address would give.
|
|
#[test]
|
|
fn camvtbl_role_6_is_the_camera_update_function() {
|
|
let Ok(p) = std::env::var("GC_EXTRACTED") else {
|
|
eprintln!("skipped: set GC_EXTRACTED");
|
|
return;
|
|
};
|
|
let Ok(ov) = std::fs::read(std::path::Path::new(&p).join("level0/overlay.bin")) else {
|
|
eprintln!("skipped: no overlay.bin");
|
|
return;
|
|
};
|
|
use gc_formats::overlay_reloc::{overlay_bytes_at, parse_overlay_sections};
|
|
let secs = parse_overlay_sections(&ov).expect("sections");
|
|
let cam = &secs[4];
|
|
let mut handler = None;
|
|
for r in 0..cam.data.len() / 12 {
|
|
let id = u32::from_le_bytes(cam.data[r * 12..r * 12 + 4].try_into().unwrap());
|
|
if id == u32::MAX {
|
|
break;
|
|
}
|
|
let h = u32::from_le_bytes(cam.data[r * 12 + 4..r * 12 + 8].try_into().unwrap());
|
|
if h == u32::MAX {
|
|
break;
|
|
}
|
|
if id == 6 {
|
|
handler = Some(h);
|
|
break;
|
|
}
|
|
}
|
|
let h = handler.expect("camvtbl has no role 6");
|
|
assert_eq!(h, 0x0042_84A8, "camvtbl role 6 moved");
|
|
let text_end = secs[6].dest + secs[6].data.len() as u32;
|
|
assert!(
|
|
(secs[6].dest..text_end).contains(&h),
|
|
"handler {h:#x} is outside .text"
|
|
);
|
|
let w = overlay_bytes_at(&ov, h).expect("in range");
|
|
assert_eq!(
|
|
u32::from_le_bytes(w[..4].try_into().unwrap()),
|
|
0x3C01_4094,
|
|
"first word at the handler changed"
|
|
);
|
|
assert_eq!(u32::from_le_bytes(w[4..8].try_into().unwrap()), 0x3463_7AE1);
|
|
assert_eq!(
|
|
u32::from_le_bytes(w[8..12].try_into().unwrap()) >> 26,
|
|
0x10,
|
|
"third word should be mtc1, got {:#x}",
|
|
w[8]
|
|
);
|
|
}
|