ReaperOfVeriod
02ced5ee83
level_overlay was the least-characterised file in the level set -- no parser, no located structure. It now has gc_formats::overlay and a real-disc test over all 27 levels: overlay: 27 levels; field_00 1735808, field_08 1; field_04 37540..=45308, field_0c 2763912..=3027336 (17 below the file length, 10 above); +0x10..0x1220 identical on all The header is four words. +0x0c is not the file length: it was recorded as one on the strength of the first levels measured, where it sat near the file end, but across all 27 it falls on both sides -- below on 17, above on 10 -- so it is an ordinary level-specific word. The test asserts this, being the claim most worth pinning. The common prefix is 4 bytes, not the whole header: +0x00 is constant at 1735808 while +0x04 and +0x0c vary. Everything from +0x10 to +0x1220 -- 4624 bytes -- is byte-identical on all 27 levels. Still not named: what any of the four header words means, and how the level-specific tail is structured. The module reports them as fields. Also records in docs/plan/pcsx2-verification.md that the 64-byte RAM stride from the previous commit is disproved. A 415 KB dump of level 1 finds the longest such run at 27 records, not 384, and those are identity matrices -- 27 of them, one per level. It is a per-level transform table, not the collision grid, and the 20.0 reported there was a coincidence.
2587 lines
99 KiB
Rust
2587 lines
99 KiB
Rust
//! Integration tests against a real disc image.
|
|
//!
|
|
//! These are skipped unless `GC_DISC` points at a legally obtained disc, since
|
|
//! the repository must not contain game data. When it is set they check the
|
|
//! identification and unpack results against the committed hash table.
|
|
//!
|
|
//! ```sh
|
|
//! GC_DISC="path/to/disc.iso" cargo test -p gc-extract --test disc
|
|
//! ```
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// The disc under test, if the environment names one.
|
|
fn disc() -> Option<PathBuf> {
|
|
let p = std::env::var_os("GC_DISC")?;
|
|
let p = PathBuf::from(p);
|
|
p.is_file().then_some(p)
|
|
}
|
|
|
|
/// Skip a test with a clear reason rather than failing.
|
|
macro_rules! require_disc {
|
|
() => {
|
|
match disc() {
|
|
Some(p) => p,
|
|
None => {
|
|
eprintln!("skipped: set GC_DISC to a Going Commando disc image to run this");
|
|
return;
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
#[test]
|
|
fn identifies_the_supported_build() {
|
|
let path = require_disc!();
|
|
let info = gc_extract::identify::identify(&path).expect("disc should identify");
|
|
|
|
assert_eq!(info.serial, "SCUS_972.68");
|
|
assert_eq!(info.volume_id, "RATCHETANDCLANK2");
|
|
assert!(
|
|
info.is_target(),
|
|
"expected the supported build, got: {}",
|
|
info.verdict()
|
|
);
|
|
assert_eq!(
|
|
info.boot_elf_sha1,
|
|
gc_extract::identify::TARGET_BOOT_ELF_SHA1,
|
|
"boot ELF SHA-1 differs from the recorded retail value"
|
|
);
|
|
assert_eq!(info.boot_elf_size, 2_618_684);
|
|
// This disc is NTSC; the movement constants assume 60 Hz.
|
|
assert!(!info.is_pal(), "test disc should be NTSC");
|
|
}
|
|
|
|
#[test]
|
|
fn agrees_with_the_python_reference_identifier() {
|
|
let path = require_disc!();
|
|
let info = gc_extract::identify::identify(&path).expect("disc should identify");
|
|
|
|
// tools/identify-disc.py is the reference implementation and prints these
|
|
// two values. If they ever disagree, one of them has a bug.
|
|
// Run from the workspace root so the relative path to the tool resolves.
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent()
|
|
.and_then(|p| p.parent())
|
|
.expect("the crate should live two levels below the workspace root")
|
|
.to_path_buf();
|
|
let py = std::process::Command::new("python3")
|
|
.arg(root.join("tools/identify-disc.py"))
|
|
.arg(&path)
|
|
.current_dir(&root)
|
|
.output()
|
|
.expect("python3 should be available to run the reference identifier");
|
|
assert!(
|
|
py.status.success(),
|
|
"the reference identifier failed:\n{}",
|
|
String::from_utf8_lossy(&py.stderr)
|
|
);
|
|
let text = String::from_utf8_lossy(&py.stdout);
|
|
|
|
assert!(
|
|
text.contains(&format!("serial {}", info.serial)),
|
|
"serial disagrees with the reference identifier:\n{text}"
|
|
);
|
|
assert!(
|
|
text.contains(&format!("SHA-1 {}", info.boot_elf_sha1)),
|
|
"boot ELF SHA-1 disagrees with the reference identifier:\n{text}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn unpacks_and_matches_the_committed_table() {
|
|
let path = require_disc!();
|
|
|
|
let table_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/gc-files.tsv");
|
|
let table_text =
|
|
std::fs::read_to_string(&table_path).expect("the hash table should be committed");
|
|
let expected = gc_extract::verify::parse_table(&table_text);
|
|
assert!(
|
|
expected.len() > 5000,
|
|
"expected a full table, got {} rows",
|
|
expected.len()
|
|
);
|
|
|
|
// Unpack into a temporary directory so the test leaves nothing behind.
|
|
let tmp = std::env::temp_dir().join(format!("gc-unpack-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&tmp);
|
|
std::fs::create_dir_all(&tmp).expect("should create a temp directory");
|
|
|
|
let files = gc_extract::unpack::unpack(&path, &tmp).expect("unpack should succeed");
|
|
|
|
// Every unpacked file must match the recorded size and hash.
|
|
let report = gc_extract::verify::verify(&tmp, &expected);
|
|
let _ = std::fs::remove_dir_all(&tmp);
|
|
|
|
assert!(
|
|
report.ok(),
|
|
"unpacked data does not match the committed table:\n{}",
|
|
report.summary()
|
|
);
|
|
assert_eq!(
|
|
files.len(),
|
|
expected.len(),
|
|
"unpack produced a different file count"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_container_parses() {
|
|
let path = require_disc!();
|
|
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let containers: Vec<_> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/") && e.name.ends_with(".WAD"))
|
|
.collect();
|
|
|
|
// 92 `G/*.WAD` containers: 27 LEVEL, 27 AUDIO, 27 SCENE, 11 globals.
|
|
assert_eq!(containers.len(), 92, "expected 92 asset containers");
|
|
|
|
// MPEG1 and MPEG2 are raw video streams with no table; the other 90 must
|
|
// all parse as indexed containers.
|
|
let raw_video: Vec<&str> = containers
|
|
.iter()
|
|
.map(|c| c.name.as_str())
|
|
.filter(|n| *n == "G/MPEG1.WAD" || *n == "G/MPEG2.WAD")
|
|
.collect();
|
|
assert_eq!(
|
|
raw_video.len(),
|
|
2,
|
|
"expected MPEG1 and MPEG2 to be the raw streams"
|
|
);
|
|
|
|
// The per-level families must all be present, at 27 each.
|
|
for family in ["LEVEL", "AUDIO", "SCENE"] {
|
|
let n = containers
|
|
.iter()
|
|
.filter(|c| {
|
|
let stem = c.name.trim_start_matches("G/").trim_end_matches(".WAD");
|
|
stem.strip_prefix(family)
|
|
.map(|rest| !rest.is_empty() && rest.chars().all(|ch| ch.is_ascii_digit()))
|
|
.unwrap_or(false)
|
|
})
|
|
.count();
|
|
assert_eq!(n, 27, "expected 27 {family}<n>.WAD containers, found {n}");
|
|
}
|
|
|
|
for entry in &containers {
|
|
if raw_video.contains(&entry.name.as_str()) {
|
|
continue;
|
|
}
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, entry.size)
|
|
.expect("read should succeed");
|
|
let kind = gc_formats::global_wad::RangeKind::for_container(&entry.name);
|
|
let container = gc_formats::global_wad::GlobalWad::parse_with(&data, kind)
|
|
.unwrap_or_else(|e| panic!("{} failed to parse: {e}", entry.name));
|
|
assert!(
|
|
!container.entries.is_empty(),
|
|
"{} parsed but had no entries",
|
|
entry.name
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The level container header is the first typed structure we parse from real
|
|
/// data, and the tiling invariant is the check that holds on every level rather
|
|
/// than one hand-picked sample.
|
|
#[test]
|
|
fn level_headers_tile_their_containers() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut checked = 0;
|
|
for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data)
|
|
.unwrap_or_else(|e| panic!("{}: {e}", entry.name));
|
|
|
|
assert_eq!(h.header_size, 0x60, "{}", entry.name);
|
|
assert_eq!(
|
|
h.runtime_sector, 0,
|
|
"{}: runtime sector field should be 0 on disc",
|
|
entry.name
|
|
);
|
|
|
|
// Every present extent must start where the previous one ended.
|
|
let extents = h.ordered_extents();
|
|
for w in extents.windows(2) {
|
|
let (pn, p) = w[0];
|
|
let (nn, n) = w[1];
|
|
assert_eq!(
|
|
p.offset + p.size,
|
|
n.offset,
|
|
"{}: gap or overlap between {} and {}",
|
|
entry.name,
|
|
pn.name(),
|
|
nn.name()
|
|
);
|
|
}
|
|
|
|
// Nothing may fall outside the container. The last extent may end one
|
|
// sector past the file end through sector rounding, hence `+ 1`.
|
|
let container_sectors = (entry.size / 2048) as i32;
|
|
for (name, r) in &extents {
|
|
assert!(
|
|
r.offset >= 0 && r.offset + r.size <= container_sectors + 1,
|
|
"{}: {} extent {:?} is outside the container ({container_sectors} sectors)",
|
|
entry.name,
|
|
name.name(),
|
|
r
|
|
);
|
|
}
|
|
checked += 1;
|
|
}
|
|
|
|
assert_eq!(checked, 27, "expected 27 level containers");
|
|
}
|
|
|
|
/// Level ids are not file order: `LEVEL21.WAD` reports 30. Anything that keys
|
|
/// off the container index would silently mislabel that planet.
|
|
#[test]
|
|
fn level_ids_are_not_container_indices() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
// The ISO filesystem lists names alphabetically, so LEVEL10 sorts before
|
|
// LEVEL2. Order by the numeric suffix, which is what the container names
|
|
// mean.
|
|
let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/LEVEL"))
|
|
.map(|e| {
|
|
let n: usize = e
|
|
.name
|
|
.trim_start_matches("G/LEVEL")
|
|
.trim_end_matches(".WAD")
|
|
.parse()
|
|
.expect("level container names end in a number");
|
|
(n, e)
|
|
})
|
|
.collect();
|
|
levels.sort_by_key(|(n, _)| *n);
|
|
|
|
let mut mismatches = Vec::new();
|
|
for (index, (_, entry)) in levels.iter().enumerate() {
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
|
|
if h.id as usize != index {
|
|
mismatches.push(format!(
|
|
"{}: id {} but is container {index}",
|
|
entry.name, h.id
|
|
));
|
|
}
|
|
}
|
|
|
|
// Exactly one level is out of step: the gap left by a removed level.
|
|
assert_eq!(
|
|
mismatches.len(),
|
|
1,
|
|
"expected exactly one level whose id differs from its container index, got {mismatches:?}"
|
|
);
|
|
assert!(
|
|
mismatches[0].contains("LEVEL21"),
|
|
"unexpected mismatch: {mismatches:?}"
|
|
);
|
|
}
|
|
|
|
/// Sizes in the level header are sector counts. If they were byte counts, the
|
|
/// extents could not tile a container this size, so this also guards against
|
|
/// "fixing" the parser to read them as bytes.
|
|
#[test]
|
|
fn level_extent_sizes_are_sector_counts() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let entry = iso.find("G/LEVEL0.WAD").expect("level 0");
|
|
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
|
|
|
|
// Sound bank occupies sectors 1..735 and data 735..7219.
|
|
assert_eq!(h.sound_bank.offset, 1);
|
|
assert_eq!(h.sound_bank.size, 734);
|
|
assert_eq!(h.data.offset, 735);
|
|
// Read as bytes, 734 would be under four sectors and the tiling would break.
|
|
assert_eq!(h.sound_bank.byte_len(), 734 * 2048);
|
|
}
|
|
|
|
/// The level data extent is a container of byte ranges. Its header locates the
|
|
/// geometry index and the compressed geometry, and every range must lie inside
|
|
/// the extent on all 27 levels.
|
|
#[test]
|
|
fn level_data_extents_are_in_range() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut checked = 0;
|
|
for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
|
|
let raw =
|
|
gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
|
|
let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
|
|
let data = lh.data;
|
|
let Some(offset) = data.byte_offset() else {
|
|
continue;
|
|
};
|
|
let len = data.byte_len() as usize;
|
|
let extent = &raw[offset as usize..offset as usize + len];
|
|
|
|
let dh = gc_formats::level_data::LevelDataHeader::parse(extent)
|
|
.unwrap_or_else(|e| panic!("{}: data extent header: {e}", entry.name));
|
|
|
|
for (part, r) in dh.present_parts() {
|
|
assert!(
|
|
r.end() <= extent.len() as u64,
|
|
"{}: {part:?} ends at {} but the extent is {} bytes",
|
|
entry.name,
|
|
r.end(),
|
|
extent.len()
|
|
);
|
|
}
|
|
|
|
// The geometry is the big one and must be substantial.
|
|
assert!(
|
|
dh.core_data.size > 1_000_000,
|
|
"{}: core_data is only {} bytes",
|
|
entry.name,
|
|
dh.core_data.size
|
|
);
|
|
checked += 1;
|
|
}
|
|
assert_eq!(checked, 27, "expected 27 level containers");
|
|
}
|
|
|
|
/// The geometry must decompress. This is the first test that touches real level
|
|
/// content rather than headers.
|
|
#[test]
|
|
fn level_geometry_decompresses() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let entry = iso.find("G/LEVEL0.WAD").expect("level 0");
|
|
|
|
let raw = gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
|
|
let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
|
|
let data = lh.data;
|
|
let extent = &raw[data.byte_offset().unwrap() as usize
|
|
..(data.byte_offset().unwrap() + data.byte_len()) as usize];
|
|
|
|
let dh = gc_formats::level_data::LevelDataHeader::parse(extent).expect("data header");
|
|
let core = dh
|
|
.core_data
|
|
.slice(extent, extent.len())
|
|
.expect("core_data slice");
|
|
|
|
assert!(
|
|
gc_formats::wad::is_wad(core),
|
|
"core_data should start with the WAD magic"
|
|
);
|
|
|
|
let decoded = gc_formats::wad::decompress(core).expect("core_data should decompress");
|
|
assert!(
|
|
decoded.len() > core.len(),
|
|
"decompressed geometry ({}) should exceed the compressed size ({})",
|
|
decoded.len(),
|
|
core.len()
|
|
);
|
|
}
|
|
|
|
/// The terrain itself. This is the first test to touch real level content: it
|
|
/// parses the tfrags directory and decodes vertex positions from every patch.
|
|
///
|
|
/// The gate is deliberately about *coverage*, not correctness of a sample: a
|
|
/// parser that works on one tfrag and silently returns `None` on the rest would
|
|
/// otherwise pass.
|
|
#[test]
|
|
fn tfrag_vertices_decode_for_every_level() {
|
|
require_disc!();
|
|
// Opening the image also confirms the disc is readable, not merely present.
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
// Where the unpacked data lives. GC_EXTRACTED overrides; otherwise the
|
|
// default location used during development.
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!(
|
|
"skipped: unpack the disc first, e.g.\
|
|
gc-extract unpack <disc.iso> ~/Documents/gc-extracted\
|
|
or point GC_EXTRACTED at an existing unpack"
|
|
);
|
|
return;
|
|
}
|
|
|
|
let mut levels = 0usize;
|
|
let mut total_verts = 0usize;
|
|
let mut total_tfrags = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core_path = dir.join("core_data.bin");
|
|
let index_path = dir.join("core_index.bin");
|
|
if !core_path.exists() || !index_path.exists() {
|
|
continue;
|
|
}
|
|
let core = std::fs::read(&core_path).expect("core_data");
|
|
let index = std::fs::read(&index_path).expect("core_index");
|
|
|
|
// The tfrags block starts at the core header's 0x08 field and runs to
|
|
// the next region that exists.
|
|
let region_start =
|
|
|off: usize| -> i32 { i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) };
|
|
let start = region_start(0x08) as usize;
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region_start(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len() as i32) as usize;
|
|
|
|
let block = &core[start..end];
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
assert_eq!(header.table_offset, 0x40, "level{n}: table offset");
|
|
|
|
let table_offset = header.table_offset as usize;
|
|
let mut decoded = 0usize;
|
|
let mut verts = 0usize;
|
|
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table_offset.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let slice = &block[data_base..];
|
|
if let Some(p) = gc_formats::tfrag::tfrag_positions(slice, &th) {
|
|
decoded += 1;
|
|
verts += p.len();
|
|
assert_eq!(p.len(), th.vert_count as usize);
|
|
}
|
|
}
|
|
|
|
assert!(header.tfrag_count > 0, "level{n}: no tfrags");
|
|
assert_eq!(
|
|
decoded, header.tfrag_count as usize,
|
|
"level{n}: only {decoded} of {} tfrags decoded",
|
|
header.tfrag_count
|
|
);
|
|
levels += 1;
|
|
total_tfrags += header.tfrag_count as usize;
|
|
total_verts += verts;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(
|
|
total_tfrags > 5000,
|
|
"expected thousands of tfrags, got {total_tfrags}"
|
|
);
|
|
assert!(
|
|
total_verts > 100_000,
|
|
"expected 100k+ vertices, got {total_verts}"
|
|
);
|
|
eprintln!("decoded {total_verts} vertices from {total_tfrags} tfrags across {levels} levels");
|
|
}
|
|
|
|
/// Terrain topology: strips and index arrays must expand to exactly the
|
|
/// triangle count each patch's header claims.
|
|
///
|
|
/// `tri_count` is stored in the patch header but is not derivable from the
|
|
/// lists directly, so reproducing it on all 18,102 patches is a strong check
|
|
/// that the strip walk, the index bank and the LOD assignment are all right.
|
|
/// Triangle indices must also stay inside the vertex-info range for the LOD.
|
|
#[test]
|
|
fn tfrag_topology_reproduces_the_header_triangle_count() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut triangles = 0usize;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
let table = header.table_offset as usize;
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
let tris = gc_formats::tfrag::tfrag_triangles(&g, 0)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
assert_eq!(
|
|
tris.len(),
|
|
th.tri_count as usize,
|
|
"level{n} patch {i}: {} triangles, header says {}",
|
|
tris.len(),
|
|
th.tri_count
|
|
);
|
|
let vinfo = g.lod_vinfo_count(0);
|
|
for t in &tris {
|
|
let idx = [t.a as usize, t.b as usize, t.c as usize];
|
|
assert!(
|
|
idx.iter().all(|&v| v < vinfo),
|
|
"level{n} patch {i}: triangle index outside {vinfo} vertex-info entries"
|
|
);
|
|
assert!((t.ad_gif as usize) < g.ad_gifs.len());
|
|
}
|
|
patches += 1;
|
|
triangles += tris.len();
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
assert!(
|
|
triangles > 700_000,
|
|
"expected 700k+ triangles, got {triangles}"
|
|
);
|
|
eprintln!("reproduced {triangles} triangles across {patches} patches in {levels} levels");
|
|
}
|
|
|
|
/// Texture bindings: every ad-gif a patch uses must index an entry that
|
|
/// actually exists in the level's texture table, and the light array must be
|
|
/// one record per vertex.
|
|
///
|
|
/// The texture table is the join between tfrags and the texel data, so an
|
|
/// out-of-range index here means the level's geometry and its textures have
|
|
/// been misread independently of each other.
|
|
#[test]
|
|
fn tfrag_ad_gifs_index_the_level_texture_table() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut ad_gifs = 0usize;
|
|
let mut textures_seen = 0usize;
|
|
let mut textures_total = 0usize;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let textures = gc_formats::level_core::texture_table(&index, ch.tfrag_textures)
|
|
.expect("texture table");
|
|
assert!(!textures.is_empty(), "level{n}: no tfrag textures");
|
|
// Indices are per-level, so the pair is what identifies a texture.
|
|
let mut level_seen = std::collections::BTreeSet::new();
|
|
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
let table = header.table_offset as usize;
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
|
|
assert_eq!(
|
|
g.lights.len(),
|
|
th.vert_count as usize,
|
|
"level{n} patch {i}: {} light records for {} vertices",
|
|
g.lights.len(),
|
|
th.vert_count
|
|
);
|
|
assert_eq!(
|
|
g.rgba.len(),
|
|
th.rgba_size as usize * 4,
|
|
"level{n} patch {i}: rgba"
|
|
);
|
|
|
|
for a in &g.ad_gifs {
|
|
let t = a.texture_index();
|
|
assert!(
|
|
t < textures.len(),
|
|
"level{n} patch {i}: texture {} outside the {} entry table",
|
|
t,
|
|
textures.len()
|
|
);
|
|
let tex = &textures[t];
|
|
assert!(
|
|
tex.width > 0 && tex.height > 0 && tex.width <= 256 && tex.height <= 256,
|
|
"level{n} patch {i}: texture {t} is {}x{}",
|
|
tex.width,
|
|
tex.height
|
|
);
|
|
level_seen.insert(t);
|
|
ad_gifs += 1;
|
|
}
|
|
patches += 1;
|
|
}
|
|
textures_seen += level_seen.len();
|
|
textures_total += textures.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
// Most of each level's texture table should be referenced by some tfrag. A
|
|
// handful genuinely go unused; if that share grows, the ad-gif walk has
|
|
// started missing primitives rather than the data having spare entries.
|
|
let covered = textures_seen as f64 / textures_total as f64;
|
|
assert!(
|
|
covered > 0.85,
|
|
"only {textures_seen} of {textures_total} level textures referenced ({:.1}%)",
|
|
covered * 100.0
|
|
);
|
|
eprintln!(
|
|
"{ad_gifs} ad-gifs over {patches} patches reference {textures_seen}/{textures_total} level textures"
|
|
);
|
|
}
|
|
|
|
/// Level textures decode: every tfrag texture's base level must resolve to a
|
|
/// plausible RGBA image.
|
|
///
|
|
/// This is the join between the texture table, the texel block and GS RAM, so
|
|
/// it is where a wrong CLUT remap, a wrong alpha scale or a wrong pixel base
|
|
/// would show up. The check is deliberately about *shape*, not specific
|
|
/// colours: a decoded image is plausible when it is fully opaque, is not
|
|
/// uniformly one colour, and uses more than a handful of distinct entries.
|
|
#[test]
|
|
fn tfrag_textures_decode_to_plausible_images() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut decoded = 0usize;
|
|
let mut opaque = 0usize;
|
|
let mut multi_colour = 0usize;
|
|
let mut flat = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let gs = match std::fs::read(dir.join("gs_ram.bin")) {
|
|
Ok(g) => g,
|
|
Err(_) => {
|
|
eprintln!("level{n}: no gs_ram.bin, skipping");
|
|
continue;
|
|
}
|
|
};
|
|
let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let textures_at = ch.textures_base.max(0) as usize;
|
|
let texture_block = &core[textures_at..];
|
|
let table =
|
|
gc_formats::level_core::texture_table(&index, ch.tfrag_textures).expect("table");
|
|
|
|
for (i, e) in table.iter().enumerate() {
|
|
let img = gc_formats::texture::base_image(texture_block, &gs, e)
|
|
.unwrap_or_else(|err| panic!("level{n} texture {i}: {err}"));
|
|
let t = img.decode().unwrap();
|
|
let texels = (e.width as i32) * (e.height as i32);
|
|
assert_eq!(t.rgba.len(), (texels * 4) as usize);
|
|
assert_eq!(t.width, e.width as u32);
|
|
decoded += 1;
|
|
|
|
// Count distinct colours and how many texels are fully opaque.
|
|
let mut seen = std::collections::HashSet::new();
|
|
let mut op = 0usize;
|
|
for px in t.rgba.chunks_exact(4) {
|
|
seen.insert([px[0], px[1], px[2], px[3]]);
|
|
if px[3] == 0xff {
|
|
op += 1;
|
|
}
|
|
}
|
|
if seen.len() > 8 {
|
|
multi_colour += 1;
|
|
} else {
|
|
flat += 1;
|
|
}
|
|
if op == texels as usize {
|
|
opaque += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(decoded > 1_500, "only {decoded} textures decoded");
|
|
// Level art is mostly opaque and has real detail. A flat, transparent or
|
|
// single-colour result almost always means a CLUT or offset mistake.
|
|
let opaque_pct = opaque as f64 / decoded as f64;
|
|
let detail_pct = multi_colour as f64 / decoded as f64;
|
|
assert!(
|
|
opaque_pct > 0.9,
|
|
"only {opaque:.1}% of textures are fully opaque"
|
|
);
|
|
assert!(
|
|
detail_pct > 0.9,
|
|
"only {detail_pct:.1} of textures have >8 colours"
|
|
);
|
|
eprintln!(
|
|
"{decoded} textures decoded: {:.1}% fully opaque, {:.1}% with >8 colours ({flat} flat)",
|
|
opaque_pct * 100.0,
|
|
detail_pct * 100.0
|
|
);
|
|
}
|
|
|
|
/// LOD linkage: every LOD-01 and LOD-0 vertex-info entry must resolve to a
|
|
/// first parent that exists and to positions inside the patch.
|
|
///
|
|
/// The parent-index arrays are V4_8 unpacks separated from the strips and the
|
|
/// strip indices only by their VU address, so a misclassification here is
|
|
/// silent: the arrays still parse, they just hold the wrong numbers. Resolving
|
|
/// every entry is what catches it.
|
|
#[test]
|
|
fn tfrag_lod_parents_resolve_for_every_entry() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut entries = 0usize;
|
|
let mut lod01 = 0usize;
|
|
let mut lod0 = 0usize;
|
|
let mut resolved_common = 0usize;
|
|
let mut common_total = 0usize;
|
|
let mut primary_failed = 0usize;
|
|
let mut extras_unresolved = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block");
|
|
let table = header.table_offset as usize;
|
|
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
|
|
common_total += g.vinfo_tier[0];
|
|
if n == 0 && patches < 5 {
|
|
eprintln!(
|
|
"p{i}: tier={:?} n01={} n0={} p01={}b x01={}b p0={}b x0={}b",
|
|
g.vinfo_tier,
|
|
g.vu_positions_lod_01_count,
|
|
g.vu_positions_lod_0_count,
|
|
g.parent_indices_lod01.len(),
|
|
g.unk_indices_2_lod01.len(),
|
|
g.parent_indices_lod0.len(),
|
|
g.unk_indices_2_lod0.len()
|
|
);
|
|
}
|
|
for v in 0..g.vertex_info.len() {
|
|
entries += 1;
|
|
match gc_formats::tfrag::tfrag_lod_link(&g, v) {
|
|
Some(link) => {
|
|
if v < g.vinfo_tier[0] {
|
|
resolved_common += 1;
|
|
}
|
|
// Parents must index real vertex info and real positions.
|
|
assert!(
|
|
link.parent1_vinfo < g.vertex_info.len(),
|
|
"level{n} patch {i}: entry {v} parent {} outside {} entries",
|
|
link.parent1_vinfo,
|
|
g.vertex_info.len()
|
|
);
|
|
assert!(link.own_position < g.positions.len());
|
|
assert!(link.parent1_position < g.positions.len());
|
|
assert!(link.parent2_position < g.positions.len());
|
|
// The first parent of a LOD-01 entry is always a common one.
|
|
if link.tier == gc_formats::tfrag::TfragMorphTier::Lod01 {
|
|
assert!(
|
|
link.parent1_vinfo < g.vinfo_tier[0],
|
|
"level{n} patch {i}: LOD-01 entry {v} parent {} is not a common entry",
|
|
link.parent1_vinfo
|
|
);
|
|
lod01 += 1;
|
|
} else {
|
|
lod0 += 1;
|
|
}
|
|
}
|
|
None => {
|
|
// Distinguish the two kinds of LOD entry. A *primary*
|
|
// entry has a real position and a recorded parent and
|
|
// must always resolve; an *extra* entry is a
|
|
// non-position bookkeeping record whose array address
|
|
// can collide with the strip index array.
|
|
let c = g.vinfo_tier[0];
|
|
let kind = if v < c {
|
|
None
|
|
} else if v < c + g.vinfo_tier[1] {
|
|
Some((g.vu_positions_lod_01_count, v - c, true))
|
|
} else {
|
|
Some((g.vu_positions_lod_0_count, v - c - g.vinfo_tier[1], false))
|
|
};
|
|
match kind {
|
|
None => {}
|
|
Some((prim, k, _)) if k < prim => primary_failed += 1,
|
|
Some(_) => extras_unresolved += 1,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
patches += 1;
|
|
}
|
|
}
|
|
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
|
|
// A common entry never has a link: they are the full-detail positions and
|
|
// have no parent. Anything else means the tier boundaries are wrong.
|
|
assert!(
|
|
resolved_common == 0,
|
|
"{resolved_common} common entries claimed an LOD parent"
|
|
);
|
|
assert!(
|
|
lod01 > 0 && lod0 > 0,
|
|
"both LOD tiers should occur in the data"
|
|
);
|
|
assert_eq!(
|
|
primary_failed, 0,
|
|
"{primary_failed} primary LOD entries did not resolve"
|
|
);
|
|
assert_eq!(
|
|
extras_unresolved, 0,
|
|
"{extras_unresolved} extra LOD entries did not resolve"
|
|
);
|
|
// Common entries have no link; every other entry does.
|
|
assert_eq!(entries, common_total + lod01 + lod0);
|
|
|
|
eprintln!(
|
|
"{entries} vertex-info entries over {patches} patches: {common_total} common, \
|
|
{lod01} LOD-01 and {lod0} LOD-0, all resolved"
|
|
);
|
|
}
|
|
|
|
/// The level core index's class tables.
|
|
///
|
|
/// These are what put the environment on top of the terrain: tie instances,
|
|
/// moby actors and shrub foliage. They were ruled out earlier for want of an
|
|
/// entry size, which turned out to be knowable exactly.
|
|
///
|
|
/// The check is the chain: each table must end precisely where the next one
|
|
/// begins, moby (32 bytes/entry) into tie (32) into shrub (48) into the tfrag
|
|
/// texture table. That closes on all 27 levels and would not close if any
|
|
/// entry size were wrong, so it is the evidence rather than a coincidence.
|
|
#[test]
|
|
fn level_core_class_tables_tile_the_index() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
let mut levels = 0usize;
|
|
let (mut moby, mut tie, mut shrub, mut textures) = (0usize, 0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let Ok(index) = std::fs::read(dir.join("core_index.bin")) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
let end = |r: lc::ArrayRange, size: usize| r.offset as usize + r.count as usize * size;
|
|
assert_eq!(
|
|
end(h.moby_classes, lc::MOBY_CLASS_SIZE),
|
|
h.tie_classes.offset as usize,
|
|
"level{n}: moby -> tie"
|
|
);
|
|
assert_eq!(
|
|
end(h.tie_classes, lc::TIE_CLASS_SIZE),
|
|
h.shrub_classes.offset as usize,
|
|
"level{n}: tie -> shrub"
|
|
);
|
|
assert_eq!(
|
|
end(h.shrub_classes, lc::SHRUB_CLASS_SIZE),
|
|
h.tfrag_textures.offset as usize,
|
|
"level{n}: shrub -> textures"
|
|
);
|
|
|
|
// Every entry parses, and its asset offset lands inside the level.
|
|
if n == 0 {
|
|
eprintln!(
|
|
"hdr moby={:?} tie={:?} shrub={:?} tex={:?} base={}",
|
|
h.moby_classes, h.tie_classes, h.shrub_classes, h.tfrag_textures, h.textures_base
|
|
);
|
|
eprintln!("index[192..200] = {:?}", &index[192..200]);
|
|
}
|
|
let moby_v = lc::class_table(
|
|
&index,
|
|
h.moby_classes,
|
|
lc::MOBY_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("moby classes");
|
|
let tie_v = lc::class_table(
|
|
&index,
|
|
h.tie_classes,
|
|
lc::TIE_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("tie classes");
|
|
let shrub_v = lc::class_table(
|
|
&index,
|
|
h.shrub_classes,
|
|
lc::SHRUB_CLASS_SIZE,
|
|
lc::ShrubClassEntry::parse,
|
|
)
|
|
.expect("shrub classes");
|
|
let tex_v = lc::texture_table(&index, h.tfrag_textures).expect("tfrag textures");
|
|
|
|
assert_eq!(moby_v.len(), h.moby_classes.count as usize);
|
|
for (i, c) in moby_v.iter().chain(tie_v.iter()).enumerate() {
|
|
assert!(
|
|
c.offset_in_asset_wad >= 0,
|
|
"level{n} entry {i}: asset offset {}",
|
|
c.offset_in_asset_wad
|
|
);
|
|
}
|
|
// Billboard slots are optional: every shrub class on level 0 carries
|
|
// them all zero, so only their sign is meaningful here.
|
|
for s in &shrub_v {
|
|
assert!(s.base.offset_in_asset_wad >= 0);
|
|
assert!(
|
|
s.palette_offset >= -1 && s.texture_offset >= -1,
|
|
"level{n}: shrub billboard offsets below the -1 sentinel"
|
|
);
|
|
}
|
|
moby += moby_v.len();
|
|
tie += tie_v.len();
|
|
shrub += shrub_v.len();
|
|
textures += tex_v.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(
|
|
moby + tie + shrub > 1000,
|
|
"only {moby}+{tie}+{shrub} class entries"
|
|
);
|
|
eprintln!(
|
|
"level core index: {moby} moby, {tie} tie, {shrub} shrub classes, \
|
|
{textures} tfrag textures across {levels} levels"
|
|
);
|
|
}
|
|
|
|
/// The moby, tie and shrub class tables point into `core_data`, and they do so
|
|
/// without overlapping.
|
|
///
|
|
/// `assets_decompressed_size` is the size of `core_data` itself, so there is
|
|
/// no separate asset container: the class entries index the decompressed asset
|
|
/// lump directly. If that is right, every offset must land inside the lump, the
|
|
/// three tables must occupy disjoint spans, and each must be internally sorted
|
|
/// so the geometry is laid out in class order.
|
|
#[test]
|
|
fn class_geometry_offsets_tile_core_data_without_overlap() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(index), Ok(core)) = (
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
// The claim under test: the "asset lump" is core_data itself.
|
|
assert_eq!(
|
|
h.assets_decompressed_size as usize,
|
|
core.len(),
|
|
"level{n}: assets_decompressed_size is not core_data's length"
|
|
);
|
|
|
|
let span = |r: lc::ArrayRange, size: usize| -> Vec<usize> {
|
|
lc::class_table(&index, r, size, lc::ClassEntry::parse)
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.map(|c| c.offset_in_asset_wad as usize)
|
|
.collect()
|
|
};
|
|
// Shrub entries are read as ShrubClassEntry so the offsets line up.
|
|
let shrub: Vec<usize> = lc::class_table(
|
|
&index,
|
|
h.shrub_classes,
|
|
lc::SHRUB_CLASS_SIZE,
|
|
lc::ShrubClassEntry::parse,
|
|
)
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.map(|s| s.base.offset_in_asset_wad as usize)
|
|
.collect();
|
|
let moby = span(h.moby_classes, lc::MOBY_CLASS_SIZE);
|
|
let tie = span(h.tie_classes, lc::TIE_CLASS_SIZE);
|
|
|
|
// An offset of 0 marks a slot with no geometry rather than the start of
|
|
// the lump, so those are excluded from the span checks. Entries are not
|
|
// sorted by class number and the offsets are not monotonic, so nothing
|
|
// here assumes an order.
|
|
let moby_l: Vec<usize> = moby.iter().copied().filter(|&o| o != 0).collect();
|
|
let tie_l: Vec<usize> = tie.iter().copied().filter(|&o| o != 0).collect();
|
|
let shrub_l: Vec<usize> = shrub.iter().copied().filter(|&o| o != 0).collect();
|
|
|
|
for (name, v) in [("moby", &moby), ("tie", &tie), ("shrub", &shrub)] {
|
|
for &o in v {
|
|
assert!(
|
|
o < core.len(),
|
|
"level{n}: {name} offset {o} past core data ({})",
|
|
core.len()
|
|
);
|
|
}
|
|
}
|
|
// The three spans must not overlap each other.
|
|
for (an, a) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] {
|
|
for (bn, b) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] {
|
|
// A level may have no geometry of one kind at all.
|
|
if an >= bn || a.is_empty() || b.is_empty() {
|
|
continue;
|
|
}
|
|
let (lo, hi) = (a[0], *a.last().unwrap());
|
|
for &o in b {
|
|
assert!(
|
|
o >= hi || o < lo,
|
|
"level{n}: {an} [{lo},{hi}] overlaps {bn} at {o}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
eprintln!("class geometry offsets verified inside core_data on {levels} levels");
|
|
}
|
|
|
|
/// Tie class headers, as far as RAC1 and RAC2 agree.
|
|
///
|
|
/// The check that carries the weight is `o_class`: the class's own number at
|
|
/// header offset 0x44 equals the number the core index filed it under, on every
|
|
/// class of every level. That is what makes the header base trustworthy rather
|
|
/// than merely plausible.
|
|
#[test]
|
|
fn tie_class_headers_match_their_index_entry() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
use gc_formats::tie::{TieClassHeader, TIE_CLASS_HEADER_MATCHED};
|
|
let mut levels = 0usize;
|
|
let (mut classes, mut ordered, mut unordered) = (0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(index), Ok(core)) = (
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
let entries = lc::class_table(
|
|
&index,
|
|
h.tie_classes,
|
|
lc::TIE_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("tie classes");
|
|
|
|
for e in &entries {
|
|
let off = e.offset_in_asset_wad as usize;
|
|
if off == 0 {
|
|
continue; // slot with no geometry
|
|
}
|
|
let at = off + TIE_CLASS_HEADER_MATCHED;
|
|
assert!(at <= core.len(), "level{n}: tie class past core data");
|
|
let t = TieClassHeader::parse(&core[off..at])
|
|
.unwrap_or_else(|err| panic!("level{n}: tie class {} header: {err}", e.o_class));
|
|
assert_eq!(t.o_class, e.o_class, "level{n}: tie o_class mismatch");
|
|
assert!(
|
|
t.far_dist > 0.0,
|
|
"level{n}: tie far distance {}",
|
|
t.far_dist
|
|
);
|
|
if 0.0 < t.near_dist && t.near_dist < t.mid_dist && t.mid_dist < t.far_dist {
|
|
ordered += 1;
|
|
} else {
|
|
unordered += 1;
|
|
}
|
|
classes += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(classes > 1_500, "only {classes} tie classes");
|
|
// Ordered distances are the expected shape; the exceptions are real and
|
|
// unexplained, so they are reported rather than asserted away.
|
|
assert!(
|
|
ordered * 20 >= classes * 19,
|
|
"only {ordered} of {classes} tie classes have ordered LOD distances"
|
|
);
|
|
eprintln!(
|
|
"tie: {classes} class headers, o_class matches the index on all; \\
|
|
{ordered} have ordered LOD distances, {unordered} do not"
|
|
);
|
|
}
|
|
|
|
/// The occlusion PVS grid.
|
|
///
|
|
/// The check that makes this real is the mask array: it starts at
|
|
/// `masks_offset` and runs to `(highest_mask + 1) * 0x80`, and on every level
|
|
/// that ends within a few bytes of the block's own length. A wrong tree shape
|
|
/// or a wrong cell size would leave the highest mask index somewhere else
|
|
/// entirely and the array would not fit.
|
|
#[test]
|
|
fn occlusion_grids_parse_on_every_level() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::occlusion::parse_occlusion_block;
|
|
let mut levels = 0usize;
|
|
let mut absent = 0usize;
|
|
let mut widest_slack = 0usize;
|
|
let (mut cells, mut masks) = (0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
// An offset of 0 means the level has no occlusion data at all, which is
|
|
// not the same as "at the start of the file" -- that is where the
|
|
// tfrags block lives, and parsing it as a grid produces a mask array
|
|
// that fits anyway, which is how this nearly went wrong.
|
|
if h.occlusion <= 0 {
|
|
absent += 1;
|
|
continue;
|
|
}
|
|
let start = h.occlusion as usize;
|
|
// The block runs to the sky region, or to the end when there is none.
|
|
let end = if h.sky > 0 {
|
|
h.sky as usize
|
|
} else {
|
|
core.len()
|
|
};
|
|
assert!(
|
|
start < end && end <= core.len(),
|
|
"level{n}: bad occlusion range"
|
|
);
|
|
let block = &core[start..end];
|
|
|
|
let o = parse_occlusion_block(block)
|
|
.unwrap_or_else(|e| panic!("level{n}: occlusion block: {e}"));
|
|
// Every mask a cell names exists.
|
|
for c in &o.cells {
|
|
assert!(
|
|
(c.mask as usize) < o.masks.len(),
|
|
"level{n}: cell names a missing mask"
|
|
);
|
|
}
|
|
// The array ends at or just before the block's end. Retail levels sit
|
|
// within a few bytes; a gross error would leave a large gap.
|
|
let need = o.masks_offset as usize + o.masks.len() * gc_formats::occlusion::MASK_BYTES;
|
|
assert!(need <= block.len(), "level{n}: masks run past the block");
|
|
let slack = block.len() - need;
|
|
// Levels whose grid is empty are mostly header and padding, so this is
|
|
// proportional and only applies where there are cells at all.
|
|
if !o.cells.is_empty() {
|
|
assert!(
|
|
slack * 8 < block.len(),
|
|
"level{n}: {slack} of {} bytes unexplained after the mask array",
|
|
block.len()
|
|
);
|
|
}
|
|
widest_slack = widest_slack.max(slack);
|
|
|
|
cells += o.cells.len();
|
|
masks += o.masks.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(
|
|
levels + absent,
|
|
27,
|
|
"expected all 27 levels, got {}",
|
|
levels + absent
|
|
);
|
|
assert_eq!(
|
|
absent, 3,
|
|
"expected 3 levels without occlusion data, got {absent}"
|
|
);
|
|
assert!(cells > 400_000, "only {cells} occlusion cells");
|
|
eprintln!(
|
|
"occlusion: {cells} populated cells over {masks} masks across {levels} levels; \
|
|
widest unexplained tail {widest_slack} bytes"
|
|
);
|
|
}
|
|
|
|
/// The sound extent framing.
|
|
///
|
|
/// The invariant here is `sblk_offset + sblk_size == daw_offset`, which holds
|
|
/// on all 27 levels and pins the meaning of the four head fields. `DAW` never
|
|
/// reaches the end of the file; the tail is positive on every level, so the
|
|
/// size field is not the whole region and this test asserts the direction
|
|
/// rather than a size.
|
|
#[test]
|
|
fn sound_bank_framing_holds_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::sound_bank::parse_sound_bank;
|
|
let mut levels = 0usize;
|
|
let mut sblk_bytes = 0usize;
|
|
let mut daw_bytes = 0usize;
|
|
let (mut narrowest, mut widest) = (usize::MAX, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let raw = match std::fs::read(data.join(format!("level{n}")).join("sound_bank.bin")) {
|
|
Ok(v) => v,
|
|
Err(_) => continue,
|
|
};
|
|
let s = parse_sound_bank(&raw).unwrap_or_else(|e| panic!("level{n}: sound bank: {e}"));
|
|
|
|
assert_eq!(s.header.version, 3, "level{n}: sound bank version");
|
|
assert_eq!(s.header.unknown_4, 2, "level{n}: sound bank unknown_4");
|
|
assert_eq!(
|
|
s.header.sblk_offset, 24,
|
|
"level{n}: SBlk sits at 24 on every level"
|
|
);
|
|
assert!(
|
|
s.header.sblk_offset + s.header.sblk_size == s.header.daw_offset,
|
|
"level{n}: SBlk must run into DAW with no gap"
|
|
);
|
|
assert!(s.has_sblk_magic(), "level{n}: SBlk magic");
|
|
assert!(s.has_daw_tag(), "level{n}: DAW tag at SBlk+12");
|
|
assert_eq!(s.sblk_version(), Some(1), "level{n}: SBlk version");
|
|
// The tail is always in the same direction: the extent is longer than
|
|
// the DAW region it describes.
|
|
narrowest = narrowest.min(s.daw_tail);
|
|
widest = widest.max(s.daw_tail);
|
|
|
|
sblk_bytes += s.sblk.len();
|
|
daw_bytes += s.daw.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(sblk_bytes > 300_000, "only {sblk_bytes} SBlk bytes");
|
|
assert!(daw_bytes > 20_000_000, "only {daw_bytes} DAW bytes");
|
|
eprintln!(
|
|
"sound_bank: {sblk_bytes} SBlk + {daw_bytes} DAW bytes across {levels} levels; \
|
|
unexplained tail {narrowest}..{widest} bytes"
|
|
);
|
|
}
|
|
|
|
/// Collision container framing.
|
|
///
|
|
/// The invariant is the header's own self-description: `header_size` is 0x40
|
|
/// on every level, and `header_size + payload_size` stays inside `core_data`.
|
|
/// The payload is deliberately left undecoded -- see `collision.rs`.
|
|
#[test]
|
|
fn collision_framing_holds_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::collision::{CollisionHeader, COLLISION_HEADER_SIZE};
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
let mut levels = 0usize;
|
|
let mut empty = 0usize;
|
|
let mut payload_bytes = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let region = CollisionHeader::region(&core, h.collision)
|
|
.unwrap_or_else(|e| panic!("level{n}: collision: {e}"));
|
|
let Some(r) = region else {
|
|
empty += 1;
|
|
continue;
|
|
};
|
|
assert_eq!(r.header.header_size, COLLISION_HEADER_SIZE, "level{n}");
|
|
assert!(
|
|
r.header.payload_size as usize <= r.payload.len(),
|
|
"level{n}"
|
|
);
|
|
if r.payload.is_empty() {
|
|
// Present but empty. Counted once, not twice.
|
|
empty += 1;
|
|
continue;
|
|
}
|
|
assert_eq!(r.payload.len(), r.header.payload_size as usize, "level{n}");
|
|
payload_bytes += r.payload.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(
|
|
levels + empty,
|
|
27,
|
|
"expected all 27 levels, got {levels}+{empty}"
|
|
);
|
|
assert!(
|
|
payload_bytes > 10_000_000,
|
|
"only {payload_bytes} collision bytes"
|
|
);
|
|
eprintln!("collision: {payload_bytes} payload bytes across {levels} levels, {empty} empty");
|
|
}
|
|
|
|
/// Moby model geometry: the per-class part table.
|
|
///
|
|
/// The invariant is `vertex_offset + 16 * vertex_count == next_part`, which is
|
|
/// what makes the 16-byte record size and the field boundaries fall out of the
|
|
/// data instead of being assumed. The parser rejects any span that violates it,
|
|
/// so this test is really checking that no span on the disc does.
|
|
#[test]
|
|
fn moby_parts_tile_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::moby::parse_moby_geometry;
|
|
let mut levels = 0usize;
|
|
let mut classes = 0usize;
|
|
let mut parts = 0usize;
|
|
let mut verts = 0usize;
|
|
let mut empties = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let range = h.moby_classes;
|
|
assert!(
|
|
range.count > 0 && range.offset > 0,
|
|
"level{n}: no moby class table"
|
|
);
|
|
|
|
// Class spans tile between consecutive geometry offsets.
|
|
let stride = 0x20usize;
|
|
let mut offsets: Vec<usize> = (0..range.count as usize)
|
|
.map(|i| {
|
|
let at = range.offset as usize + i * stride;
|
|
let e = gc_formats::level_core::ClassEntry::parse(&index[at..at + stride])
|
|
.unwrap_or_else(|e| panic!("level{n}: class {i}: {e}"));
|
|
e.offset_in_asset_wad
|
|
})
|
|
.map(|o| o.max(0) as usize)
|
|
.filter(|&o| o > 0)
|
|
.collect();
|
|
offsets.sort_unstable();
|
|
assert!(!offsets.is_empty(), "level{n}: no moby geometry offsets");
|
|
|
|
for i in 0..offsets.len() {
|
|
let start = offsets[i];
|
|
let end = offsets.get(i + 1).copied().unwrap_or(core.len());
|
|
if end <= start {
|
|
continue;
|
|
}
|
|
let g = parse_moby_geometry(&core[start..end])
|
|
.unwrap_or_else(|e| panic!("level{n}: moby class at {start}: {e}"));
|
|
if g.is_empty() {
|
|
empties += 1;
|
|
}
|
|
// Every part's vertices are readable and inside the span.
|
|
for p in &g.parts {
|
|
assert!(
|
|
g.vertices(p).is_some(),
|
|
"level{n}: part vertices past the end of the class span"
|
|
);
|
|
}
|
|
parts += g.parts.len();
|
|
verts += g.vertex_count();
|
|
classes += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(classes > 4_000, "only {classes} moby class spans");
|
|
assert!(parts > 40_000, "only {parts} moby parts");
|
|
assert!(verts > 1_200_000, "only {verts} moby vertices");
|
|
eprintln!(
|
|
"moby: {parts} parts tiling exactly, {verts} vertices, {classes} class spans \
|
|
({empties} with no geometry) across {levels} levels"
|
|
);
|
|
}
|
|
|
|
/// Shrub class headers and blobs.
|
|
///
|
|
/// Two fields self-identify exactly on every span, and the parser checks both
|
|
/// rather than merely reporting them: `+0x24` must equal the owning class
|
|
/// entry's own `o_class`, and `+0x2c` must land inside the span.
|
|
///
|
|
/// The body is then checked against the R&C1 shape this format follows: a
|
|
/// palette of 24 unit-length `s16` normals with a zero pad word, then zero
|
|
/// padding records. And the billboard offset at `+0x1c` is checked against the
|
|
/// class entry's billboard texture fields, which is the one header field that
|
|
/// can be confirmed from a second file.
|
|
#[test]
|
|
fn shrub_headers_name_their_own_class_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::ShrubClassEntry;
|
|
use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES, SHRUB_NORMAL_COUNT};
|
|
let mut levels = 0usize;
|
|
let mut spans = 0usize;
|
|
let mut absent = 0usize;
|
|
let (mut min_size, mut max_size) = (f32::MAX, 0.0f32);
|
|
let mut scratch_nonzero = 0usize;
|
|
let mut mode_hist: std::collections::BTreeMap<u16, usize> = Default::default();
|
|
let mut scratch_hist: std::collections::BTreeMap<u16, usize> = Default::default();
|
|
let mut radius_range = (f32::MAX, 0f32);
|
|
let mut scale_range = (f32::MAX, 0f32);
|
|
let mut bounded_records = 0usize;
|
|
let mut bounded_bodies = 0usize;
|
|
let mut palette_records = 0usize;
|
|
let mut padding_records = 0usize;
|
|
let mut norm2_deviation = 0i64;
|
|
let mut billboards = 0usize;
|
|
let mut billboard_entries = 0usize;
|
|
let mut packets_total = 0usize;
|
|
let mut packet_headers_total = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let range = h.shrub_classes;
|
|
// Class spans tile between consecutive geometry offsets, with the last
|
|
// one bounded by the chunk container. It is found by signature, since
|
|
// the core index carries no successor offset for shrub; without it the
|
|
// last class absorbs the container and the body check measures its
|
|
// neighbours.
|
|
let mut offsets: Vec<usize> = Vec::new();
|
|
for i in 0..range.count.max(0) as usize {
|
|
let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
|
|
.unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
|
|
if e.base.offset_in_asset_wad > 0 {
|
|
offsets.push(e.base.offset_in_asset_wad as usize);
|
|
}
|
|
}
|
|
offsets.sort_unstable();
|
|
// Levels without shrub geometry have no span to bound. The container
|
|
// that follows tie on those levels is checked by
|
|
// `chunk_container_closes_the_last_class_region`.
|
|
let container = match offsets.last() {
|
|
Some(&last) if !offsets.is_empty() => {
|
|
gc_formats::chunks::chunk_region_start(&core, last).unwrap_or_else(|| {
|
|
panic!("level{n}: no chunk container after the shrub classes")
|
|
})
|
|
}
|
|
_ => core.len(),
|
|
};
|
|
let mut span_end_of: std::collections::HashMap<usize, usize> =
|
|
std::collections::HashMap::new();
|
|
for (i, &o) in offsets.iter().enumerate() {
|
|
let end = offsets.get(i + 1).copied().unwrap_or(container);
|
|
span_end_of.insert(o, end);
|
|
}
|
|
for i in 0..range.count.max(0) as usize {
|
|
let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
|
|
.unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
|
|
let (span_end, has_successor) =
|
|
match span_end_of.get(&(e.base.offset_in_asset_wad.max(0) as usize)) {
|
|
Some(&e2) => (e2, true),
|
|
None => (core.len(), false),
|
|
};
|
|
match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class, span_end) {
|
|
Ok(Some(mut s)) => {
|
|
s.span_end_is_a_successor = has_successor;
|
|
// Every class is bounded now, so every body must be whole
|
|
// 8-byte records. That was not assertable while the last
|
|
// class ran to the lump's end, and it is the check the
|
|
// boundary earns its keep for.
|
|
assert!(
|
|
s.body_is_well_formed(),
|
|
"level{n} class {i}: shrub body is not whole 8-byte records"
|
|
);
|
|
bounded_bodies += 1;
|
|
bounded_records += s.body_record_count();
|
|
assert_eq!(
|
|
s.header.o_class,
|
|
e.base.o_class.max(0) as u32,
|
|
"level{n} class {i}"
|
|
);
|
|
assert!(!s.data.is_empty(), "level{n} class {i}: empty shrub data");
|
|
assert!(
|
|
s.header.sphere.iter().all(|v| v.is_finite()),
|
|
"level{n} class {i}: non-finite shrub sphere"
|
|
);
|
|
assert!(
|
|
s.header.size.is_finite() && s.header.size > 0.0,
|
|
"level{n} class {i}: shrub size {}",
|
|
s.header.size
|
|
);
|
|
// The scratch word is zero on every span; the parser
|
|
// refuses anything else. `mode_bits` is 4 on 16 spans and
|
|
// is reported rather than asserted. On those 16 the word
|
|
// at +0x14 reads as an f32 denormal, which the Emotion
|
|
// Engine flushes to zero like everything else.
|
|
if s.header.scratch != 0 {
|
|
scratch_nonzero += 1;
|
|
}
|
|
*mode_hist.entry(s.header.mode_bits).or_default() += 1;
|
|
*scratch_hist.entry(s.header.scratch).or_default() += 1;
|
|
radius_range.0 = radius_range.0.min(s.header.sphere[3]);
|
|
radius_range.1 = radius_range.1.max(s.header.sphere[3]);
|
|
scale_range.0 = scale_range.0.min(s.header.scale);
|
|
scale_range.1 = scale_range.1.max(s.header.scale);
|
|
min_size = min_size.min(s.header.size);
|
|
max_size = max_size.max(s.header.size);
|
|
|
|
// The palette: 24 unit normals, then zero padding.
|
|
let normals: Vec<_> = s.normals().collect();
|
|
assert_eq!(
|
|
normals.len(),
|
|
SHRUB_NORMAL_COUNT,
|
|
"level{n} class {i}: palette is {} entries, not {SHRUB_NORMAL_COUNT}",
|
|
normals.len()
|
|
);
|
|
for (k, normal) in normals.iter().enumerate() {
|
|
assert!(
|
|
normal.is_unit(),
|
|
"level{n} class {i}: palette entry {k} {normal:?} is not a unit normal"
|
|
);
|
|
let n2 = (normal.x as i64).pow(2)
|
|
+ (normal.y as i64).pow(2)
|
|
+ (normal.z as i64).pow(2);
|
|
norm2_deviation = norm2_deviation.max((n2 - 32767i64.pow(2)).abs());
|
|
}
|
|
palette_records += normals.len();
|
|
for (k, record) in s.body_records().skip(SHRUB_NORMAL_COUNT).enumerate() {
|
|
assert!(
|
|
record.iter().all(|&b| b == 0),
|
|
"level{n} class {i}: padding record {k} is not all zero"
|
|
);
|
|
}
|
|
padding_records += s.padding_record_count();
|
|
|
|
// The packets: the table at +0x40 tiles the span from
|
|
// align16 of its own end to the palette, or to the
|
|
// billboard when the class has one. This is what puts the
|
|
// mesh geometry in the blob, and it is the identity that
|
|
// makes `count` the packet count rather than a guess.
|
|
let packets = s.packets().unwrap_or_else(|| {
|
|
panic!("level{n} class {i}: packet table does not tile")
|
|
});
|
|
assert_eq!(
|
|
packets.len(),
|
|
s.header.count as usize,
|
|
"level{n} class {i}: packet count disagrees with the header"
|
|
);
|
|
packets_total += packets.len();
|
|
|
|
// The packet headers themselves. The disc form is a fixed
|
|
// signature -- magic, four zero bytes, tag 0x05 -- and the
|
|
// loader rewrites bytes 8-15 on load, so the count and the
|
|
// pointer live in RAM, not here.
|
|
let hdrs: Vec<_> = s.packet_headers().collect();
|
|
assert_eq!(
|
|
hdrs.len(),
|
|
packets.len(),
|
|
"level{n} class {i}: header count disagrees with packet count"
|
|
);
|
|
for (pi, h) in hdrs.iter().enumerate() {
|
|
assert!(
|
|
h.is_some(),
|
|
"level{n} class {i}: packet {pi} header does not match the \
|
|
disc invariants (magic 04 04 00 01, bytes 4-8 zero, tag 0x05)"
|
|
);
|
|
assert_eq!(
|
|
packets[pi].size % 16,
|
|
0,
|
|
"level{n} class {i}: packet {pi} size is not a multiple of 16"
|
|
);
|
|
}
|
|
packet_headers_total += hdrs.len();
|
|
|
|
// The billboard: present exactly when the class entry
|
|
// carries billboard texture fields.
|
|
let entry_has_billboard = e.billboard_width != 0 || e.billboard_height != 0;
|
|
match s.billboard().expect("billboard parses") {
|
|
Some(b) => {
|
|
billboards += 1;
|
|
assert!(
|
|
entry_has_billboard,
|
|
"level{n} class {i}: billboard record without billboard texture fields"
|
|
);
|
|
assert_eq!(b.tex1_addr, 0x14, "level{n} class {i}: TEX1 address");
|
|
assert_eq!(b.tex0_addr, 0x06, "level{n} class {i}: TEX0 address");
|
|
assert_eq!(b.miptbp_addr, 0x34, "level{n} class {i}: MIPTBP address");
|
|
assert_eq!(b.mmin, 4, "level{n} class {i}: MMIN");
|
|
}
|
|
None => assert!(
|
|
!entry_has_billboard,
|
|
"level{n} class {i}: billboard texture fields with no billboard record"
|
|
),
|
|
}
|
|
if entry_has_billboard {
|
|
billboard_entries += 1;
|
|
}
|
|
spans += 1;
|
|
}
|
|
Ok(None) => absent += 1,
|
|
Err(err) => panic!("level{n}: shrub class {i}: {err}"),
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
spans + absent,
|
|
358,
|
|
"expected 358 shrub classes, got {}",
|
|
spans + absent
|
|
);
|
|
assert!(spans > 340, "only {spans} shrub spans with geometry");
|
|
// The container bounds every class now, so every body is checkable.
|
|
assert_eq!(bounded_bodies, spans, "every shrub class should be bounded");
|
|
assert_eq!(scratch_nonzero, 0, "the scratch word is zero on every span");
|
|
assert_eq!(
|
|
billboards, billboard_entries,
|
|
"billboard records and billboard class entries disagree: \
|
|
{billboards} records vs {billboard_entries} entries"
|
|
);
|
|
eprintln!(
|
|
"shrub: {spans} headers naming their own class, {absent} without geometry; \
|
|
all {bounded_bodies} bodies bounded, {bounded_records} records, \
|
|
{palette_records} of them palette normals and {padding_records} zero padding; \
|
|
{billboards} billboards, all with texture fields; \
|
|
{packets_total} packets across all blobs, {packet_headers_total} headers; \
|
|
mode_bits {mode_hist:?}, radius {radius_range:?}, scale {scale_range:?}, \
|
|
largest palette deviation from unit length {norm2_deviation} of a possible 65535"
|
|
);
|
|
}
|
|
|
|
/// Region bounds derived from successor boundaries, on every level.
|
|
///
|
|
/// Every region is closed by the next region's start, or -- for the last class
|
|
/// region -- by the located chunk container, never by `core_data.len()`, so no
|
|
/// span silently absorbs the tail of the lump the way the last shrub class was
|
|
/// doing.
|
|
#[test]
|
|
fn core_regions_chain_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES};
|
|
let mut levels = 0usize;
|
|
let mut unresolved_shrub = 0usize;
|
|
let mut unresolved_chunks = 0usize;
|
|
let mut bounded = 0usize;
|
|
let mut shrub_bytes = 0usize;
|
|
let mut chunk_bytes = 0usize;
|
|
let (mut first, mut last) = (usize::MAX, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let core_len = core.len();
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let regions = core_layout(&index, &core, &h);
|
|
|
|
// Regions appear in order, never overlap, and never run past the lump.
|
|
for pair in regions.windows(2) {
|
|
assert!(
|
|
pair[0].start <= pair[1].start,
|
|
"level{n}: {} vs {}",
|
|
pair[0].name,
|
|
pair[1].name
|
|
);
|
|
if let Some(end) = pair[0].end {
|
|
assert!(
|
|
end <= pair[1].start,
|
|
"level{n}: {} ends after {}",
|
|
pair[0].name,
|
|
pair[1].name
|
|
);
|
|
}
|
|
}
|
|
for r in ®ions {
|
|
assert!(
|
|
r.start < core_len,
|
|
"level{n}: {} starts past the lump",
|
|
r.name
|
|
);
|
|
if let Some(end) = r.end {
|
|
assert!(end <= core_len, "level{n}: {} ends past the lump", r.name);
|
|
}
|
|
}
|
|
|
|
// Regions appear in the canonical order, but not every level has every
|
|
// region -- level 10 has no occlusion and no shrub at all.
|
|
const ORDER: [&str; 8] = [
|
|
"tfrags",
|
|
"occlusion",
|
|
"sky",
|
|
"collision",
|
|
"moby",
|
|
"tie",
|
|
"shrub",
|
|
"chunks",
|
|
];
|
|
let names: Vec<_> = regions.iter().map(|r| r.name).collect();
|
|
let ranks: Vec<usize> = names
|
|
.iter()
|
|
.map(|name| {
|
|
ORDER
|
|
.iter()
|
|
.position(|o| o == name)
|
|
.unwrap_or_else(|| panic!("level{n}: unknown region {name}"))
|
|
})
|
|
.collect();
|
|
assert!(
|
|
ranks.windows(2).all(|w| w[0] < w[1]),
|
|
"level{n}: regions out of order: {names:?}"
|
|
);
|
|
if let Some(s) = regions.iter().find(|r| r.name == "shrub") {
|
|
match s.end {
|
|
Some(e) => {
|
|
// The container's start, not the trailer and not the lump's
|
|
// end: the trailer is the chunk region's business.
|
|
assert!(e < core_len - CORE_DATA_TRAILER_BYTES);
|
|
shrub_bytes += s.len().unwrap_or(0);
|
|
bounded += 1;
|
|
}
|
|
None => unresolved_shrub += 1,
|
|
}
|
|
first = first.min(s.start);
|
|
last = last.max(s.start);
|
|
}
|
|
if let Some(c) = regions.iter().find(|r| r.name == "chunks") {
|
|
match c.end {
|
|
Some(e) => {
|
|
// The trailer is the only thing past the chunk container.
|
|
assert_eq!(e, core_len - CORE_DATA_TRAILER_BYTES);
|
|
chunk_bytes += c.len().unwrap_or(0);
|
|
}
|
|
None => unresolved_chunks += 1,
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
unresolved_shrub, 0,
|
|
"every level with shrub should bound it at the chunk container"
|
|
);
|
|
assert_eq!(bounded, 24, "expected 24 levels with a shrub region");
|
|
assert_eq!(
|
|
unresolved_chunks, 0,
|
|
"every chunk region should end at the trailer"
|
|
);
|
|
eprintln!(
|
|
"regions: shrub starts {first}..{last}; {shrub_bytes} bytes of shrub and \
|
|
{chunk_bytes} bytes of chunk data, shrub closed by the located container on \
|
|
all {bounded} levels"
|
|
);
|
|
}
|
|
|
|
/// The post-shrub chunk container: chunks carry the `0xf0`-tagged offset
|
|
/// signature on every level, and their offsets resolve to records with a
|
|
/// shared field layout.
|
|
///
|
|
/// The region is bounded properly rather than by the fixed window this test
|
|
/// first used: it runs from the marker-paired chunk signature that closes the
|
|
/// last class region to the constant trailer. The window approach worked and
|
|
/// then over-worked -- a fixed 900,000 bytes from the end reaches past the
|
|
/// container on some levels and stops inside it on others, which is how 27
|
|
/// chunks came to be truncated at a "window edge" that is not a format
|
|
/// boundary. The locator is asserted by
|
|
/// `chunk_container_closes_the_last_class_region`.
|
|
///
|
|
/// The layout facts asserted here are measured, not assumed:
|
|
///
|
|
/// - every tagged offset has top byte `0xf0`, on all 27 levels;
|
|
/// - resolving records at `chunk_start + low` gives 17-21% byte positions
|
|
/// constant across that chunk's own records, which does not happen by
|
|
/// chance and is what distinguishes this from a coincidental byte pattern;
|
|
/// - stride is a per-chunk property, not a region-wide one.
|
|
#[test]
|
|
fn chunk_containers_resolve_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES};
|
|
|
|
let mut levels = 0usize;
|
|
let mut chunks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut strides = std::collections::BTreeSet::new();
|
|
let mut truncated = 0usize;
|
|
let mut constant_total = 0usize;
|
|
let mut stride_total = 0usize;
|
|
let mut frac: Vec<f64> = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
|
|
// The region runs from the chunk signature that closes the last class
|
|
// region to the constant trailer, both measured and identical in size
|
|
// on all 27 levels. core_layout locates the start through the same
|
|
// signature the chunk parser uses.
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let Some(region) = core_layout(&index, &core, &h)
|
|
.into_iter()
|
|
.find(|r| r.name == "chunks")
|
|
else {
|
|
continue;
|
|
};
|
|
let Some(end) = region.end else {
|
|
panic!("level{n}: chunk region is unbounded");
|
|
};
|
|
assert_eq!(end, core.len() - CORE_DATA_TRAILER_BYTES);
|
|
assert!(!region.is_empty(), "level{n}: empty chunk region");
|
|
let region = &core[region.start..end];
|
|
|
|
let found = gc_formats::chunks::parse_all_chunks(region).expect("chunks should parse");
|
|
if found.is_empty() {
|
|
continue;
|
|
}
|
|
levels += 1;
|
|
|
|
for chunk in &found {
|
|
assert!(
|
|
!chunk.chunk.offsets.is_empty(),
|
|
"level{n}: chunk at {} has no offsets",
|
|
chunk.chunk.offset
|
|
);
|
|
assert!(chunk.chunk.stride > 0, "level{n}: zero stride");
|
|
// The one decoded header field.
|
|
assert_eq!(
|
|
chunk.chunk.count as usize,
|
|
chunk.chunk.offsets.len(),
|
|
"level{n}: chunk at {} header count {} but {} offsets",
|
|
chunk.chunk.offset,
|
|
chunk.chunk.count,
|
|
chunk.chunk.offsets.len()
|
|
);
|
|
chunks += 1;
|
|
records += chunk.records.len();
|
|
strides.insert(chunk.chunk.stride);
|
|
|
|
// Records resolve unless the last one would run past the region
|
|
// end, which happens for the chunk sitting at the trailer end:
|
|
// the container's records stop there, so its final record or two
|
|
// fall outside. Truncation is counted rather than asserted away.
|
|
assert!(
|
|
chunk.records.len() <= chunk.chunk.offsets.len(),
|
|
"level{n}: chunk at {} resolved MORE records than offsets",
|
|
chunk.chunk.offset
|
|
);
|
|
if chunk.records.len() < chunk.chunk.offsets.len() {
|
|
truncated += 1;
|
|
}
|
|
|
|
// The invariant that makes this a container and not a coincidence.
|
|
// Accumulated rather than asserted per chunk: individual chunks
|
|
// range from ~7% to ~21% constant positions, so a per-chunk
|
|
// threshold would be asserting a property the format does not have.
|
|
let constant = constant_positions(&chunk.records);
|
|
constant_total += constant;
|
|
stride_total += chunk.chunk.stride;
|
|
frac.push(constant as f64 / chunk.chunk.stride as f64);
|
|
}
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(chunks > 500, "only {chunks} chunks resolved");
|
|
assert!(records > 5_000, "only {records} records resolved");
|
|
assert!(
|
|
strides.len() > 5,
|
|
"stride should vary per chunk, saw {}",
|
|
strides.len()
|
|
);
|
|
// Chance would give a constant position a probability of 1/256^n, so even
|
|
// 5% across tens of thousands of records is far above noise.
|
|
let mean_frac = constant_total as f64 / stride_total as f64;
|
|
assert!(
|
|
mean_frac > 0.05,
|
|
"constant positions {constant_total}/{stride_total} = {mean_frac:.3} is near chance"
|
|
);
|
|
eprintln!(
|
|
"chunks: {chunks} over {levels} levels, {records} records, {} distinct strides, \
|
|
constant {constant_total}/{stride_total} ({:.1}%), per-chunk min {:.1}% max {:.1}%, \
|
|
{truncated} chunks truncated at the trailer end",
|
|
strides.len(),
|
|
mean_frac * 100.0,
|
|
frac.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0,
|
|
frac.iter().cloned().fold(0.0, f64::max) * 100.0,
|
|
);
|
|
}
|
|
|
|
/// The chunk container closes the last class region, on every level.
|
|
///
|
|
/// The core index carries no successor offset for shrub -- unlike moby and
|
|
/// tie, whose boundaries fall out of the class tables -- so the boundary is
|
|
/// located in `core_data`: the constant marker pair sitting immediately in
|
|
/// front of a chunk signature. Both halves of that signature are weak alone,
|
|
/// which is why the test measures them separately:
|
|
///
|
|
/// - the marker pair `f6 7c` occurs 25 to 103 times in a 5 to 17 MB lump, so
|
|
/// on its own it means nothing;
|
|
/// - the chunk shape (16-byte header ending `0xff`, 10 zeros, tagged offsets)
|
|
/// matches random data often enough to need four tagged offsets before it
|
|
/// means anything.
|
|
///
|
|
/// Together they are exact: on all 27 levels the marker sits exactly two bytes
|
|
/// before the first chunk header. Three checks pin that down per level:
|
|
///
|
|
/// 1. the located boundary carries the marker;
|
|
/// 2. the last shrub body walks to 8-byte records that end exactly where the
|
|
/// signature begins, with no chunk signature inside the body itself;
|
|
/// 3. the container's records fill the region up to the constant trailer.
|
|
///
|
|
/// Levels without shrub are the control for the locator: there the container
|
|
/// closes tie instead, and the same marker precedes it.
|
|
#[test]
|
|
fn chunk_container_closes_the_last_class_region() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{self, CHUNK_REGION_MARKER};
|
|
use gc_formats::level_core::{LevelCoreHeader, ShrubClassEntry};
|
|
use gc_formats::regions::{core_layout, last_geometry, CORE_DATA_TRAILER_BYTES};
|
|
use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES};
|
|
|
|
let mut levels = 0usize;
|
|
let mut with_shrub = 0usize;
|
|
let mut marker_wide_max = 0usize;
|
|
let mut body_records = 0usize;
|
|
let mut container_bytes = 0usize;
|
|
let mut fill_gap_max = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
panic!("level{n}: missing core_data or core_index");
|
|
};
|
|
let trailer = core.len() - CORE_DATA_TRAILER_BYTES;
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
let last_shrub = last_geometry(&index, h.shrub_classes, SHRUB_CLASS_ENTRY_BYTES);
|
|
let last_tie = last_geometry(&index, h.tie_classes, 0x20);
|
|
// The container follows the last of the shrub and tie classes, so the
|
|
// signature search starts after the class bodies rather than inside
|
|
// them.
|
|
let scan_from = last_shrub
|
|
.or(last_tie)
|
|
.unwrap_or_else(|| panic!("level{n}: no class geometry to follow"));
|
|
let container = chunks::chunk_region_start(&core, scan_from)
|
|
.unwrap_or_else(|| panic!("level{n}: no chunk container after {scan_from}"));
|
|
|
|
// 1. The marker, and how common it is without the signature behind it.
|
|
assert_eq!(
|
|
&core[container - 2..container],
|
|
&CHUNK_REGION_MARKER,
|
|
"level{n}: container at {container} is not preceded by the marker"
|
|
);
|
|
marker_wide_max = marker_wide_max.max(
|
|
core.windows(2)
|
|
.filter(|w| *w == CHUNK_REGION_MARKER)
|
|
.count(),
|
|
);
|
|
|
|
// 2. The last shrub body ends exactly at the marker. This is the check
|
|
// that the boundary is the successor of shrub rather than a coincident
|
|
// pair inside it.
|
|
if let Some(last) = last_shrub {
|
|
let o_class = (0..h.shrub_classes.count.max(0) as usize)
|
|
.filter_map(|i| {
|
|
let at = h.shrub_classes.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(
|
|
index
|
|
.get(at..at + SHRUB_CLASS_ENTRY_BYTES)
|
|
.unwrap_or_default(),
|
|
)
|
|
.ok()?;
|
|
(e.base.offset_in_asset_wad == last as i32).then_some(e.base.o_class)
|
|
})
|
|
.next()
|
|
.unwrap_or_else(|| panic!("level{n}: no shrub class at {last}"));
|
|
let span = ShrubHeader::region(&core, last as i32, o_class, trailer)
|
|
.expect("last shrub span parses")
|
|
.expect("last shrub span present");
|
|
let data_start = last + span.header.data_offset as usize;
|
|
|
|
// Walk 8-byte records, bounded by the container so the walk can
|
|
// never continue into the chunk region itself.
|
|
let mut at = data_start;
|
|
while at + 8 <= container && core[at + 6] == 0 && core[at + 7] == 0 {
|
|
at += 8;
|
|
}
|
|
assert_eq!(
|
|
at + 2,
|
|
container,
|
|
"level{n}: shrub body at {data_start} does not end at the marker"
|
|
);
|
|
assert!(at > data_start, "level{n}: last shrub body is empty");
|
|
body_records += (at - data_start) / 8;
|
|
|
|
// No chunk signature inside the body: the search for the boundary
|
|
// must not match one it should have skipped.
|
|
let sig_inside = (last..container - 2).any(|off| chunks::is_chunk_start(&core, off, 4));
|
|
assert!(
|
|
!sig_inside,
|
|
"level{n}: chunk signature inside the last shrub body"
|
|
);
|
|
with_shrub += 1;
|
|
} else {
|
|
// Without shrub the container closes tie, and the marker precedes
|
|
// it the same way -- the locator is not shrub-specific.
|
|
let tie = core_layout(&index, &core, &h)
|
|
.into_iter()
|
|
.find(|r| r.name == "tie")
|
|
.unwrap_or_else(|| panic!("level{n}: no tie region to close"));
|
|
assert_eq!(
|
|
tie.end,
|
|
Some(container),
|
|
"level{n}: tie must end at the container"
|
|
);
|
|
}
|
|
|
|
// 3. The container fills its region up to the constant trailer: the
|
|
// furthest record should reach within a few KB of it, never inside it.
|
|
let region = &core[container..trailer];
|
|
let found = chunks::parse_all_chunks(region).expect("chunks should parse");
|
|
let region_ptr = region.as_ptr() as usize;
|
|
let furthest = found
|
|
.iter()
|
|
.flat_map(|c| {
|
|
c.records
|
|
.iter()
|
|
.map(move |r| r.as_ptr() as usize + r.len() - region_ptr)
|
|
})
|
|
.max()
|
|
.unwrap_or(0);
|
|
assert!(
|
|
region.len() - furthest < 4096,
|
|
"level{n}: chunk records stop {} bytes short of the trailer",
|
|
region.len() - furthest
|
|
);
|
|
container_bytes += region.len();
|
|
fill_gap_max = fill_gap_max.max(region.len() - furthest);
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
with_shrub, 24,
|
|
"expected 24 levels with shrub, got {with_shrub}"
|
|
);
|
|
eprintln!(
|
|
"container: {levels} levels ({with_shrub} closing shrub), \
|
|
{body_records} 8-byte records in last-shrub bodies, \
|
|
{container_bytes} bytes of chunk data, largest trailer gap {fill_gap_max}, \
|
|
marker pair occurs at most {marker_wide_max} times per lump"
|
|
);
|
|
}
|
|
|
|
/// Byte positions that hold the same value in every supplied record.
|
|
fn constant_positions(records: &[&[u8]]) -> usize {
|
|
let Some(first) = records.first() else {
|
|
return 0;
|
|
};
|
|
(0..first.len())
|
|
.filter(|&i| records.iter().all(|r| r[i] == first[i]))
|
|
.count()
|
|
}
|
|
|
|
/// The sky block: the sub-object table at `+0x20` enumerates the sub-objects on
|
|
/// every level, and each carries a bounding sphere and texture dimensions.
|
|
///
|
|
/// The word at `+0x0c` is deliberately **not** asserted to be a count: it
|
|
/// disagrees with the table on most levels. See `gc_formats::sky`.
|
|
#[test]
|
|
fn sky_sub_objects_parse_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut levels = 0usize;
|
|
let mut total = 0usize;
|
|
let mut radii = (f32::MAX, f32::MIN);
|
|
let mut counts = (u32::MAX, 0u32);
|
|
let mut dims = (u16::MAX, 0u16);
|
|
let mut nonzero_radius = 0usize;
|
|
let mut reserved_nonzero = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let hdr = match gc_formats::level_core::LevelCoreHeader::parse(&index) {
|
|
Ok(h) => h,
|
|
Err(_) => continue,
|
|
};
|
|
let sky = hdr.sky as usize;
|
|
if sky == 0 || sky >= core.len() {
|
|
continue;
|
|
}
|
|
let end = [
|
|
hdr.collision as usize,
|
|
hdr.moby_classes.offset as usize,
|
|
core.len(),
|
|
]
|
|
.into_iter()
|
|
.filter(|&v| v > sky)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let region = &core[sky..end.min(core.len())];
|
|
|
|
let parsed = gc_formats::sky::Sky::parse(region).expect("sky should parse");
|
|
assert!(
|
|
!parsed.sub_objects.is_empty(),
|
|
"level{n}: sky region has no sub-objects"
|
|
);
|
|
levels += 1;
|
|
total += parsed.sub_objects.len();
|
|
|
|
for s in &parsed.sub_objects {
|
|
if s.reserved != [0u8; 12] {
|
|
reserved_nonzero += 1;
|
|
}
|
|
assert!(s.count > 0, "level{n}: sky sub-object count is {}", s.count);
|
|
assert!(
|
|
s.width > 0 && s.height > 0,
|
|
"level{n}: sky texture dims are {}x{}",
|
|
s.width,
|
|
s.height
|
|
);
|
|
assert!(
|
|
s.sphere.iter().all(|f| f.is_finite()),
|
|
"level{n}: sky sphere is not finite: {:?}",
|
|
s.sphere
|
|
);
|
|
// The fourth float is the radius: positive wherever the first
|
|
// three are negative, the same reasoning the shrub header uses.
|
|
if s.sphere[3] > 0.0 {
|
|
nonzero_radius += 1;
|
|
radii.0 = radii.0.min(s.sphere[3]);
|
|
radii.1 = radii.1.max(s.sphere[3]);
|
|
}
|
|
counts.0 = counts.0.min(s.count);
|
|
counts.1 = counts.1.max(s.count);
|
|
dims.0 = dims.0.min(s.width);
|
|
dims.1 = dims.1.max(s.width);
|
|
}
|
|
}
|
|
|
|
assert!(levels >= 20, "only {levels} levels had a sky region");
|
|
assert!(total >= 100, "only {total} sky sub-objects parsed");
|
|
eprintln!(
|
|
"sky: {total} sub-objects over {levels} levels; count {}..={}, width {}..={}, radius {:.2}..={:.2} on {nonzero_radius}, {reserved_nonzero} with a non-zero reserved word",
|
|
counts.0, counts.1, dims.0, dims.1, radii.0, radii.1
|
|
);
|
|
}
|
|
|
|
/// The per-level overlay: the header is stable, `+0x10..+0x1220` is identical
|
|
/// on every level, and `+0x0c` is not the file length.
|
|
///
|
|
/// See `gc_formats::overlay`. This pins the header and the constant block's
|
|
/// extent; it does not name any field.
|
|
#[test]
|
|
fn overlay_header_and_constant_block_are_stable() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay::{parse_overlay, OVERLAY_CONSTANT_BLOCK_END, OVERLAY_HEADER_SIZE};
|
|
|
|
let mut levels = 0usize;
|
|
let mut constant_block: Option<Vec<u8>> = None;
|
|
let mut f00 = 0u32;
|
|
let mut f08 = 0u32;
|
|
let (mut min_f04, mut max_f04) = (u32::MAX, 0u32);
|
|
let (mut min_f0c, mut max_f0c) = (u32::MAX, 0u32);
|
|
let mut f0c_below = 0usize;
|
|
let mut f0c_above = 0usize;
|
|
let mut levels_seen = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let p = data.join(format!("level{n}/overlay.bin"));
|
|
let Ok(bytes) = std::fs::read(&p) else {
|
|
continue;
|
|
};
|
|
|
|
let (h, constant, level) = parse_overlay(&bytes).expect("overlay should parse");
|
|
|
|
assert!(
|
|
bytes.len() > OVERLAY_CONSTANT_BLOCK_END,
|
|
"level{n}: overlay is {} bytes, too short for the constant block",
|
|
bytes.len()
|
|
);
|
|
assert!(
|
|
!level.bytes.is_empty(),
|
|
"level{n}: overlay has no level data"
|
|
);
|
|
assert_eq!(
|
|
constant.bytes.len(),
|
|
OVERLAY_CONSTANT_BLOCK_END - OVERLAY_HEADER_SIZE
|
|
);
|
|
|
|
if f00 == 0 {
|
|
f00 = h.field_00;
|
|
}
|
|
assert_eq!(h.field_00, f00, "level{n}: field_00 differs");
|
|
if f08 == 0 {
|
|
f08 = h.field_08;
|
|
}
|
|
assert_eq!(h.field_08, f08, "level{n}: field_08 differs");
|
|
|
|
assert_eq!(
|
|
h.field_04 % 4,
|
|
0,
|
|
"level{n}: field_04 {} is not a multiple of 4",
|
|
h.field_04
|
|
);
|
|
min_f04 = min_f04.min(h.field_04);
|
|
max_f04 = max_f04.max(h.field_04);
|
|
|
|
min_f0c = min_f0c.min(h.field_0c);
|
|
max_f0c = max_f0c.max(h.field_0c);
|
|
// The claim being pinned: field_0c is not the file length.
|
|
if (h.field_0c as usize) < bytes.len() {
|
|
f0c_below += 1
|
|
} else {
|
|
f0c_above += 1
|
|
}
|
|
|
|
match &constant_block {
|
|
None => constant_block = Some(constant.bytes.to_vec()),
|
|
Some(prev) => assert_eq!(
|
|
prev.as_slice(),
|
|
constant.bytes,
|
|
"level{n}: the constant block +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} differs"
|
|
),
|
|
}
|
|
|
|
levels += 1;
|
|
levels_seen.push(n);
|
|
}
|
|
|
|
assert!(
|
|
levels >= 27,
|
|
"only {levels} levels had an overlay.bin, saw {levels_seen:?}"
|
|
);
|
|
assert!(f0c_below > 0 && f0c_above > 0,
|
|
"field_0c must fall on both sides of the file length to show it is not one: below {f0c_below}, above {f0c_above}");
|
|
assert_eq!(levels, 27, "expected all levels, got {levels_seen:?}");
|
|
assert!(
|
|
max_f04 - min_f04 > 1000,
|
|
"field_04 range collapsed to {min_f04}..={max_f04}"
|
|
);
|
|
assert!(
|
|
max_f0c - min_f0c > 100000,
|
|
"field_0c range collapsed to {min_f0c}..={max_f0c}"
|
|
);
|
|
|
|
eprintln!(
|
|
"overlay: {levels} levels; field_00 {f00}, field_08 {f08}; \
|
|
field_04 {min_f04}..={max_f04}, field_0c {min_f0c}..={max_f0c} \
|
|
({f0c_below} below the file length, {f0c_above} above); \
|
|
+0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} identical on all"
|
|
);
|
|
}
|