ReaperOfVeriod
9f938f47de
The i16[3] high-byte finding -- the most recent and most subtle result of this stretch, and the one I corrected twice -- lived only in prose. Two commits is exactly how long a claim can sit unasserted before it drifts. It now has a test: tie_vertex_layout_is_selected_by_i16_3_high_byte which reproduces the figures exactly and, importantly, asserts both directions: (4,5) and (6,7) are above 0.85 and 0.80 in the ordinary group, (6,7) is BELOW 0.99 so it cannot silently become the constant I once mistook it for, (0,1) and (2,3) are under 0.20, and every pair in the non-zero group is under 0.15. A future change in either layout now fails loudly. Also audited the rest of this session's claims against the test suite and recorded the result in the format doc: tie vertex element is 16 bytes asserted layout selected by i16[3] HIGH byte asserted (this commit) blocks self-delimiting on 0xf6 asserted per-block packets == 0 means no block at that LOD asserted packets[0] == 256 on 1,864 of 1,864 asserted chunk record length uniform per chunk asserted moby class IDs in one body slot per type asserted collision payload verbatim in RAM NOT assertable here -- needs a savestate tfrag patch bytes all rewritten by LOD same morphing is unobservable in that image same The split is deliberate: disc-readable claims are asserted against the disc, savestate-dependent ones live in plan/pcsx2-verification.md with their method. What is not acceptable is a disc-readable claim living only in prose -- which is how the tie i16[3] finding spent two commits before it had a test, and how several older claims drifted before the roadmap reconciliation caught them.
3695 lines
142 KiB
Rust
3695 lines
142 KiB
Rust
//! Integration tests against a real disc image.
|
|
//!
|
|
//! These are skipped unless `GC_DISC` points at a legally obtained disc, since
|
|
//! the repository must not contain game data. When it is set they check the
|
|
//! identification and unpack results against the committed hash table.
|
|
//!
|
|
//! ```sh
|
|
//! GC_DISC="path/to/disc.iso" cargo test -p gc-extract --test disc
|
|
//! ```
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// The disc under test, if the environment names one.
|
|
fn disc() -> Option<PathBuf> {
|
|
let p = std::env::var_os("GC_DISC")?;
|
|
let p = PathBuf::from(p);
|
|
p.is_file().then_some(p)
|
|
}
|
|
|
|
/// Skip a test with a clear reason rather than failing.
|
|
macro_rules! require_disc {
|
|
() => {
|
|
match disc() {
|
|
Some(p) => p,
|
|
None => {
|
|
eprintln!("skipped: set GC_DISC to a Going Commando disc image to run this");
|
|
return;
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
#[test]
|
|
fn identifies_the_supported_build() {
|
|
let path = require_disc!();
|
|
let info = gc_extract::identify::identify(&path).expect("disc should identify");
|
|
|
|
assert_eq!(info.serial, "SCUS_972.68");
|
|
assert_eq!(info.volume_id, "RATCHETANDCLANK2");
|
|
assert!(
|
|
info.is_target(),
|
|
"expected the supported build, got: {}",
|
|
info.verdict()
|
|
);
|
|
assert_eq!(
|
|
info.boot_elf_sha1,
|
|
gc_extract::identify::TARGET_BOOT_ELF_SHA1,
|
|
"boot ELF SHA-1 differs from the recorded retail value"
|
|
);
|
|
assert_eq!(info.boot_elf_size, 2_618_684);
|
|
// This disc is NTSC; the movement constants assume 60 Hz.
|
|
assert!(!info.is_pal(), "test disc should be NTSC");
|
|
}
|
|
|
|
#[test]
|
|
fn agrees_with_the_python_reference_identifier() {
|
|
let path = require_disc!();
|
|
let info = gc_extract::identify::identify(&path).expect("disc should identify");
|
|
|
|
// tools/identify-disc.py is the reference implementation and prints these
|
|
// two values. If they ever disagree, one of them has a bug.
|
|
// Run from the workspace root so the relative path to the tool resolves.
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent()
|
|
.and_then(|p| p.parent())
|
|
.expect("the crate should live two levels below the workspace root")
|
|
.to_path_buf();
|
|
let py = std::process::Command::new("python3")
|
|
.arg(root.join("tools/identify-disc.py"))
|
|
.arg(&path)
|
|
.current_dir(&root)
|
|
.output()
|
|
.expect("python3 should be available to run the reference identifier");
|
|
assert!(
|
|
py.status.success(),
|
|
"the reference identifier failed:\n{}",
|
|
String::from_utf8_lossy(&py.stderr)
|
|
);
|
|
let text = String::from_utf8_lossy(&py.stdout);
|
|
|
|
assert!(
|
|
text.contains(&format!("serial {}", info.serial)),
|
|
"serial disagrees with the reference identifier:\n{text}"
|
|
);
|
|
assert!(
|
|
text.contains(&format!("SHA-1 {}", info.boot_elf_sha1)),
|
|
"boot ELF SHA-1 disagrees with the reference identifier:\n{text}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn unpacks_and_matches_the_committed_table() {
|
|
let path = require_disc!();
|
|
|
|
let table_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/gc-files.tsv");
|
|
let table_text =
|
|
std::fs::read_to_string(&table_path).expect("the hash table should be committed");
|
|
let expected = gc_extract::verify::parse_table(&table_text);
|
|
assert!(
|
|
expected.len() > 5000,
|
|
"expected a full table, got {} rows",
|
|
expected.len()
|
|
);
|
|
|
|
// Unpack into a temporary directory so the test leaves nothing behind.
|
|
let tmp = std::env::temp_dir().join(format!("gc-unpack-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&tmp);
|
|
std::fs::create_dir_all(&tmp).expect("should create a temp directory");
|
|
|
|
let files = gc_extract::unpack::unpack(&path, &tmp).expect("unpack should succeed");
|
|
|
|
// Every unpacked file must match the recorded size and hash.
|
|
let report = gc_extract::verify::verify(&tmp, &expected);
|
|
let _ = std::fs::remove_dir_all(&tmp);
|
|
|
|
assert!(
|
|
report.ok(),
|
|
"unpacked data does not match the committed table:\n{}",
|
|
report.summary()
|
|
);
|
|
assert_eq!(
|
|
files.len(),
|
|
expected.len(),
|
|
"unpack produced a different file count"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_container_parses() {
|
|
let path = require_disc!();
|
|
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let containers: Vec<_> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/") && e.name.ends_with(".WAD"))
|
|
.collect();
|
|
|
|
// 92 `G/*.WAD` containers: 27 LEVEL, 27 AUDIO, 27 SCENE, 11 globals.
|
|
assert_eq!(containers.len(), 92, "expected 92 asset containers");
|
|
|
|
// MPEG1 and MPEG2 are raw video streams with no table; the other 90 must
|
|
// all parse as indexed containers.
|
|
let raw_video: Vec<&str> = containers
|
|
.iter()
|
|
.map(|c| c.name.as_str())
|
|
.filter(|n| *n == "G/MPEG1.WAD" || *n == "G/MPEG2.WAD")
|
|
.collect();
|
|
assert_eq!(
|
|
raw_video.len(),
|
|
2,
|
|
"expected MPEG1 and MPEG2 to be the raw streams"
|
|
);
|
|
|
|
// The per-level families must all be present, at 27 each.
|
|
for family in ["LEVEL", "AUDIO", "SCENE"] {
|
|
let n = containers
|
|
.iter()
|
|
.filter(|c| {
|
|
let stem = c.name.trim_start_matches("G/").trim_end_matches(".WAD");
|
|
stem.strip_prefix(family)
|
|
.map(|rest| !rest.is_empty() && rest.chars().all(|ch| ch.is_ascii_digit()))
|
|
.unwrap_or(false)
|
|
})
|
|
.count();
|
|
assert_eq!(n, 27, "expected 27 {family}<n>.WAD containers, found {n}");
|
|
}
|
|
|
|
for entry in &containers {
|
|
if raw_video.contains(&entry.name.as_str()) {
|
|
continue;
|
|
}
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, entry.size)
|
|
.expect("read should succeed");
|
|
let kind = gc_formats::global_wad::RangeKind::for_container(&entry.name);
|
|
let container = gc_formats::global_wad::GlobalWad::parse_with(&data, kind)
|
|
.unwrap_or_else(|e| panic!("{} failed to parse: {e}", entry.name));
|
|
assert!(
|
|
!container.entries.is_empty(),
|
|
"{} parsed but had no entries",
|
|
entry.name
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The level container header is the first typed structure we parse from real
|
|
/// data, and the tiling invariant is the check that holds on every level rather
|
|
/// than one hand-picked sample.
|
|
#[test]
|
|
fn level_headers_tile_their_containers() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut checked = 0;
|
|
for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data)
|
|
.unwrap_or_else(|e| panic!("{}: {e}", entry.name));
|
|
|
|
assert_eq!(h.header_size, 0x60, "{}", entry.name);
|
|
assert_eq!(
|
|
h.runtime_sector, 0,
|
|
"{}: runtime sector field should be 0 on disc",
|
|
entry.name
|
|
);
|
|
|
|
// Every present extent must start where the previous one ended.
|
|
let extents = h.ordered_extents();
|
|
for w in extents.windows(2) {
|
|
let (pn, p) = w[0];
|
|
let (nn, n) = w[1];
|
|
assert_eq!(
|
|
p.offset + p.size,
|
|
n.offset,
|
|
"{}: gap or overlap between {} and {}",
|
|
entry.name,
|
|
pn.name(),
|
|
nn.name()
|
|
);
|
|
}
|
|
|
|
// Nothing may fall outside the container. The last extent may end one
|
|
// sector past the file end through sector rounding, hence `+ 1`.
|
|
let container_sectors = (entry.size / 2048) as i32;
|
|
for (name, r) in &extents {
|
|
assert!(
|
|
r.offset >= 0 && r.offset + r.size <= container_sectors + 1,
|
|
"{}: {} extent {:?} is outside the container ({container_sectors} sectors)",
|
|
entry.name,
|
|
name.name(),
|
|
r
|
|
);
|
|
}
|
|
checked += 1;
|
|
}
|
|
|
|
assert_eq!(checked, 27, "expected 27 level containers");
|
|
}
|
|
|
|
/// Level ids are not file order: `LEVEL21.WAD` reports 30. Anything that keys
|
|
/// off the container index would silently mislabel that planet.
|
|
#[test]
|
|
fn level_ids_are_not_container_indices() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
// The ISO filesystem lists names alphabetically, so LEVEL10 sorts before
|
|
// LEVEL2. Order by the numeric suffix, which is what the container names
|
|
// mean.
|
|
let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso
|
|
.files
|
|
.iter()
|
|
.filter(|e| e.name.starts_with("G/LEVEL"))
|
|
.map(|e| {
|
|
let n: usize = e
|
|
.name
|
|
.trim_start_matches("G/LEVEL")
|
|
.trim_end_matches(".WAD")
|
|
.parse()
|
|
.expect("level container names end in a number");
|
|
(n, e)
|
|
})
|
|
.collect();
|
|
levels.sort_by_key(|(n, _)| *n);
|
|
|
|
let mut mismatches = Vec::new();
|
|
for (index, (_, entry)) in levels.iter().enumerate() {
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
|
|
if h.id as usize != index {
|
|
mismatches.push(format!(
|
|
"{}: id {} but is container {index}",
|
|
entry.name, h.id
|
|
));
|
|
}
|
|
}
|
|
|
|
// Exactly one level is out of step: the gap left by a removed level.
|
|
assert_eq!(
|
|
mismatches.len(),
|
|
1,
|
|
"expected exactly one level whose id differs from its container index, got {mismatches:?}"
|
|
);
|
|
assert!(
|
|
mismatches[0].contains("LEVEL21"),
|
|
"unexpected mismatch: {mismatches:?}"
|
|
);
|
|
}
|
|
|
|
/// Sizes in the level header are sector counts. If they were byte counts, the
|
|
/// extents could not tile a container this size, so this also guards against
|
|
/// "fixing" the parser to read them as bytes.
|
|
#[test]
|
|
fn level_extent_sizes_are_sector_counts() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let entry = iso.find("G/LEVEL0.WAD").expect("level 0");
|
|
|
|
let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
|
|
let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
|
|
|
|
// Sound bank occupies sectors 1..735 and data 735..7219.
|
|
assert_eq!(h.sound_bank.offset, 1);
|
|
assert_eq!(h.sound_bank.size, 734);
|
|
assert_eq!(h.data.offset, 735);
|
|
// Read as bytes, 734 would be under four sectors and the tiling would break.
|
|
assert_eq!(h.sound_bank.byte_len(), 734 * 2048);
|
|
}
|
|
|
|
/// The level data extent is a container of byte ranges. Its header locates the
|
|
/// geometry index and the compressed geometry, and every range must lie inside
|
|
/// the extent on all 27 levels.
|
|
#[test]
|
|
fn level_data_extents_are_in_range() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
|
|
let mut checked = 0;
|
|
for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
|
|
let raw =
|
|
gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
|
|
let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
|
|
let data = lh.data;
|
|
let Some(offset) = data.byte_offset() else {
|
|
continue;
|
|
};
|
|
let len = data.byte_len() as usize;
|
|
let extent = &raw[offset as usize..offset as usize + len];
|
|
|
|
let dh = gc_formats::level_data::LevelDataHeader::parse(extent)
|
|
.unwrap_or_else(|e| panic!("{}: data extent header: {e}", entry.name));
|
|
|
|
for (part, r) in dh.present_parts() {
|
|
assert!(
|
|
r.end() <= extent.len() as u64,
|
|
"{}: {part:?} ends at {} but the extent is {} bytes",
|
|
entry.name,
|
|
r.end(),
|
|
extent.len()
|
|
);
|
|
}
|
|
|
|
// The geometry is the big one and must be substantial.
|
|
assert!(
|
|
dh.core_data.size > 1_000_000,
|
|
"{}: core_data is only {} bytes",
|
|
entry.name,
|
|
dh.core_data.size
|
|
);
|
|
checked += 1;
|
|
}
|
|
assert_eq!(checked, 27, "expected 27 level containers");
|
|
}
|
|
|
|
/// The geometry must decompress. This is the first test that touches real level
|
|
/// content rather than headers.
|
|
#[test]
|
|
fn level_geometry_decompresses() {
|
|
let path = require_disc!();
|
|
let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
|
|
let entry = iso.find("G/LEVEL0.WAD").expect("level 0");
|
|
|
|
let raw = gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
|
|
let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
|
|
let data = lh.data;
|
|
let extent = &raw[data.byte_offset().unwrap() as usize
|
|
..(data.byte_offset().unwrap() + data.byte_len()) as usize];
|
|
|
|
let dh = gc_formats::level_data::LevelDataHeader::parse(extent).expect("data header");
|
|
let core = dh
|
|
.core_data
|
|
.slice(extent, extent.len())
|
|
.expect("core_data slice");
|
|
|
|
assert!(
|
|
gc_formats::wad::is_wad(core),
|
|
"core_data should start with the WAD magic"
|
|
);
|
|
|
|
let decoded = gc_formats::wad::decompress(core).expect("core_data should decompress");
|
|
assert!(
|
|
decoded.len() > core.len(),
|
|
"decompressed geometry ({}) should exceed the compressed size ({})",
|
|
decoded.len(),
|
|
core.len()
|
|
);
|
|
}
|
|
|
|
/// The terrain itself. This is the first test to touch real level content: it
|
|
/// parses the tfrags directory and decodes vertex positions from every patch.
|
|
///
|
|
/// The gate is deliberately about *coverage*, not correctness of a sample: a
|
|
/// parser that works on one tfrag and silently returns `None` on the rest would
|
|
/// otherwise pass.
|
|
#[test]
|
|
fn tfrag_vertices_decode_for_every_level() {
|
|
require_disc!();
|
|
// Opening the image also confirms the disc is readable, not merely present.
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
// Where the unpacked data lives. GC_EXTRACTED overrides; otherwise the
|
|
// default location used during development.
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!(
|
|
"skipped: unpack the disc first, e.g.\
|
|
gc-extract unpack <disc.iso> ~/Documents/gc-extracted\
|
|
or point GC_EXTRACTED at an existing unpack"
|
|
);
|
|
return;
|
|
}
|
|
|
|
let mut levels = 0usize;
|
|
let mut total_verts = 0usize;
|
|
let mut total_tfrags = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core_path = dir.join("core_data.bin");
|
|
let index_path = dir.join("core_index.bin");
|
|
if !core_path.exists() || !index_path.exists() {
|
|
continue;
|
|
}
|
|
let core = std::fs::read(&core_path).expect("core_data");
|
|
let index = std::fs::read(&index_path).expect("core_index");
|
|
|
|
// The tfrags block starts at the core header's 0x08 field and runs to
|
|
// the next region that exists.
|
|
let region_start =
|
|
|off: usize| -> i32 { i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) };
|
|
let start = region_start(0x08) as usize;
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region_start(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len() as i32) as usize;
|
|
|
|
let block = &core[start..end];
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
assert_eq!(header.table_offset, 0x40, "level{n}: table offset");
|
|
|
|
let table_offset = header.table_offset as usize;
|
|
let mut decoded = 0usize;
|
|
let mut verts = 0usize;
|
|
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table_offset.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let slice = &block[data_base..];
|
|
if let Some(p) = gc_formats::tfrag::tfrag_positions(slice, &th) {
|
|
decoded += 1;
|
|
verts += p.len();
|
|
assert_eq!(p.len(), th.vert_count as usize);
|
|
}
|
|
}
|
|
|
|
assert!(header.tfrag_count > 0, "level{n}: no tfrags");
|
|
assert_eq!(
|
|
decoded, header.tfrag_count as usize,
|
|
"level{n}: only {decoded} of {} tfrags decoded",
|
|
header.tfrag_count
|
|
);
|
|
levels += 1;
|
|
total_tfrags += header.tfrag_count as usize;
|
|
total_verts += verts;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(
|
|
total_tfrags > 5000,
|
|
"expected thousands of tfrags, got {total_tfrags}"
|
|
);
|
|
assert!(
|
|
total_verts > 100_000,
|
|
"expected 100k+ vertices, got {total_verts}"
|
|
);
|
|
eprintln!("decoded {total_verts} vertices from {total_tfrags} tfrags across {levels} levels");
|
|
}
|
|
|
|
/// Terrain topology: strips and index arrays must expand to exactly the
|
|
/// triangle count each patch's header claims.
|
|
///
|
|
/// `tri_count` is stored in the patch header but is not derivable from the
|
|
/// lists directly, so reproducing it on all 18,102 patches is a strong check
|
|
/// that the strip walk, the index bank and the LOD assignment are all right.
|
|
/// Triangle indices must also stay inside the vertex-info range for the LOD.
|
|
#[test]
|
|
fn tfrag_topology_reproduces_the_header_triangle_count() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut triangles = 0usize;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
let table = header.table_offset as usize;
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
let tris = gc_formats::tfrag::tfrag_triangles(&g, 0)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
assert_eq!(
|
|
tris.len(),
|
|
th.tri_count as usize,
|
|
"level{n} patch {i}: {} triangles, header says {}",
|
|
tris.len(),
|
|
th.tri_count
|
|
);
|
|
let vinfo = g.lod_vinfo_count(0);
|
|
for t in &tris {
|
|
let idx = [t.a as usize, t.b as usize, t.c as usize];
|
|
assert!(
|
|
idx.iter().all(|&v| v < vinfo),
|
|
"level{n} patch {i}: triangle index outside {vinfo} vertex-info entries"
|
|
);
|
|
assert!((t.ad_gif as usize) < g.ad_gifs.len());
|
|
}
|
|
patches += 1;
|
|
triangles += tris.len();
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
assert!(
|
|
triangles > 700_000,
|
|
"expected 700k+ triangles, got {triangles}"
|
|
);
|
|
eprintln!("reproduced {triangles} triangles across {patches} patches in {levels} levels");
|
|
}
|
|
|
|
/// Texture bindings: every ad-gif a patch uses must index an entry that
|
|
/// actually exists in the level's texture table, and the light array must be
|
|
/// one record per vertex.
|
|
///
|
|
/// The texture table is the join between tfrags and the texel data, so an
|
|
/// out-of-range index here means the level's geometry and its textures have
|
|
/// been misread independently of each other.
|
|
#[test]
|
|
fn tfrag_ad_gifs_index_the_level_texture_table() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut ad_gifs = 0usize;
|
|
let mut textures_seen = 0usize;
|
|
let mut textures_total = 0usize;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let textures = gc_formats::level_core::texture_table(&index, ch.tfrag_textures)
|
|
.expect("texture table");
|
|
assert!(!textures.is_empty(), "level{n}: no tfrag textures");
|
|
// Indices are per-level, so the pair is what identifies a texture.
|
|
let mut level_seen = std::collections::BTreeSet::new();
|
|
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
|
|
let table = header.table_offset as usize;
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
|
|
assert_eq!(
|
|
g.lights.len(),
|
|
th.vert_count as usize,
|
|
"level{n} patch {i}: {} light records for {} vertices",
|
|
g.lights.len(),
|
|
th.vert_count
|
|
);
|
|
assert_eq!(
|
|
g.rgba.len(),
|
|
th.rgba_size as usize * 4,
|
|
"level{n} patch {i}: rgba"
|
|
);
|
|
|
|
for a in &g.ad_gifs {
|
|
let t = a.texture_index();
|
|
assert!(
|
|
t < textures.len(),
|
|
"level{n} patch {i}: texture {} outside the {} entry table",
|
|
t,
|
|
textures.len()
|
|
);
|
|
let tex = &textures[t];
|
|
assert!(
|
|
tex.width > 0 && tex.height > 0 && tex.width <= 256 && tex.height <= 256,
|
|
"level{n} patch {i}: texture {t} is {}x{}",
|
|
tex.width,
|
|
tex.height
|
|
);
|
|
level_seen.insert(t);
|
|
ad_gifs += 1;
|
|
}
|
|
patches += 1;
|
|
}
|
|
textures_seen += level_seen.len();
|
|
textures_total += textures.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels > 0, "no unpacked levels were found");
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
// Most of each level's texture table should be referenced by some tfrag. A
|
|
// handful genuinely go unused; if that share grows, the ad-gif walk has
|
|
// started missing primitives rather than the data having spare entries.
|
|
let covered = textures_seen as f64 / textures_total as f64;
|
|
assert!(
|
|
covered > 0.85,
|
|
"only {textures_seen} of {textures_total} level textures referenced ({:.1}%)",
|
|
covered * 100.0
|
|
);
|
|
eprintln!(
|
|
"{ad_gifs} ad-gifs over {patches} patches reference {textures_seen}/{textures_total} level textures"
|
|
);
|
|
}
|
|
|
|
/// Level textures decode: every tfrag texture's base level must resolve to a
|
|
/// plausible RGBA image.
|
|
///
|
|
/// This is the join between the texture table, the texel block and GS RAM, so
|
|
/// it is where a wrong CLUT remap, a wrong alpha scale or a wrong pixel base
|
|
/// would show up. The check is deliberately about *shape*, not specific
|
|
/// colours: a decoded image is plausible when it is fully opaque, is not
|
|
/// uniformly one colour, and uses more than a handful of distinct entries.
|
|
#[test]
|
|
fn tfrag_textures_decode_to_plausible_images() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut decoded = 0usize;
|
|
let mut opaque = 0usize;
|
|
let mut multi_colour = 0usize;
|
|
let mut flat = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let gs = match std::fs::read(dir.join("gs_ram.bin")) {
|
|
Ok(g) => g,
|
|
Err(_) => {
|
|
eprintln!("level{n}: no gs_ram.bin, skipping");
|
|
continue;
|
|
}
|
|
};
|
|
let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let textures_at = ch.textures_base.max(0) as usize;
|
|
let texture_block = &core[textures_at..];
|
|
let table =
|
|
gc_formats::level_core::texture_table(&index, ch.tfrag_textures).expect("table");
|
|
|
|
for (i, e) in table.iter().enumerate() {
|
|
let img = gc_formats::texture::base_image(texture_block, &gs, e)
|
|
.unwrap_or_else(|err| panic!("level{n} texture {i}: {err}"));
|
|
let t = img.decode().unwrap();
|
|
let texels = (e.width as i32) * (e.height as i32);
|
|
assert_eq!(t.rgba.len(), (texels * 4) as usize);
|
|
assert_eq!(t.width, e.width as u32);
|
|
decoded += 1;
|
|
|
|
// Count distinct colours and how many texels are fully opaque.
|
|
let mut seen = std::collections::HashSet::new();
|
|
let mut op = 0usize;
|
|
for px in t.rgba.chunks_exact(4) {
|
|
seen.insert([px[0], px[1], px[2], px[3]]);
|
|
if px[3] == 0xff {
|
|
op += 1;
|
|
}
|
|
}
|
|
if seen.len() > 8 {
|
|
multi_colour += 1;
|
|
} else {
|
|
flat += 1;
|
|
}
|
|
if op == texels as usize {
|
|
opaque += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(decoded > 1_500, "only {decoded} textures decoded");
|
|
// Level art is mostly opaque and has real detail. A flat, transparent or
|
|
// single-colour result almost always means a CLUT or offset mistake.
|
|
let opaque_pct = opaque as f64 / decoded as f64;
|
|
let detail_pct = multi_colour as f64 / decoded as f64;
|
|
assert!(
|
|
opaque_pct > 0.9,
|
|
"only {opaque:.1}% of textures are fully opaque"
|
|
);
|
|
assert!(
|
|
detail_pct > 0.9,
|
|
"only {detail_pct:.1} of textures have >8 colours"
|
|
);
|
|
eprintln!(
|
|
"{decoded} textures decoded: {:.1}% fully opaque, {:.1}% with >8 colours ({flat} flat)",
|
|
opaque_pct * 100.0,
|
|
detail_pct * 100.0
|
|
);
|
|
}
|
|
|
|
/// LOD linkage: every LOD-01 and LOD-0 vertex-info entry must resolve to a
|
|
/// first parent that exists and to positions inside the patch.
|
|
///
|
|
/// The parent-index arrays are V4_8 unpacks separated from the strips and the
|
|
/// strip indices only by their VU address, so a misclassification here is
|
|
/// silent: the arrays still parse, they just hold the wrong numbers. Resolving
|
|
/// every entry is what catches it.
|
|
#[test]
|
|
fn tfrag_lod_parents_resolve_for_every_entry() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut patches = 0usize;
|
|
let mut entries = 0usize;
|
|
let mut lod01 = 0usize;
|
|
let mut lod0 = 0usize;
|
|
let mut resolved_common = 0usize;
|
|
let mut common_total = 0usize;
|
|
let mut primary_failed = 0usize;
|
|
let mut extras_unresolved = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let region =
|
|
|off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
|
|
let start = region(0x08);
|
|
let end = [0x0c_usize, 0x10, 0x14]
|
|
.iter()
|
|
.map(|&o| region(o))
|
|
.filter(|&v| v > 0)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let block = &core[start..end];
|
|
let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block");
|
|
let table = header.table_offset as usize;
|
|
|
|
for i in 0..header.tfrag_count as usize {
|
|
let th = header.header(block, i).expect("tfrag header");
|
|
let data_base = table.wrapping_add(th.data as usize);
|
|
if data_base >= block.len() {
|
|
continue;
|
|
}
|
|
let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
|
|
.unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
|
|
|
|
common_total += g.vinfo_tier[0];
|
|
if n == 0 && patches < 5 {
|
|
eprintln!(
|
|
"p{i}: tier={:?} n01={} n0={} p01={}b x01={}b p0={}b x0={}b",
|
|
g.vinfo_tier,
|
|
g.vu_positions_lod_01_count,
|
|
g.vu_positions_lod_0_count,
|
|
g.parent_indices_lod01.len(),
|
|
g.unk_indices_2_lod01.len(),
|
|
g.parent_indices_lod0.len(),
|
|
g.unk_indices_2_lod0.len()
|
|
);
|
|
}
|
|
for v in 0..g.vertex_info.len() {
|
|
entries += 1;
|
|
match gc_formats::tfrag::tfrag_lod_link(&g, v) {
|
|
Some(link) => {
|
|
if v < g.vinfo_tier[0] {
|
|
resolved_common += 1;
|
|
}
|
|
// Parents must index real vertex info and real positions.
|
|
assert!(
|
|
link.parent1_vinfo < g.vertex_info.len(),
|
|
"level{n} patch {i}: entry {v} parent {} outside {} entries",
|
|
link.parent1_vinfo,
|
|
g.vertex_info.len()
|
|
);
|
|
assert!(link.own_position < g.positions.len());
|
|
assert!(link.parent1_position < g.positions.len());
|
|
assert!(link.parent2_position < g.positions.len());
|
|
// The first parent of a LOD-01 entry is always a common one.
|
|
if link.tier == gc_formats::tfrag::TfragMorphTier::Lod01 {
|
|
assert!(
|
|
link.parent1_vinfo < g.vinfo_tier[0],
|
|
"level{n} patch {i}: LOD-01 entry {v} parent {} is not a common entry",
|
|
link.parent1_vinfo
|
|
);
|
|
lod01 += 1;
|
|
} else {
|
|
lod0 += 1;
|
|
}
|
|
}
|
|
None => {
|
|
// Distinguish the two kinds of LOD entry. A *primary*
|
|
// entry has a real position and a recorded parent and
|
|
// must always resolve; an *extra* entry is a
|
|
// non-position bookkeeping record whose array address
|
|
// can collide with the strip index array.
|
|
let c = g.vinfo_tier[0];
|
|
let kind = if v < c {
|
|
None
|
|
} else if v < c + g.vinfo_tier[1] {
|
|
Some((g.vu_positions_lod_01_count, v - c, true))
|
|
} else {
|
|
Some((g.vu_positions_lod_0_count, v - c - g.vinfo_tier[1], false))
|
|
};
|
|
match kind {
|
|
None => {}
|
|
Some((prim, k, _)) if k < prim => primary_failed += 1,
|
|
Some(_) => extras_unresolved += 1,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
patches += 1;
|
|
}
|
|
}
|
|
|
|
assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
|
|
|
|
// A common entry never has a link: they are the full-detail positions and
|
|
// have no parent. Anything else means the tier boundaries are wrong.
|
|
assert!(
|
|
resolved_common == 0,
|
|
"{resolved_common} common entries claimed an LOD parent"
|
|
);
|
|
assert!(
|
|
lod01 > 0 && lod0 > 0,
|
|
"both LOD tiers should occur in the data"
|
|
);
|
|
assert_eq!(
|
|
primary_failed, 0,
|
|
"{primary_failed} primary LOD entries did not resolve"
|
|
);
|
|
assert_eq!(
|
|
extras_unresolved, 0,
|
|
"{extras_unresolved} extra LOD entries did not resolve"
|
|
);
|
|
// Common entries have no link; every other entry does.
|
|
assert_eq!(entries, common_total + lod01 + lod0);
|
|
|
|
eprintln!(
|
|
"{entries} vertex-info entries over {patches} patches: {common_total} common, \
|
|
{lod01} LOD-01 and {lod0} LOD-0, all resolved"
|
|
);
|
|
}
|
|
|
|
/// The level core index's class tables.
|
|
///
|
|
/// These are what put the environment on top of the terrain: tie instances,
|
|
/// moby actors and shrub foliage. They were ruled out earlier for want of an
|
|
/// entry size, which turned out to be knowable exactly.
|
|
///
|
|
/// The check is the chain: each table must end precisely where the next one
|
|
/// begins, moby (32 bytes/entry) into tie (32) into shrub (48) into the tfrag
|
|
/// texture table. That closes on all 27 levels and would not close if any
|
|
/// entry size were wrong, so it is the evidence rather than a coincidence.
|
|
#[test]
|
|
fn level_core_class_tables_tile_the_index() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
let mut levels = 0usize;
|
|
let (mut moby, mut tie, mut shrub, mut textures) = (0usize, 0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let Ok(index) = std::fs::read(dir.join("core_index.bin")) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
let end = |r: lc::ArrayRange, size: usize| r.offset as usize + r.count as usize * size;
|
|
assert_eq!(
|
|
end(h.moby_classes, lc::MOBY_CLASS_SIZE),
|
|
h.tie_classes.offset as usize,
|
|
"level{n}: moby -> tie"
|
|
);
|
|
assert_eq!(
|
|
end(h.tie_classes, lc::TIE_CLASS_SIZE),
|
|
h.shrub_classes.offset as usize,
|
|
"level{n}: tie -> shrub"
|
|
);
|
|
assert_eq!(
|
|
end(h.shrub_classes, lc::SHRUB_CLASS_SIZE),
|
|
h.tfrag_textures.offset as usize,
|
|
"level{n}: shrub -> textures"
|
|
);
|
|
|
|
// Every entry parses, and its asset offset lands inside the level.
|
|
if n == 0 {
|
|
eprintln!(
|
|
"hdr moby={:?} tie={:?} shrub={:?} tex={:?} base={}",
|
|
h.moby_classes, h.tie_classes, h.shrub_classes, h.tfrag_textures, h.textures_base
|
|
);
|
|
eprintln!("index[192..200] = {:?}", &index[192..200]);
|
|
}
|
|
let moby_v = lc::class_table(
|
|
&index,
|
|
h.moby_classes,
|
|
lc::MOBY_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("moby classes");
|
|
let tie_v = lc::class_table(
|
|
&index,
|
|
h.tie_classes,
|
|
lc::TIE_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("tie classes");
|
|
let shrub_v = lc::class_table(
|
|
&index,
|
|
h.shrub_classes,
|
|
lc::SHRUB_CLASS_SIZE,
|
|
lc::ShrubClassEntry::parse,
|
|
)
|
|
.expect("shrub classes");
|
|
let tex_v = lc::texture_table(&index, h.tfrag_textures).expect("tfrag textures");
|
|
|
|
assert_eq!(moby_v.len(), h.moby_classes.count as usize);
|
|
for (i, c) in moby_v.iter().chain(tie_v.iter()).enumerate() {
|
|
assert!(
|
|
c.offset_in_asset_wad >= 0,
|
|
"level{n} entry {i}: asset offset {}",
|
|
c.offset_in_asset_wad
|
|
);
|
|
}
|
|
// Billboard slots are optional: every shrub class on level 0 carries
|
|
// them all zero, so only their sign is meaningful here.
|
|
for s in &shrub_v {
|
|
assert!(s.base.offset_in_asset_wad >= 0);
|
|
assert!(
|
|
s.palette_offset >= -1 && s.texture_offset >= -1,
|
|
"level{n}: shrub billboard offsets below the -1 sentinel"
|
|
);
|
|
}
|
|
moby += moby_v.len();
|
|
tie += tie_v.len();
|
|
shrub += shrub_v.len();
|
|
textures += tex_v.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(
|
|
moby + tie + shrub > 1000,
|
|
"only {moby}+{tie}+{shrub} class entries"
|
|
);
|
|
eprintln!(
|
|
"level core index: {moby} moby, {tie} tie, {shrub} shrub classes, \
|
|
{textures} tfrag textures across {levels} levels"
|
|
);
|
|
}
|
|
|
|
/// The moby, tie and shrub class tables point into `core_data`, and they do so
|
|
/// without overlapping.
|
|
///
|
|
/// `assets_decompressed_size` is the size of `core_data` itself, so there is
|
|
/// no separate asset container: the class entries index the decompressed asset
|
|
/// lump directly. If that is right, every offset must land inside the lump, the
|
|
/// three tables must occupy disjoint spans, and each must be internally sorted
|
|
/// so the geometry is laid out in class order.
|
|
#[test]
|
|
fn class_geometry_offsets_tile_core_data_without_overlap() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
let mut levels = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(index), Ok(core)) = (
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
// The claim under test: the "asset lump" is core_data itself.
|
|
assert_eq!(
|
|
h.assets_decompressed_size as usize,
|
|
core.len(),
|
|
"level{n}: assets_decompressed_size is not core_data's length"
|
|
);
|
|
|
|
let span = |r: lc::ArrayRange, size: usize| -> Vec<usize> {
|
|
lc::class_table(&index, r, size, lc::ClassEntry::parse)
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.map(|c| c.offset_in_asset_wad as usize)
|
|
.collect()
|
|
};
|
|
// Shrub entries are read as ShrubClassEntry so the offsets line up.
|
|
let shrub: Vec<usize> = lc::class_table(
|
|
&index,
|
|
h.shrub_classes,
|
|
lc::SHRUB_CLASS_SIZE,
|
|
lc::ShrubClassEntry::parse,
|
|
)
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.map(|s| s.base.offset_in_asset_wad as usize)
|
|
.collect();
|
|
let moby = span(h.moby_classes, lc::MOBY_CLASS_SIZE);
|
|
let tie = span(h.tie_classes, lc::TIE_CLASS_SIZE);
|
|
|
|
// An offset of 0 marks a slot with no geometry rather than the start of
|
|
// the lump, so those are excluded from the span checks. Entries are not
|
|
// sorted by class number and the offsets are not monotonic, so nothing
|
|
// here assumes an order.
|
|
let moby_l: Vec<usize> = moby.iter().copied().filter(|&o| o != 0).collect();
|
|
let tie_l: Vec<usize> = tie.iter().copied().filter(|&o| o != 0).collect();
|
|
let shrub_l: Vec<usize> = shrub.iter().copied().filter(|&o| o != 0).collect();
|
|
|
|
for (name, v) in [("moby", &moby), ("tie", &tie), ("shrub", &shrub)] {
|
|
for &o in v {
|
|
assert!(
|
|
o < core.len(),
|
|
"level{n}: {name} offset {o} past core data ({})",
|
|
core.len()
|
|
);
|
|
}
|
|
}
|
|
// The three spans must not overlap each other.
|
|
for (an, a) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] {
|
|
for (bn, b) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] {
|
|
// A level may have no geometry of one kind at all.
|
|
if an >= bn || a.is_empty() || b.is_empty() {
|
|
continue;
|
|
}
|
|
let (lo, hi) = (a[0], *a.last().unwrap());
|
|
for &o in b {
|
|
assert!(
|
|
o >= hi || o < lo,
|
|
"level{n}: {an} [{lo},{hi}] overlaps {bn} at {o}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
eprintln!("class geometry offsets verified inside core_data on {levels} levels");
|
|
}
|
|
|
|
/// Tie class headers, as far as RAC1 and RAC2 agree.
|
|
///
|
|
/// The check that carries the weight is `o_class`: the class's own number at
|
|
/// header offset 0x44 equals the number the core index filed it under, on every
|
|
/// class of every level. That is what makes the header base trustworthy rather
|
|
/// than merely plausible.
|
|
#[test]
|
|
fn tie_class_headers_match_their_index_entry() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core as lc;
|
|
use gc_formats::tie::{TieClassHeader, TIE_CLASS_HEADER_MATCHED};
|
|
let mut levels = 0usize;
|
|
let (mut classes, mut ordered, mut unordered) = (0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(index), Ok(core)) = (
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = lc::LevelCoreHeader::parse(&index).expect("core header");
|
|
let entries = lc::class_table(
|
|
&index,
|
|
h.tie_classes,
|
|
lc::TIE_CLASS_SIZE,
|
|
lc::ClassEntry::parse,
|
|
)
|
|
.expect("tie classes");
|
|
|
|
for e in &entries {
|
|
let off = e.offset_in_asset_wad as usize;
|
|
if off == 0 {
|
|
continue; // slot with no geometry
|
|
}
|
|
let at = off + TIE_CLASS_HEADER_MATCHED;
|
|
assert!(at <= core.len(), "level{n}: tie class past core data");
|
|
let t = TieClassHeader::parse(&core[off..at])
|
|
.unwrap_or_else(|err| panic!("level{n}: tie class {} header: {err}", e.o_class));
|
|
assert_eq!(t.o_class, e.o_class, "level{n}: tie o_class mismatch");
|
|
assert!(
|
|
t.far_dist > 0.0,
|
|
"level{n}: tie far distance {}",
|
|
t.far_dist
|
|
);
|
|
if 0.0 < t.near_dist && t.near_dist < t.mid_dist && t.mid_dist < t.far_dist {
|
|
ordered += 1;
|
|
} else {
|
|
unordered += 1;
|
|
}
|
|
classes += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(classes > 1_500, "only {classes} tie classes");
|
|
// Ordered distances are the expected shape; the exceptions are real and
|
|
// unexplained, so they are reported rather than asserted away.
|
|
assert!(
|
|
ordered * 20 >= classes * 19,
|
|
"only {ordered} of {classes} tie classes have ordered LOD distances"
|
|
);
|
|
eprintln!(
|
|
"tie: {classes} class headers, o_class matches the index on all; \\
|
|
{ordered} have ordered LOD distances, {unordered} do not"
|
|
);
|
|
}
|
|
|
|
/// The occlusion PVS grid.
|
|
///
|
|
/// The check that makes this real is the mask array: it starts at
|
|
/// `masks_offset` and runs to `(highest_mask + 1) * 0x80`, and on every level
|
|
/// that ends within a few bytes of the block's own length. A wrong tree shape
|
|
/// or a wrong cell size would leave the highest mask index somewhere else
|
|
/// entirely and the array would not fit.
|
|
#[test]
|
|
fn occlusion_grids_parse_on_every_level() {
|
|
require_disc!();
|
|
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
|
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
|
))
|
|
.expect("ISO should open");
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::occlusion::parse_occlusion_block;
|
|
let mut levels = 0usize;
|
|
let mut absent = 0usize;
|
|
let mut widest_slack = 0usize;
|
|
let (mut cells, mut masks) = (0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
// An offset of 0 means the level has no occlusion data at all, which is
|
|
// not the same as "at the start of the file" -- that is where the
|
|
// tfrags block lives, and parsing it as a grid produces a mask array
|
|
// that fits anyway, which is how this nearly went wrong.
|
|
if h.occlusion <= 0 {
|
|
absent += 1;
|
|
continue;
|
|
}
|
|
let start = h.occlusion as usize;
|
|
// The block runs to the sky region, or to the end when there is none.
|
|
let end = if h.sky > 0 {
|
|
h.sky as usize
|
|
} else {
|
|
core.len()
|
|
};
|
|
assert!(
|
|
start < end && end <= core.len(),
|
|
"level{n}: bad occlusion range"
|
|
);
|
|
let block = &core[start..end];
|
|
|
|
let o = parse_occlusion_block(block)
|
|
.unwrap_or_else(|e| panic!("level{n}: occlusion block: {e}"));
|
|
// Every mask a cell names exists.
|
|
for c in &o.cells {
|
|
assert!(
|
|
(c.mask as usize) < o.masks.len(),
|
|
"level{n}: cell names a missing mask"
|
|
);
|
|
}
|
|
// The array ends at or just before the block's end. Retail levels sit
|
|
// within a few bytes; a gross error would leave a large gap.
|
|
let need = o.masks_offset as usize + o.masks.len() * gc_formats::occlusion::MASK_BYTES;
|
|
assert!(need <= block.len(), "level{n}: masks run past the block");
|
|
let slack = block.len() - need;
|
|
// Levels whose grid is empty are mostly header and padding, so this is
|
|
// proportional and only applies where there are cells at all.
|
|
if !o.cells.is_empty() {
|
|
assert!(
|
|
slack * 8 < block.len(),
|
|
"level{n}: {slack} of {} bytes unexplained after the mask array",
|
|
block.len()
|
|
);
|
|
}
|
|
widest_slack = widest_slack.max(slack);
|
|
|
|
cells += o.cells.len();
|
|
masks += o.masks.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(
|
|
levels + absent,
|
|
27,
|
|
"expected all 27 levels, got {}",
|
|
levels + absent
|
|
);
|
|
assert_eq!(
|
|
absent, 3,
|
|
"expected 3 levels without occlusion data, got {absent}"
|
|
);
|
|
assert!(cells > 400_000, "only {cells} occlusion cells");
|
|
eprintln!(
|
|
"occlusion: {cells} populated cells over {masks} masks across {levels} levels; \
|
|
widest unexplained tail {widest_slack} bytes"
|
|
);
|
|
}
|
|
|
|
/// The sound extent framing.
|
|
///
|
|
/// The invariant here is `sblk_offset + sblk_size == daw_offset`, which holds
|
|
/// on all 27 levels and pins the meaning of the four head fields. `DAW` never
|
|
/// reaches the end of the file; the tail is positive on every level, so the
|
|
/// size field is not the whole region and this test asserts the direction
|
|
/// rather than a size.
|
|
#[test]
|
|
fn sound_bank_framing_holds_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::sound_bank::parse_sound_bank;
|
|
let mut levels = 0usize;
|
|
let mut sblk_bytes = 0usize;
|
|
let mut daw_bytes = 0usize;
|
|
let (mut narrowest, mut widest) = (usize::MAX, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let raw = match std::fs::read(data.join(format!("level{n}")).join("sound_bank.bin")) {
|
|
Ok(v) => v,
|
|
Err(_) => continue,
|
|
};
|
|
let s = parse_sound_bank(&raw).unwrap_or_else(|e| panic!("level{n}: sound bank: {e}"));
|
|
|
|
assert_eq!(s.header.version, 3, "level{n}: sound bank version");
|
|
assert_eq!(s.header.unknown_4, 2, "level{n}: sound bank unknown_4");
|
|
assert_eq!(
|
|
s.header.sblk_offset, 24,
|
|
"level{n}: SBlk sits at 24 on every level"
|
|
);
|
|
assert!(
|
|
s.header.sblk_offset + s.header.sblk_size == s.header.daw_offset,
|
|
"level{n}: SBlk must run into DAW with no gap"
|
|
);
|
|
assert!(s.has_sblk_magic(), "level{n}: SBlk magic");
|
|
assert!(s.has_daw_tag(), "level{n}: DAW tag at SBlk+12");
|
|
assert_eq!(s.sblk_version(), Some(1), "level{n}: SBlk version");
|
|
// The tail is always in the same direction: the extent is longer than
|
|
// the DAW region it describes.
|
|
narrowest = narrowest.min(s.daw_tail);
|
|
widest = widest.max(s.daw_tail);
|
|
|
|
sblk_bytes += s.sblk.len();
|
|
daw_bytes += s.daw.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(sblk_bytes > 300_000, "only {sblk_bytes} SBlk bytes");
|
|
assert!(daw_bytes > 20_000_000, "only {daw_bytes} DAW bytes");
|
|
eprintln!(
|
|
"sound_bank: {sblk_bytes} SBlk + {daw_bytes} DAW bytes across {levels} levels; \
|
|
unexplained tail {narrowest}..{widest} bytes"
|
|
);
|
|
}
|
|
|
|
/// Collision container framing.
|
|
///
|
|
/// The invariant is the header's own self-description: `header_size` is 0x40
|
|
/// on every level, and `header_size + payload_size` stays inside `core_data`.
|
|
/// The payload is deliberately left undecoded -- see `collision.rs`.
|
|
#[test]
|
|
fn collision_framing_holds_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::collision::{CollisionHeader, COLLISION_HEADER_SIZE};
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
let mut levels = 0usize;
|
|
let mut empty = 0usize;
|
|
let mut payload_bytes = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let region = CollisionHeader::region(&core, h.collision)
|
|
.unwrap_or_else(|e| panic!("level{n}: collision: {e}"));
|
|
let Some(r) = region else {
|
|
empty += 1;
|
|
continue;
|
|
};
|
|
assert_eq!(r.header.header_size, COLLISION_HEADER_SIZE, "level{n}");
|
|
assert!(
|
|
r.header.payload_size as usize <= r.payload.len(),
|
|
"level{n}"
|
|
);
|
|
if r.payload.is_empty() {
|
|
// Present but empty. Counted once, not twice.
|
|
empty += 1;
|
|
continue;
|
|
}
|
|
assert_eq!(r.payload.len(), r.header.payload_size as usize, "level{n}");
|
|
payload_bytes += r.payload.len();
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(
|
|
levels + empty,
|
|
27,
|
|
"expected all 27 levels, got {levels}+{empty}"
|
|
);
|
|
assert!(
|
|
payload_bytes > 10_000_000,
|
|
"only {payload_bytes} collision bytes"
|
|
);
|
|
eprintln!("collision: {payload_bytes} payload bytes across {levels} levels, {empty} empty");
|
|
}
|
|
|
|
/// Moby model geometry: the per-class part table.
|
|
///
|
|
/// The invariant is `vertex_offset + 16 * vertex_count == next_part`, which is
|
|
/// what makes the 16-byte record size and the field boundaries fall out of the
|
|
/// data instead of being assumed. The parser rejects any span that violates it,
|
|
/// so this test is really checking that no span on the disc does.
|
|
#[test]
|
|
fn moby_parts_tile_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::moby::parse_moby_geometry;
|
|
let mut levels = 0usize;
|
|
let mut classes = 0usize;
|
|
let mut parts = 0usize;
|
|
let mut verts = 0usize;
|
|
let mut empties = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let range = h.moby_classes;
|
|
assert!(
|
|
range.count > 0 && range.offset > 0,
|
|
"level{n}: no moby class table"
|
|
);
|
|
|
|
// Class spans tile between consecutive geometry offsets.
|
|
let stride = 0x20usize;
|
|
let mut offsets: Vec<usize> = (0..range.count as usize)
|
|
.map(|i| {
|
|
let at = range.offset as usize + i * stride;
|
|
let e = gc_formats::level_core::ClassEntry::parse(&index[at..at + stride])
|
|
.unwrap_or_else(|e| panic!("level{n}: class {i}: {e}"));
|
|
e.offset_in_asset_wad
|
|
})
|
|
.map(|o| o.max(0) as usize)
|
|
.filter(|&o| o > 0)
|
|
.collect();
|
|
offsets.sort_unstable();
|
|
assert!(!offsets.is_empty(), "level{n}: no moby geometry offsets");
|
|
|
|
for i in 0..offsets.len() {
|
|
let start = offsets[i];
|
|
let end = offsets.get(i + 1).copied().unwrap_or(core.len());
|
|
if end <= start {
|
|
continue;
|
|
}
|
|
let g = parse_moby_geometry(&core[start..end])
|
|
.unwrap_or_else(|e| panic!("level{n}: moby class at {start}: {e}"));
|
|
if g.is_empty() {
|
|
empties += 1;
|
|
}
|
|
// Every part's vertices are readable and inside the span.
|
|
for p in &g.parts {
|
|
assert!(
|
|
g.vertices(p).is_some(),
|
|
"level{n}: part vertices past the end of the class span"
|
|
);
|
|
}
|
|
parts += g.parts.len();
|
|
verts += g.vertex_count();
|
|
classes += 1;
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(classes > 4_000, "only {classes} moby class spans");
|
|
assert!(parts > 40_000, "only {parts} moby parts");
|
|
assert!(verts > 1_200_000, "only {verts} moby vertices");
|
|
eprintln!(
|
|
"moby: {parts} parts tiling exactly, {verts} vertices, {classes} class spans \
|
|
({empties} with no geometry) across {levels} levels"
|
|
);
|
|
}
|
|
|
|
/// Shrub class headers and blobs.
|
|
///
|
|
/// Two fields self-identify exactly on every span, and the parser checks both
|
|
/// rather than merely reporting them: `+0x24` must equal the owning class
|
|
/// entry's own `o_class`, and `+0x2c` must land inside the span.
|
|
///
|
|
/// The body is then checked against the R&C1 shape this format follows: a
|
|
/// palette of 24 unit-length `s16` normals with a zero pad word, then zero
|
|
/// padding records. And the billboard offset at `+0x1c` is checked against the
|
|
/// class entry's billboard texture fields, which is the one header field that
|
|
/// can be confirmed from a second file.
|
|
#[test]
|
|
fn shrub_headers_name_their_own_class_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::ShrubClassEntry;
|
|
use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES, SHRUB_NORMAL_COUNT};
|
|
let mut levels = 0usize;
|
|
let mut spans = 0usize;
|
|
let mut absent = 0usize;
|
|
let (mut min_size, mut max_size) = (f32::MAX, 0.0f32);
|
|
let mut scratch_nonzero = 0usize;
|
|
let mut mode_hist: std::collections::BTreeMap<u16, usize> = Default::default();
|
|
let mut scratch_hist: std::collections::BTreeMap<u16, usize> = Default::default();
|
|
let mut radius_range = (f32::MAX, 0f32);
|
|
let mut scale_range = (f32::MAX, 0f32);
|
|
let mut bounded_records = 0usize;
|
|
let mut bounded_bodies = 0usize;
|
|
let mut palette_records = 0usize;
|
|
let mut padding_records = 0usize;
|
|
let mut norm2_deviation = 0i64;
|
|
let mut billboards = 0usize;
|
|
let mut billboard_entries = 0usize;
|
|
let mut packets_total = 0usize;
|
|
let mut packet_headers_total = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
|
|
let range = h.shrub_classes;
|
|
// Class spans tile between consecutive geometry offsets, with the last
|
|
// one bounded by the chunk container. It is found by signature, since
|
|
// the core index carries no successor offset for shrub; without it the
|
|
// last class absorbs the container and the body check measures its
|
|
// neighbours.
|
|
let mut offsets: Vec<usize> = Vec::new();
|
|
for i in 0..range.count.max(0) as usize {
|
|
let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
|
|
.unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
|
|
if e.base.offset_in_asset_wad > 0 {
|
|
offsets.push(e.base.offset_in_asset_wad as usize);
|
|
}
|
|
}
|
|
offsets.sort_unstable();
|
|
// Levels without shrub geometry have no span to bound. The container
|
|
// that follows tie on those levels is checked by
|
|
// `chunk_container_closes_the_last_class_region`.
|
|
let container = match offsets.last() {
|
|
Some(&last) if !offsets.is_empty() => {
|
|
gc_formats::chunks::chunk_region_start(&core, last).unwrap_or_else(|| {
|
|
panic!("level{n}: no chunk container after the shrub classes")
|
|
})
|
|
}
|
|
_ => core.len(),
|
|
};
|
|
let mut span_end_of: std::collections::HashMap<usize, usize> =
|
|
std::collections::HashMap::new();
|
|
for (i, &o) in offsets.iter().enumerate() {
|
|
let end = offsets.get(i + 1).copied().unwrap_or(container);
|
|
span_end_of.insert(o, end);
|
|
}
|
|
for i in 0..range.count.max(0) as usize {
|
|
let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES])
|
|
.unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}"));
|
|
let (span_end, has_successor) =
|
|
match span_end_of.get(&(e.base.offset_in_asset_wad.max(0) as usize)) {
|
|
Some(&e2) => (e2, true),
|
|
None => (core.len(), false),
|
|
};
|
|
match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class, span_end) {
|
|
Ok(Some(mut s)) => {
|
|
s.span_end_is_a_successor = has_successor;
|
|
// Every class is bounded now, so every body must be whole
|
|
// 8-byte records. That was not assertable while the last
|
|
// class ran to the lump's end, and it is the check the
|
|
// boundary earns its keep for.
|
|
assert!(
|
|
s.body_is_well_formed(),
|
|
"level{n} class {i}: shrub body is not whole 8-byte records"
|
|
);
|
|
bounded_bodies += 1;
|
|
bounded_records += s.body_record_count();
|
|
assert_eq!(
|
|
s.header.o_class,
|
|
e.base.o_class.max(0) as u32,
|
|
"level{n} class {i}"
|
|
);
|
|
assert!(!s.data.is_empty(), "level{n} class {i}: empty shrub data");
|
|
assert!(
|
|
s.header.sphere.iter().all(|v| v.is_finite()),
|
|
"level{n} class {i}: non-finite shrub sphere"
|
|
);
|
|
assert!(
|
|
s.header.size.is_finite() && s.header.size > 0.0,
|
|
"level{n} class {i}: shrub size {}",
|
|
s.header.size
|
|
);
|
|
// The scratch word is zero on every span; the parser
|
|
// refuses anything else. `mode_bits` is 4 on 16 spans and
|
|
// is reported rather than asserted. On those 16 the word
|
|
// at +0x14 reads as an f32 denormal, which the Emotion
|
|
// Engine flushes to zero like everything else.
|
|
if s.header.scratch != 0 {
|
|
scratch_nonzero += 1;
|
|
}
|
|
*mode_hist.entry(s.header.mode_bits).or_default() += 1;
|
|
*scratch_hist.entry(s.header.scratch).or_default() += 1;
|
|
radius_range.0 = radius_range.0.min(s.header.sphere[3]);
|
|
radius_range.1 = radius_range.1.max(s.header.sphere[3]);
|
|
scale_range.0 = scale_range.0.min(s.header.scale);
|
|
scale_range.1 = scale_range.1.max(s.header.scale);
|
|
min_size = min_size.min(s.header.size);
|
|
max_size = max_size.max(s.header.size);
|
|
|
|
// The palette: 24 unit normals, then zero padding.
|
|
let normals: Vec<_> = s.normals().collect();
|
|
assert_eq!(
|
|
normals.len(),
|
|
SHRUB_NORMAL_COUNT,
|
|
"level{n} class {i}: palette is {} entries, not {SHRUB_NORMAL_COUNT}",
|
|
normals.len()
|
|
);
|
|
for (k, normal) in normals.iter().enumerate() {
|
|
assert!(
|
|
normal.is_unit(),
|
|
"level{n} class {i}: palette entry {k} {normal:?} is not a unit normal"
|
|
);
|
|
let n2 = (normal.x as i64).pow(2)
|
|
+ (normal.y as i64).pow(2)
|
|
+ (normal.z as i64).pow(2);
|
|
norm2_deviation = norm2_deviation.max((n2 - 32767i64.pow(2)).abs());
|
|
}
|
|
palette_records += normals.len();
|
|
for (k, record) in s.body_records().skip(SHRUB_NORMAL_COUNT).enumerate() {
|
|
assert!(
|
|
record.iter().all(|&b| b == 0),
|
|
"level{n} class {i}: padding record {k} is not all zero"
|
|
);
|
|
}
|
|
padding_records += s.padding_record_count();
|
|
|
|
// The packets: the table at +0x40 tiles the span from
|
|
// align16 of its own end to the palette, or to the
|
|
// billboard when the class has one. This is what puts the
|
|
// mesh geometry in the blob, and it is the identity that
|
|
// makes `count` the packet count rather than a guess.
|
|
let packets = s.packets().unwrap_or_else(|| {
|
|
panic!("level{n} class {i}: packet table does not tile")
|
|
});
|
|
assert_eq!(
|
|
packets.len(),
|
|
s.header.count as usize,
|
|
"level{n} class {i}: packet count disagrees with the header"
|
|
);
|
|
packets_total += packets.len();
|
|
|
|
// The packet headers themselves. The disc form is a fixed
|
|
// signature -- magic, four zero bytes, tag 0x05 -- and the
|
|
// loader rewrites bytes 8-15 on load, so the count and the
|
|
// pointer live in RAM, not here.
|
|
let hdrs: Vec<_> = s.packet_headers().collect();
|
|
assert_eq!(
|
|
hdrs.len(),
|
|
packets.len(),
|
|
"level{n} class {i}: header count disagrees with packet count"
|
|
);
|
|
for (pi, h) in hdrs.iter().enumerate() {
|
|
assert!(
|
|
h.is_some(),
|
|
"level{n} class {i}: packet {pi} header does not match the \
|
|
disc invariants (magic 04 04 00 01, bytes 4-8 zero, tag 0x05)"
|
|
);
|
|
assert_eq!(
|
|
packets[pi].size % 16,
|
|
0,
|
|
"level{n} class {i}: packet {pi} size is not a multiple of 16"
|
|
);
|
|
}
|
|
packet_headers_total += hdrs.len();
|
|
|
|
// The billboard: present exactly when the class entry
|
|
// carries billboard texture fields.
|
|
let entry_has_billboard = e.billboard_width != 0 || e.billboard_height != 0;
|
|
match s.billboard().expect("billboard parses") {
|
|
Some(b) => {
|
|
billboards += 1;
|
|
assert!(
|
|
entry_has_billboard,
|
|
"level{n} class {i}: billboard record without billboard texture fields"
|
|
);
|
|
assert_eq!(b.tex1_addr, 0x14, "level{n} class {i}: TEX1 address");
|
|
assert_eq!(b.tex0_addr, 0x06, "level{n} class {i}: TEX0 address");
|
|
assert_eq!(b.miptbp_addr, 0x34, "level{n} class {i}: MIPTBP address");
|
|
assert_eq!(b.mmin, 4, "level{n} class {i}: MMIN");
|
|
}
|
|
None => assert!(
|
|
!entry_has_billboard,
|
|
"level{n} class {i}: billboard texture fields with no billboard record"
|
|
),
|
|
}
|
|
if entry_has_billboard {
|
|
billboard_entries += 1;
|
|
}
|
|
spans += 1;
|
|
}
|
|
Ok(None) => absent += 1,
|
|
Err(err) => panic!("level{n}: shrub class {i}: {err}"),
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
spans + absent,
|
|
358,
|
|
"expected 358 shrub classes, got {}",
|
|
spans + absent
|
|
);
|
|
assert!(spans > 340, "only {spans} shrub spans with geometry");
|
|
// The container bounds every class now, so every body is checkable.
|
|
assert_eq!(bounded_bodies, spans, "every shrub class should be bounded");
|
|
assert_eq!(scratch_nonzero, 0, "the scratch word is zero on every span");
|
|
assert_eq!(
|
|
billboards, billboard_entries,
|
|
"billboard records and billboard class entries disagree: \
|
|
{billboards} records vs {billboard_entries} entries"
|
|
);
|
|
eprintln!(
|
|
"shrub: {spans} headers naming their own class, {absent} without geometry; \
|
|
all {bounded_bodies} bodies bounded, {bounded_records} records, \
|
|
{palette_records} of them palette normals and {padding_records} zero padding; \
|
|
{billboards} billboards, all with texture fields; \
|
|
{packets_total} packets across all blobs, {packet_headers_total} headers; \
|
|
mode_bits {mode_hist:?}, radius {radius_range:?}, scale {scale_range:?}, \
|
|
largest palette deviation from unit length {norm2_deviation} of a possible 65535"
|
|
);
|
|
}
|
|
|
|
/// Region bounds derived from successor boundaries, on every level.
|
|
///
|
|
/// Every region is closed by the next region's start, or -- for the last class
|
|
/// region -- by the located chunk container, never by `core_data.len()`, so no
|
|
/// span silently absorbs the tail of the lump the way the last shrub class was
|
|
/// doing.
|
|
#[test]
|
|
fn core_regions_chain_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES};
|
|
let mut levels = 0usize;
|
|
let mut unresolved_shrub = 0usize;
|
|
let mut unresolved_chunks = 0usize;
|
|
let mut bounded = 0usize;
|
|
let mut shrub_bytes = 0usize;
|
|
let mut chunk_bytes = 0usize;
|
|
let (mut first, mut last) = (usize::MAX, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
let core_len = core.len();
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let regions = core_layout(&index, &core, &h);
|
|
|
|
// Regions appear in order, never overlap, and never run past the lump.
|
|
for pair in regions.windows(2) {
|
|
assert!(
|
|
pair[0].start <= pair[1].start,
|
|
"level{n}: {} vs {}",
|
|
pair[0].name,
|
|
pair[1].name
|
|
);
|
|
if let Some(end) = pair[0].end {
|
|
assert!(
|
|
end <= pair[1].start,
|
|
"level{n}: {} ends after {}",
|
|
pair[0].name,
|
|
pair[1].name
|
|
);
|
|
}
|
|
}
|
|
for r in ®ions {
|
|
assert!(
|
|
r.start < core_len,
|
|
"level{n}: {} starts past the lump",
|
|
r.name
|
|
);
|
|
if let Some(end) = r.end {
|
|
assert!(end <= core_len, "level{n}: {} ends past the lump", r.name);
|
|
}
|
|
}
|
|
|
|
// Regions appear in the canonical order, but not every level has every
|
|
// region -- level 10 has no occlusion and no shrub at all.
|
|
const ORDER: [&str; 8] = [
|
|
"tfrags",
|
|
"occlusion",
|
|
"sky",
|
|
"collision",
|
|
"moby",
|
|
"tie",
|
|
"shrub",
|
|
"chunks",
|
|
];
|
|
let names: Vec<_> = regions.iter().map(|r| r.name).collect();
|
|
let ranks: Vec<usize> = names
|
|
.iter()
|
|
.map(|name| {
|
|
ORDER
|
|
.iter()
|
|
.position(|o| o == name)
|
|
.unwrap_or_else(|| panic!("level{n}: unknown region {name}"))
|
|
})
|
|
.collect();
|
|
assert!(
|
|
ranks.windows(2).all(|w| w[0] < w[1]),
|
|
"level{n}: regions out of order: {names:?}"
|
|
);
|
|
if let Some(s) = regions.iter().find(|r| r.name == "shrub") {
|
|
match s.end {
|
|
Some(e) => {
|
|
// The container's start, not the trailer and not the lump's
|
|
// end: the trailer is the chunk region's business.
|
|
assert!(e < core_len - CORE_DATA_TRAILER_BYTES);
|
|
shrub_bytes += s.len().unwrap_or(0);
|
|
bounded += 1;
|
|
}
|
|
None => unresolved_shrub += 1,
|
|
}
|
|
first = first.min(s.start);
|
|
last = last.max(s.start);
|
|
}
|
|
if let Some(c) = regions.iter().find(|r| r.name == "chunks") {
|
|
match c.end {
|
|
Some(e) => {
|
|
// The trailer is the only thing past the chunk container.
|
|
assert_eq!(e, core_len - CORE_DATA_TRAILER_BYTES);
|
|
chunk_bytes += c.len().unwrap_or(0);
|
|
}
|
|
None => unresolved_chunks += 1,
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
unresolved_shrub, 0,
|
|
"every level with shrub should bound it at the chunk container"
|
|
);
|
|
assert_eq!(bounded, 24, "expected 24 levels with a shrub region");
|
|
assert_eq!(
|
|
unresolved_chunks, 0,
|
|
"every chunk region should end at the trailer"
|
|
);
|
|
eprintln!(
|
|
"regions: shrub starts {first}..{last}; {shrub_bytes} bytes of shrub and \
|
|
{chunk_bytes} bytes of chunk data, shrub closed by the located container on \
|
|
all {bounded} levels"
|
|
);
|
|
}
|
|
|
|
/// The post-shrub chunk container: chunks carry the `0xf0`-tagged offset
|
|
/// signature on every level, and their offsets resolve to records with a
|
|
/// shared field layout.
|
|
///
|
|
/// The region is bounded properly rather than by the fixed window this test
|
|
/// first used: it runs from the marker-paired chunk signature that closes the
|
|
/// last class region to the constant trailer. The window approach worked and
|
|
/// then over-worked -- a fixed 900,000 bytes from the end reaches past the
|
|
/// container on some levels and stops inside it on others, which is how 27
|
|
/// chunks came to be truncated at a "window edge" that is not a format
|
|
/// boundary. The locator is asserted by
|
|
/// `chunk_container_closes_the_last_class_region`.
|
|
///
|
|
/// The layout facts asserted here are measured, not assumed:
|
|
///
|
|
/// - every tagged offset has top byte `0xf0`, on all 27 levels;
|
|
/// - resolving records at `chunk_start + low` gives 17-21% byte positions
|
|
/// constant across that chunk's own records, which does not happen by
|
|
/// chance and is what distinguishes this from a coincidental byte pattern;
|
|
/// - stride is a per-chunk property, not a region-wide one.
|
|
#[test]
|
|
fn chunk_containers_resolve_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES};
|
|
|
|
let mut levels = 0usize;
|
|
let mut chunks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut strides = std::collections::BTreeSet::new();
|
|
let mut truncated = 0usize;
|
|
let mut constant_total = 0usize;
|
|
let mut stride_total = 0usize;
|
|
let mut frac: Vec<f64> = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
continue;
|
|
};
|
|
|
|
// The region runs from the chunk signature that closes the last class
|
|
// region to the constant trailer, both measured and identical in size
|
|
// on all 27 levels. core_layout locates the start through the same
|
|
// signature the chunk parser uses.
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
let Some(region) = core_layout(&index, &core, &h)
|
|
.into_iter()
|
|
.find(|r| r.name == "chunks")
|
|
else {
|
|
continue;
|
|
};
|
|
let Some(end) = region.end else {
|
|
panic!("level{n}: chunk region is unbounded");
|
|
};
|
|
assert_eq!(end, core.len() - CORE_DATA_TRAILER_BYTES);
|
|
assert!(!region.is_empty(), "level{n}: empty chunk region");
|
|
let region = &core[region.start..end];
|
|
|
|
let found = gc_formats::chunks::parse_all_chunks(region).expect("chunks should parse");
|
|
if found.is_empty() {
|
|
continue;
|
|
}
|
|
levels += 1;
|
|
|
|
for chunk in &found {
|
|
assert!(
|
|
!chunk.chunk.offsets.is_empty(),
|
|
"level{n}: chunk at {} has no offsets",
|
|
chunk.chunk.offset
|
|
);
|
|
assert!(chunk.chunk.stride > 0, "level{n}: zero stride");
|
|
// The one decoded header field.
|
|
assert_eq!(
|
|
chunk.chunk.count as usize,
|
|
chunk.chunk.offsets.len(),
|
|
"level{n}: chunk at {} header count {} but {} offsets",
|
|
chunk.chunk.offset,
|
|
chunk.chunk.count,
|
|
chunk.chunk.offsets.len()
|
|
);
|
|
chunks += 1;
|
|
records += chunk.records.len();
|
|
strides.insert(chunk.chunk.stride);
|
|
|
|
// Records resolve unless the last one would run past the region
|
|
// end, which happens for the chunk sitting at the trailer end:
|
|
// the container's records stop there, so its final record or two
|
|
// fall outside. Truncation is counted rather than asserted away.
|
|
assert!(
|
|
chunk.records.len() <= chunk.chunk.offsets.len(),
|
|
"level{n}: chunk at {} resolved MORE records than offsets",
|
|
chunk.chunk.offset
|
|
);
|
|
if chunk.records.len() < chunk.chunk.offsets.len() {
|
|
truncated += 1;
|
|
}
|
|
|
|
// The invariant that makes this a container and not a coincidence.
|
|
// Accumulated rather than asserted per chunk: individual chunks
|
|
// range from ~7% to ~21% constant positions, so a per-chunk
|
|
// threshold would be asserting a property the format does not have.
|
|
let constant = constant_positions(&chunk.records);
|
|
constant_total += constant;
|
|
stride_total += chunk.chunk.stride;
|
|
frac.push(constant as f64 / chunk.chunk.stride as f64);
|
|
}
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert!(chunks > 500, "only {chunks} chunks resolved");
|
|
assert!(records > 5_000, "only {records} records resolved");
|
|
assert!(
|
|
strides.len() > 5,
|
|
"stride should vary per chunk, saw {}",
|
|
strides.len()
|
|
);
|
|
// Chance would give a constant position a probability of 1/256^n, so even
|
|
// 5% across tens of thousands of records is far above noise.
|
|
let mean_frac = constant_total as f64 / stride_total as f64;
|
|
assert!(
|
|
mean_frac > 0.05,
|
|
"constant positions {constant_total}/{stride_total} = {mean_frac:.3} is near chance"
|
|
);
|
|
eprintln!(
|
|
"chunks: {chunks} over {levels} levels, {records} records, {} distinct strides, \
|
|
constant {constant_total}/{stride_total} ({:.1}%), per-chunk min {:.1}% max {:.1}%, \
|
|
{truncated} chunks truncated at the trailer end",
|
|
strides.len(),
|
|
mean_frac * 100.0,
|
|
frac.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0,
|
|
frac.iter().cloned().fold(0.0, f64::max) * 100.0,
|
|
);
|
|
}
|
|
|
|
/// The chunk container closes the last class region, on every level.
|
|
///
|
|
/// The core index carries no successor offset for shrub -- unlike moby and
|
|
/// tie, whose boundaries fall out of the class tables -- so the boundary is
|
|
/// located in `core_data`: the constant marker pair sitting immediately in
|
|
/// front of a chunk signature. Both halves of that signature are weak alone,
|
|
/// which is why the test measures them separately:
|
|
///
|
|
/// - the marker pair `f6 7c` occurs 25 to 103 times in a 5 to 17 MB lump, so
|
|
/// on its own it means nothing;
|
|
/// - the chunk shape (16-byte header ending `0xff`, 10 zeros, tagged offsets)
|
|
/// matches random data often enough to need four tagged offsets before it
|
|
/// means anything.
|
|
///
|
|
/// Together they are exact: on all 27 levels the marker sits exactly two bytes
|
|
/// before the first chunk header. Three checks pin that down per level:
|
|
///
|
|
/// 1. the located boundary carries the marker;
|
|
/// 2. the last shrub body walks to 8-byte records that end exactly where the
|
|
/// signature begins, with no chunk signature inside the body itself;
|
|
/// 3. the container's records fill the region up to the constant trailer.
|
|
///
|
|
/// Levels without shrub are the control for the locator: there the container
|
|
/// closes tie instead, and the same marker precedes it.
|
|
#[test]
|
|
fn chunk_container_closes_the_last_class_region() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{self, CHUNK_REGION_MARKER};
|
|
use gc_formats::level_core::{LevelCoreHeader, ShrubClassEntry};
|
|
use gc_formats::regions::{core_layout, last_geometry, CORE_DATA_TRAILER_BYTES};
|
|
use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES};
|
|
|
|
let mut levels = 0usize;
|
|
let mut with_shrub = 0usize;
|
|
let mut marker_wide_max = 0usize;
|
|
let mut body_records = 0usize;
|
|
let mut container_bytes = 0usize;
|
|
let mut fill_gap_max = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(dir.join("core_data.bin")),
|
|
std::fs::read(dir.join("core_index.bin")),
|
|
) else {
|
|
panic!("level{n}: missing core_data or core_index");
|
|
};
|
|
let trailer = core.len() - CORE_DATA_TRAILER_BYTES;
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
|
|
let last_shrub = last_geometry(&index, h.shrub_classes, SHRUB_CLASS_ENTRY_BYTES);
|
|
let last_tie = last_geometry(&index, h.tie_classes, 0x20);
|
|
// The container follows the last of the shrub and tie classes, so the
|
|
// signature search starts after the class bodies rather than inside
|
|
// them.
|
|
let scan_from = last_shrub
|
|
.or(last_tie)
|
|
.unwrap_or_else(|| panic!("level{n}: no class geometry to follow"));
|
|
let container = chunks::chunk_region_start(&core, scan_from)
|
|
.unwrap_or_else(|| panic!("level{n}: no chunk container after {scan_from}"));
|
|
|
|
// 1. The marker, and how common it is without the signature behind it.
|
|
assert_eq!(
|
|
&core[container - 2..container],
|
|
&CHUNK_REGION_MARKER,
|
|
"level{n}: container at {container} is not preceded by the marker"
|
|
);
|
|
marker_wide_max = marker_wide_max.max(
|
|
core.windows(2)
|
|
.filter(|w| *w == CHUNK_REGION_MARKER)
|
|
.count(),
|
|
);
|
|
|
|
// 2. The last shrub body ends exactly at the marker. This is the check
|
|
// that the boundary is the successor of shrub rather than a coincident
|
|
// pair inside it.
|
|
if let Some(last) = last_shrub {
|
|
let o_class = (0..h.shrub_classes.count.max(0) as usize)
|
|
.filter_map(|i| {
|
|
let at = h.shrub_classes.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES;
|
|
let e = ShrubClassEntry::parse(
|
|
index
|
|
.get(at..at + SHRUB_CLASS_ENTRY_BYTES)
|
|
.unwrap_or_default(),
|
|
)
|
|
.ok()?;
|
|
(e.base.offset_in_asset_wad == last as i32).then_some(e.base.o_class)
|
|
})
|
|
.next()
|
|
.unwrap_or_else(|| panic!("level{n}: no shrub class at {last}"));
|
|
let span = ShrubHeader::region(&core, last as i32, o_class, trailer)
|
|
.expect("last shrub span parses")
|
|
.expect("last shrub span present");
|
|
let data_start = last + span.header.data_offset as usize;
|
|
|
|
// Walk 8-byte records, bounded by the container so the walk can
|
|
// never continue into the chunk region itself.
|
|
let mut at = data_start;
|
|
while at + 8 <= container && core[at + 6] == 0 && core[at + 7] == 0 {
|
|
at += 8;
|
|
}
|
|
assert_eq!(
|
|
at + 2,
|
|
container,
|
|
"level{n}: shrub body at {data_start} does not end at the marker"
|
|
);
|
|
assert!(at > data_start, "level{n}: last shrub body is empty");
|
|
body_records += (at - data_start) / 8;
|
|
|
|
// No chunk signature inside the body: the search for the boundary
|
|
// must not match one it should have skipped.
|
|
let sig_inside = (last..container - 2).any(|off| chunks::is_chunk_start(&core, off, 4));
|
|
assert!(
|
|
!sig_inside,
|
|
"level{n}: chunk signature inside the last shrub body"
|
|
);
|
|
with_shrub += 1;
|
|
} else {
|
|
// Without shrub the container closes tie, and the marker precedes
|
|
// it the same way -- the locator is not shrub-specific.
|
|
let tie = core_layout(&index, &core, &h)
|
|
.into_iter()
|
|
.find(|r| r.name == "tie")
|
|
.unwrap_or_else(|| panic!("level{n}: no tie region to close"));
|
|
assert_eq!(
|
|
tie.end,
|
|
Some(container),
|
|
"level{n}: tie must end at the container"
|
|
);
|
|
}
|
|
|
|
// 3. The container fills its region up to the constant trailer: the
|
|
// furthest record should reach within a few KB of it, never inside it.
|
|
let region = &core[container..trailer];
|
|
let found = chunks::parse_all_chunks(region).expect("chunks should parse");
|
|
let region_ptr = region.as_ptr() as usize;
|
|
let furthest = found
|
|
.iter()
|
|
.flat_map(|c| {
|
|
c.records
|
|
.iter()
|
|
.map(move |r| r.as_ptr() as usize + r.len() - region_ptr)
|
|
})
|
|
.max()
|
|
.unwrap_or(0);
|
|
assert!(
|
|
region.len() - furthest < 4096,
|
|
"level{n}: chunk records stop {} bytes short of the trailer",
|
|
region.len() - furthest
|
|
);
|
|
container_bytes += region.len();
|
|
fill_gap_max = fill_gap_max.max(region.len() - furthest);
|
|
levels += 1;
|
|
}
|
|
|
|
assert_eq!(levels, 27, "expected all 27 levels, got {levels}");
|
|
assert_eq!(
|
|
with_shrub, 24,
|
|
"expected 24 levels with shrub, got {with_shrub}"
|
|
);
|
|
eprintln!(
|
|
"container: {levels} levels ({with_shrub} closing shrub), \
|
|
{body_records} 8-byte records in last-shrub bodies, \
|
|
{container_bytes} bytes of chunk data, largest trailer gap {fill_gap_max}, \
|
|
marker pair occurs at most {marker_wide_max} times per lump"
|
|
);
|
|
}
|
|
|
|
/// The chunk record's leading word is fixed per type.
|
|
///
|
|
/// A record is `[leading u16][two index bytes][payload]`. Measured on the
|
|
/// disc: the word is the same at every record of a chunk on 1,439 of 1,509
|
|
/// chunks (`None` on the 70 that vary, which the parser reports rather than
|
|
/// averaging), and fixed within every chunk type, taking only the values
|
|
/// 1, 2, 4, 6, 8 and 10.
|
|
///
|
|
/// Not named. What the value counts is not established, and two readings were
|
|
/// measured and disproved: it is not the payload's entry count -- the payload
|
|
/// length divided by it leaves 57 distinct fractional "sizes" -- and it is not
|
|
/// the number of distinct leading bytes, which agrees on 0 of 1,506 chunks.
|
|
/// See `docs/formats/core-data-regions.md`.
|
|
#[test]
|
|
fn chunk_record_leading_word_is_fixed_per_type() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::parse_all_chunks;
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
// header -> the words its chunks carry, and the value histogram. Chunks
|
|
// whose own records disagree are counted apart rather than folded in.
|
|
let mut by_type: BTreeMap<[u8; 16], BTreeSet<u16>> = BTreeMap::new();
|
|
let mut histogram: BTreeMap<u16, usize> = BTreeMap::new();
|
|
let mut levels = 0usize;
|
|
let mut chunks = 0usize;
|
|
let mut varying = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let core = match std::fs::read(data.join(format!("level{n}/core_data.bin"))) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let found = parse_all_chunks(&core).expect("chunks should parse");
|
|
if found.is_empty() {
|
|
continue;
|
|
}
|
|
levels += 1;
|
|
for c in &found {
|
|
chunks += 1;
|
|
match c.chunk.prefix {
|
|
Some(p) => {
|
|
by_type.entry(c.chunk.header).or_default().insert(p);
|
|
*histogram.entry(p).or_default() += 1;
|
|
}
|
|
None => varying += 1,
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(levels >= 25, "only {levels} levels carry chunks");
|
|
assert!(chunks > 1400, "only {chunks} chunks parsed");
|
|
// The word is uniform within a chunk on all but a small minority.
|
|
let uniform = chunks - varying;
|
|
assert!(
|
|
uniform > 1400,
|
|
"only {uniform} of {chunks} chunks carry a uniform leading word"
|
|
);
|
|
// Fixed within every type: no type's chunks carry two different words.
|
|
let varying_types: Vec<_> = by_type.iter().filter(|(_, v)| v.len() > 1).collect();
|
|
assert!(
|
|
varying_types.is_empty(),
|
|
"{} types carry more than one leading word",
|
|
varying_types.len()
|
|
);
|
|
// The word is never zero and never large, and it is not the record count:
|
|
// 621 chunks carry 2, 520 carry 4, and the rest are spread thin.
|
|
assert!(
|
|
histogram.keys().all(|&p| p > 0 && p <= 16),
|
|
"prefix values {histogram:?}"
|
|
);
|
|
let two = histogram.get(&2).copied().unwrap_or(0);
|
|
let four = histogram.get(&4).copied().unwrap_or(0);
|
|
assert!(two > 500, "only {two} chunks carry the word 2");
|
|
assert!(four > 400, "only {four} chunks carry the word 4");
|
|
eprintln!(
|
|
"chunk leading word: {uniform} of {chunks} chunks uniform ({varying} vary), \
|
|
{} types, values {histogram:?}",
|
|
by_type.len()
|
|
);
|
|
}
|
|
|
|
/// The tie record is R&C1's `TiePacketHeader`, and its rule holds.
|
|
///
|
|
/// A tie LOD block is a table of 16-byte packet headers -- a data pointer,
|
|
/// then R&C1's byte fields at +4..+0x10. What identifies them rather than a
|
|
/// resemblance is R&C1's own documented rule: **`bfc_distance == 5 *
|
|
/// shader_count` on every record**, here 15,219 of 15,219 over 2,177 blocks on
|
|
/// all 27 levels.
|
|
///
|
|
/// Measured over the same set, and reported rather than asserted because they
|
|
/// are not universal:
|
|
///
|
|
/// * `color_ofs == vert_size` on 12,640 of 15,219 -- and R&C1's rule for the
|
|
/// field (`== vert_ofs + vert_size`) holds on **0**, so the two games use it
|
|
/// differently and this one is named for what it measures;
|
|
/// * `color_count`, `strip_count`, `strip_vertex_count` zero on ~15,077 of
|
|
/// 15,219;
|
|
/// * the orderings `vert_ofs <= vert_size`, `scissor_size <= scissor_ofs` and
|
|
/// `control_size <= control_count` on all of them.
|
|
///
|
|
/// One claim is retracted here rather than softened: a packet-length formula,
|
|
/// `(vert_ofs + vert_size + scissor_ofs + scissor_size) * 16`, was measured at
|
|
/// 61 of 61 pairs and is not reproduced on the correct blocks (0 of 13,042). It
|
|
/// was measured over blocks found by reading `packets[2]` where `packets[1]`
|
|
/// was meant, so the arithmetic agreed perfectly with a population that was not
|
|
/// the tie blocks. The block set was the error; see `AGENTS.md` rule 3.
|
|
#[test]
|
|
fn tie_records_are_packet_headers() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::tie::{parse_tie_block, resolve_tie_class, TieRecord};
|
|
|
|
let mut levels = 0usize;
|
|
let mut blocks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut bfc_ok = 0usize;
|
|
let mut invariants_ok = 0usize;
|
|
let (mut color_eq_size, mut color_zero, mut strips_zero, mut strip_verts_zero) =
|
|
(0usize, 0usize, 0usize, 0usize);
|
|
|
|
for n in 0..27 {
|
|
let (Ok(core), Ok(index)) = (
|
|
std::fs::read(data.join(format!("level{n}/core_data.bin"))),
|
|
std::fs::read(data.join(format!("level{n}/core_index.bin"))),
|
|
) else {
|
|
continue;
|
|
};
|
|
let h = LevelCoreHeader::parse(&index).expect("core header");
|
|
for c in 0..h.tie_classes.count.max(0) as usize {
|
|
let Ok(class) = resolve_tie_class(&index, &core, h.tie_classes, c) else {
|
|
continue;
|
|
};
|
|
for lod in 1..=2usize {
|
|
let off = class.header.packets[lod];
|
|
if off <= 0 {
|
|
continue;
|
|
}
|
|
let start = class.entry.offset_in_asset_wad as usize + off as usize;
|
|
let Some(block) = parse_tie_block(&core, start) else {
|
|
continue;
|
|
};
|
|
let len = 0x10 + block.payload.len() + block.padding.len();
|
|
let Some(slice) = core.get(start..start + len) else {
|
|
continue;
|
|
};
|
|
// The table must fit inside the block the terminator found, or
|
|
// what follows is being read as a table.
|
|
if block.header.data_offset as usize > 0x10 + block.payload.len() {
|
|
continue;
|
|
}
|
|
blocks += 1;
|
|
for r in block
|
|
.header
|
|
.records(slice)
|
|
.map(|r| TieRecord::parse(r).expect("record parses"))
|
|
{
|
|
records += 1;
|
|
bfc_ok += usize::from(r.bfc_distance == 5 * r.shader_count);
|
|
invariants_ok += usize::from(r.invariants_hold());
|
|
color_eq_size += usize::from(r.color_ofs == r.vert_size);
|
|
color_zero += usize::from(r.color_count == 0);
|
|
strips_zero += usize::from(r.strip_count == 0);
|
|
strip_verts_zero += usize::from(r.strip_vertex_count == 0);
|
|
}
|
|
}
|
|
}
|
|
levels += 1;
|
|
}
|
|
|
|
assert!(levels >= 25, "only {levels} levels were read");
|
|
assert_eq!(blocks, 2_177, "expected 2,177 tie blocks, got {blocks}");
|
|
assert!(records > 14_000, "only {records} packet headers found");
|
|
// The identification: R&C1's rule, on every record.
|
|
assert_eq!(
|
|
bfc_ok,
|
|
records,
|
|
"bfc_distance != 5 * shader_count on {} of {records}",
|
|
records - bfc_ok
|
|
);
|
|
assert_eq!(
|
|
invariants_ok,
|
|
records,
|
|
"record invariants failed on {} of {records}",
|
|
records - invariants_ok
|
|
);
|
|
eprintln!(
|
|
"tie packet headers: {records} over {levels} levels, {blocks} blocks; \
|
|
bfc_distance == 5*shader_count on all; orderings on all; \
|
|
color_ofs == vert_size on {color_eq_size}/{records}; \
|
|
color_count/strip_count/strip_vertex_count zero on \
|
|
{color_zero}/{strips_zero}/{strip_verts_zero}"
|
|
);
|
|
}
|
|
|
|
/// Byte positions that hold the same value in every supplied record.
|
|
fn constant_positions(records: &[&[u8]]) -> usize {
|
|
let Some(first) = records.first() else {
|
|
return 0;
|
|
};
|
|
(0..first.len())
|
|
.filter(|&i| records.iter().all(|r| r[i] == first[i]))
|
|
.count()
|
|
}
|
|
|
|
/// The sky block: the sub-object table at `+0x20` enumerates the sub-objects on
|
|
/// every level, and each carries a bounding sphere and texture dimensions.
|
|
///
|
|
/// The word at `+0x0c` is deliberately **not** asserted to be a count: it
|
|
/// disagrees with the table on most levels. See `gc_formats::sky`.
|
|
#[test]
|
|
fn sky_sub_objects_parse_on_every_level() {
|
|
require_disc!();
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
let mut levels = 0usize;
|
|
let mut total = 0usize;
|
|
let mut radii = (f32::MAX, f32::MIN);
|
|
let mut counts = (u32::MAX, 0u32);
|
|
let mut dims = (u16::MAX, 0u16);
|
|
let mut nonzero_radius = 0usize;
|
|
let mut reserved_nonzero = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let dir = data.join(format!("level{n}"));
|
|
let core = match std::fs::read(dir.join("core_data.bin")) {
|
|
Ok(c) => c,
|
|
Err(_) => continue,
|
|
};
|
|
let index = match std::fs::read(dir.join("core_index.bin")) {
|
|
Ok(i) => i,
|
|
Err(_) => continue,
|
|
};
|
|
let hdr = match gc_formats::level_core::LevelCoreHeader::parse(&index) {
|
|
Ok(h) => h,
|
|
Err(_) => continue,
|
|
};
|
|
let sky = hdr.sky as usize;
|
|
if sky == 0 || sky >= core.len() {
|
|
continue;
|
|
}
|
|
let end = [
|
|
hdr.collision as usize,
|
|
hdr.moby_classes.offset as usize,
|
|
core.len(),
|
|
]
|
|
.into_iter()
|
|
.filter(|&v| v > sky)
|
|
.min()
|
|
.unwrap_or(core.len());
|
|
let region = &core[sky..end.min(core.len())];
|
|
|
|
let parsed = gc_formats::sky::Sky::parse(region).expect("sky should parse");
|
|
assert!(
|
|
!parsed.sub_objects.is_empty(),
|
|
"level{n}: sky region has no sub-objects"
|
|
);
|
|
levels += 1;
|
|
total += parsed.sub_objects.len();
|
|
|
|
for s in &parsed.sub_objects {
|
|
if s.reserved != [0u8; 12] {
|
|
reserved_nonzero += 1;
|
|
}
|
|
assert!(s.count > 0, "level{n}: sky sub-object count is {}", s.count);
|
|
assert!(
|
|
s.width > 0 && s.height > 0,
|
|
"level{n}: sky texture dims are {}x{}",
|
|
s.width,
|
|
s.height
|
|
);
|
|
assert!(
|
|
s.sphere.iter().all(|f| f.is_finite()),
|
|
"level{n}: sky sphere is not finite: {:?}",
|
|
s.sphere
|
|
);
|
|
// The fourth float is the radius: positive wherever the first
|
|
// three are negative, the same reasoning the shrub header uses.
|
|
if s.sphere[3] > 0.0 {
|
|
nonzero_radius += 1;
|
|
radii.0 = radii.0.min(s.sphere[3]);
|
|
radii.1 = radii.1.max(s.sphere[3]);
|
|
}
|
|
counts.0 = counts.0.min(s.count);
|
|
counts.1 = counts.1.max(s.count);
|
|
dims.0 = dims.0.min(s.width);
|
|
dims.1 = dims.1.max(s.width);
|
|
}
|
|
}
|
|
|
|
assert!(levels >= 20, "only {levels} levels had a sky region");
|
|
assert!(total >= 100, "only {total} sky sub-objects parsed");
|
|
eprintln!(
|
|
"sky: {total} sub-objects over {levels} levels; count {}..={}, width {}..={}, radius {:.2}..={:.2} on {nonzero_radius}, {reserved_nonzero} with a non-zero reserved word",
|
|
counts.0, counts.1, dims.0, dims.1, radii.0, radii.1
|
|
);
|
|
}
|
|
|
|
/// The per-level overlay: the header is stable, `+0x10..+0x1220` is identical
|
|
/// on every level, and `+0x0c` is not the file length.
|
|
///
|
|
/// See `gc_formats::overlay`. This pins the header and the constant block's
|
|
/// extent; it does not name any field.
|
|
#[test]
|
|
fn overlay_header_and_constant_block_are_stable() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::overlay::{parse_overlay, OVERLAY_CONSTANT_BLOCK_END, OVERLAY_HEADER_SIZE};
|
|
|
|
let mut levels = 0usize;
|
|
let mut constant_block: Option<Vec<u8>> = None;
|
|
let mut f00 = 0u32;
|
|
let mut f08 = 0u32;
|
|
let (mut min_f04, mut max_f04) = (u32::MAX, 0u32);
|
|
let (mut min_f0c, mut max_f0c) = (u32::MAX, 0u32);
|
|
let mut f0c_below = 0usize;
|
|
let mut f0c_above = 0usize;
|
|
let mut levels_seen = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let p = data.join(format!("level{n}/overlay.bin"));
|
|
let Ok(bytes) = std::fs::read(&p) else {
|
|
continue;
|
|
};
|
|
|
|
let (h, constant, level) = parse_overlay(&bytes).expect("overlay should parse");
|
|
|
|
assert!(
|
|
bytes.len() > OVERLAY_CONSTANT_BLOCK_END,
|
|
"level{n}: overlay is {} bytes, too short for the constant block",
|
|
bytes.len()
|
|
);
|
|
assert!(
|
|
!level.bytes.is_empty(),
|
|
"level{n}: overlay has no level data"
|
|
);
|
|
assert_eq!(
|
|
constant.bytes.len(),
|
|
OVERLAY_CONSTANT_BLOCK_END - OVERLAY_HEADER_SIZE
|
|
);
|
|
|
|
if f00 == 0 {
|
|
f00 = h.field_00;
|
|
}
|
|
assert_eq!(h.field_00, f00, "level{n}: field_00 differs");
|
|
if f08 == 0 {
|
|
f08 = h.field_08;
|
|
}
|
|
assert_eq!(h.field_08, f08, "level{n}: field_08 differs");
|
|
|
|
assert_eq!(
|
|
h.field_04 % 4,
|
|
0,
|
|
"level{n}: field_04 {} is not a multiple of 4",
|
|
h.field_04
|
|
);
|
|
min_f04 = min_f04.min(h.field_04);
|
|
max_f04 = max_f04.max(h.field_04);
|
|
|
|
min_f0c = min_f0c.min(h.field_0c);
|
|
max_f0c = max_f0c.max(h.field_0c);
|
|
// The claim being pinned: field_0c is not the file length.
|
|
if (h.field_0c as usize) < bytes.len() {
|
|
f0c_below += 1
|
|
} else {
|
|
f0c_above += 1
|
|
}
|
|
|
|
match &constant_block {
|
|
None => constant_block = Some(constant.bytes.to_vec()),
|
|
Some(prev) => assert_eq!(
|
|
prev.as_slice(),
|
|
constant.bytes,
|
|
"level{n}: the constant block +0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} differs"
|
|
),
|
|
}
|
|
|
|
levels += 1;
|
|
levels_seen.push(n);
|
|
}
|
|
|
|
assert!(
|
|
levels >= 27,
|
|
"only {levels} levels had an overlay.bin, saw {levels_seen:?}"
|
|
);
|
|
assert!(f0c_below > 0 && f0c_above > 0,
|
|
"field_0c must fall on both sides of the file length to show it is not one: below {f0c_below}, above {f0c_above}");
|
|
assert_eq!(levels, 27, "expected all levels, got {levels_seen:?}");
|
|
assert!(
|
|
max_f04 - min_f04 > 1000,
|
|
"field_04 range collapsed to {min_f04}..={max_f04}"
|
|
);
|
|
assert!(
|
|
max_f0c - min_f0c > 100000,
|
|
"field_0c range collapsed to {min_f0c}..={max_f0c}"
|
|
);
|
|
|
|
eprintln!(
|
|
"overlay: {levels} levels; field_00 {f00}, field_08 {f08}; \
|
|
field_04 {min_f04}..={max_f04}, field_0c {min_f0c}..={max_f0c} \
|
|
({f0c_below} below the file length, {f0c_above} above); \
|
|
+0x10..0x{OVERLAY_CONSTANT_BLOCK_END:x} identical on all"
|
|
);
|
|
}
|
|
|
|
/// The two tie indirections resolve on every class on every level, and the
|
|
/// LOD block offsets land inside `core_data`.
|
|
///
|
|
/// See `gc_formats::tie`. This pins the resolution chain and the header; the
|
|
/// block's internal layout is *not* decoded, and one tempting reading of it is
|
|
/// disproved below.
|
|
#[test]
|
|
fn tie_classes_resolve_through_both_indirections() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use gc_formats::tie::resolve_tie_class;
|
|
|
|
let mut classes = 0usize;
|
|
let mut o_class_agree = 0usize;
|
|
let mut lod_blocks = 0usize;
|
|
let mut lod_blocks_in_range = 0usize;
|
|
let mut monotonic_u32_0 = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let t = h.tie_classes;
|
|
if t.count == 0 {
|
|
continue;
|
|
}
|
|
for c in 0..t.count as usize {
|
|
let r = resolve_tie_class(&idx, &core, t, c).expect("tie class should resolve");
|
|
classes += 1;
|
|
if r.header.o_class == r.entry.o_class {
|
|
o_class_agree += 1;
|
|
}
|
|
|
|
let base = r.entry.offset_in_asset_wad as usize;
|
|
for (i, p) in r.header.packets.iter().enumerate() {
|
|
let abs = base.wrapping_add(*p as usize);
|
|
if abs <= core.len() {
|
|
lod_blocks_in_range += 1;
|
|
}
|
|
lod_blocks += 1;
|
|
if *p >= 0 && abs + 16 <= core.len() {
|
|
// Is the word at the block start strictly increasing across
|
|
// 16-byte units? It looked like it on the first twelve units
|
|
// of one class. It is not a property of the format.
|
|
let next = if i + 1 < 3 {
|
|
r.header.packets[i + 1]
|
|
} else {
|
|
*p
|
|
};
|
|
if next > *p {
|
|
let end = base + next as usize;
|
|
if end <= core.len() {
|
|
let n16 = (end - abs) / 16;
|
|
let mut prev = 0i64;
|
|
let mut ok = true;
|
|
for k in 0..n16 {
|
|
let q = abs + k * 16;
|
|
let a =
|
|
u32::from_le_bytes(core[q..q + 4].try_into().unwrap()) as i64;
|
|
if a <= prev {
|
|
ok = false;
|
|
break;
|
|
}
|
|
prev = a;
|
|
}
|
|
if ok && n16 > 0 {
|
|
monotonic_u32_0 += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// packets[0] is a constant, not an offset, and the other two order only
|
|
// sometimes -- both are why the block layout is only partly understood.
|
|
let mut packets0_constant = 0usize;
|
|
let mut packets_increasing = 0usize;
|
|
let mut consecutive_div16 = 0usize;
|
|
let mut consecutive_total = 0usize;
|
|
|
|
for n in 0..27 {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let r = resolve_tie_class(&idx, &core, t, c).expect("tie class");
|
|
if r.header.packets[0] == gc_formats::tie::TIE_PACKETS0 {
|
|
packets0_constant += 1;
|
|
}
|
|
let p = r.header.packets;
|
|
if p[0] < p[1] && p[1] < p[2] {
|
|
packets_increasing += 1;
|
|
for i in 0..2 {
|
|
consecutive_total += 1;
|
|
if ((p[i + 1] - p[i]) as usize) % 16 == 0 {
|
|
consecutive_div16 += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tie LOD blocks are self-delimiting, which is what makes them parseable
|
|
// without trusting packets[1] to bound the block.
|
|
use gc_formats::tie::{
|
|
parse_tie_block, TIE_BLOCK_HEADER_SIZE, TIE_BLOCK_TERMINATOR, TIE_PACKED_TEXTURE_VALUES,
|
|
};
|
|
|
|
let mut blocks = 0usize;
|
|
let mut absent = 0usize;
|
|
let mut hdr_ok = 0usize;
|
|
let mut count_mult16 = 0usize;
|
|
let mut reserved_02_zero = 0usize;
|
|
let mut tables_bounded = 0usize;
|
|
let mut packed_ok = 0usize;
|
|
let mut padding_all_zero = 0usize;
|
|
let mut packed_seen: Vec<u16> = Vec::new();
|
|
|
|
for n in 0..27 {
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("core header");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let r = resolve_tie_class(&idx, &core, t, c).expect("tie class");
|
|
let base = r.entry.offset_in_asset_wad as usize;
|
|
for lod in [1usize, 2] {
|
|
let off = r.header.packets[lod] as i64;
|
|
if off == 0 {
|
|
// packets == 0 means this class has no block at this LOD.
|
|
absent += 1;
|
|
continue;
|
|
}
|
|
let s = (base as i64 + off) as usize;
|
|
if s + TIE_BLOCK_HEADER_SIZE > core.len() {
|
|
continue;
|
|
}
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
|
|
blocks += 1;
|
|
// The word at +0 is the data offset -- the record table's
|
|
// length -- so it is a non-zero multiple of 16, and it bounds
|
|
// the table.
|
|
if blk.header.data_offset % 16 == 0 && blk.header.data_offset != 0 {
|
|
count_mult16 += 1
|
|
}
|
|
let table = blk.header.records(&core[s..]).count();
|
|
if table == blk.header.record_count() && table > 0 {
|
|
tables_bounded += 1
|
|
}
|
|
if blk.header.reserved_02 == 0 {
|
|
reserved_02_zero += 1
|
|
}
|
|
if TIE_PACKED_TEXTURE_VALUES.contains(&blk.header.packed_texture) {
|
|
packed_ok += 1
|
|
}
|
|
if blk.header.invariants_hold() {
|
|
hdr_ok += 1
|
|
}
|
|
if blk.padding.iter().all(|&b| b == 0) {
|
|
padding_all_zero += 1
|
|
}
|
|
assert_eq!(
|
|
blk.payload.last(),
|
|
Some(&TIE_BLOCK_TERMINATOR),
|
|
"level{n} class{c} lod{lod}: payload does not end on the terminator"
|
|
);
|
|
if !packed_seen.contains(&blk.header.packed_texture) {
|
|
packed_seen.push(blk.header.packed_texture);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
packed_seen.sort_unstable();
|
|
|
|
assert!(blocks >= 2100, "only {blocks} tie blocks located");
|
|
assert!(
|
|
absent > 1000,
|
|
"only {absent} absent (packets == 0) LOD blocks"
|
|
);
|
|
assert_eq!(blocks + absent, 2 * classes,
|
|
"expected two LOD slots per class, got {blocks} present + {absent} absent for {classes} classes");
|
|
assert_eq!(
|
|
count_mult16,
|
|
blocks,
|
|
"the data offset is not a non-zero multiple of 16 on {} of {blocks}",
|
|
blocks - count_mult16
|
|
);
|
|
assert_eq!(
|
|
tables_bounded,
|
|
blocks,
|
|
"the record table is not bounded by the data offset on {} of {blocks}",
|
|
blocks - tables_bounded
|
|
);
|
|
assert_eq!(
|
|
reserved_02_zero,
|
|
blocks,
|
|
"reserved_02 non-zero on {} of {blocks}",
|
|
blocks - reserved_02_zero
|
|
);
|
|
assert_eq!(
|
|
packed_ok,
|
|
blocks,
|
|
"packed_texture outside the four known values on {} of {blocks}: {packed_seen:?}",
|
|
blocks - packed_ok
|
|
);
|
|
assert_eq!(
|
|
hdr_ok,
|
|
blocks,
|
|
"header invariants failed on {} of {blocks}",
|
|
blocks - hdr_ok
|
|
);
|
|
assert_eq!(
|
|
padding_all_zero,
|
|
blocks,
|
|
"non-zero padding on {} of {blocks}",
|
|
blocks - padding_all_zero
|
|
);
|
|
assert_eq!(
|
|
packed_seen.len(),
|
|
TIE_PACKED_TEXTURE_VALUES.len(),
|
|
"packed_texture took {} distinct values, expected the four {packed_seen:?}",
|
|
packed_seen.len()
|
|
);
|
|
|
|
eprintln!(
|
|
"tie blocks: {blocks} present, {absent} absent (packets == 0); all end on 0xf6 with zero padding; \
|
|
count %16 on all; reserved_02 == 0 on all; packed_texture in {packed_seen:?} on all"
|
|
);
|
|
|
|
assert!(classes >= 1800, "only {classes} tie classes resolved");
|
|
assert_eq!(
|
|
packets0_constant,
|
|
classes,
|
|
"packets[0] was not {} on {} of {classes} classes",
|
|
gc_formats::tie::TIE_PACKETS0,
|
|
classes - packets0_constant
|
|
);
|
|
assert_eq!(
|
|
consecutive_div16,
|
|
consecutive_total,
|
|
"{} of {consecutive_total} consecutive tie blocks are not a multiple of 16 bytes",
|
|
consecutive_total - consecutive_div16
|
|
);
|
|
assert!(
|
|
packets_increasing < classes,
|
|
"packets increase on all {classes} classes; the block-end reasoning in the docs needs revisiting"
|
|
);
|
|
assert_eq!(
|
|
o_class_agree,
|
|
classes,
|
|
"header o_class disagreed with the class entry on {} of {classes}",
|
|
classes - o_class_agree
|
|
);
|
|
assert_eq!(
|
|
lod_blocks_in_range,
|
|
lod_blocks,
|
|
"{} of {lod_blocks} LOD block offsets fell outside core_data",
|
|
lod_blocks - lod_blocks_in_range
|
|
);
|
|
assert_eq!(
|
|
monotonic_u32_0, 0,
|
|
"the word at +0 of a 16-byte tie unit is strictly increasing on {monotonic_u32_0} blocks; \
|
|
if that ever becomes true the disproved reading below needs revisiting"
|
|
);
|
|
|
|
eprintln!(
|
|
"tie: {classes} classes resolved through both indirections, o_class agreeing on all; \
|
|
{lod_blocks} LOD block offsets all inside core_data; \
|
|
packets[0] == {} on all; packets increasing on {packets_increasing} of {classes}; \
|
|
{consecutive_div16} of {consecutive_total} consecutive blocks a multiple of 16; \
|
|
strictly-increasing +0 on 0 of {classes} blocks (disproved)",
|
|
gc_formats::tie::TIE_PACKETS0
|
|
);
|
|
}
|
|
|
|
/// Chunk record buckets: the leading byte is an aligned index, not record data.
|
|
///
|
|
/// The leading byte is not decoded -- what it indexes is not established. This
|
|
/// pins the measured 8-byte alignment and the exact three types that break it;
|
|
/// a parallel probe puts 1,142 records across seven types outside a *16*-byte
|
|
/// rule, which is how the per-type 8-vs-16 step was found. See
|
|
/// `gc_formats::chunks::CHUNK_RECORD_ALIGNMENT`.
|
|
#[test]
|
|
fn chunk_records_start_16_byte_aligned() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::{parse_all_chunks, record_is_aligned};
|
|
use std::collections::BTreeMap;
|
|
|
|
let mut chunks = 0usize;
|
|
let mut records = 0usize;
|
|
let mut aligned = 0usize;
|
|
let mut unaligned: BTreeMap<u32, usize> = BTreeMap::new();
|
|
|
|
for n in 0..27 {
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
chunks += 1;
|
|
let t = u32::from_le_bytes([
|
|
c.chunk.header[0],
|
|
c.chunk.header[1],
|
|
c.chunk.header[2],
|
|
c.chunk.header[3],
|
|
]);
|
|
for r in &c.records {
|
|
if r.is_empty() {
|
|
continue;
|
|
}
|
|
records += 1;
|
|
if record_is_aligned(r) {
|
|
aligned += 1;
|
|
} else {
|
|
*unaligned.entry(t).or_default() += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(chunks >= 1500, "only {chunks} chunks parsed");
|
|
assert!(records >= 34000, "only {records} chunk records");
|
|
// Every chunk has a uniform record length -- length varies between chunks
|
|
// of a type, never within one -- and on most chunks the leading byte is a
|
|
// per-chunk phase counter rather than data.
|
|
let mut uniform = 0usize;
|
|
let mut ruled = 0usize;
|
|
let mut indexed = 0usize;
|
|
for n in 0..27 {
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
if c.has_uniform_record_length() {
|
|
uniform += 1;
|
|
}
|
|
if c.bucket_rule().is_some() {
|
|
ruled += 1;
|
|
}
|
|
if c.index_rule().is_some() {
|
|
indexed += 1;
|
|
}
|
|
}
|
|
}
|
|
assert_eq!(uniform, 1509, "uniform-record-length count moved off 1509");
|
|
assert_eq!(ruled, 1262, "bucket-rule count moved off 1262");
|
|
assert_eq!(indexed, 1235, "index-rule count moved off 1235");
|
|
|
|
// Not universal. Three chunk types break even the 8-byte rule, all of them
|
|
// the variable-length or 8-stepped ones. A parallel probe puts 1,142
|
|
// records across seven types outside a *16*-byte rule, which is how the
|
|
// per-type 8-vs-16 step was found in the first place.
|
|
let bad: usize = unaligned.values().sum();
|
|
assert_eq!(
|
|
bad, 59,
|
|
"unaligned record count moved off 59: {unaligned:?}"
|
|
);
|
|
assert_eq!(
|
|
unaligned.len(),
|
|
3,
|
|
"unaligned records now span {} chunk types, not the 3 known: {unaligned:?}",
|
|
unaligned.len()
|
|
);
|
|
assert_eq!(
|
|
records - bad,
|
|
34_432,
|
|
"multiple-of-8 count moved off 34,432"
|
|
);
|
|
|
|
eprintln!(
|
|
"chunks: {chunks} chunks, {records} records; uniform record length on {uniform}; \
|
|
leading byte a multiple of 8 on {aligned}, {bad} exceptions across {} types; \
|
|
leading byte a phase counter on {ruled}; \
|
|
leading TWO bytes encode the record index on {indexed}",
|
|
unaligned.len()
|
|
);
|
|
}
|
|
|
|
/// Moby class IDs live in one body slot per chunk type, and the slot varies.
|
|
///
|
|
/// Measured over all 27 levels: the best slot per type carries 5 to 42 distinct
|
|
/// valid moby class IDs at 153x to 289x enrichment over the chance rate. The
|
|
/// `distinct >= 4` filter is load-bearing -- without it the top slots are
|
|
/// constants that read as 305x because any value below the moby count counts as
|
|
/// a hit. See `docs/formats/core-data-regions.md`.
|
|
#[test]
|
|
fn moby_class_ids_sit_in_one_slot_per_chunk_type() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::chunks::parse_all_chunks;
|
|
use gc_formats::level_core::LevelCoreHeader;
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
// (type, slot) -> (nonzero, hits, expected, distinct)
|
|
let mut stat: BTreeMap<(u32, usize), (u64, u64, f64, usize)> = BTreeMap::new();
|
|
let mut seen: BTreeMap<(u32, usize), BTreeSet<u16>> = BTreeMap::new();
|
|
|
|
for n in 0..27 {
|
|
let core = std::fs::read(data.join(format!("level{n}/core_data.bin"))).expect("core_data");
|
|
let idx = std::fs::read(data.join(format!("level{n}/core_index.bin"))).expect("core_index");
|
|
let nm = LevelCoreHeader::parse(&idx)
|
|
.expect("hdr")
|
|
.moby_classes
|
|
.count as u64;
|
|
if nm == 0 {
|
|
continue;
|
|
}
|
|
for c in parse_all_chunks(&core).expect("chunks") {
|
|
let t = u32::from_le_bytes([
|
|
c.chunk.header[0],
|
|
c.chunk.header[1],
|
|
c.chunk.header[2],
|
|
c.chunk.header[3],
|
|
]);
|
|
let len = c.records[0].len();
|
|
if c.records.iter().any(|r| r.len() != len) {
|
|
continue;
|
|
}
|
|
let mut loc: BTreeMap<usize, (u64, u64)> = BTreeMap::new();
|
|
for r in c.records.iter() {
|
|
for k in 1..len / 2 {
|
|
let v = u16::from_le_bytes([r[k * 2], r[k * 2 + 1]]);
|
|
if v == 0 {
|
|
continue;
|
|
}
|
|
seen.entry((t, k)).or_default().insert(v);
|
|
let e = loc.entry(k).or_default();
|
|
e.0 += 1;
|
|
if (v as u64) < nm {
|
|
e.1 += 1
|
|
}
|
|
}
|
|
}
|
|
for (k, (nn, hh)) in loc {
|
|
let e = stat.entry((t, k)).or_default();
|
|
e.0 += nn;
|
|
e.1 += hh;
|
|
e.2 += nn as f64 * (nm as f64 / 65536.0);
|
|
}
|
|
}
|
|
}
|
|
for (key, s) in seen.iter() {
|
|
stat.entry(*key).or_default().3 = s.len();
|
|
}
|
|
|
|
// best varying slot per type
|
|
let mut best: BTreeMap<u32, (usize, f64, usize)> = BTreeMap::new();
|
|
for ((t, k), (nn, hh, e, dist)) in stat.iter() {
|
|
if *nn < 8 || *e < 1.0 || *dist < 4 {
|
|
continue;
|
|
}
|
|
let r = (*hh as f64) / (*e);
|
|
if best.get(t).is_none_or(|(_, ce, _)| r > *ce) {
|
|
best.insert(*t, (*k, *hh as f64 / *e, *dist));
|
|
}
|
|
}
|
|
|
|
assert!(
|
|
best.len() >= 20,
|
|
"only {} chunk types carry a varying class-ID slot",
|
|
best.len()
|
|
);
|
|
let strong: Vec<_> = best.values().filter(|(_, r, _)| *r >= 100.0).collect();
|
|
assert!(
|
|
strong.len() >= 20,
|
|
"only {} types reach 100x enrichment",
|
|
strong.len()
|
|
);
|
|
|
|
let mut all: Vec<_> = best
|
|
.iter()
|
|
.map(|(t, (k, r, dist))| (*t, *k, *r, *dist))
|
|
.collect();
|
|
all.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap());
|
|
assert!(!all.is_empty(), "no varying class-ID slots found");
|
|
let mn = all
|
|
.iter()
|
|
.map(|a| a.2)
|
|
.fold(f64::MAX, |g: f64, v: f64| f64::min(g, v));
|
|
let mx = all
|
|
.iter()
|
|
.map(|a| a.2)
|
|
.fold(0.0f64, |g: f64, v: f64| f64::max(g, v));
|
|
let dmin = all.iter().map(|a| a.3).min().unwrap();
|
|
let dmax = all.iter().map(|a| a.3).max().unwrap();
|
|
|
|
eprintln!(
|
|
"chunk class-ID slots: {} types; best-slot enrichment {mn:.0}x..{mx:.0}x; \
|
|
distinct IDs per slot {dmin}..{dmax}",
|
|
best.len()
|
|
);
|
|
eprintln!(
|
|
" strongest (type, byte, ratio, distinct): {:?}",
|
|
&all[..all.len().min(6)]
|
|
);
|
|
}
|
|
|
|
/// The tie vertex element is 16 bytes, with 12-bit fixed-point texcoords.
|
|
///
|
|
/// Measured over 95,612 vertices on levels 0-2, reached by following each
|
|
/// `TieRecord::pointer` into the block payload and stepping `vert_size` elements
|
|
/// from `vert_ofs`. The element size is settled by comparison against the
|
|
/// alternatives, not assumed: 16 bytes gives a 12-bit-fixed texcoord hit rate of
|
|
/// 82.5%, against 47.8% at 8 bytes and 45.4% at 12.
|
|
///
|
|
/// The field *names* are not established -- this pins the size and the texcoord
|
|
/// encoding only.
|
|
#[test]
|
|
fn tie_vertex_element_is_16_bytes_with_fixed_texcoords() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
let mut verts = 0usize;
|
|
let mut fixed16 = 0usize;
|
|
let mut fixed8 = 0usize;
|
|
let mut fixed12 = 0usize;
|
|
|
|
for lv in 0..3usize {
|
|
let idx =
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
let recs: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in recs.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start = s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
for v in 0..r.vert_size as usize {
|
|
let o = start + v * 16;
|
|
if o + 16 > core.len() {
|
|
break;
|
|
}
|
|
verts += 1;
|
|
// Compare element sizes properly: each strides its own
|
|
// way through the payload, with its texcoords at that
|
|
// element's own offset.
|
|
for stride in [8usize, 12, 16] {
|
|
let q = start + v * stride;
|
|
if q + stride > core.len() {
|
|
continue;
|
|
}
|
|
let tc = if stride >= 16 { (8, 10) } else { (4, 6) };
|
|
let a = i16::from_le_bytes(
|
|
core[q + tc.0..q + tc.0 + 2].try_into().unwrap(),
|
|
);
|
|
let b = i16::from_le_bytes(
|
|
core[q + tc.1..q + tc.1 + 2].try_into().unwrap(),
|
|
);
|
|
if a % 4096 == 0 && b % 4096 == 0 {
|
|
match stride {
|
|
16 => fixed16 += 1,
|
|
12 => fixed12 += 1,
|
|
_ => fixed8 += 1,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Which i16 slot pair is the texcoord? Scored per pair on level 0.
|
|
let core0 = std::fs::read(data.join("level0/core_data.bin")).expect("core_data");
|
|
let idx0 = std::fs::read(data.join("level0/core_index.bin")).expect("core_index");
|
|
let t0 = LevelCoreHeader::parse(&idx0).expect("hdr").tie_classes;
|
|
let (mut pair_hits, mut pair_tot) = ([0usize; 4], 0usize);
|
|
for c in 0..t0.count as usize {
|
|
let o = t0.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx0[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core0.len() {
|
|
continue;
|
|
}
|
|
let Ok(hdr) = TieClassHeader::parse(&core0[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hdr.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core0, s) else {
|
|
continue;
|
|
};
|
|
let rr: Vec<&[u8]> = blk.header.records(&core0[s..]).collect();
|
|
for (ri, rec) in rr.iter().enumerate() {
|
|
let Ok(tr) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start = s + 16 + ri * 16 + tr.pointer as usize + tr.vert_ofs as usize * 16;
|
|
for v in 0..tr.vert_size as usize {
|
|
let q = start + v * 16;
|
|
if q + 16 > core0.len() {
|
|
break;
|
|
}
|
|
pair_tot += 1;
|
|
for p in 0..4usize {
|
|
let a =
|
|
i16::from_le_bytes(core0[q + p * 4..q + p * 4 + 2].try_into().unwrap());
|
|
let b = i16::from_le_bytes(
|
|
core0[q + p * 4 + 2..q + p * 4 + 4].try_into().unwrap(),
|
|
);
|
|
if a % 4096 == 0 && b % 4096 == 0 {
|
|
pair_hits[p] += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// (4,5) is the texcoord field. (6,7) scores *higher* only because it is the
|
|
// near-constant 4096/0 pair, which is trivially a pair of multiples of 4096
|
|
// -- the same artifact a constant would produce anywhere.
|
|
assert!(pair_tot > 30_000, "only {pair_tot} level-0 vertices scored");
|
|
let r = |p: usize| pair_hits[p] as f64 / pair_tot as f64;
|
|
assert!(r(2) > 0.60, "texcoord pair (4,5) fell to {:.3}", r(2));
|
|
assert!(
|
|
r(0) < 0.05,
|
|
"pair (0,1) looks like a texcoord at {:.3}",
|
|
r(0)
|
|
);
|
|
assert!(
|
|
r(1) < 0.15,
|
|
"pair (2,3) looks like a texcoord at {:.3}",
|
|
r(1)
|
|
);
|
|
|
|
eprintln!(
|
|
"tie vertex slot pairs, level 0 (n={pair_tot}): (0,1) {:.1}% (2,3) {:.1}% \
|
|
(4,5) {:.1}% <- texcoords (6,7) {:.1}% <- the constant 4096/0 pair",
|
|
r(0) * 100.0,
|
|
r(1) * 100.0,
|
|
r(2) * 100.0,
|
|
r(3) * 100.0
|
|
);
|
|
|
|
assert!(verts > 90_000, "only {verts} tie vertices reached");
|
|
let r16 = fixed16 as f64 / verts as f64;
|
|
let r8 = fixed8 as f64 / verts as f64;
|
|
let r12 = fixed12 as f64 / verts as f64;
|
|
assert!(r16 > 0.80, "16-byte element hit rate fell to {r16:.3}");
|
|
assert!(
|
|
r16 > r8 * 1.5,
|
|
"16-byte element ({r16:.3}) no longer beats 8-byte ({r8:.3})"
|
|
);
|
|
assert!(
|
|
r16 > r12 * 1.5,
|
|
"16-byte element ({r16:.3}) no longer beats 12-byte ({r12:.3})"
|
|
);
|
|
|
|
eprintln!(
|
|
"tie vertices: {verts} on levels 0-2; 12-bit-fixed texcoord hit rate \
|
|
16B {:.1}%, 8B {:.1}%, 12B {:.1}% -- element size is 16 bytes",
|
|
r16 * 100.0,
|
|
r8 * 100.0,
|
|
r12 * 100.0
|
|
);
|
|
}
|
|
|
|
/// The tie vertex element's layout is selected by the HIGH byte of `i16[3]`.
|
|
///
|
|
/// Measured over the tie vertices of levels 0-2, grouped by that high byte:
|
|
///
|
|
/// - high byte zero (the ordinary case, 89,341 vertices): slot pairs `(4,5)` and
|
|
/// `(6,7)` are both 12-bit fixed, at 92.9% and 89.5% for a divisor of 4, while
|
|
/// `(0,1)` and `(2,3)` sit at baseline.
|
|
/// - high byte non-zero (6,271 vertices): **no** pair is fixed; the best is 7.5%
|
|
/// against a 2-5% baseline.
|
|
///
|
|
/// A constant would score 100%, so `(6,7)` at 89.5% is a real pair rather than
|
|
/// the `4096, 0` artifact it looks like. See `docs/formats/core-data-regions.md`.
|
|
#[test]
|
|
fn tie_vertex_layout_is_selected_by_i16_3_high_byte() {
|
|
require_disc!();
|
|
|
|
let data = std::env::var_os("GC_EXTRACTED")
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| {
|
|
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
|
PathBuf::from(home).join("Documents/gc-extracted")
|
|
});
|
|
if !data.exists() {
|
|
eprintln!("skipped: unpack the disc first");
|
|
return;
|
|
}
|
|
|
|
use gc_formats::level_core::{ClassEntry, LevelCoreHeader};
|
|
use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord};
|
|
|
|
// [pair][k] where k indexes divisor 1, 4, 16, 256
|
|
let divisors = [1i32, 4, 16, 256];
|
|
let mut zero = [[0u64; 4]; 4];
|
|
let mut nonzero = [[0u64; 4]; 4];
|
|
let (mut n_zero, mut n_nonzero) = (0u64, 0u64);
|
|
|
|
for lv in 0..3usize {
|
|
let idx =
|
|
std::fs::read(data.join(format!("level{lv}/core_index.bin"))).expect("core_index");
|
|
let core = std::fs::read(data.join(format!("level{lv}/core_data.bin"))).expect("core_data");
|
|
let h = LevelCoreHeader::parse(&idx).expect("hdr");
|
|
let t = h.tie_classes;
|
|
for c in 0..t.count as usize {
|
|
let o = t.offset as usize + c * 0x20;
|
|
let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry");
|
|
let base = e.offset_in_asset_wad as usize;
|
|
if base == 0 || base + 0x48 > core.len() {
|
|
continue;
|
|
}
|
|
let Ok(hd) = TieClassHeader::parse(&core[base..base + 0x48]) else {
|
|
continue;
|
|
};
|
|
for lod in [1usize, 2] {
|
|
let off = hd.packets[lod] as usize;
|
|
if off == 0 {
|
|
continue;
|
|
}
|
|
let s = base + off;
|
|
let Some(blk) = parse_tie_block(&core, s) else {
|
|
continue;
|
|
};
|
|
let recs: Vec<&[u8]> = blk.header.records(&core[s..]).collect();
|
|
for (ri, rec) in recs.iter().enumerate() {
|
|
let Ok(r) = TieRecord::parse(rec) else {
|
|
continue;
|
|
};
|
|
let start = s + 16 + ri * 16 + r.pointer as usize + r.vert_ofs as usize * 16;
|
|
for v in 0..r.vert_size as usize {
|
|
let q = start + v * 16;
|
|
if q + 16 > core.len() {
|
|
break;
|
|
}
|
|
let hi = (i16::from_le_bytes(core[q + 6..q + 8].try_into().unwrap())
|
|
as u32
|
|
>> 8)
|
|
!= 0;
|
|
if hi {
|
|
n_nonzero += 1
|
|
} else {
|
|
n_zero += 1
|
|
}
|
|
let tbl = if hi { &mut nonzero } else { &mut zero };
|
|
for pair in 0..4usize {
|
|
let a = i16::from_le_bytes(
|
|
core[q + pair * 4..q + pair * 4 + 2].try_into().unwrap(),
|
|
) as i32;
|
|
let b = i16::from_le_bytes(
|
|
core[q + pair * 4 + 2..q + pair * 4 + 4].try_into().unwrap(),
|
|
) as i32;
|
|
for (k, dv) in divisors.iter().enumerate() {
|
|
if a % dv == 0 && b % dv == 0 {
|
|
tbl[pair][k] += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
assert!(n_zero > 80_000, "only {n_zero} ordinary tie vertices");
|
|
assert!(
|
|
n_nonzero > 5_000,
|
|
"only {n_nonzero} vertices with a non-zero i16[3] high byte"
|
|
);
|
|
let rate = |t: &[[u64; 4]; 4], pair: usize, k: usize, n: u64| t[pair][k] as f64 / n as f64;
|
|
|
|
// The ordinary case: (4,5) and (6,7) are both fixed-point pairs.
|
|
assert!(
|
|
rate(&zero, 2, 1, n_zero) > 0.85,
|
|
"(4,5) fell to {:.2}",
|
|
rate(&zero, 2, 1, n_zero)
|
|
);
|
|
assert!(
|
|
rate(&zero, 3, 1, n_zero) > 0.80,
|
|
"(6,7) fell to {:.2}",
|
|
rate(&zero, 3, 1, n_zero)
|
|
);
|
|
// A constant would be 1.0; below 1.0 means it is not one.
|
|
assert!(
|
|
rate(&zero, 3, 1, n_zero) < 0.99,
|
|
"(6,7) is exactly constant again; revisit the pair claim"
|
|
);
|
|
// (0,1) and (2,3) are baseline.
|
|
assert!(
|
|
rate(&zero, 0, 1, n_zero) < 0.20,
|
|
"(0,1) rose to {:.2}",
|
|
rate(&zero, 0, 1, n_zero)
|
|
);
|
|
assert!(
|
|
rate(&zero, 1, 1, n_zero) < 0.20,
|
|
"(2,3) rose to {:.2}",
|
|
rate(&zero, 1, 1, n_zero)
|
|
);
|
|
// The non-zero case: nothing is fixed.
|
|
for pair in 0..4usize {
|
|
assert!(
|
|
rate(&nonzero, pair, 1, n_nonzero) < 0.15,
|
|
"pair ({},{}) rose to {:.2} in the non-zero group",
|
|
pair * 2,
|
|
pair * 2 + 1,
|
|
rate(&nonzero, pair, 1, n_nonzero)
|
|
);
|
|
}
|
|
|
|
eprintln!(
|
|
"tie vertex layout by i16[3] high byte, levels 0-2: \
|
|
zero {n_zero} -> (4,5) {:.1}%, (6,7) {:.1}%, (0,1) {:.1}%, (2,3) {:.1}%; \
|
|
non-zero {n_nonzero} -> best pair {:.1}%",
|
|
rate(&zero, 2, 1, n_zero) * 100.0,
|
|
rate(&zero, 3, 1, n_zero) * 100.0,
|
|
rate(&zero, 0, 1, n_zero) * 100.0,
|
|
rate(&zero, 1, 1, n_zero) * 100.0,
|
|
(0..4)
|
|
.map(|p| rate(&nonzero, p, 1, n_nonzero))
|
|
.fold(0.0f64, f64::max)
|
|
* 100.0
|
|
);
|
|
}
|