//! Integration tests against a real disc image. //! //! These are skipped unless `GC_DISC` points at a legally obtained disc, since //! the repository must not contain game data. When it is set they check the //! identification and unpack results against the committed hash table. //! //! ```sh //! GC_DISC="path/to/disc.iso" cargo test -p gc-extract --test disc //! ``` use std::path::{Path, PathBuf}; /// The disc under test, if the environment names one. fn disc() -> Option { let p = std::env::var_os("GC_DISC")?; let p = PathBuf::from(p); p.is_file().then_some(p) } /// Skip a test with a clear reason rather than failing. macro_rules! require_disc { () => { match disc() { Some(p) => p, None => { eprintln!("skipped: set GC_DISC to a Going Commando disc image to run this"); return; } } }; } #[test] fn identifies_the_supported_build() { let path = require_disc!(); let info = gc_extract::identify::identify(&path).expect("disc should identify"); assert_eq!(info.serial, "SCUS_972.68"); assert_eq!(info.volume_id, "RATCHETANDCLANK2"); assert!( info.is_target(), "expected the supported build, got: {}", info.verdict() ); assert_eq!( info.boot_elf_sha1, gc_extract::identify::TARGET_BOOT_ELF_SHA1, "boot ELF SHA-1 differs from the recorded retail value" ); assert_eq!(info.boot_elf_size, 2_618_684); // This disc is NTSC; the movement constants assume 60 Hz. assert!(!info.is_pal(), "test disc should be NTSC"); } #[test] fn agrees_with_the_python_reference_identifier() { let path = require_disc!(); let info = gc_extract::identify::identify(&path).expect("disc should identify"); // tools/identify-disc.py is the reference implementation and prints these // two values. If they ever disagree, one of them has a bug. // Run from the workspace root so the relative path to the tool resolves. let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .and_then(|p| p.parent()) .expect("the crate should live two levels below the workspace root") .to_path_buf(); let py = std::process::Command::new("python3") .arg(root.join("tools/identify-disc.py")) .arg(&path) .current_dir(&root) .output() .expect("python3 should be available to run the reference identifier"); assert!( py.status.success(), "the reference identifier failed:\n{}", String::from_utf8_lossy(&py.stderr) ); let text = String::from_utf8_lossy(&py.stdout); assert!( text.contains(&format!("serial {}", info.serial)), "serial disagrees with the reference identifier:\n{text}" ); assert!( text.contains(&format!("SHA-1 {}", info.boot_elf_sha1)), "boot ELF SHA-1 disagrees with the reference identifier:\n{text}" ); } #[test] fn unpacks_and_matches_the_committed_table() { let path = require_disc!(); let table_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/gc-files.tsv"); let table_text = std::fs::read_to_string(&table_path).expect("the hash table should be committed"); let expected = gc_extract::verify::parse_table(&table_text); assert!( expected.len() > 5000, "expected a full table, got {} rows", expected.len() ); // Unpack into a temporary directory so the test leaves nothing behind. let tmp = std::env::temp_dir().join(format!("gc-unpack-{}", std::process::id())); let _ = std::fs::remove_dir_all(&tmp); std::fs::create_dir_all(&tmp).expect("should create a temp directory"); let files = gc_extract::unpack::unpack(&path, &tmp).expect("unpack should succeed"); // Every unpacked file must match the recorded size and hash. let report = gc_extract::verify::verify(&tmp, &expected); let _ = std::fs::remove_dir_all(&tmp); assert!( report.ok(), "unpacked data does not match the committed table:\n{}", report.summary() ); assert_eq!( files.len(), expected.len(), "unpack produced a different file count" ); } #[test] fn every_container_parses() { let path = require_disc!(); let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); let containers: Vec<_> = iso .files .iter() .filter(|e| e.name.starts_with("G/") && e.name.ends_with(".WAD")) .collect(); // 92 `G/*.WAD` containers: 27 LEVEL, 27 AUDIO, 27 SCENE, 11 globals. assert_eq!(containers.len(), 92, "expected 92 asset containers"); // MPEG1 and MPEG2 are raw video streams with no table; the other 90 must // all parse as indexed containers. let raw_video: Vec<&str> = containers .iter() .map(|c| c.name.as_str()) .filter(|n| *n == "G/MPEG1.WAD" || *n == "G/MPEG2.WAD") .collect(); assert_eq!( raw_video.len(), 2, "expected MPEG1 and MPEG2 to be the raw streams" ); // The per-level families must all be present, at 27 each. for family in ["LEVEL", "AUDIO", "SCENE"] { let n = containers .iter() .filter(|c| { let stem = c.name.trim_start_matches("G/").trim_end_matches(".WAD"); stem.strip_prefix(family) .map(|rest| !rest.is_empty() && rest.chars().all(|ch| ch.is_ascii_digit())) .unwrap_or(false) }) .count(); assert_eq!(n, 27, "expected 27 {family}.WAD containers, found {n}"); } for entry in &containers { if raw_video.contains(&entry.name.as_str()) { continue; } let data = gc_formats::iso9660::read_at(&path, entry.lba, entry.size) .expect("read should succeed"); let kind = gc_formats::global_wad::RangeKind::for_container(&entry.name); let container = gc_formats::global_wad::GlobalWad::parse_with(&data, kind) .unwrap_or_else(|e| panic!("{} failed to parse: {e}", entry.name)); assert!( !container.entries.is_empty(), "{} parsed but had no entries", entry.name ); } } /// The level container header is the first typed structure we parse from real /// data, and the tiling invariant is the check that holds on every level rather /// than one hand-picked sample. #[test] fn level_headers_tile_their_containers() { let path = require_disc!(); let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); let mut checked = 0; for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) { let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header"); let h = gc_formats::level_wad::LevelHeader::parse(&data) .unwrap_or_else(|e| panic!("{}: {e}", entry.name)); assert_eq!(h.header_size, 0x60, "{}", entry.name); assert_eq!( h.runtime_sector, 0, "{}: runtime sector field should be 0 on disc", entry.name ); // Every present extent must start where the previous one ended. let extents = h.ordered_extents(); for w in extents.windows(2) { let (pn, p) = w[0]; let (nn, n) = w[1]; assert_eq!( p.offset + p.size, n.offset, "{}: gap or overlap between {} and {}", entry.name, pn.name(), nn.name() ); } // Nothing may fall outside the container. The last extent may end one // sector past the file end through sector rounding, hence `+ 1`. let container_sectors = (entry.size / 2048) as i32; for (name, r) in &extents { assert!( r.offset >= 0 && r.offset + r.size <= container_sectors + 1, "{}: {} extent {:?} is outside the container ({container_sectors} sectors)", entry.name, name.name(), r ); } checked += 1; } assert_eq!(checked, 27, "expected 27 level containers"); } /// Level ids are not file order: `LEVEL21.WAD` reports 30. Anything that keys /// off the container index would silently mislabel that planet. #[test] fn level_ids_are_not_container_indices() { let path = require_disc!(); let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); // The ISO filesystem lists names alphabetically, so LEVEL10 sorts before // LEVEL2. Order by the numeric suffix, which is what the container names // mean. let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso .files .iter() .filter(|e| e.name.starts_with("G/LEVEL")) .map(|e| { let n: usize = e .name .trim_start_matches("G/LEVEL") .trim_end_matches(".WAD") .parse() .expect("level container names end in a number"); (n, e) }) .collect(); levels.sort_by_key(|(n, _)| *n); let mut mismatches = Vec::new(); for (index, (_, entry)) in levels.iter().enumerate() { let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header"); let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse"); if h.id as usize != index { mismatches.push(format!( "{}: id {} but is container {index}", entry.name, h.id )); } } // Exactly one level is out of step: the gap left by a removed level. assert_eq!( mismatches.len(), 1, "expected exactly one level whose id differs from its container index, got {mismatches:?}" ); assert!( mismatches[0].contains("LEVEL21"), "unexpected mismatch: {mismatches:?}" ); } /// Sizes in the level header are sector counts. If they were byte counts, the /// extents could not tile a container this size, so this also guards against /// "fixing" the parser to read them as bytes. #[test] fn level_extent_sizes_are_sector_counts() { let path = require_disc!(); let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); let entry = iso.find("G/LEVEL0.WAD").expect("level 0"); let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header"); let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse"); // Sound bank occupies sectors 1..735 and data 735..7219. assert_eq!(h.sound_bank.offset, 1); assert_eq!(h.sound_bank.size, 734); assert_eq!(h.data.offset, 735); // Read as bytes, 734 would be under four sectors and the tiling would break. assert_eq!(h.sound_bank.byte_len(), 734 * 2048); } /// The level data extent is a container of byte ranges. Its header locates the /// geometry index and the compressed geometry, and every range must lie inside /// the extent on all 27 levels. #[test] fn level_data_extents_are_in_range() { let path = require_disc!(); let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); let mut checked = 0; for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) { let raw = gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container"); let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header"); let data = lh.data; let Some(offset) = data.byte_offset() else { continue; }; let len = data.byte_len() as usize; let extent = &raw[offset as usize..offset as usize + len]; let dh = gc_formats::level_data::LevelDataHeader::parse(extent) .unwrap_or_else(|e| panic!("{}: data extent header: {e}", entry.name)); for (part, r) in dh.present_parts() { assert!( r.end() <= extent.len() as u64, "{}: {part:?} ends at {} but the extent is {} bytes", entry.name, r.end(), extent.len() ); } // The geometry is the big one and must be substantial. assert!( dh.core_data.size > 1_000_000, "{}: core_data is only {} bytes", entry.name, dh.core_data.size ); checked += 1; } assert_eq!(checked, 27, "expected 27 level containers"); } /// The geometry must decompress. This is the first test that touches real level /// content rather than headers. #[test] fn level_geometry_decompresses() { let path = require_disc!(); let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); let entry = iso.find("G/LEVEL0.WAD").expect("level 0"); let raw = gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container"); let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header"); let data = lh.data; let extent = &raw[data.byte_offset().unwrap() as usize ..(data.byte_offset().unwrap() + data.byte_len()) as usize]; let dh = gc_formats::level_data::LevelDataHeader::parse(extent).expect("data header"); let core = dh .core_data .slice(extent, extent.len()) .expect("core_data slice"); assert!( gc_formats::wad::is_wad(core), "core_data should start with the WAD magic" ); let decoded = gc_formats::wad::decompress(core).expect("core_data should decompress"); assert!( decoded.len() > core.len(), "decompressed geometry ({}) should exceed the compressed size ({})", decoded.len(), core.len() ); } /// The terrain itself. This is the first test to touch real level content: it /// parses the tfrags directory and decodes vertex positions from every patch. /// /// The gate is deliberately about *coverage*, not correctness of a sample: a /// parser that works on one tfrag and silently returns `None` on the rest would /// otherwise pass. #[test] fn tfrag_vertices_decode_for_every_level() { require_disc!(); // Opening the image also confirms the disc is readable, not merely present. gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); // Where the unpacked data lives. GC_EXTRACTED overrides; otherwise the // default location used during development. let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!( "skipped: unpack the disc first, e.g.\ gc-extract unpack ~/Documents/gc-extracted\ or point GC_EXTRACTED at an existing unpack" ); return; } let mut levels = 0usize; let mut total_verts = 0usize; let mut total_tfrags = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); let core_path = dir.join("core_data.bin"); let index_path = dir.join("core_index.bin"); if !core_path.exists() || !index_path.exists() { continue; } let core = std::fs::read(&core_path).expect("core_data"); let index = std::fs::read(&index_path).expect("core_index"); // The tfrags block starts at the core header's 0x08 field and runs to // the next region that exists. let region_start = |off: usize| -> i32 { i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) }; let start = region_start(0x08) as usize; let end = [0x0c_usize, 0x10, 0x14] .iter() .map(|&o| region_start(o)) .filter(|&v| v > 0) .min() .unwrap_or(core.len() as i32) as usize; let block = &core[start..end]; let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header"); assert_eq!(header.table_offset, 0x40, "level{n}: table offset"); let table_offset = header.table_offset as usize; let mut decoded = 0usize; let mut verts = 0usize; for i in 0..header.tfrag_count as usize { let th = header.header(block, i).expect("tfrag header"); let data_base = table_offset.wrapping_add(th.data as usize); if data_base >= block.len() { continue; } let slice = &block[data_base..]; if let Some(p) = gc_formats::tfrag::tfrag_positions(slice, &th) { decoded += 1; verts += p.len(); assert_eq!(p.len(), th.vert_count as usize); } } assert!(header.tfrag_count > 0, "level{n}: no tfrags"); assert_eq!( decoded, header.tfrag_count as usize, "level{n}: only {decoded} of {} tfrags decoded", header.tfrag_count ); levels += 1; total_tfrags += header.tfrag_count as usize; total_verts += verts; } assert!(levels > 0, "no unpacked levels were found"); assert!( total_tfrags > 5000, "expected thousands of tfrags, got {total_tfrags}" ); assert!( total_verts > 100_000, "expected 100k+ vertices, got {total_verts}" ); eprintln!("decoded {total_verts} vertices from {total_tfrags} tfrags across {levels} levels"); } /// Terrain topology: strips and index arrays must expand to exactly the /// triangle count each patch's header claims. /// /// `tri_count` is stored in the patch header but is not derivable from the /// lists directly, so reproducing it on all 18,102 patches is a strong check /// that the strip walk, the index bank and the LOD assignment are all right. /// Triangle indices must also stay inside the vertex-info range for the LOD. #[test] fn tfrag_topology_reproduces_the_header_triangle_count() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } let mut patches = 0usize; let mut triangles = 0usize; let mut levels = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); let core = match std::fs::read(dir.join("core_data.bin")) { Ok(c) => c, Err(_) => continue, }; let index = match std::fs::read(dir.join("core_index.bin")) { Ok(i) => i, Err(_) => continue, }; let region = |off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize; let start = region(0x08); let end = [0x0c_usize, 0x10, 0x14] .iter() .map(|&o| region(o)) .filter(|&v| v > 0) .min() .unwrap_or(core.len()); let block = &core[start..end]; let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header"); let table = header.table_offset as usize; for i in 0..header.tfrag_count as usize { let th = header.header(block, i).expect("tfrag header"); let data_base = table.wrapping_add(th.data as usize); if data_base >= block.len() { continue; } let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th) .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}")); let tris = gc_formats::tfrag::tfrag_triangles(&g, 0) .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}")); assert_eq!( tris.len(), th.tri_count as usize, "level{n} patch {i}: {} triangles, header says {}", tris.len(), th.tri_count ); let vinfo = g.lod_vinfo_count(0); for t in &tris { let idx = [t.a as usize, t.b as usize, t.c as usize]; assert!( idx.iter().all(|&v| v < vinfo), "level{n} patch {i}: triangle index outside {vinfo} vertex-info entries" ); assert!((t.ad_gif as usize) < g.ad_gifs.len()); } patches += 1; triangles += tris.len(); } levels += 1; } assert!(levels > 0, "no unpacked levels were found"); assert!(patches > 18_000, "expected 18k+ patches, got {patches}"); assert!( triangles > 700_000, "expected 700k+ triangles, got {triangles}" ); eprintln!("reproduced {triangles} triangles across {patches} patches in {levels} levels"); } /// Texture bindings: every ad-gif a patch uses must index an entry that /// actually exists in the level's texture table, and the light array must be /// one record per vertex. /// /// The texture table is the join between tfrags and the texel data, so an /// out-of-range index here means the level's geometry and its textures have /// been misread independently of each other. #[test] fn tfrag_ad_gifs_index_the_level_texture_table() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } let mut patches = 0usize; let mut ad_gifs = 0usize; let mut textures_seen = 0usize; let mut textures_total = 0usize; let mut levels = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); let core = match std::fs::read(dir.join("core_data.bin")) { Ok(c) => c, Err(_) => continue, }; let index = match std::fs::read(dir.join("core_index.bin")) { Ok(i) => i, Err(_) => continue, }; let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header"); let textures = gc_formats::level_core::texture_table(&index, ch.tfrag_textures) .expect("texture table"); assert!(!textures.is_empty(), "level{n}: no tfrag textures"); // Indices are per-level, so the pair is what identifies a texture. let mut level_seen = std::collections::BTreeSet::new(); let region = |off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize; let start = region(0x08); let end = [0x0c_usize, 0x10, 0x14] .iter() .map(|&o| region(o)) .filter(|&v| v > 0) .min() .unwrap_or(core.len()); let block = &core[start..end]; let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header"); let table = header.table_offset as usize; for i in 0..header.tfrag_count as usize { let th = header.header(block, i).expect("tfrag header"); let data_base = table.wrapping_add(th.data as usize); if data_base >= block.len() { continue; } let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th) .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}")); assert_eq!( g.lights.len(), th.vert_count as usize, "level{n} patch {i}: {} light records for {} vertices", g.lights.len(), th.vert_count ); assert_eq!( g.rgba.len(), th.rgba_size as usize * 4, "level{n} patch {i}: rgba" ); for a in &g.ad_gifs { let t = a.texture_index(); assert!( t < textures.len(), "level{n} patch {i}: texture {} outside the {} entry table", t, textures.len() ); let tex = &textures[t]; assert!( tex.width > 0 && tex.height > 0 && tex.width <= 256 && tex.height <= 256, "level{n} patch {i}: texture {t} is {}x{}", tex.width, tex.height ); level_seen.insert(t); ad_gifs += 1; } patches += 1; } textures_seen += level_seen.len(); textures_total += textures.len(); levels += 1; } assert!(levels > 0, "no unpacked levels were found"); assert!(patches > 18_000, "expected 18k+ patches, got {patches}"); // Most of each level's texture table should be referenced by some tfrag. A // handful genuinely go unused; if that share grows, the ad-gif walk has // started missing primitives rather than the data having spare entries. let covered = textures_seen as f64 / textures_total as f64; assert!( covered > 0.85, "only {textures_seen} of {textures_total} level textures referenced ({:.1}%)", covered * 100.0 ); eprintln!( "{ad_gifs} ad-gifs over {patches} patches reference {textures_seen}/{textures_total} level textures" ); } /// Level textures decode: every tfrag texture's base level must resolve to a /// plausible RGBA image. /// /// This is the join between the texture table, the texel block and GS RAM, so /// it is where a wrong CLUT remap, a wrong alpha scale or a wrong pixel base /// would show up. The check is deliberately about *shape*, not specific /// colours: a decoded image is plausible when it is fully opaque, is not /// uniformly one colour, and uses more than a handful of distinct entries. #[test] fn tfrag_textures_decode_to_plausible_images() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } let mut decoded = 0usize; let mut opaque = 0usize; let mut multi_colour = 0usize; let mut flat = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); let core = match std::fs::read(dir.join("core_data.bin")) { Ok(c) => c, Err(_) => continue, }; let index = match std::fs::read(dir.join("core_index.bin")) { Ok(i) => i, Err(_) => continue, }; let gs = match std::fs::read(dir.join("gs_ram.bin")) { Ok(g) => g, Err(_) => { eprintln!("level{n}: no gs_ram.bin, skipping"); continue; } }; let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header"); let textures_at = ch.textures_base.max(0) as usize; let texture_block = &core[textures_at..]; let table = gc_formats::level_core::texture_table(&index, ch.tfrag_textures).expect("table"); for (i, e) in table.iter().enumerate() { let img = gc_formats::texture::base_image(texture_block, &gs, e) .unwrap_or_else(|err| panic!("level{n} texture {i}: {err}")); let t = img.decode().unwrap(); let texels = (e.width as i32) * (e.height as i32); assert_eq!(t.rgba.len(), (texels * 4) as usize); assert_eq!(t.width, e.width as u32); decoded += 1; // Count distinct colours and how many texels are fully opaque. let mut seen = std::collections::HashSet::new(); let mut op = 0usize; for px in t.rgba.chunks_exact(4) { seen.insert([px[0], px[1], px[2], px[3]]); if px[3] == 0xff { op += 1; } } if seen.len() > 8 { multi_colour += 1; } else { flat += 1; } if op == texels as usize { opaque += 1; } } } assert!(decoded > 1_500, "only {decoded} textures decoded"); // Level art is mostly opaque and has real detail. A flat, transparent or // single-colour result almost always means a CLUT or offset mistake. let opaque_pct = opaque as f64 / decoded as f64; let detail_pct = multi_colour as f64 / decoded as f64; assert!( opaque_pct > 0.9, "only {opaque:.1}% of textures are fully opaque" ); assert!( detail_pct > 0.9, "only {detail_pct:.1} of textures have >8 colours" ); eprintln!( "{decoded} textures decoded: {:.1}% fully opaque, {:.1}% with >8 colours ({flat} flat)", opaque_pct * 100.0, detail_pct * 100.0 ); } /// LOD linkage: every LOD-01 and LOD-0 vertex-info entry must resolve to a /// first parent that exists and to positions inside the patch. /// /// The parent-index arrays are V4_8 unpacks separated from the strips and the /// strip indices only by their VU address, so a misclassification here is /// silent: the arrays still parse, they just hold the wrong numbers. Resolving /// every entry is what catches it. #[test] fn tfrag_lod_parents_resolve_for_every_entry() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } let mut patches = 0usize; let mut entries = 0usize; let mut lod01 = 0usize; let mut lod0 = 0usize; let mut resolved_common = 0usize; let mut common_total = 0usize; let mut primary_failed = 0usize; let mut extras_unresolved = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); let core = match std::fs::read(dir.join("core_data.bin")) { Ok(c) => c, Err(_) => continue, }; let index = match std::fs::read(dir.join("core_index.bin")) { Ok(i) => i, Err(_) => continue, }; let region = |off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize; let start = region(0x08); let end = [0x0c_usize, 0x10, 0x14] .iter() .map(|&o| region(o)) .filter(|&v| v > 0) .min() .unwrap_or(core.len()); let block = &core[start..end]; let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block"); let table = header.table_offset as usize; for i in 0..header.tfrag_count as usize { let th = header.header(block, i).expect("tfrag header"); let data_base = table.wrapping_add(th.data as usize); if data_base >= block.len() { continue; } let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th) .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}")); common_total += g.vinfo_tier[0]; if n == 0 && patches < 5 { eprintln!( "p{i}: tier={:?} n01={} n0={} p01={}b x01={}b p0={}b x0={}b", g.vinfo_tier, g.vu_positions_lod_01_count, g.vu_positions_lod_0_count, g.parent_indices_lod01.len(), g.unk_indices_2_lod01.len(), g.parent_indices_lod0.len(), g.unk_indices_2_lod0.len() ); } for v in 0..g.vertex_info.len() { entries += 1; match gc_formats::tfrag::tfrag_lod_link(&g, v) { Some(link) => { if v < g.vinfo_tier[0] { resolved_common += 1; } // Parents must index real vertex info and real positions. assert!( link.parent1_vinfo < g.vertex_info.len(), "level{n} patch {i}: entry {v} parent {} outside {} entries", link.parent1_vinfo, g.vertex_info.len() ); assert!(link.own_position < g.positions.len()); assert!(link.parent1_position < g.positions.len()); assert!(link.parent2_position < g.positions.len()); // The first parent of a LOD-01 entry is always a common one. if link.tier == gc_formats::tfrag::TfragMorphTier::Lod01 { assert!( link.parent1_vinfo < g.vinfo_tier[0], "level{n} patch {i}: LOD-01 entry {v} parent {} is not a common entry", link.parent1_vinfo ); lod01 += 1; } else { lod0 += 1; } } None => { // Distinguish the two kinds of LOD entry. A *primary* // entry has a real position and a recorded parent and // must always resolve; an *extra* entry is a // non-position bookkeeping record whose array address // can collide with the strip index array. let c = g.vinfo_tier[0]; let kind = if v < c { None } else if v < c + g.vinfo_tier[1] { Some((g.vu_positions_lod_01_count, v - c, true)) } else { Some((g.vu_positions_lod_0_count, v - c - g.vinfo_tier[1], false)) }; match kind { None => {} Some((prim, k, _)) if k < prim => primary_failed += 1, Some(_) => extras_unresolved += 1, } } } } patches += 1; } } assert!(patches > 18_000, "expected 18k+ patches, got {patches}"); // A common entry never has a link: they are the full-detail positions and // have no parent. Anything else means the tier boundaries are wrong. assert!( resolved_common == 0, "{resolved_common} common entries claimed an LOD parent" ); assert!( lod01 > 0 && lod0 > 0, "both LOD tiers should occur in the data" ); assert_eq!( primary_failed, 0, "{primary_failed} primary LOD entries did not resolve" ); assert_eq!( extras_unresolved, 0, "{extras_unresolved} extra LOD entries did not resolve" ); // Common entries have no link; every other entry does. assert_eq!(entries, common_total + lod01 + lod0); eprintln!( "{entries} vertex-info entries over {patches} patches: {common_total} common, \ {lod01} LOD-01 and {lod0} LOD-0, all resolved" ); } /// The level core index's class tables. /// /// These are what put the environment on top of the terrain: tie instances, /// moby actors and shrub foliage. They were ruled out earlier for want of an /// entry size, which turned out to be knowable exactly. /// /// The check is the chain: each table must end precisely where the next one /// begins, moby (32 bytes/entry) into tie (32) into shrub (48) into the tfrag /// texture table. That closes on all 27 levels and would not close if any /// entry size were wrong, so it is the evidence rather than a coincidence. #[test] fn level_core_class_tables_tile_the_index() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } use gc_formats::level_core as lc; let mut levels = 0usize; let (mut moby, mut tie, mut shrub, mut textures) = (0usize, 0usize, 0usize, 0usize); for n in 0..27 { let dir = data.join(format!("level{n}")); let Ok(index) = std::fs::read(dir.join("core_index.bin")) else { continue; }; let h = lc::LevelCoreHeader::parse(&index).expect("core header"); let end = |r: lc::ArrayRange, size: usize| r.offset as usize + r.count as usize * size; assert_eq!( end(h.moby_classes, lc::MOBY_CLASS_SIZE), h.tie_classes.offset as usize, "level{n}: moby -> tie" ); assert_eq!( end(h.tie_classes, lc::TIE_CLASS_SIZE), h.shrub_classes.offset as usize, "level{n}: tie -> shrub" ); assert_eq!( end(h.shrub_classes, lc::SHRUB_CLASS_SIZE), h.tfrag_textures.offset as usize, "level{n}: shrub -> textures" ); // Every entry parses, and its asset offset lands inside the level. if n == 0 { eprintln!( "hdr moby={:?} tie={:?} shrub={:?} tex={:?} base={}", h.moby_classes, h.tie_classes, h.shrub_classes, h.tfrag_textures, h.textures_base ); eprintln!("index[192..200] = {:?}", &index[192..200]); } let moby_v = lc::class_table( &index, h.moby_classes, lc::MOBY_CLASS_SIZE, lc::ClassEntry::parse, ) .expect("moby classes"); let tie_v = lc::class_table( &index, h.tie_classes, lc::TIE_CLASS_SIZE, lc::ClassEntry::parse, ) .expect("tie classes"); let shrub_v = lc::class_table( &index, h.shrub_classes, lc::SHRUB_CLASS_SIZE, lc::ShrubClassEntry::parse, ) .expect("shrub classes"); let tex_v = lc::texture_table(&index, h.tfrag_textures).expect("tfrag textures"); assert_eq!(moby_v.len(), h.moby_classes.count as usize); for (i, c) in moby_v.iter().chain(tie_v.iter()).enumerate() { assert!( c.offset_in_asset_wad >= 0, "level{n} entry {i}: asset offset {}", c.offset_in_asset_wad ); } // Billboard slots are optional: every shrub class on level 0 carries // them all zero, so only their sign is meaningful here. for s in &shrub_v { assert!(s.base.offset_in_asset_wad >= 0); assert!( s.palette_offset >= -1 && s.texture_offset >= -1, "level{n}: shrub billboard offsets below the -1 sentinel" ); } moby += moby_v.len(); tie += tie_v.len(); shrub += shrub_v.len(); textures += tex_v.len(); levels += 1; } assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); assert!( moby + tie + shrub > 1000, "only {moby}+{tie}+{shrub} class entries" ); eprintln!( "level core index: {moby} moby, {tie} tie, {shrub} shrub classes, \ {textures} tfrag textures across {levels} levels" ); } /// The moby, tie and shrub class tables point into `core_data`, and they do so /// without overlapping. /// /// `assets_decompressed_size` is the size of `core_data` itself, so there is /// no separate asset container: the class entries index the decompressed asset /// lump directly. If that is right, every offset must land inside the lump, the /// three tables must occupy disjoint spans, and each must be internally sorted /// so the geometry is laid out in class order. #[test] fn class_geometry_offsets_tile_core_data_without_overlap() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } use gc_formats::level_core as lc; let mut levels = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); let (Ok(index), Ok(core)) = ( std::fs::read(dir.join("core_index.bin")), std::fs::read(dir.join("core_data.bin")), ) else { continue; }; let h = lc::LevelCoreHeader::parse(&index).expect("core header"); // The claim under test: the "asset lump" is core_data itself. assert_eq!( h.assets_decompressed_size as usize, core.len(), "level{n}: assets_decompressed_size is not core_data's length" ); let span = |r: lc::ArrayRange, size: usize| -> Vec { lc::class_table(&index, r, size, lc::ClassEntry::parse) .unwrap_or_default() .iter() .map(|c| c.offset_in_asset_wad as usize) .collect() }; // Shrub entries are read as ShrubClassEntry so the offsets line up. let shrub: Vec = lc::class_table( &index, h.shrub_classes, lc::SHRUB_CLASS_SIZE, lc::ShrubClassEntry::parse, ) .unwrap_or_default() .iter() .map(|s| s.base.offset_in_asset_wad as usize) .collect(); let moby = span(h.moby_classes, lc::MOBY_CLASS_SIZE); let tie = span(h.tie_classes, lc::TIE_CLASS_SIZE); // An offset of 0 marks a slot with no geometry rather than the start of // the lump, so those are excluded from the span checks. Entries are not // sorted by class number and the offsets are not monotonic, so nothing // here assumes an order. let moby_l: Vec = moby.iter().copied().filter(|&o| o != 0).collect(); let tie_l: Vec = tie.iter().copied().filter(|&o| o != 0).collect(); let shrub_l: Vec = shrub.iter().copied().filter(|&o| o != 0).collect(); for (name, v) in [("moby", &moby), ("tie", &tie), ("shrub", &shrub)] { for &o in v { assert!( o < core.len(), "level{n}: {name} offset {o} past core data ({})", core.len() ); } } // The three spans must not overlap each other. for (an, a) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] { for (bn, b) in [("moby", &moby_l), ("tie", &tie_l), ("shrub", &shrub_l)] { // A level may have no geometry of one kind at all. if an >= bn || a.is_empty() || b.is_empty() { continue; } let (lo, hi) = (a[0], *a.last().unwrap()); for &o in b { assert!( o >= hi || o < lo, "level{n}: {an} [{lo},{hi}] overlaps {bn} at {o}" ); } } } levels += 1; } assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); eprintln!("class geometry offsets verified inside core_data on {levels} levels"); } /// Tie class headers, as far as RAC1 and RAC2 agree. /// /// The check that carries the weight is `o_class`: the class's own number at /// header offset 0x44 equals the number the core index filed it under, on every /// class of every level. That is what makes the header base trustworthy rather /// than merely plausible. #[test] fn tie_class_headers_match_their_index_entry() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } use gc_formats::level_core as lc; use gc_formats::tie::{TieClassHeader, TIE_CLASS_HEADER_MATCHED}; let mut levels = 0usize; let (mut classes, mut ordered, mut unordered) = (0usize, 0usize, 0usize); for n in 0..27 { let dir = data.join(format!("level{n}")); let (Ok(index), Ok(core)) = ( std::fs::read(dir.join("core_index.bin")), std::fs::read(dir.join("core_data.bin")), ) else { continue; }; let h = lc::LevelCoreHeader::parse(&index).expect("core header"); let entries = lc::class_table( &index, h.tie_classes, lc::TIE_CLASS_SIZE, lc::ClassEntry::parse, ) .expect("tie classes"); for e in &entries { let off = e.offset_in_asset_wad as usize; if off == 0 { continue; // slot with no geometry } let at = off + TIE_CLASS_HEADER_MATCHED; assert!(at <= core.len(), "level{n}: tie class past core data"); let t = TieClassHeader::parse(&core[off..at]) .unwrap_or_else(|err| panic!("level{n}: tie class {} header: {err}", e.o_class)); assert_eq!(t.o_class, e.o_class, "level{n}: tie o_class mismatch"); assert!( t.far_dist > 0.0, "level{n}: tie far distance {}", t.far_dist ); if 0.0 < t.near_dist && t.near_dist < t.mid_dist && t.mid_dist < t.far_dist { ordered += 1; } else { unordered += 1; } classes += 1; } levels += 1; } assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); assert!(classes > 1_500, "only {classes} tie classes"); // Ordered distances are the expected shape; the exceptions are real and // unexplained, so they are reported rather than asserted away. assert!( ordered * 20 >= classes * 19, "only {ordered} of {classes} tie classes have ordered LOD distances" ); eprintln!( "tie: {classes} class headers, o_class matches the index on all; \\ {ordered} have ordered LOD distances, {unordered} do not" ); } /// The occlusion PVS grid. /// /// The check that makes this real is the mask array: it starts at /// `masks_offset` and runs to `(highest_mask + 1) * 0x80`, and on every level /// that ends within a few bytes of the block's own length. A wrong tree shape /// or a wrong cell size would leave the highest mask index somewhere else /// entirely and the array would not fit. #[test] fn occlusion_grids_parse_on_every_level() { require_disc!(); gc_formats::iso9660::IsoImage::open(std::path::Path::new( &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), )) .expect("ISO should open"); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } use gc_formats::level_core::LevelCoreHeader; use gc_formats::occlusion::parse_occlusion_block; let mut levels = 0usize; let mut absent = 0usize; let mut widest_slack = 0usize; let (mut cells, mut masks) = (0usize, 0usize); for n in 0..27 { let dir = data.join(format!("level{n}")); let (Ok(core), Ok(index)) = ( std::fs::read(dir.join("core_data.bin")), std::fs::read(dir.join("core_index.bin")), ) else { continue; }; let h = LevelCoreHeader::parse(&index).expect("core header"); // An offset of 0 means the level has no occlusion data at all, which is // not the same as "at the start of the file" -- that is where the // tfrags block lives, and parsing it as a grid produces a mask array // that fits anyway, which is how this nearly went wrong. if h.occlusion <= 0 { absent += 1; continue; } let start = h.occlusion as usize; // The block runs to the sky region, or to the end when there is none. let end = if h.sky > 0 { h.sky as usize } else { core.len() }; assert!( start < end && end <= core.len(), "level{n}: bad occlusion range" ); let block = &core[start..end]; let o = parse_occlusion_block(block) .unwrap_or_else(|e| panic!("level{n}: occlusion block: {e}")); // Every mask a cell names exists. for c in &o.cells { assert!( (c.mask as usize) < o.masks.len(), "level{n}: cell names a missing mask" ); } // The array ends at or just before the block's end. Retail levels sit // within a few bytes; a gross error would leave a large gap. let need = o.masks_offset as usize + o.masks.len() * gc_formats::occlusion::MASK_BYTES; assert!(need <= block.len(), "level{n}: masks run past the block"); let slack = block.len() - need; // Levels whose grid is empty are mostly header and padding, so this is // proportional and only applies where there are cells at all. if !o.cells.is_empty() { assert!( slack * 8 < block.len(), "level{n}: {slack} of {} bytes unexplained after the mask array", block.len() ); } widest_slack = widest_slack.max(slack); cells += o.cells.len(); masks += o.masks.len(); levels += 1; } assert_eq!( levels + absent, 27, "expected all 27 levels, got {}", levels + absent ); assert_eq!( absent, 3, "expected 3 levels without occlusion data, got {absent}" ); assert!(cells > 400_000, "only {cells} occlusion cells"); eprintln!( "occlusion: {cells} populated cells over {masks} masks across {levels} levels; \ widest unexplained tail {widest_slack} bytes" ); } /// The sound extent framing. /// /// The invariant here is `sblk_offset + sblk_size == daw_offset`, which holds /// on all 27 levels and pins the meaning of the four head fields. `DAW` never /// reaches the end of the file; the tail is positive on every level, so the /// size field is not the whole region and this test asserts the direction /// rather than a size. #[test] fn sound_bank_framing_holds_on_every_level() { require_disc!(); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { eprintln!("skipped: unpack the disc first"); return; } use gc_formats::sound_bank::parse_sound_bank; let mut levels = 0usize; let mut sblk_bytes = 0usize; let mut daw_bytes = 0usize; let (mut narrowest, mut widest) = (usize::MAX, 0usize); for n in 0..27 { let raw = match std::fs::read(data.join(format!("level{n}")).join("sound_bank.bin")) { Ok(v) => v, Err(_) => continue, }; let s = parse_sound_bank(&raw).unwrap_or_else(|e| panic!("level{n}: sound bank: {e}")); assert_eq!(s.header.version, 3, "level{n}: sound bank version"); assert_eq!(s.header.unknown_4, 2, "level{n}: sound bank unknown_4"); assert_eq!( s.header.sblk_offset, 24, "level{n}: SBlk sits at 24 on every level" ); assert!( s.header.sblk_offset + s.header.sblk_size == s.header.daw_offset, "level{n}: SBlk must run into DAW with no gap" ); assert!(s.has_sblk_magic(), "level{n}: SBlk magic"); assert!(s.has_daw_tag(), "level{n}: DAW tag at SBlk+12"); assert_eq!(s.sblk_version(), Some(1), "level{n}: SBlk version"); // The tail is always in the same direction: the extent is longer than // the DAW region it describes. narrowest = narrowest.min(s.daw_tail); widest = widest.max(s.daw_tail); sblk_bytes += s.sblk.len(); daw_bytes += s.daw.len(); levels += 1; } assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); assert!(sblk_bytes > 300_000, "only {sblk_bytes} SBlk bytes"); assert!(daw_bytes > 20_000_000, "only {daw_bytes} DAW bytes"); eprintln!( "sound_bank: {sblk_bytes} SBlk + {daw_bytes} DAW bytes across {levels} levels; \ unexplained tail {narrowest}..{widest} bytes" ); }