Going-Commando/crates/gc-extract/tests/disc.rs

//! Integration tests against a real disc image.
//!
//! These are skipped unless `GC_DISC` points at a legally obtained disc, since
//! the repository must not contain game data. When it is set they check the
//! identification and unpack results against the committed hash table.
//!
//! ```sh
//! GC_DISC="path/to/disc.iso" cargo test -p gc-extract --test disc
//! ```

use std::path::{Path, PathBuf};

/// The disc under test, if the environment names one.
fn disc() -> Option<PathBuf> {
    let p = std::env::var_os("GC_DISC")?;
    let p = PathBuf::from(p);
    p.is_file().then_some(p)
}

/// Skip a test with a clear reason rather than failing.
macro_rules! require_disc {
    () => {
        match disc() {
            Some(p) => p,
            None => {
                eprintln!("skipped: set GC_DISC to a Going Commando disc image to run this");
                return;
            }
        }
    };
}

#[test]
fn identifies_the_supported_build() {
    let path = require_disc!();
    let info = gc_extract::identify::identify(&path).expect("disc should identify");

    assert_eq!(info.serial, "SCUS_972.68");
    assert_eq!(info.volume_id, "RATCHETANDCLANK2");
    assert!(
        info.is_target(),
        "expected the supported build, got: {}",
        info.verdict()
    );
    assert_eq!(
        info.boot_elf_sha1,
        gc_extract::identify::TARGET_BOOT_ELF_SHA1,
        "boot ELF SHA-1 differs from the recorded retail value"
    );
    assert_eq!(info.boot_elf_size, 2_618_684);
    // This disc is NTSC; the movement constants assume 60 Hz.
    assert!(!info.is_pal(), "test disc should be NTSC");
}

#[test]
fn agrees_with_the_python_reference_identifier() {
    let path = require_disc!();
    let info = gc_extract::identify::identify(&path).expect("disc should identify");

    // tools/identify-disc.py is the reference implementation and prints these
    // two values. If they ever disagree, one of them has a bug.
    // Run from the workspace root so the relative path to the tool resolves.
    let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .and_then(|p| p.parent())
        .expect("the crate should live two levels below the workspace root")
        .to_path_buf();
    let py = std::process::Command::new("python3")
        .arg(root.join("tools/identify-disc.py"))
        .arg(&path)
        .current_dir(&root)
        .output()
        .expect("python3 should be available to run the reference identifier");
    assert!(
        py.status.success(),
        "the reference identifier failed:\n{}",
        String::from_utf8_lossy(&py.stderr)
    );
    let text = String::from_utf8_lossy(&py.stdout);

    assert!(
        text.contains(&format!("serial        {}", info.serial)),
        "serial disagrees with the reference identifier:\n{text}"
    );
    assert!(
        text.contains(&format!("SHA-1         {}", info.boot_elf_sha1)),
        "boot ELF SHA-1 disagrees with the reference identifier:\n{text}"
    );
}

#[test]
fn unpacks_and_matches_the_committed_table() {
    let path = require_disc!();

    let table_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("data/gc-files.tsv");
    let table_text =
        std::fs::read_to_string(&table_path).expect("the hash table should be committed");
    let expected = gc_extract::verify::parse_table(&table_text);
    assert!(
        expected.len() > 5000,
        "expected a full table, got {} rows",
        expected.len()
    );

    // Unpack into a temporary directory so the test leaves nothing behind.
    let tmp = std::env::temp_dir().join(format!("gc-unpack-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&tmp);
    std::fs::create_dir_all(&tmp).expect("should create a temp directory");

    let files = gc_extract::unpack::unpack(&path, &tmp).expect("unpack should succeed");

    // Every unpacked file must match the recorded size and hash.
    let report = gc_extract::verify::verify(&tmp, &expected);
    let _ = std::fs::remove_dir_all(&tmp);

    assert!(
        report.ok(),
        "unpacked data does not match the committed table:\n{}",
        report.summary()
    );
    assert_eq!(
        files.len(),
        expected.len(),
        "unpack produced a different file count"
    );
}

#[test]
fn every_container_parses() {
    let path = require_disc!();

    let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
    let containers: Vec<_> = iso
        .files
        .iter()
        .filter(|e| e.name.starts_with("G/") && e.name.ends_with(".WAD"))
        .collect();

    // 92 `G/*.WAD` containers: 27 LEVEL, 27 AUDIO, 27 SCENE, 11 globals.
    assert_eq!(containers.len(), 92, "expected 92 asset containers");

    // MPEG1 and MPEG2 are raw video streams with no table; the other 90 must
    // all parse as indexed containers.
    let raw_video: Vec<&str> = containers
        .iter()
        .map(|c| c.name.as_str())
        .filter(|n| *n == "G/MPEG1.WAD" || *n == "G/MPEG2.WAD")
        .collect();
    assert_eq!(
        raw_video.len(),
        2,
        "expected MPEG1 and MPEG2 to be the raw streams"
    );

    // The per-level families must all be present, at 27 each.
    for family in ["LEVEL", "AUDIO", "SCENE"] {
        let n = containers
            .iter()
            .filter(|c| {
                let stem = c.name.trim_start_matches("G/").trim_end_matches(".WAD");
                stem.strip_prefix(family)
                    .map(|rest| !rest.is_empty() && rest.chars().all(|ch| ch.is_ascii_digit()))
                    .unwrap_or(false)
            })
            .count();
        assert_eq!(n, 27, "expected 27 {family}<n>.WAD containers, found {n}");
    }

    for entry in &containers {
        if raw_video.contains(&entry.name.as_str()) {
            continue;
        }
        let data = gc_formats::iso9660::read_at(&path, entry.lba, entry.size)
            .expect("read should succeed");
        let kind = gc_formats::global_wad::RangeKind::for_container(&entry.name);
        let container = gc_formats::global_wad::GlobalWad::parse_with(&data, kind)
            .unwrap_or_else(|e| panic!("{} failed to parse: {e}", entry.name));
        assert!(
            !container.entries.is_empty(),
            "{} parsed but had no entries",
            entry.name
        );
    }
}

/// The level container header is the first typed structure we parse from real
/// data, and the tiling invariant is the check that holds on every level rather
/// than one hand-picked sample.
#[test]
fn level_headers_tile_their_containers() {
    let path = require_disc!();
    let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");

    let mut checked = 0;
    for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
        let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
        let h = gc_formats::level_wad::LevelHeader::parse(&data)
            .unwrap_or_else(|e| panic!("{}: {e}", entry.name));

        assert_eq!(h.header_size, 0x60, "{}", entry.name);
        assert_eq!(
            h.runtime_sector, 0,
            "{}: runtime sector field should be 0 on disc",
            entry.name
        );

        // Every present extent must start where the previous one ended.
        let extents = h.ordered_extents();
        for w in extents.windows(2) {
            let (pn, p) = w[0];
            let (nn, n) = w[1];
            assert_eq!(
                p.offset + p.size,
                n.offset,
                "{}: gap or overlap between {} and {}",
                entry.name,
                pn.name(),
                nn.name()
            );
        }

        // Nothing may fall outside the container. The last extent may end one
        // sector past the file end through sector rounding, hence `+ 1`.
        let container_sectors = (entry.size / 2048) as i32;
        for (name, r) in &extents {
            assert!(
                r.offset >= 0 && r.offset + r.size <= container_sectors + 1,
                "{}: {} extent {:?} is outside the container ({container_sectors} sectors)",
                entry.name,
                name.name(),
                r
            );
        }
        checked += 1;
    }

    assert_eq!(checked, 27, "expected 27 level containers");
}

/// Level ids are not file order: `LEVEL21.WAD` reports 30. Anything that keys
/// off the container index would silently mislabel that planet.
#[test]
fn level_ids_are_not_container_indices() {
    let path = require_disc!();
    let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");

    // The ISO filesystem lists names alphabetically, so LEVEL10 sorts before
    // LEVEL2. Order by the numeric suffix, which is what the container names
    // mean.
    let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso
        .files
        .iter()
        .filter(|e| e.name.starts_with("G/LEVEL"))
        .map(|e| {
            let n: usize = e
                .name
                .trim_start_matches("G/LEVEL")
                .trim_end_matches(".WAD")
                .parse()
                .expect("level container names end in a number");
            (n, e)
        })
        .collect();
    levels.sort_by_key(|(n, _)| *n);

    let mut mismatches = Vec::new();
    for (index, (_, entry)) in levels.iter().enumerate() {
        let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
        let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");
        if h.id as usize != index {
            mismatches.push(format!(
                "{}: id {} but is container {index}",
                entry.name, h.id
            ));
        }
    }

    // Exactly one level is out of step: the gap left by a removed level.
    assert_eq!(
        mismatches.len(),
        1,
        "expected exactly one level whose id differs from its container index, got {mismatches:?}"
    );
    assert!(
        mismatches[0].contains("LEVEL21"),
        "unexpected mismatch: {mismatches:?}"
    );
}

/// Sizes in the level header are sector counts. If they were byte counts, the
/// extents could not tile a container this size, so this also guards against
/// "fixing" the parser to read them as bytes.
#[test]
fn level_extent_sizes_are_sector_counts() {
    let path = require_disc!();
    let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
    let entry = iso.find("G/LEVEL0.WAD").expect("level 0");

    let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header");
    let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse");

    // Sound bank occupies sectors 1..735 and data 735..7219.
    assert_eq!(h.sound_bank.offset, 1);
    assert_eq!(h.sound_bank.size, 734);
    assert_eq!(h.data.offset, 735);
    // Read as bytes, 734 would be under four sectors and the tiling would break.
    assert_eq!(h.sound_bank.byte_len(), 734 * 2048);
}

/// The level data extent is a container of byte ranges. Its header locates the
/// geometry index and the compressed geometry, and every range must lie inside
/// the extent on all 27 levels.
#[test]
fn level_data_extents_are_in_range() {
    let path = require_disc!();
    let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");

    let mut checked = 0;
    for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) {
        let raw =
            gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
        let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
        let data = lh.data;
        let Some(offset) = data.byte_offset() else {
            continue;
        };
        let len = data.byte_len() as usize;
        let extent = &raw[offset as usize..offset as usize + len];

        let dh = gc_formats::level_data::LevelDataHeader::parse(extent)
            .unwrap_or_else(|e| panic!("{}: data extent header: {e}", entry.name));

        for (part, r) in dh.present_parts() {
            assert!(
                r.end() <= extent.len() as u64,
                "{}: {part:?} ends at {} but the extent is {} bytes",
                entry.name,
                r.end(),
                extent.len()
            );
        }

        // The geometry is the big one and must be substantial.
        assert!(
            dh.core_data.size > 1_000_000,
            "{}: core_data is only {} bytes",
            entry.name,
            dh.core_data.size
        );
        checked += 1;
    }
    assert_eq!(checked, 27, "expected 27 level containers");
}

/// The geometry must decompress. This is the first test that touches real level
/// content rather than headers.
#[test]
fn level_geometry_decompresses() {
    let path = require_disc!();
    let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open");
    let entry = iso.find("G/LEVEL0.WAD").expect("level 0");

    let raw = gc_formats::iso9660::read_at(&path, entry.lba, entry.size).expect("read container");
    let lh = gc_formats::level_wad::LevelHeader::parse(&raw).expect("level header");
    let data = lh.data;
    let extent = &raw[data.byte_offset().unwrap() as usize
        ..(data.byte_offset().unwrap() + data.byte_len()) as usize];

    let dh = gc_formats::level_data::LevelDataHeader::parse(extent).expect("data header");
    let core = dh
        .core_data
        .slice(extent, extent.len())
        .expect("core_data slice");

    assert!(
        gc_formats::wad::is_wad(core),
        "core_data should start with the WAD magic"
    );

    let decoded = gc_formats::wad::decompress(core).expect("core_data should decompress");
    assert!(
        decoded.len() > core.len(),
        "decompressed geometry ({}) should exceed the compressed size ({})",
        decoded.len(),
        core.len()
    );
}

/// The terrain itself. This is the first test to touch real level content: it
/// parses the tfrags directory and decodes vertex positions from every patch.
///
/// The gate is deliberately about *coverage*, not correctness of a sample: a
/// parser that works on one tfrag and silently returns `None` on the rest would
/// otherwise pass.
#[test]
fn tfrag_vertices_decode_for_every_level() {
    require_disc!();
    // Opening the image also confirms the disc is readable, not merely present.
    gc_formats::iso9660::IsoImage::open(std::path::Path::new(
        &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
    ))
    .expect("ISO should open");
    // Where the unpacked data lives. GC_EXTRACTED overrides; otherwise the
    // default location used during development.
    let data = std::env::var_os("GC_EXTRACTED")
        .map(PathBuf::from)
        .unwrap_or_else(|| {
            let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
            PathBuf::from(home).join("Documents/gc-extracted")
        });
    if !data.exists() {
        eprintln!(
            "skipped: unpack the disc first, e.g.\
             gc-extract unpack <disc.iso> ~/Documents/gc-extracted\
             or point GC_EXTRACTED at an existing unpack"
        );
        return;
    }

    let mut levels = 0usize;
    let mut total_verts = 0usize;
    let mut total_tfrags = 0usize;

    for n in 0..27 {
        let dir = data.join(format!("level{n}"));
        let core_path = dir.join("core_data.bin");
        let index_path = dir.join("core_index.bin");
        if !core_path.exists() || !index_path.exists() {
            continue;
        }
        let core = std::fs::read(&core_path).expect("core_data");
        let index = std::fs::read(&index_path).expect("core_index");

        // The tfrags block starts at the core header's 0x08 field and runs to
        // the next region that exists.
        let region_start =
            |off: usize| -> i32 { i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) };
        let start = region_start(0x08) as usize;
        let end = [0x0c_usize, 0x10, 0x14]
            .iter()
            .map(|&o| region_start(o))
            .filter(|&v| v > 0)
            .min()
            .unwrap_or(core.len() as i32) as usize;

        let block = &core[start..end];
        let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
        assert_eq!(header.table_offset, 0x40, "level{n}: table offset");

        let table_offset = header.table_offset as usize;
        let mut decoded = 0usize;
        let mut verts = 0usize;

        for i in 0..header.tfrag_count as usize {
            let th = header.header(block, i).expect("tfrag header");
            let data_base = table_offset.wrapping_add(th.data as usize);
            if data_base >= block.len() {
                continue;
            }
            let slice = &block[data_base..];
            if let Some(p) = gc_formats::tfrag::tfrag_positions(slice, &th) {
                decoded += 1;
                verts += p.len();
                assert_eq!(p.len(), th.vert_count as usize);
            }
        }

        assert!(header.tfrag_count > 0, "level{n}: no tfrags");
        assert_eq!(
            decoded, header.tfrag_count as usize,
            "level{n}: only {decoded} of {} tfrags decoded",
            header.tfrag_count
        );
        levels += 1;
        total_tfrags += header.tfrag_count as usize;
        total_verts += verts;
    }

    assert!(levels > 0, "no unpacked levels were found");
    assert!(
        total_tfrags > 5000,
        "expected thousands of tfrags, got {total_tfrags}"
    );
    assert!(
        total_verts > 100_000,
        "expected 100k+ vertices, got {total_verts}"
    );
    eprintln!("decoded {total_verts} vertices from {total_tfrags} tfrags across {levels} levels");
}

/// Terrain topology: strips and index arrays must expand to exactly the
/// triangle count each patch's header claims.
///
/// `tri_count` is stored in the patch header but is not derivable from the
/// lists directly, so reproducing it on all 18,102 patches is a strong check
/// that the strip walk, the index bank and the LOD assignment are all right.
/// Triangle indices must also stay inside the vertex-info range for the LOD.
#[test]
fn tfrag_topology_reproduces_the_header_triangle_count() {
    require_disc!();
    gc_formats::iso9660::IsoImage::open(std::path::Path::new(
        &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
    ))
    .expect("ISO should open");
    let data = std::env::var_os("GC_EXTRACTED")
        .map(PathBuf::from)
        .unwrap_or_else(|| {
            let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
            PathBuf::from(home).join("Documents/gc-extracted")
        });
    if !data.exists() {
        eprintln!("skipped: unpack the disc first");
        return;
    }

    let mut patches = 0usize;
    let mut triangles = 0usize;
    let mut levels = 0usize;

    for n in 0..27 {
        let dir = data.join(format!("level{n}"));
        let core = match std::fs::read(dir.join("core_data.bin")) {
            Ok(c) => c,
            Err(_) => continue,
        };
        let index = match std::fs::read(dir.join("core_index.bin")) {
            Ok(i) => i,
            Err(_) => continue,
        };
        let region =
            |off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
        let start = region(0x08);
        let end = [0x0c_usize, 0x10, 0x14]
            .iter()
            .map(|&o| region(o))
            .filter(|&v| v > 0)
            .min()
            .unwrap_or(core.len());
        let block = &core[start..end];

        let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
        let table = header.table_offset as usize;
        for i in 0..header.tfrag_count as usize {
            let th = header.header(block, i).expect("tfrag header");
            let data_base = table.wrapping_add(th.data as usize);
            if data_base >= block.len() {
                continue;
            }
            let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
                .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
            let tris = gc_formats::tfrag::tfrag_triangles(&g, 0)
                .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));
            assert_eq!(
                tris.len(),
                th.tri_count as usize,
                "level{n} patch {i}: {} triangles, header says {}",
                tris.len(),
                th.tri_count
            );
            let vinfo = g.lod_vinfo_count(0);
            for t in &tris {
                let idx = [t.a as usize, t.b as usize, t.c as usize];
                assert!(
                    idx.iter().all(|&v| v < vinfo),
                    "level{n} patch {i}: triangle index outside {vinfo} vertex-info entries"
                );
                assert!((t.ad_gif as usize) < g.ad_gifs.len());
            }
            patches += 1;
            triangles += tris.len();
        }
        levels += 1;
    }

    assert!(levels > 0, "no unpacked levels were found");
    assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
    assert!(
        triangles > 700_000,
        "expected 700k+ triangles, got {triangles}"
    );
    eprintln!("reproduced {triangles} triangles across {patches} patches in {levels} levels");
}

/// Texture bindings: every ad-gif a patch uses must index an entry that
/// actually exists in the level's texture table, and the light array must be
/// one record per vertex.
///
/// The texture table is the join between tfrags and the texel data, so an
/// out-of-range index here means the level's geometry and its textures have
/// been misread independently of each other.
#[test]
fn tfrag_ad_gifs_index_the_level_texture_table() {
    require_disc!();
    gc_formats::iso9660::IsoImage::open(std::path::Path::new(
        &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
    ))
    .expect("ISO should open");
    let data = std::env::var_os("GC_EXTRACTED")
        .map(PathBuf::from)
        .unwrap_or_else(|| {
            let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
            PathBuf::from(home).join("Documents/gc-extracted")
        });
    if !data.exists() {
        eprintln!("skipped: unpack the disc first");
        return;
    }

    let mut patches = 0usize;
    let mut ad_gifs = 0usize;
    let mut textures_seen = 0usize;
    let mut textures_total = 0usize;
    let mut levels = 0usize;

    for n in 0..27 {
        let dir = data.join(format!("level{n}"));
        let core = match std::fs::read(dir.join("core_data.bin")) {
            Ok(c) => c,
            Err(_) => continue,
        };
        let index = match std::fs::read(dir.join("core_index.bin")) {
            Ok(i) => i,
            Err(_) => continue,
        };
        let ch = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header");
        let textures = gc_formats::level_core::texture_table(&index, ch.tfrag_textures)
            .expect("texture table");
        assert!(!textures.is_empty(), "level{n}: no tfrag textures");
        // Indices are per-level, so the pair is what identifies a texture.
        let mut level_seen = std::collections::BTreeSet::new();

        let region =
            |off: usize| i32::from_le_bytes(index[off..off + 4].try_into().unwrap()) as usize;
        let start = region(0x08);
        let end = [0x0c_usize, 0x10, 0x14]
            .iter()
            .map(|&o| region(o))
            .filter(|&v| v > 0)
            .min()
            .unwrap_or(core.len());
        let block = &core[start..end];

        let header = gc_formats::tfrag::TfragsBlock::parse(block).expect("tfrags block header");
        let table = header.table_offset as usize;
        for i in 0..header.tfrag_count as usize {
            let th = header.header(block, i).expect("tfrag header");
            let data_base = table.wrapping_add(th.data as usize);
            if data_base >= block.len() {
                continue;
            }
            let g = gc_formats::tfrag::parse_tfrag(&block[data_base..], &th)
                .unwrap_or_else(|e| panic!("level{n} patch {i}: {e}"));

            assert_eq!(
                g.lights.len(),
                th.vert_count as usize,
                "level{n} patch {i}: {} light records for {} vertices",
                g.lights.len(),
                th.vert_count
            );
            assert_eq!(
                g.rgba.len(),
                th.rgba_size as usize * 4,
                "level{n} patch {i}: rgba"
            );

            for a in &g.ad_gifs {
                let t = a.texture_index();
                assert!(
                    t < textures.len(),
                    "level{n} patch {i}: texture {} outside the {} entry table",
                    t,
                    textures.len()
                );
                let tex = &textures[t];
                assert!(
                    tex.width > 0 && tex.height > 0 && tex.width <= 256 && tex.height <= 256,
                    "level{n} patch {i}: texture {t} is {}x{}",
                    tex.width,
                    tex.height
                );
                level_seen.insert(t);
                ad_gifs += 1;
            }
            patches += 1;
        }
        textures_seen += level_seen.len();
        textures_total += textures.len();
        levels += 1;
    }

    assert!(levels > 0, "no unpacked levels were found");
    assert!(patches > 18_000, "expected 18k+ patches, got {patches}");
    // Most of each level's texture table should be referenced by some tfrag. A
    // handful genuinely go unused; if that share grows, the ad-gif walk has
    // started missing primitives rather than the data having spare entries.
    let covered = textures_seen as f64 / textures_total as f64;
    assert!(
        covered > 0.85,
        "only {textures_seen} of {textures_total} level textures referenced ({:.1}%)",
        covered * 100.0
    );
    eprintln!(
        "{ad_gifs} ad-gifs over {patches} patches reference {textures_seen}/{textures_total} level textures"
    );
}