ReaperOfVeriod
16c04db559
A defect in the Phase 1 extractor, found by reading the data extent's contents: level extents are declared in sector counts, the generic G/ pair table uses byte counts, and level containers were being routed through the generic path. So LEVEL0.WAD's data extent -- 6,484 sectors -- was taken as 6,484 bytes. Every test still passed, because a truncated file is still a file. The extent's own header gave it away: it declares an uncompressed total of 13,277,920 bytes, and 13,277,920 / 6,484 is 2047.7. A ratio landing on 2048 is a unit error, not a compression ratio. Across 27 levels this had been discarding roughly 300 MB of level content. Level containers now go through the typed LevelHeader instead. Level 0 yields: data.bin 13,279,232 the data extent core_data.wad 9,738,678 the geometry, compressed core_data.bin 16,969,296 the geometry, decompressed core_index.bin 31,600 the geometry directory gameplay.bin 659,456 sound_bank.bin 1,503,232 core_data is a WAD LZ77 stream -- the format already implemented for the game -- and it decompresses cleanly. The first real level content this project has recovered. New: LevelDataHeader parses the data extent's own header, a table of byte ranges naming the level overlay executable, the geometry directory, the GS RAM image, the HUD banks and the geometry. Note this is a third meaning for a "size" field on the same disc, after sector counts in the level header and byte counts in the generic table; all three are documented together in level-data.md so the next parser does not rediscover this the hard way. The geometry directory's offsets were checked against the geometry rather than taken on trust: all twelve land inside the 16.9 MB of decompressed core_data, which is what confirms the header is being read correctly. The class tables themselves do not yet decode under the documented layout, and that is recorded as open work in plan/level-assets.md rather than papered over. 75 unit tests, 9 integration tests against the real disc. Hash table regenerated at 5,159 files and 5,671,915,799 bytes -- hashes only, no content. Extraction now writes to ~/Documents/gc-extracted rather than /tmp, which is 16 GB and did not have room for a 5.3 GB output.
325 lines
13 KiB
Rust
325 lines
13 KiB
Rust
//! Stage 2: unpack the disc into asset files.
|
|
//!
|
|
//! R&C2 lists its assets in the ISO 9660 filesystem, unlike R&C1, so
|
|
//! enumeration is straightforward: every entry under `G/` is a container whose
|
|
//! header holds a table of sector ranges, and each range is written out as a
|
|
//! file.
|
|
//!
|
|
//! Output goes outside the repository. `AGENTS.md` explains why, and the
|
|
//! pre-commit hook enforces it.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use gc_formats::global_wad::{GlobalWad, RangeKind};
|
|
use gc_formats::iso9660::IsoImage;
|
|
use gc_formats::level_data::{LevelDataHeader, LevelDataPart};
|
|
use gc_formats::level_wad::LevelHeader;
|
|
|
|
/// One unpacked asset, with the identity we record for verification.
|
|
pub struct Extracted {
|
|
/// Path relative to the output directory, e.g. `armor/0000.bin`.
|
|
pub path: String,
|
|
pub size: u64,
|
|
pub sha1: String,
|
|
}
|
|
|
|
/// Unpack `iso_path` into `out_dir`.
|
|
///
|
|
/// Returns every file written, sorted by path, so the caller can build a hash
|
|
/// table or verify against one.
|
|
pub fn unpack(iso_path: &Path, out_dir: &Path) -> Result<Vec<Extracted>, String> {
|
|
let iso = IsoImage::open(iso_path).map_err(|e| format!("cannot read the disc: {e}"))?;
|
|
std::fs::create_dir_all(out_dir)
|
|
.map_err(|e| format!("cannot create {}: {e}", out_dir.display()))?;
|
|
|
|
let mut written: BTreeMap<String, Extracted> = BTreeMap::new();
|
|
let mut skipped: Vec<(String, String)> = Vec::new();
|
|
|
|
for entry in &iso.files {
|
|
if entry.is_dir {
|
|
continue;
|
|
}
|
|
// `Z6TAIL.DUP` is zero-length padding the ISO 9660 filesystem needs.
|
|
if entry.name.starts_with("Z6TAIL") {
|
|
continue;
|
|
}
|
|
|
|
let relative = entry.name.trim_end_matches('/').to_string();
|
|
|
|
if !relative.starts_with("G/") {
|
|
// Not a container: the boot ELF, SYSTEM.CNF, RC2.HDR and the IOP
|
|
// image are copied through verbatim.
|
|
let data = gc_formats::iso9660::read_at(iso_path, entry.lba, entry.size)
|
|
.map_err(|e| format!("cannot read {}: {e}", entry.name))?;
|
|
write_file(out_dir, &relative, &data, &mut written);
|
|
continue;
|
|
}
|
|
|
|
// A container: write out each range its header declares.
|
|
let raw = gc_formats::iso9660::read_at(iso_path, entry.lba, entry.size)
|
|
.map_err(|e| format!("cannot read {}: {e}", entry.name))?;
|
|
|
|
let stem = relative
|
|
.trim_start_matches("G/")
|
|
.trim_end_matches(".WAD")
|
|
.to_lowercase();
|
|
|
|
// MPEG1 and MPEG2 are raw concatenated video, not indexed containers:
|
|
// their first word is video payload, not a header size. Copy them
|
|
// through; the video phase splits them properly.
|
|
if stem.starts_with("mpeg1") || stem.starts_with("mpeg2") {
|
|
write_file(out_dir, &format!("{stem}.bin"), &raw, &mut written);
|
|
println!(
|
|
" {:<16} {} bytes, raw video stream (not indexed)",
|
|
stem,
|
|
raw.len()
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// Level containers have a typed header whose extents use sector counts,
|
|
// not the byte counts of the generic `G/` pair table. Reading them
|
|
// through the generic path would take a 6,484-sector extent as 6,484
|
|
// bytes and silently lose 13 MB of every level, so they go through the
|
|
// typed parser.
|
|
if stem.starts_with("level")
|
|
&& stem
|
|
.trim_start_matches("level")
|
|
.chars()
|
|
.all(|c| c.is_ascii_digit())
|
|
{
|
|
match LevelHeader::parse(&raw) {
|
|
Ok(header) => {
|
|
let mut parts = 0usize;
|
|
for (part, range) in header.ordered_extents() {
|
|
let Some(offset) = range.byte_offset() else {
|
|
continue;
|
|
};
|
|
let len = range.byte_len() as usize;
|
|
if offset + len as u64 > raw.len() as u64 {
|
|
eprintln!(
|
|
" {stem}/{}: runs past the end of the container; skipped",
|
|
part.name()
|
|
);
|
|
continue;
|
|
}
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/{}.bin", part.name()),
|
|
&raw[offset as usize..offset as usize + len],
|
|
&mut written,
|
|
);
|
|
parts += 1;
|
|
}
|
|
// The `data` extent is itself a container of byte ranges.
|
|
// Split out the geometry directory and the geometry, so the
|
|
// latter is usable without re-deriving the layout later.
|
|
// `data_extent` is the `data` slice; `raw` is the whole
|
|
// level container and its offsets are unrelated.
|
|
let data_extent: &[u8] = &raw[header.data.offset as usize * 2048
|
|
..(header.data.offset as usize + header.data.size as usize) * 2048];
|
|
match LevelDataHeader::parse(data_extent) {
|
|
Ok(data_header) => {
|
|
let core = data_header.part(LevelDataPart::CoreData);
|
|
let index = data_header.part(LevelDataPart::CoreIndex);
|
|
if let Ok(bytes) = core.slice(data_extent, data_extent.len()) {
|
|
// `core_data` is a WAD stream: keep both the
|
|
// compressed original and the decoded form,
|
|
// because the hash table covers the original.
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/core_data.wad"),
|
|
bytes,
|
|
&mut written,
|
|
);
|
|
match gc_formats::wad::decompress(bytes) {
|
|
Ok(decoded) => {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/core_data.bin"),
|
|
&decoded,
|
|
&mut written,
|
|
);
|
|
}
|
|
Err(e) => eprintln!(
|
|
" {stem}: core_data did not decompress ({e}); the \
|
|
compressed form was kept"
|
|
),
|
|
}
|
|
parts += 1;
|
|
}
|
|
if let Ok(bytes) = index.slice(data_extent, data_extent.len()) {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/core_index.bin"),
|
|
bytes,
|
|
&mut written,
|
|
);
|
|
parts += 1;
|
|
}
|
|
}
|
|
Err(e) => eprintln!(" {stem}: data extent header did not parse: {e}"),
|
|
}
|
|
|
|
println!(
|
|
" {:<16} {} bytes, level id {} -> {} parts",
|
|
stem,
|
|
raw.len(),
|
|
header.id,
|
|
parts
|
|
);
|
|
}
|
|
Err(e) => {
|
|
skipped.push((relative.clone(), e.to_string()));
|
|
write_file(out_dir, &relative, &raw, &mut written);
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
let container = match GlobalWad::parse_with(&raw, RangeKind::for_container(&relative)) {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
// Some containers have a header this generic parser does not
|
|
// model (the scene containers nest further). Copy the file
|
|
// through untouched rather than failing the whole run: the
|
|
// bytes are preserved and a typed parser can be added later.
|
|
skipped.push((relative.clone(), e.to_string()));
|
|
write_file(out_dir, &relative, &raw, &mut written);
|
|
continue;
|
|
}
|
|
};
|
|
let present = container.present().count();
|
|
let mut out_of_range = 0usize;
|
|
|
|
for (index, offset, byte_len) in container.present() {
|
|
let len = byte_len as usize;
|
|
|
|
// Bounds-check against the container we actually read, so a corrupt
|
|
// header cannot make us read past the end of the file.
|
|
if offset + len as u64 > raw.len() as u64 {
|
|
out_of_range += 1;
|
|
continue;
|
|
}
|
|
let data = &raw[offset as usize..offset as usize + len];
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/{index:04}.bin"),
|
|
data,
|
|
&mut written,
|
|
);
|
|
}
|
|
if out_of_range > 0 {
|
|
eprintln!(
|
|
" {:<16} {} bytes, {} entries -> {} present, {out_of_range} out of range (header describes \
|
|
sectors beyond the file; see docs/plan/disc-pipeline.md)",
|
|
stem, raw.len(), container.entries.len(), present
|
|
);
|
|
} else {
|
|
println!(
|
|
" {:<16} {} bytes, {} entries -> {} present",
|
|
stem,
|
|
raw.len(),
|
|
container.entries.len(),
|
|
present
|
|
);
|
|
}
|
|
}
|
|
|
|
for (name, why) in &skipped {
|
|
eprintln!(" {name}: not an indexed container ({why}); copied verbatim");
|
|
}
|
|
|
|
Ok(written.into_values().collect())
|
|
}
|
|
|
|
fn write_file(
|
|
out_dir: &Path,
|
|
relative: &str,
|
|
data: &[u8],
|
|
written: &mut BTreeMap<String, Extracted>,
|
|
) {
|
|
use crate::identify::sha1_hex;
|
|
|
|
let dest = out_dir.join(relative);
|
|
if let Some(parent) = dest.parent() {
|
|
if let Err(e) = std::fs::create_dir_all(parent) {
|
|
eprintln!("warning: cannot create {}: {e}", parent.display());
|
|
return;
|
|
}
|
|
}
|
|
if let Err(e) = std::fs::write(&dest, data) {
|
|
eprintln!("warning: cannot write {}: {e}", dest.display());
|
|
return;
|
|
}
|
|
written.insert(
|
|
relative.to_string(),
|
|
Extracted {
|
|
path: relative.to_string(),
|
|
size: data.len() as u64,
|
|
sha1: sha1_hex(data),
|
|
},
|
|
);
|
|
}
|
|
|
|
/// Default output directory: `extracted/` beside the repository root.
|
|
pub fn default_out_dir() -> PathBuf {
|
|
PathBuf::from("extracted")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// A container with three entries, the second of which is absent.
|
|
fn container_bytes() -> Vec<u8> {
|
|
let header_size = 8 + 3 * 8;
|
|
let mut v = Vec::new();
|
|
v.extend_from_slice(&(header_size as u32).to_le_bytes());
|
|
v.extend_from_slice(&10u32.to_le_bytes());
|
|
v.extend_from_slice(&1i32.to_le_bytes());
|
|
v.extend_from_slice(&1i32.to_le_bytes());
|
|
v.extend_from_slice(&2i32.to_le_bytes());
|
|
v.extend_from_slice(&0i32.to_le_bytes()); // absent
|
|
v.extend_from_slice(&2i32.to_le_bytes());
|
|
v.extend_from_slice(&1i32.to_le_bytes());
|
|
v.resize(header_size + 3 * 2048, 0xAB);
|
|
v
|
|
}
|
|
|
|
#[test]
|
|
fn parses_a_container_and_lists_present_entries() {
|
|
let raw = container_bytes();
|
|
let c = GlobalWad::parse(&raw).unwrap();
|
|
assert_eq!(c.entries.len(), 3);
|
|
let present: Vec<usize> = c.present().map(|(i, _, _)| i).collect();
|
|
assert_eq!(present, vec![0, 2]);
|
|
// The default reading treats the second field as a byte count, so an
|
|
// entry of size 1 is one byte.
|
|
assert_eq!(c.present().next().unwrap().2, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn every_declared_range_is_inside_the_container() {
|
|
let raw = container_bytes();
|
|
let c = GlobalWad::parse(&raw).unwrap();
|
|
for (_, offset, len) in c.present() {
|
|
assert!(
|
|
offset + len <= raw.len() as u64,
|
|
"range at {offset} runs past the end"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_range_past_the_end_would_be_caught() {
|
|
let mut raw = container_bytes();
|
|
// Push the first entry's offset far beyond the file.
|
|
raw[8..12].copy_from_slice(&9999i32.to_le_bytes());
|
|
let c = GlobalWad::parse(&raw).unwrap();
|
|
let (_, offset, len) = c.present().next().unwrap();
|
|
assert!(offset + len > raw.len() as u64);
|
|
}
|
|
}
|