ReaperOfVeriod
4c4a78d402
assets_compressed_size and assets_decompressed_size at index offsets 0x88 and 0x8c are 9,738,678 and 16,969,296 on level 0 -- exactly the sizes of core_data before and after decompression. There is no separate asset container. The "asset WAD" is core_data, and offset_in_asset_wad indexes the decompressed form directly. Checked rather than assumed, on all 27 levels: the decompressed size equals core_data's exact length, every class offset is either 0 or inside it, and the moby, tie and shrub spans do not overlap each other. An offset of 0 means the slot has no geometry, not the start of the lump; level 0's ninth moby class is 4909 at offset 0. Two assumptions worth discarding while here -- the tables are not sorted by class number (level 0's run 0, 1, 2, 14 ... 4909) and the offsets within a table are not monotonic. Neither is needed, but both look true from the first few entries. The chunk descriptors after the named extents are not it: they exist on only 3 of 27 levels, because level 0's four named extents tile the container exactly. An asset blob there would be missing almost everywhere. What sits at the offsets is raw GS/VIF packet data rather than a further container -- moby[0] begins 00 00 00 00 3f 16 02 55 6f 6f and tie[0] 00 01 00 00 70 50 00 00, which read as packet headers rather than a magic number. None of the three is decoded yet and this does not start that. Moby is skeletal and is the largest single piece left before the rest of a level can draw. Also: the pipeline now writes each level's chunk descriptors where they exist, and LevelCoreHeader decodes the asset lump's two sizes.
396 lines
16 KiB
Rust
396 lines
16 KiB
Rust
//! Stage 2: unpack the disc into asset files.
|
|
//!
|
|
//! R&C2 lists its assets in the ISO 9660 filesystem, unlike R&C1, so
|
|
//! enumeration is straightforward: every entry under `G/` is a container whose
|
|
//! header holds a table of sector ranges, and each range is written out as a
|
|
//! file.
|
|
//!
|
|
//! Output goes outside the repository. `AGENTS.md` explains why, and the
|
|
//! pre-commit hook enforces it.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use gc_formats::global_wad::{GlobalWad, RangeKind};
|
|
use gc_formats::iso9660::IsoImage;
|
|
use gc_formats::level_data::{LevelDataHeader, LevelDataPart};
|
|
use gc_formats::level_wad::LevelHeader;
|
|
|
|
/// One unpacked asset, with the identity we record for verification.
|
|
pub struct Extracted {
|
|
/// Path relative to the output directory, e.g. `armor/0000.bin`.
|
|
pub path: String,
|
|
pub size: u64,
|
|
pub sha1: String,
|
|
}
|
|
|
|
/// Unpack `iso_path` into `out_dir`.
|
|
///
|
|
/// Returns every file written, sorted by path, so the caller can build a hash
|
|
/// table or verify against one.
|
|
pub fn unpack(iso_path: &Path, out_dir: &Path) -> Result<Vec<Extracted>, String> {
|
|
let iso = IsoImage::open(iso_path).map_err(|e| format!("cannot read the disc: {e}"))?;
|
|
std::fs::create_dir_all(out_dir)
|
|
.map_err(|e| format!("cannot create {}: {e}", out_dir.display()))?;
|
|
|
|
let mut written: BTreeMap<String, Extracted> = BTreeMap::new();
|
|
let mut skipped: Vec<(String, String)> = Vec::new();
|
|
|
|
for entry in &iso.files {
|
|
if entry.is_dir {
|
|
continue;
|
|
}
|
|
// `Z6TAIL.DUP` is zero-length padding the ISO 9660 filesystem needs.
|
|
if entry.name.starts_with("Z6TAIL") {
|
|
continue;
|
|
}
|
|
|
|
let relative = entry.name.trim_end_matches('/').to_string();
|
|
|
|
if !relative.starts_with("G/") {
|
|
// Not a container: the boot ELF, SYSTEM.CNF, RC2.HDR and the IOP
|
|
// image are copied through verbatim.
|
|
let data = gc_formats::iso9660::read_at(iso_path, entry.lba, entry.size)
|
|
.map_err(|e| format!("cannot read {}: {e}", entry.name))?;
|
|
write_file(out_dir, &relative, &data, &mut written);
|
|
continue;
|
|
}
|
|
|
|
// A container: write out each range its header declares.
|
|
let raw = gc_formats::iso9660::read_at(iso_path, entry.lba, entry.size)
|
|
.map_err(|e| format!("cannot read {}: {e}", entry.name))?;
|
|
|
|
let stem = relative
|
|
.trim_start_matches("G/")
|
|
.trim_end_matches(".WAD")
|
|
.to_lowercase();
|
|
|
|
// MPEG1 and MPEG2 are raw concatenated video, not indexed containers:
|
|
// their first word is video payload, not a header size. Copy them
|
|
// through; the video phase splits them properly.
|
|
if stem.starts_with("mpeg1") || stem.starts_with("mpeg2") {
|
|
write_file(out_dir, &format!("{stem}.bin"), &raw, &mut written);
|
|
println!(
|
|
" {:<16} {} bytes, raw video stream (not indexed)",
|
|
stem,
|
|
raw.len()
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// Level containers have a typed header whose extents use sector counts,
|
|
// not the byte counts of the generic `G/` pair table. Reading them
|
|
// through the generic path would take a 6,484-sector extent as 6,484
|
|
// bytes and silently lose 13 MB of every level, so they go through the
|
|
// typed parser.
|
|
if stem.starts_with("level")
|
|
&& stem
|
|
.trim_start_matches("level")
|
|
.chars()
|
|
.all(|c| c.is_ascii_digit())
|
|
{
|
|
match LevelHeader::parse(&raw) {
|
|
Ok(header) => {
|
|
let mut parts = 0usize;
|
|
for (part, range) in header.ordered_extents() {
|
|
let Some(offset) = range.byte_offset() else {
|
|
continue;
|
|
};
|
|
let len = range.byte_len() as usize;
|
|
if offset + len as u64 > raw.len() as u64 {
|
|
eprintln!(
|
|
" {stem}/{}: runs past the end of the container; skipped",
|
|
part.name()
|
|
);
|
|
continue;
|
|
}
|
|
let slice = &raw[offset as usize..offset as usize + len];
|
|
// `gameplay` is a WAD stream, like `core_data`. It was
|
|
// being written compressed, which is why the light bank
|
|
// inside it did not parse: a WAD header where a count
|
|
// was expected.
|
|
if matches!(part, gc_formats::level_wad::LevelPart::Gameplay)
|
|
&& gc_formats::wad::is_wad(slice)
|
|
{
|
|
match gc_formats::wad::decompress(slice) {
|
|
Ok(decoded) => {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/{}.bin", part.name()),
|
|
&decoded,
|
|
&mut written,
|
|
);
|
|
}
|
|
Err(e) => eprintln!(
|
|
" {stem}/{}: WAD did not decompress ({e}); the compressed \
|
|
form was kept",
|
|
part.name()
|
|
),
|
|
}
|
|
} else {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/{}.bin", part.name()),
|
|
slice,
|
|
&mut written,
|
|
);
|
|
}
|
|
parts += 1;
|
|
}
|
|
// Chunk descriptors follow the named extents. One of them
|
|
// is the level's asset WAD, which the moby/tie/shrub class
|
|
// tables point into by byte offset.
|
|
for (i, range) in header.present_chunks() {
|
|
let Some(offset) = range.byte_offset() else {
|
|
continue;
|
|
};
|
|
let len = range.byte_len() as usize;
|
|
if offset + len as u64 > raw.len() as u64 {
|
|
continue;
|
|
}
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/chunk{i}.bin"),
|
|
&raw[offset as usize..offset as usize + len],
|
|
&mut written,
|
|
);
|
|
parts += 1;
|
|
}
|
|
|
|
// The `data` extent is itself a container of byte ranges.
|
|
// Split out the geometry directory and the geometry, so the
|
|
// latter is usable without re-deriving the layout later.
|
|
// `data_extent` is the `data` slice; `raw` is the whole
|
|
// level container and its offsets are unrelated.
|
|
let data_extent: &[u8] = &raw[header.data.offset as usize * 2048
|
|
..(header.data.offset as usize + header.data.size as usize) * 2048];
|
|
match LevelDataHeader::parse(data_extent) {
|
|
Ok(data_header) => {
|
|
let core = data_header.part(LevelDataPart::CoreData);
|
|
let index = data_header.part(LevelDataPart::CoreIndex);
|
|
if let Ok(bytes) = core.slice(data_extent, data_extent.len()) {
|
|
// `core_data` is a WAD stream: keep both the
|
|
// compressed original and the decoded form,
|
|
// because the hash table covers the original.
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/core_data.wad"),
|
|
bytes,
|
|
&mut written,
|
|
);
|
|
match gc_formats::wad::decompress(bytes) {
|
|
Ok(decoded) => {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/core_data.bin"),
|
|
&decoded,
|
|
&mut written,
|
|
);
|
|
}
|
|
Err(e) => eprintln!(
|
|
" {stem}: core_data did not decompress ({e}); the \
|
|
compressed form was kept"
|
|
),
|
|
}
|
|
parts += 1;
|
|
}
|
|
if let Ok(bytes) = index.slice(data_extent, data_extent.len()) {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/core_index.bin"),
|
|
bytes,
|
|
&mut written,
|
|
);
|
|
parts += 1;
|
|
}
|
|
// The level's executable. It carries the normal
|
|
// table the tfrag lighting microcode indexes, so
|
|
// it is not optional background.
|
|
let overlay = data_header.part(LevelDataPart::Overlay);
|
|
if let Ok(bytes) = overlay.slice(data_extent, data_extent.len()) {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/overlay.bin"),
|
|
bytes,
|
|
&mut written,
|
|
);
|
|
parts += 1;
|
|
}
|
|
// Graphics Synthesizer RAM holds the CLUTs the tfrag
|
|
// texture table's palette indices point into, so it
|
|
// is needed to decode any level texture.
|
|
let gs = data_header.part(LevelDataPart::GsRam);
|
|
if let Ok(bytes) = gs.slice(data_extent, data_extent.len()) {
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/gs_ram.bin"),
|
|
bytes,
|
|
&mut written,
|
|
);
|
|
parts += 1;
|
|
}
|
|
}
|
|
Err(e) => eprintln!(" {stem}: data extent header did not parse: {e}"),
|
|
}
|
|
|
|
println!(
|
|
" {:<16} {} bytes, level id {} -> {} parts",
|
|
stem,
|
|
raw.len(),
|
|
header.id,
|
|
parts
|
|
);
|
|
}
|
|
Err(e) => {
|
|
skipped.push((relative.clone(), e.to_string()));
|
|
write_file(out_dir, &relative, &raw, &mut written);
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
let container = match GlobalWad::parse_with(&raw, RangeKind::for_container(&relative)) {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
// Some containers have a header this generic parser does not
|
|
// model (the scene containers nest further). Copy the file
|
|
// through untouched rather than failing the whole run: the
|
|
// bytes are preserved and a typed parser can be added later.
|
|
skipped.push((relative.clone(), e.to_string()));
|
|
write_file(out_dir, &relative, &raw, &mut written);
|
|
continue;
|
|
}
|
|
};
|
|
let present = container.present().count();
|
|
let mut out_of_range = 0usize;
|
|
|
|
for (index, offset, byte_len) in container.present() {
|
|
let len = byte_len as usize;
|
|
|
|
// Bounds-check against the container we actually read, so a corrupt
|
|
// header cannot make us read past the end of the file.
|
|
if offset + len as u64 > raw.len() as u64 {
|
|
out_of_range += 1;
|
|
continue;
|
|
}
|
|
let data = &raw[offset as usize..offset as usize + len];
|
|
write_file(
|
|
out_dir,
|
|
&format!("{stem}/{index:04}.bin"),
|
|
data,
|
|
&mut written,
|
|
);
|
|
}
|
|
if out_of_range > 0 {
|
|
eprintln!(
|
|
" {:<16} {} bytes, {} entries -> {} present, {out_of_range} out of range (header describes \
|
|
sectors beyond the file; see docs/plan/disc-pipeline.md)",
|
|
stem, raw.len(), container.entries.len(), present
|
|
);
|
|
} else {
|
|
println!(
|
|
" {:<16} {} bytes, {} entries -> {} present",
|
|
stem,
|
|
raw.len(),
|
|
container.entries.len(),
|
|
present
|
|
);
|
|
}
|
|
}
|
|
|
|
for (name, why) in &skipped {
|
|
eprintln!(" {name}: not an indexed container ({why}); copied verbatim");
|
|
}
|
|
|
|
Ok(written.into_values().collect())
|
|
}
|
|
|
|
fn write_file(
|
|
out_dir: &Path,
|
|
relative: &str,
|
|
data: &[u8],
|
|
written: &mut BTreeMap<String, Extracted>,
|
|
) {
|
|
use crate::identify::sha1_hex;
|
|
|
|
let dest = out_dir.join(relative);
|
|
if let Some(parent) = dest.parent() {
|
|
if let Err(e) = std::fs::create_dir_all(parent) {
|
|
eprintln!("warning: cannot create {}: {e}", parent.display());
|
|
return;
|
|
}
|
|
}
|
|
if let Err(e) = std::fs::write(&dest, data) {
|
|
eprintln!("warning: cannot write {}: {e}", dest.display());
|
|
return;
|
|
}
|
|
written.insert(
|
|
relative.to_string(),
|
|
Extracted {
|
|
path: relative.to_string(),
|
|
size: data.len() as u64,
|
|
sha1: sha1_hex(data),
|
|
},
|
|
);
|
|
}
|
|
|
|
/// Default output directory: `extracted/` beside the repository root.
|
|
pub fn default_out_dir() -> PathBuf {
|
|
PathBuf::from("extracted")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// A container with three entries, the second of which is absent.
|
|
fn container_bytes() -> Vec<u8> {
|
|
let header_size = 8 + 3 * 8;
|
|
let mut v = Vec::new();
|
|
v.extend_from_slice(&(header_size as u32).to_le_bytes());
|
|
v.extend_from_slice(&10u32.to_le_bytes());
|
|
v.extend_from_slice(&1i32.to_le_bytes());
|
|
v.extend_from_slice(&1i32.to_le_bytes());
|
|
v.extend_from_slice(&2i32.to_le_bytes());
|
|
v.extend_from_slice(&0i32.to_le_bytes()); // absent
|
|
v.extend_from_slice(&2i32.to_le_bytes());
|
|
v.extend_from_slice(&1i32.to_le_bytes());
|
|
v.resize(header_size + 3 * 2048, 0xAB);
|
|
v
|
|
}
|
|
|
|
#[test]
|
|
fn parses_a_container_and_lists_present_entries() {
|
|
let raw = container_bytes();
|
|
let c = GlobalWad::parse(&raw).unwrap();
|
|
assert_eq!(c.entries.len(), 3);
|
|
let present: Vec<usize> = c.present().map(|(i, _, _)| i).collect();
|
|
assert_eq!(present, vec![0, 2]);
|
|
// The default reading treats the second field as a byte count, so an
|
|
// entry of size 1 is one byte.
|
|
assert_eq!(c.present().next().unwrap().2, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn every_declared_range_is_inside_the_container() {
|
|
let raw = container_bytes();
|
|
let c = GlobalWad::parse(&raw).unwrap();
|
|
for (_, offset, len) in c.present() {
|
|
assert!(
|
|
offset + len <= raw.len() as u64,
|
|
"range at {offset} runs past the end"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_range_past_the_end_would_be_caught() {
|
|
let mut raw = container_bytes();
|
|
// Push the first entry's offset far beyond the file.
|
|
raw[8..12].copy_from_slice(&9999i32.to_le_bytes());
|
|
let c = GlobalWad::parse(&raw).unwrap();
|
|
let (_, offset, len) = c.present().next().unwrap();
|
|
assert!(offset + len > raw.len() as u64);
|
|
}
|
|
}
|