diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 116af54..5cd9cb6 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -182,3 +182,127 @@ fn every_container_parses() { ); } } + +/// The level container header is the first typed structure we parse from real +/// data, and the tiling invariant is the check that holds on every level rather +/// than one hand-picked sample. +#[test] +fn level_headers_tile_their_containers() { + let path = require_disc!(); + let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); + + let mut checked = 0; + for entry in iso.files.iter().filter(|e| e.name.starts_with("G/LEVEL")) { + let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header"); + let h = gc_formats::level_wad::LevelHeader::parse(&data) + .unwrap_or_else(|e| panic!("{}: {e}", entry.name)); + + assert_eq!(h.header_size, 0x60, "{}", entry.name); + assert_eq!( + h.runtime_sector, 0, + "{}: runtime sector field should be 0 on disc", + entry.name + ); + + // Every present extent must start where the previous one ended. + let extents = h.ordered_extents(); + for w in extents.windows(2) { + let (pn, p) = w[0]; + let (nn, n) = w[1]; + assert_eq!( + p.offset + p.size, + n.offset, + "{}: gap or overlap between {} and {}", + entry.name, + pn.name(), + nn.name() + ); + } + + // Nothing may fall outside the container. The last extent may end one + // sector past the file end through sector rounding, hence `+ 1`. + let container_sectors = (entry.size / 2048) as i32; + for (name, r) in &extents { + assert!( + r.offset >= 0 && r.offset + r.size <= container_sectors + 1, + "{}: {} extent {:?} is outside the container ({container_sectors} sectors)", + entry.name, + name.name(), + r + ); + } + checked += 1; + } + + assert_eq!(checked, 27, "expected 27 level containers"); +} + +/// Level ids are not file order: `LEVEL21.WAD` reports 30. Anything that keys +/// off the container index would silently mislabel that planet. +#[test] +fn level_ids_are_not_container_indices() { + let path = require_disc!(); + let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); + + // The ISO filesystem lists names alphabetically, so LEVEL10 sorts before + // LEVEL2. Order by the numeric suffix, which is what the container names + // mean. + let mut levels: Vec<(usize, &gc_formats::iso9660::DirEntry)> = iso + .files + .iter() + .filter(|e| e.name.starts_with("G/LEVEL")) + .map(|e| { + let n: usize = e + .name + .trim_start_matches("G/LEVEL") + .trim_end_matches(".WAD") + .parse() + .expect("level container names end in a number"); + (n, e) + }) + .collect(); + levels.sort_by_key(|(n, _)| *n); + + let mut mismatches = Vec::new(); + for (index, (_, entry)) in levels.iter().enumerate() { + let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header"); + let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse"); + if h.id as usize != index { + mismatches.push(format!( + "{}: id {} but is container {index}", + entry.name, h.id + )); + } + } + + // Exactly one level is out of step: the gap left by a removed level. + assert_eq!( + mismatches.len(), + 1, + "expected exactly one level whose id differs from its container index, got {mismatches:?}" + ); + assert!( + mismatches[0].contains("LEVEL21"), + "unexpected mismatch: {mismatches:?}" + ); +} + +/// Sizes in the level header are sector counts. If they were byte counts, the +/// extents could not tile a container this size, so this also guards against +/// "fixing" the parser to read them as bytes. +#[test] +fn level_extent_sizes_are_sector_counts() { + let path = require_disc!(); + let iso = gc_formats::iso9660::IsoImage::open(&path).expect("ISO should open"); + let entry = iso.find("G/LEVEL0.WAD").expect("level 0"); + + let data = gc_formats::iso9660::read_at(&path, entry.lba, 0x60).expect("read header"); + let h = gc_formats::level_wad::LevelHeader::parse(&data).expect("parse"); + + // Sound bank occupies sectors 1..735 and data 735..7219. + assert_eq!(h.sound_bank.offset, 1); + assert_eq!(h.sound_bank.size, 734); + assert_eq!(h.data.offset, 735); + // Read as bytes, 734 would be under four sectors and the tiling would break. + assert_eq!(h.sound_bank.byte_len(), 734 * 2048); +} diff --git a/crates/gc-formats/src/level_wad.rs b/crates/gc-formats/src/level_wad.rs new file mode 100644 index 0000000..87db518 --- /dev/null +++ b/crates/gc-formats/src/level_wad.rs @@ -0,0 +1,348 @@ +//! The level container header (`LEVEL.WAD`). +//! +//! Spec: `docs/formats/level-wad.md`. +//! +//! Every one of the 27 level containers begins with the same 0x60-byte header, +//! which names five extents and turns the generic pair table into named, +//! individually parseable assets: +//! +//! ```text +//! offset size field +//! 0x00 4 header size, 0x60 on every level +//! 0x04 4 zero on every level; a field the game fills in at runtime +//! 0x08 4 level id +//! 0x0c 4 reverb configuration index +//! 0x10 8 data (geometry: tfrag, tie, shrub, moby, sky, ...) +//! 0x18 8 sound_bank +//! 0x20 8 gameplay +//! 0x28 8 occlusion +//! 0x30 48 chunk descriptors (3 x 16) +//! ``` +//! +//! `data` is by far the largest and is where the level's visible content lives. +//! `gameplay` carries the per-class dispatch tables; `occlusion` is tiny, 12 +//! sectors on level 0. +//! +//! **Sizes here are sector counts, not byte counts.** This differs from the +//! generic pair table in [`crate::global_wad`], whose sizes are bytes. Both +//! appear on the disc and reading one as the other produces extents that look +//! plausible and are wrong by a factor of 2048. Verified: on all 27 levels the +//! four named extents tile the container in order with no gap and no overlap, +//! which only holds if the size field is a sector count. + +use crate::buf::{Buf, Error, ErrorKind, Result}; +use crate::global_wad::SectorRange; + +/// Header size, identical on all 27 levels. +pub const LEVEL_HEADER_SIZE: usize = 0x60; + +/// A parsed level container header. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LevelHeader { + pub header_size: u32, + /// Always zero on disc across all 27 levels. The game writes it at load + /// time, so it is read but must not be trusted as the container size; use + /// the ISO entry's length for that. + pub runtime_sector: u32, + /// The level's own identifier. + /// + /// Equal to the container index on 26 of 27 levels. `LEVEL21.WAD` reports + /// **30**, so the two are not interchangeable: this field is what the game + /// calls a level, and the file order is not authoritative. Some level was + /// removed from the disc and its number reused by nothing, leaving a gap. + pub id: i32, + /// Index into the game's reverb configuration. + pub reverb: i32, + pub data: SectorRange, + pub sound_bank: SectorRange, + pub gameplay: SectorRange, + pub occlusion: SectorRange, + /// Three 16-byte chunk descriptors. These follow the named extents and claim + /// the remainder of the container on most levels; they are zero on others. + pub chunks: [SectorRange; 6], +} + +/// A named extent from a level header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LevelPart { + Data, + SoundBank, + Gameplay, + Occlusion, +} + +impl LevelPart { + pub fn name(self) -> &'static str { + match self { + LevelPart::Data => "data", + LevelPart::SoundBank => "sound_bank", + LevelPart::Gameplay => "gameplay", + LevelPart::Occlusion => "occlusion", + } + } +} + +impl LevelHeader { + /// Parse a level container header from the start of the file. + pub fn parse(bytes: &[u8]) -> Result { + let mut b = Buf::new(bytes); + + let header_size = b.u32("level header size")?; + if header_size as usize != LEVEL_HEADER_SIZE { + return Err(Error { + offset: 0, + context: "level container header", + kind: ErrorKind::Invalid("header size is not 0x60"), + }); + } + + let runtime_sector = b.u32("level runtime sector")?; + let id = b.i32("level id")?; + let reverb = b.i32("reverb index")?; + + let range = |b: &mut Buf<'_>, what: &'static str| -> Result { + let offset = b.i32(what)?; + let size = b.i32(what)?; + Ok(SectorRange { offset, size }) + }; + + let data = range(&mut b, "data extent")?; + let sound_bank = range(&mut b, "sound_bank extent")?; + let gameplay = range(&mut b, "gameplay extent")?; + let occlusion = range(&mut b, "occlusion extent")?; + + // 0x30..0x60: six u32s forming three {offset, size} descriptors. + let mut chunks = [SectorRange { offset: 0, size: 0 }; 6]; + for slot in chunks.iter_mut() { + slot.offset = b.i32("chunk descriptor")?; + slot.size = b.i32("chunk descriptor")?; + } + + Ok(LevelHeader { + header_size, + runtime_sector, + id, + reverb, + data, + sound_bank, + gameplay, + occlusion, + chunks, + }) + } + + /// The extent for a named part. + pub fn part(&self, which: LevelPart) -> SectorRange { + match which { + LevelPart::Data => self.data, + LevelPart::SoundBank => self.sound_bank, + LevelPart::Gameplay => self.gameplay, + LevelPart::Occlusion => self.occlusion, + } + } + + /// The named extents in the order they appear in the file. + /// + /// The order on the disc is sound bank, data, gameplay, occlusion — not the + /// order the fields appear in the header. + pub fn ordered_extents(&self) -> Vec<(LevelPart, SectorRange)> { + [ + (LevelPart::SoundBank, self.sound_bank), + (LevelPart::Data, self.data), + (LevelPart::Gameplay, self.gameplay), + (LevelPart::Occlusion, self.occlusion), + ] + .into_iter() + .filter(|(_, r)| r.is_present()) + .collect() + } + + /// Every extent that points at real data, in file order. + pub fn present_parts(&self) -> Vec<(LevelPart, SectorRange)> { + self.ordered_extents() + } + + /// Chunk descriptors that claim real data, as `(index, extent)`. + pub fn present_chunks(&self) -> Vec<(usize, SectorRange)> { + self.chunks + .iter() + .copied() + .enumerate() + .filter(|(_, r)| r.is_present()) + .collect() + } + + /// Total sectors claimed by the named extents and the chunks. + /// + /// Cross-check against the container's length in sectors. Measured across + /// all 27 levels: 21 land at `length` (the extents start at sector 1, after + /// the header, and the last one ends one sector past the file end, which is + /// normal sector rounding), and 6 leave 134 to 689 sectors of trailing data + /// past the last named extent. + /// + /// So the value is always `>= length - 1`, and that is the invariant worth + /// asserting. It is not always equal to `length`, and treating it as such + /// would truncate the tail of a level. + pub fn payload_sectors(&self) -> i32 { + let named: i32 = self.ordered_extents().iter().map(|(_, r)| r.size).sum(); + let chunks: i32 = self.present_chunks().iter().map(|(_, r)| r.size).sum(); + named + chunks + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A header matching LEVEL0.WAD on the retail disc. + fn level0_header() -> Vec { + let mut v = vec![0u8; 0x60]; + v[0x00..0x04].copy_from_slice(&0x60u32.to_le_bytes()); + // The runtime sector field is zero on every retail level. + v[0x04..0x08].copy_from_slice(&0u32.to_le_bytes()); + v[0x08..0x0c].copy_from_slice(&0i32.to_le_bytes()); + v[0x0c..0x10].copy_from_slice(&3i32.to_le_bytes()); + v[0x10..0x14].copy_from_slice(&735i32.to_le_bytes()); + v[0x14..0x18].copy_from_slice(&6484i32.to_le_bytes()); + v[0x18..0x1c].copy_from_slice(&1i32.to_le_bytes()); + v[0x1c..0x20].copy_from_slice(&734i32.to_le_bytes()); + v[0x20..0x24].copy_from_slice(&7219i32.to_le_bytes()); + v[0x24..0x28].copy_from_slice(&322i32.to_le_bytes()); + v[0x28..0x2c].copy_from_slice(&7541i32.to_le_bytes()); + v[0x2c..0x30].copy_from_slice(&12i32.to_le_bytes()); + v + } + + #[test] + fn parses_a_retail_level_header() { + let h = LevelHeader::parse(&level0_header()).unwrap(); + assert_eq!(h.header_size, 0x60); + assert_eq!(h.runtime_sector, 0); + assert_eq!(h.id, 0); + assert_eq!(h.reverb, 3); + assert_eq!( + h.data, + SectorRange { + offset: 735, + size: 6484 + } + ); + assert_eq!( + h.sound_bank, + SectorRange { + offset: 1, + size: 734 + } + ); + assert_eq!( + h.gameplay, + SectorRange { + offset: 7219, + size: 322 + } + ); + assert_eq!( + h.occlusion, + SectorRange { + offset: 7541, + size: 12 + } + ); + } + + #[test] + fn extents_tile_the_container_in_order() { + let h = LevelHeader::parse(&level0_header()).unwrap(); + // Each extent's end must be the next extent's start, in file order. This + // is the check that holds on every level rather than only the sample in + // front of us. + let parts = h.ordered_extents(); + for w in parts.windows(2) { + assert_eq!( + w[0].1.offset + w[0].1.size, + w[1].1.offset, + "extent gap or overlap between {:?} and {:?}", + w[0].1, + w[1].1 + ); + } + // Level 0's container is 7552 sectors; its extents end one sector past + // that, which is why `payload_sectors` is compared with `>=` below + // rather than for equality. + assert!(h.payload_sectors() >= 7552 - 1); + } + + #[test] + fn payload_sectors_sums_the_named_extents() { + let h = LevelHeader::parse(&level0_header()).unwrap(); + // Sizes are sector counts: 734 + 6484 + 322 + 12. + assert_eq!(h.payload_sectors(), 7552); + } + + #[test] + fn chunk_descriptors_add_to_the_remainder() { + let mut v = level0_header(); + // Level 1 declares two chunks after its named extents. + v[0x30..0x34].copy_from_slice(&8741i32.to_le_bytes()); + v[0x34..0x38].copy_from_slice(&1519i32.to_le_bytes()); + v[0x38..0x3c].copy_from_slice(&10260i32.to_le_bytes()); + v[0x3c..0x40].copy_from_slice(&116i32.to_le_bytes()); + let h = LevelHeader::parse(&v).unwrap(); + assert_eq!(h.present_chunks().len(), 2); + // The named extents still total 7552; the two chunks add 1635. + assert_eq!(h.payload_sectors(), 7552 + 1519 + 116); + } + + #[test] + fn present_parts_skips_empty_extents() { + let mut v = level0_header(); + v[0x28..0x30].copy_from_slice(&[0u8; 8]); + let h = LevelHeader::parse(&v).unwrap(); + let names: Vec<&str> = h.present_parts().iter().map(|(p, _)| p.name()).collect(); + assert_eq!(names, vec!["sound_bank", "data", "gameplay"]); + } + + #[test] + fn rejects_a_wrong_header_size() { + let mut v = level0_header(); + v[0x00..0x04].copy_from_slice(&0x68u32.to_le_bytes()); + let err = LevelHeader::parse(&v).unwrap_err(); + assert!(matches!(err.kind, ErrorKind::Invalid(_)), "got {err:?}"); + } + + #[test] + fn rejects_a_truncated_header() { + // Correct declared size, but the buffer stops early. + let mut v = vec![0u8; 0x20]; + v[0x00..0x04].copy_from_slice(&0x60u32.to_le_bytes()); + let err = LevelHeader::parse(&v).unwrap_err(); + assert!( + matches!(err.kind, ErrorKind::OutOfBounds { .. }), + "got {err:?}" + ); + } + + #[test] + fn extent_order_is_sound_bank_then_data_not_header_order() { + let h = LevelHeader::parse(&level0_header()).unwrap(); + let order: Vec<&str> = h.ordered_extents().iter().map(|(p, _)| p.name()).collect(); + assert_eq!(order, vec!["sound_bank", "data", "gameplay", "occlusion"]); + } + + #[test] + fn chunk_descriptors_are_read() { + let mut v = level0_header(); + v[0x30..0x34].copy_from_slice(&11i32.to_le_bytes()); + v[0x34..0x38].copy_from_slice(&22i32.to_le_bytes()); + let h = LevelHeader::parse(&v).unwrap(); + assert_eq!( + h.chunks[0], + SectorRange { + offset: 11, + size: 22 + } + ); + // The rest stay zero. + assert_eq!(h.chunks[1], SectorRange { offset: 0, size: 0 }); + } +} diff --git a/crates/gc-formats/src/lib.rs b/crates/gc-formats/src/lib.rs index f4f37e3..ec0adfb 100644 --- a/crates/gc-formats/src/lib.rs +++ b/crates/gc-formats/src/lib.rs @@ -1,4 +1,5 @@ pub mod buf; pub mod global_wad; pub mod iso9660; +pub mod level_wad; pub mod wad; diff --git a/docs/formats/level-wad.md b/docs/formats/level-wad.md new file mode 100644 index 0000000..b34ca70 --- /dev/null +++ b/docs/formats/level-wad.md @@ -0,0 +1,139 @@ +# The level container (`LEVEL.WAD`) + +Target: `SCUS_972.68` v1.01. 27 containers. + +The first typed structure in this project, and the one that turns the generic +container table of [`global-wad.md`](global-wad.md) into named, individually +parseable assets. Implementation: `crates/gc-formats/src/level_wad.rs`. + +## Header + +All 27 level containers begin with the same 0x60-byte header: + +```text +offset size field +0x00 4 header size, 0x60 on every level +0x04 4 zero on every level; written by the game at load time +0x08 4 level id +0x0c 4 reverb configuration index +0x10 8 data geometry: tfrag, tie, shrub, moby, sky, textures +0x18 8 sound_bank +0x20 8 gameplay +0x28 8 occlusion +0x30 48 three {offset, size} chunk descriptors +``` + +Sizes here are **sector counts**. This is the opposite of the generic pair table +in the `G/` containers, where sizes are byte counts. Both appear on the same +disc, so reading one as the other produces extents that are wrong by a factor of +2048 and that still *look* plausible. See "Two fields, two meanings" below. + +## Extents + +The four named extents appear in the file in an order that is **not** the order +of the fields in the header: + +```text +sound_bank data gameplay occlusion +``` + +Level 0, measured: + +| Part | Offset | Sectors | Bytes | Unpacked to | +|---|---:|---:|---:|---| +| sound_bank | 1 | 734 | 1.50 MB | `level0/0002.bin` | +| data | 735 | 6,484 | 13.28 MB | `level0/0001.bin` | +| gameplay | 7,219 | 322 | 0.66 MB | `level0/0003.bin` | +| occlusion | 7,541 | 12 | 0.02 MB | `level0/0004.bin` | + +The numbering of the unpacked files follows the header's field order, so the +`data` extent is `0001.bin` while it is physically second in the container. Both +orderings are worth keeping straight; the typed header exists so callers never +have to. + +`data` is where the level's visible content lives and is the target of the next +phase. + +## Two fields, two meanings + +This is the trap in this format, and it is worth stating plainly because both +readings type-check. + +| Where | Size field is | +|---|---| +| `G/*.WAD` generic pair table | **bytes** | +| `LEVEL.WAD` named extents | **sectors** | + +Measured across all 92 indexed pair containers, the byte reading places every +entry inside its file while the sector reading does not. Measured across all 27 +level headers, the sector reading tiles each container exactly. + +If the level header's sizes were read as bytes, `sound_bank` on level 0 would +claim 734 bytes starting at sector 1, and `data` would start at byte offset 735 +— inside the sound bank. The extents would overlap, which is the symptom to +look for. + +## Invariants, as measured on all 27 levels + +These are the checks the parser is validated against, and each holds everywhere +rather than on one sample: + +1. **Extent order and tiling.** Each extent's end is the next extent's start. + 27 of 27. +2. **The runtime sector field at `0x04` is zero.** 27 of 27. It is not the + container size and must not be used as one — the container's length comes + from the ISO directory entry. +3. **Extent coverage.** `sound_bank + data + gameplay + occlusion + chunks` + always reaches at least the end of the container. 21 of 27 land on exactly + the container's sector length; 6 leave between 134 and 689 sectors of trailing + data past the last named extent (levels 1, 2, 4, 11, 19, 20). + + The 21-level case ends *one sector past* the file end. The header occupies + sector 0 and the extents begin at sector 1, so this is ordinary sector + rounding rather than an error. The correct assertion is therefore `>=`, not + equality — asserting equality would truncate the tail of a level. + +4. **Level id equals container index on 26 of 27 levels.** `LEVEL21.WAD` + reports **id 30**. + +## Level ids are not file order + +Point 4 matters enough to state separately. `LEVEL21.WAD` has `id = 30`, so the +container's position in the directory is not the level's identity. Some level +was cut from the disc and its number is not reused, leaving a gap. + +Anything that keys off file order — a save slot, a planet select screen, the +moby class table — would silently mislabel that planet. Use the header's `id`. + +The ISO filesystem also sorts names alphabetically, so `LEVEL10.WAD` precedes +`LEVEL2.WAD`. Tests and tools that care about order must sort numerically. + +## Chunk descriptors + +`0x30..0x60` holds three `{offset, size}` pairs in sector counts, following the +named extents. They are populated on 6 levels (1, 2, 4, 7, 8, 11, 19, 20 — of +which some are the trailing-data levels) and zero elsewhere. Their contents are +not yet identified. + +## Not yet parsed + +- **`data`** — the geometry. This is the next target and where the visible + level comes from: tfrag terrain, tie instances, shrub foliage, moby classes, + sky shells and textures. +- **`gameplay`** — begins `WAD` + a per-level id byte + a count, and behaves + like an LZ77 stream but does **not** decode with the WAD decoder (all 27 + levels fail with "adjacent literal packets", which the format forbids). It is + a different compression, or a different framing. Recorded in + [`../plan/level-assets.md`](../plan/level-assets.md). +- **`sound_bank`** — begins with `SBlk` at `+24` and `DAW` at `+36`; both + signatures are present in all 27 levels. +- **`occlusion`** — 2 to 25 sectors. Small enough to read fully. + +## Provenance + +The header layout matches the `GcUyaLevelWadHeader` structure documented by the +Wrench project (, GPLv3). **No Wrench code +is copied** — see `PROVENANCE.md`. The structure here is written from the +specification and every field is verified against all 27 levels on this disc. +The two facts that the documentation does not state — the sector-count sizes and +the `LEVEL21` id divergence — were measured here. \ No newline at end of file diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md new file mode 100644 index 0000000..b7e4dab --- /dev/null +++ b/docs/plan/level-assets.md @@ -0,0 +1,117 @@ +# Level assets + +Phase 2 work record. What has been established about the four extents inside +each `LEVEL.WAD`, and what remains. + +## Done: the container header + +`crates/gc-formats/src/level_wad.rs` parses the 0x60-byte header on all 27 +levels and names the four extents. See +[`../formats/level-wad.md`](../formats/level-wad.md) for the layout and the +measured invariants. + +Two facts worth carrying forward: + +- **Sizes in this header are sector counts**, unlike the generic `G/*.WAD` pair + table where they are byte counts. Both are on the same disc. +- **`LEVEL21.WAD` reports level id 30.** Container order is not level identity. + +## `gameplay`: a compression we have not identified + +This is the one genuinely puzzling result so far, and it is worth recording in +full because the next attempt will otherwise repeat the work. + +The extent begins: + +```text +offset 0 57 41 44 "WAD" +offset 3 79 0d 0a 0d <- varies per level +offset 4+ +``` + +The variation is the tell. For the LZ77 WAD container the fourth byte is part of +a 32-bit little-endian length; here byte 3 differs per level (0x79, 0x3a, 0xd8, +0xa9, ...) while byte 4 is a small count (13, 88, 13, 105, 163, ...) and bytes 6 +and 7 are always zero. + +**Attempt: treat it as the known WAD LZ77 stream.** Decoded all 27 levels with +the decoder in `wad.rs`, starting after an 8-byte header. Every one produces +plausible output for the first packet — `0000 0000 0000 0750 0100 0020 2b0c...` +looks like real level data — and then fails on all 27 with "adjacent literal +packets", which the format's encoder never emits. + +A uniform failure on all 27 inputs means the input is wrong, not the decoder. +Ruled out: + +- **Bit order.** The WAD container is a flag-byte packet stream, not a bit + stream, so no ordering applies. Both orderings fail identically. +- **Header length.** Tried 8 bytes. A different skip would not fix a failure + that occurs uniformly in the first few packets. +- **The WAD format at all.** The first packet (flag `0x00`) decodes cleanly as a + literal run of 18 bytes, which is *why* it looks plausible, but the run + immediately after breaks the format's own rule. Whatever this is, it is not + WAD LZ77. + +Next to try, in order of cheapness: + +1. Whether the payload is uncompressed and the header is a table. `(len - 8)` + is 196..497 across the levels and divides by neither 8 nor a plausible stride, + which argues against a flat pair table — but a *variable-length* table + (counts of counts) fits that range. +2. Whether byte 4 is an entry count and bytes 5.. are records of a fixed but + non-obvious width. +3. Whether the LZ77 variant differs only in the "adjacent literals" rule. + +None of these is expensive to test; each is a few lines against all 27 levels at +once. The right move is to write a throwaway script that tries many hypotheses +across all 27 and reports which produce self-consistent results — the approach +that worked for the container pair table, where "does this hold on every level" +was the discriminator that the sector/byte ambiguity failed. + +## `sound_bank`: signatures found, structure unknown + +```text ++0x18 "SBlk" present in all 27 levels ++0x24 "DAW" present in all 27 levels +``` + +Sizes 458..833 bytes. Too small to be the level's audio; likely an index into +the global `AUDIO.WAD`. + +## `occlusion`: smallest, and unread + +2 to 25 sectors. Small enough to dump whole and stare at. + +## `data`: the real target, untouched + +6,484 sectors on level 0, down to 2,102 on level 24 — 4.2 to 21.9 MB. This is +the level's geometry and the largest thing in the container. Next target. + +What is known from the community reference and from Insomniac's engine family: +it splits into four geometry subsystems — `tfrag` (terrain fragments), +`tie` (instanced environment structures), `shrub` (foliage) and `moby` (dynamic +actors) — plus sky shells, collision data and textures. ReRAC's documentation of +the equivalent structures in the first game is the closest reference available, +though R&C2's spherical worlds change how they are used. + +## Order of work + +1. `gameplay` — smallest, fully bounded, and the format is identified enough to + make progress. A win here validates the approach before the large effort. +2. `occlusion` — tiny. +3. `sound_bank` — small. +4. `data` — the large one, and the one that decides whether anything renders. + +Doing the small extents first is not just convenience: each one is a contained +reverse-engineering problem that either resolves or tells us something, whereas +`data` will keep producing ambiguous readings until we have something to check +them against. + +## Method note + +Every format question this phase has been settled the same way: write the +hypothesis, test it against **all 27 levels at once**, and require an invariant +that holds everywhere. The sector/byte ambiguity in Phase 1 survived because it +happened to be correct on the one container checked by hand. The level header +was fixed in a single pass because the tiling check failed immediately on +levels 1, 2, 4, 7, 8, 11, 19 and 20. \ No newline at end of file diff --git a/docs/roadmap.md b/docs/roadmap.md index 63b4d51..9399545 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -16,7 +16,10 @@ early, while they are still cheap to retire. - [x] **Phase 1 complete.** The disc pipeline runs end to end and is verified against a committed hash table: 5,135 files, 4,500,106,960 bytes, every one of the 92 asset containers parsing. -- [ ] Phase 2 next: the asset format parsers. +- [x] **Phase 2 started.** The level container header is parsed and typed, which + names the `data`, `sound_bank`, `gameplay` and `occlusion` extents inside + each of the 27 level containers. +- [ ] Phase 2 next: the `gameplay` extent, then `data`. The disc being the exact build the reference projects pin is the single biggest favourable fact so far: no disc conversion, no regional timing work, @@ -85,8 +88,26 @@ failing, which is the failure mode worth guarding against. disc data. Phase 1 hands over the extents; this phase works out what each one contains. -`global_wad.md` records which pair in each container corresponds to which asset, -and that table starts empty. + +**Done so far:** + +- [x] `level_wad.rs` — the 0x60-byte header on all 27 level containers, naming + four extents plus three chunk descriptors. Verified: extents tile each + container in file order (27 of 27). +- [x] Two facts the format documentation does not state, both measured here: + sizes in this header are **sector counts** where the generic `G/` table + uses byte counts, and **`LEVEL21.WAD` reports level id 30**, so container + order is not level identity. + +**In progress:** + +- [ ] `gameplay` — begins `WAD` + a per-level byte, and does *not* decode with + the WAD LZ77 decoder. All 27 levels fail identically, so the input is + wrong rather than the decoder. Ruled out: bit order, header length, the + LZ77 framing. Hypotheses queued in `plan/level-assets.md`. +- [ ] `occlusion` — 2 to 25 sectors, small enough to read whole. +- [ ] `sound_bank` — `SBlk` and `DAW` signatures found in all 27. +- [ ] `data` — the large one: 4.2 to 21.9 MB of geometry per level. - [ ] `tfrag` — terrain: LOD chains, strips, morph targets - [ ] `tie` — instanced environment geometry, fat vertices