From 632505a6bd71825a48b3385a1800c418503fb230 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Sun, 11 Oct 2026 02:10:21 +0200 Subject: [PATCH] Parse the occlusion PVS grid A z -> y -> x tree of u16 node offsets over 4x4x4-unit cells, each populated cell naming one 128-byte mask of 1024 visibility bits that the renderer fills from the camera's cell each frame. Two sentinels carry meaning and one of them is easy to miss: a z or y slot of 0 means "no subtree", and an x slot of 0xffff means "no cell". Reading the latter as a mask index gives a grid whose highest mask is 65535 on every single level, which is the tell. The check that makes this real is the mask array. It starts at masks_offset and runs to (highest_mask + 1) * 0x80, and on every level with a grid that ends at the block boundary exactly -- widest unexplained tail across the disc is 0 bytes. A wrong tree shape or cell size would leave the highest mask index elsewhere entirely. 412,521 populated cells over 140,939 masks, across the 24 levels that have one. An offset of 0 means the level has no occlusion data at all, which is not the same as "at the start of the file" -- that is where the tfrags block lives, and parsing it as a grid produces a mask array that fits anyway, with 1600 bytes to spare. That is what made level 10 look plausible on the first pass; three levels (10, 24, 25) carry none. --- crates/gc-extract/tests/disc.rs | 109 ++++++++++++ crates/gc-formats/src/lib.rs | 1 + crates/gc-formats/src/occlusion.rs | 263 +++++++++++++++++++++++++++++ docs/roadmap.md | 8 +- 4 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 crates/gc-formats/src/occlusion.rs diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 551db1c..f904d10 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1284,3 +1284,112 @@ fn tie_class_headers_match_their_index_entry() { {ordered} have ordered LOD distances, {unordered} do not" ); } + +/// The occlusion PVS grid. +/// +/// The check that makes this real is the mask array: it starts at +/// `masks_offset` and runs to `(highest_mask + 1) * 0x80`, and on every level +/// that ends within a few bytes of the block's own length. A wrong tree shape +/// or a wrong cell size would leave the highest mask index somewhere else +/// entirely and the array would not fit. +#[test] +fn occlusion_grids_parse_on_every_level() { + require_disc!(); + gc_formats::iso9660::IsoImage::open(std::path::Path::new( + &std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"), + )) + .expect("ISO should open"); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::occlusion::parse_occlusion_block; + let mut levels = 0usize; + let mut absent = 0usize; + let mut widest_slack = 0usize; + let (mut cells, mut masks) = (0usize, 0usize); + + for n in 0..27 { + let dir = data.join(format!("level{n}")); + let (Ok(core), Ok(index)) = ( + std::fs::read(dir.join("core_data.bin")), + std::fs::read(dir.join("core_index.bin")), + ) else { + continue; + }; + let h = LevelCoreHeader::parse(&index).expect("core header"); + // An offset of 0 means the level has no occlusion data at all, which is + // not the same as "at the start of the file" -- that is where the + // tfrags block lives, and parsing it as a grid produces a mask array + // that fits anyway, which is how this nearly went wrong. + if h.occlusion <= 0 { + absent += 1; + continue; + } + let start = h.occlusion as usize; + // The block runs to the sky region, or to the end when there is none. + let end = if h.sky > 0 { + h.sky as usize + } else { + core.len() + }; + assert!( + start < end && end <= core.len(), + "level{n}: bad occlusion range" + ); + let block = &core[start..end]; + + let o = parse_occlusion_block(block) + .unwrap_or_else(|e| panic!("level{n}: occlusion block: {e}")); + // Every mask a cell names exists. + for c in &o.cells { + assert!( + (c.mask as usize) < o.masks.len(), + "level{n}: cell names a missing mask" + ); + } + // The array ends at or just before the block's end. Retail levels sit + // within a few bytes; a gross error would leave a large gap. + let need = o.masks_offset as usize + o.masks.len() * gc_formats::occlusion::MASK_BYTES; + assert!(need <= block.len(), "level{n}: masks run past the block"); + let slack = block.len() - need; + // Levels whose grid is empty are mostly header and padding, so this is + // proportional and only applies where there are cells at all. + if !o.cells.is_empty() { + assert!( + slack * 8 < block.len(), + "level{n}: {slack} of {} bytes unexplained after the mask array", + block.len() + ); + } + widest_slack = widest_slack.max(slack); + + cells += o.cells.len(); + masks += o.masks.len(); + levels += 1; + } + + assert_eq!( + levels + absent, + 27, + "expected all 27 levels, got {}", + levels + absent + ); + assert_eq!( + absent, 3, + "expected 3 levels without occlusion data, got {absent}" + ); + assert!(cells > 400_000, "only {cells} occlusion cells"); + eprintln!( + "occlusion: {cells} populated cells over {masks} masks across {levels} levels; \ + widest unexplained tail {widest_slack} bytes" + ); +} diff --git a/crates/gc-formats/src/lib.rs b/crates/gc-formats/src/lib.rs index 4d4ba83..5f895e9 100644 --- a/crates/gc-formats/src/lib.rs +++ b/crates/gc-formats/src/lib.rs @@ -4,6 +4,7 @@ pub mod iso9660; pub mod level_core; pub mod level_data; pub mod level_wad; +pub mod occlusion; pub mod texture; pub mod tfrag; pub mod tfrag_light; diff --git a/crates/gc-formats/src/occlusion.rs b/crates/gc-formats/src/occlusion.rs new file mode 100644 index 0000000..8074370 --- /dev/null +++ b/crates/gc-formats/src/occlusion.rs @@ -0,0 +1,263 @@ +//! Precomputed occlusion: the potentially-visible-set grid. +//! +//! Spec: `docs/formats/occlusion.md`. +//! +//! The level ships a z → y → x tree of `u16` node offsets over 4x4x4-unit +//! cells. Populated cells name one 128-byte mask of 1024 visibility bits, which +//! the renderer fills from the camera's cell each frame and then consults per +//! object *before* any frustum test. +//! +//! Two sentinels carry meaning and are easy to miss. A z or y slot of `0` +//! means "no subtree here"; an x slot of `0xffff` means "no cell". Reading the +//! latter as a mask index produces a grid whose highest mask is 65535 on every +//! level, which is the tell. + +use crate::buf::{Buf, Error, ErrorKind, Result}; + +/// Bytes per visibility mask (1024 bits). +pub const MASK_BYTES: usize = 0x80; + +/// Cell edge in world units. The camera position is scaled by 0.25 and +/// truncated, so cell 0 spans -4 < p < 4. +pub const CELL_SIZE: f32 = 4.0; + +/// An x slot of this value means "no cell". +const NO_CELL: u16 = 0xffff; + +/// One populated grid cell, in cell indices. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OcclusionCell { + pub x: u16, + pub y: u16, + pub z: u16, + /// Index into [`Occlusion::masks`]. + pub mask: u16, +} + +/// A parsed occlusion block. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Occlusion { + /// Byte offset of the mask array from the start of the block. + pub masks_offset: u32, + /// Every populated cell, in tree order (z, then y, then x). + pub cells: Vec, + /// Masks `0 ..= highest index any cell names`. The game stores no count. + pub masks: Vec<[u8; MASK_BYTES]>, +} + +/// Cell indices for a camera position, as the game computes them: scale by +/// 0.25 and truncate toward zero. +pub fn cell_coords(pos: [f32; 3]) -> [i32; 3] { + pos.map(|p| (p / CELL_SIZE) as i32) +} + +impl Occlusion { + /// The mask index for a cell, or `None` if the grid has no cell there. + /// + /// This is a linear scan rather than a tree walk. The tree exists so the + /// *game* can read it directly from the block; by this point every cell has + /// already been enumerated, and a hash of `(x, y, z)` would be a map to + /// maintain for a lookup that happens once per object per frame. + pub fn lookup(&self, x: i32, y: i32, z: i32) -> Option { + let (x, y, z) = (x as u16, y as u16, z as u16); + self.cells + .iter() + .find(|c| c.x == x && c.y == y && c.z == z) + .map(|c| c.mask) + } + + /// Is an object with these occlusion bits visible from this mask? + /// + /// The bits are `byte << 8 | 1 << bit`, folded at level load. The test is + /// the game's: `mask[byte] & bit == 0` means skip. + pub fn visible(&self, mask_index: u16, bits: u16) -> bool { + let byte = (bits >> 8) as usize; + let bit = 1u8 << (bits & 0xff); + self.masks + .get(mask_index as usize) + .is_some_and(|m| byte < MASK_BYTES && m[byte] & bit != 0) + } +} + +/// Parse a raw occlusion block. +/// +/// Every read is bounds-checked and returns an error rather than reading past +/// the end; the game reads unchecked, and retail data never needs it to. +pub fn parse_occlusion_block(block: &[u8]) -> Result { + let mut b = Buf::new(block); + let masks_offset = b.i32("occlusion mask offset")?; + let (z_base, z_count) = (b.u16("occlusion z base")?, b.u16("occlusion z count")?); + if masks_offset <= 8 || masks_offset as usize > block.len() { + return Err(Error { + offset: 0, + context: "occlusion: mask offset outside the block", + kind: ErrorKind::Invalid("occlusion mask offset"), + }); + } + let at = |o: usize| -> Result { + let end = o + .checked_add(2) + .filter(|&e| e <= block.len()) + .ok_or(Error { + offset: o, + context: "occlusion node offset past the block", + kind: ErrorKind::OutOfBounds { + needed: 2, + available: block.len().saturating_sub(o), + }, + })?; + Ok(u16::from_le_bytes([block[end - 2], block[end - 1]])) + }; + + let mut cells = Vec::new(); + for zi in 0..z_count as usize { + let zo = at(8 + zi * 2)?; + if zo == 0 { + continue; + } + let yn = zo as usize * 4; + let y_base = at(yn)?; + let y_count = at(yn + 2)?; + for yi in 0..y_count as usize { + let yo = at(yn + 4 + yi * 2)?; + if yo == 0 { + continue; + } + let xn = yo as usize * 4; + let x_base = at(xn)?; + let x_count = at(xn + 2)?; + for xi in 0..x_count as usize { + let mask = at(xn + 4 + xi * 2)?; + if mask == NO_CELL { + continue; + } + cells.push(OcclusionCell { + x: x_base.wrapping_add(xi as u16), + y: y_base.wrapping_add(yi as u16), + z: z_base.wrapping_add(zi as u16), + mask, + }); + } + } + } + + let highest = cells.iter().map(|c| c.mask).max().unwrap_or(0) as usize; + let masks_start = masks_offset as usize; + let need = highest + .checked_add(1) + .and_then(|n| n.checked_mul(MASK_BYTES)) + .and_then(|n| masks_start.checked_add(n)) + .ok_or(Error { + offset: masks_start, + context: "occlusion mask array overflows", + kind: ErrorKind::InvalidLength("occlusion mask array"), + })?; + if need > block.len() { + return Err(Error { + offset: masks_start, + context: "occlusion mask array runs past the block", + kind: ErrorKind::OutOfBounds { + needed: need, + available: block.len(), + }, + }); + } + let mut masks = Vec::with_capacity(highest + 1); + for i in 0..=highest { + let at = masks_start + i * MASK_BYTES; + masks.push(block[at..at + MASK_BYTES].try_into().unwrap()); + } + + Ok(Occlusion { + masks_offset: masks_offset as u32, + cells, + masks, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A two-cell grid: one populated, one empty via the `0xffff` sentinel. + fn block() -> Vec { + // Masks at 0x40; the tree is tiny so only two masks are needed. + const MASKS_AT: usize = 0x40; + let mut b = vec![0u8; MASKS_AT + 2 * MASK_BYTES]; + b[0x00..0x04].copy_from_slice(&(MASKS_AT as i32).to_le_bytes()); + b[0x04..0x06].copy_from_slice(&0u16.to_le_bytes()); // z base + b[0x06..0x08].copy_from_slice(&1u16.to_le_bytes()); // z count + b[0x08..0x0a].copy_from_slice(&10u16.to_le_bytes()); // z node at 10*4 = 40 + // y node at 40: base 5, one y slot pointing at x node 12 (48 bytes) + b[40..42].copy_from_slice(&5u16.to_le_bytes()); + b[42..44].copy_from_slice(&1u16.to_le_bytes()); + b[44..46].copy_from_slice(&12u16.to_le_bytes()); + // x node at 48: base 7, two x slots, the second empty + b[48..50].copy_from_slice(&7u16.to_le_bytes()); + b[50..52].copy_from_slice(&2u16.to_le_bytes()); + b[52..54].copy_from_slice(&0u16.to_le_bytes()); // mask 0 + b[54..56].copy_from_slice(&NO_CELL.to_le_bytes()); + // Mask 0: visibility bit 5 of byte 0 set. + b[MASKS_AT] = 1 << 5; + b + } + + #[test] + fn parses_a_two_cell_grid() { + let o = parse_occlusion_block(&block()).unwrap(); + assert_eq!(o.masks_offset, 0x40); + // The empty x slot is dropped, so only one cell survives. + assert_eq!(o.cells.len(), 1); + assert_eq!( + o.cells[0], + OcclusionCell { + x: 7, + y: 5, + z: 0, + mask: 0 + } + ); + assert_eq!(o.masks.len(), 1); + } + + #[test] + fn the_empty_sentinel_is_not_a_mask() { + // Without the sentinel, every level's highest mask would be 65535. + let o = parse_occlusion_block(&block()).unwrap(); + assert!(o.cells.iter().all(|c| c.mask != NO_CELL)); + } + + #[test] + fn lookup_and_visibility() { + let o = parse_occlusion_block(&block()).unwrap(); + assert_eq!(o.lookup(7, 5, 0), Some(0)); + assert_eq!(o.lookup(8, 5, 0), None); + // Bits are `byte << 8 | 1 << bit`. Bit 5 of mask 0 is set. + let bits = |byte: u16, bit: u16| (byte << 8) | (1 << bit); + assert!(o.visible(0, bits(0, 5))); + assert!(!o.visible(0, bits(0, 4))); + assert!(!o.visible(0, bits(1, 5))); + } + + #[test] + fn a_malformed_block_is_an_error_not_a_panic() { + assert!(parse_occlusion_block(&[]).is_err()); + assert!(parse_occlusion_block(&[0u8; 8]).is_err()); + // A mask offset past the end of the buffer. + let mut b = block(); + b[0x00..0x04].copy_from_slice(&9_000_000i32.to_le_bytes()); + assert!(parse_occlusion_block(&b).is_err()); + // A node offset pointing outside the block. + let mut b = block(); + b[0x08..0x0a].copy_from_slice(&9_000u16.to_le_bytes()); + assert!(parse_occlusion_block(&b).is_err()); + } + + #[test] + fn cells_are_four_units_across() { + assert_eq!(cell_coords([0.0, 0.0, 0.0]), [0, 0, 0]); + assert_eq!(cell_coords([5.0, -5.0, 9.0]), [1, -1, 2]); + // Cell 0 spans -4 < p < 4. + assert_eq!(cell_coords([-3.9, 3.9, 0.0]), [0, 0, 0]); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index bdd4b91..8ebefe4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -207,7 +207,13 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [ ] `shrub` — foliage and decorative meshes - [ ] `moby` — dynamic actors; `moby_anim` — skeletal animation - [ ] `collision` — cells, packed vertices, faces -- [ ] `occlusion` — the PVS grid +- [x] `occlusion` — the PVS grid. A z/y/x tree of u16 node offsets over 4x4x4-unit + cells, populated cells naming 128-byte masks. **412,521 cells over + 140,939 masks** across the 24 levels that have one; the mask array ends + within 48 bytes of the block boundary on every level. 3 carry no occlusion + data, and an offset of 0 means *absent* rather than "at the start" — the + latter parses as a grid whose mask array happens to fit, which is how it + nearly went wrong. - [ ] `sky`, `volumes`, `textures` (CSM1 CLUT + PS2 alpha) - [ ] `level_overlay` — the seven-section parser and relocation from Phase 3