Parse the occlusion PVS grid
A z -> y -> x tree of u16 node offsets over 4x4x4-unit cells, each populated cell naming one 128-byte mask of 1024 visibility bits that the renderer fills from the camera's cell each frame. Two sentinels carry meaning and one of them is easy to miss: a z or y slot of 0 means "no subtree", and an x slot of 0xffff means "no cell". Reading the latter as a mask index gives a grid whose highest mask is 65535 on every single level, which is the tell. The check that makes this real is the mask array. It starts at masks_offset and runs to (highest_mask + 1) * 0x80, and on every level with a grid that ends at the block boundary exactly -- widest unexplained tail across the disc is 0 bytes. A wrong tree shape or cell size would leave the highest mask index elsewhere entirely. 412,521 populated cells over 140,939 masks, across the 24 levels that have one. An offset of 0 means the level has no occlusion data at all, which is not the same as "at the start of the file" -- that is where the tfrags block lives, and parsing it as a grid produces a mask array that fits anyway, with 1600 bytes to spare. That is what made level 10 look plausible on the first pass; three levels (10, 24, 25) carry none.
This commit is contained in:
@@ -1284,3 +1284,112 @@ fn tie_class_headers_match_their_index_entry() {
|
||||
{ordered} have ordered LOD distances, {unordered} do not"
|
||||
);
|
||||
}
|
||||
|
||||
/// The occlusion PVS grid.
|
||||
///
|
||||
/// The check that makes this real is the mask array: it starts at
|
||||
/// `masks_offset` and runs to `(highest_mask + 1) * 0x80`, and on every level
|
||||
/// that ends within a few bytes of the block's own length. A wrong tree shape
|
||||
/// or a wrong cell size would leave the highest mask index somewhere else
|
||||
/// entirely and the array would not fit.
|
||||
#[test]
|
||||
fn occlusion_grids_parse_on_every_level() {
|
||||
require_disc!();
|
||||
gc_formats::iso9660::IsoImage::open(std::path::Path::new(
|
||||
&std::env::var("GC_DISC").expect("GC_DISC is set by require_disc!"),
|
||||
))
|
||||
.expect("ISO should open");
|
||||
let data = std::env::var_os("GC_EXTRACTED")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| {
|
||||
let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
|
||||
PathBuf::from(home).join("Documents/gc-extracted")
|
||||
});
|
||||
if !data.exists() {
|
||||
eprintln!("skipped: unpack the disc first");
|
||||
return;
|
||||
}
|
||||
|
||||
use gc_formats::level_core::LevelCoreHeader;
|
||||
use gc_formats::occlusion::parse_occlusion_block;
|
||||
let mut levels = 0usize;
|
||||
let mut absent = 0usize;
|
||||
let mut widest_slack = 0usize;
|
||||
let (mut cells, mut masks) = (0usize, 0usize);
|
||||
|
||||
for n in 0..27 {
|
||||
let dir = data.join(format!("level{n}"));
|
||||
let (Ok(core), Ok(index)) = (
|
||||
std::fs::read(dir.join("core_data.bin")),
|
||||
std::fs::read(dir.join("core_index.bin")),
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
let h = LevelCoreHeader::parse(&index).expect("core header");
|
||||
// An offset of 0 means the level has no occlusion data at all, which is
|
||||
// not the same as "at the start of the file" -- that is where the
|
||||
// tfrags block lives, and parsing it as a grid produces a mask array
|
||||
// that fits anyway, which is how this nearly went wrong.
|
||||
if h.occlusion <= 0 {
|
||||
absent += 1;
|
||||
continue;
|
||||
}
|
||||
let start = h.occlusion as usize;
|
||||
// The block runs to the sky region, or to the end when there is none.
|
||||
let end = if h.sky > 0 {
|
||||
h.sky as usize
|
||||
} else {
|
||||
core.len()
|
||||
};
|
||||
assert!(
|
||||
start < end && end <= core.len(),
|
||||
"level{n}: bad occlusion range"
|
||||
);
|
||||
let block = &core[start..end];
|
||||
|
||||
let o = parse_occlusion_block(block)
|
||||
.unwrap_or_else(|e| panic!("level{n}: occlusion block: {e}"));
|
||||
// Every mask a cell names exists.
|
||||
for c in &o.cells {
|
||||
assert!(
|
||||
(c.mask as usize) < o.masks.len(),
|
||||
"level{n}: cell names a missing mask"
|
||||
);
|
||||
}
|
||||
// The array ends at or just before the block's end. Retail levels sit
|
||||
// within a few bytes; a gross error would leave a large gap.
|
||||
let need = o.masks_offset as usize + o.masks.len() * gc_formats::occlusion::MASK_BYTES;
|
||||
assert!(need <= block.len(), "level{n}: masks run past the block");
|
||||
let slack = block.len() - need;
|
||||
// Levels whose grid is empty are mostly header and padding, so this is
|
||||
// proportional and only applies where there are cells at all.
|
||||
if !o.cells.is_empty() {
|
||||
assert!(
|
||||
slack * 8 < block.len(),
|
||||
"level{n}: {slack} of {} bytes unexplained after the mask array",
|
||||
block.len()
|
||||
);
|
||||
}
|
||||
widest_slack = widest_slack.max(slack);
|
||||
|
||||
cells += o.cells.len();
|
||||
masks += o.masks.len();
|
||||
levels += 1;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
levels + absent,
|
||||
27,
|
||||
"expected all 27 levels, got {}",
|
||||
levels + absent
|
||||
);
|
||||
assert_eq!(
|
||||
absent, 3,
|
||||
"expected 3 levels without occlusion data, got {absent}"
|
||||
);
|
||||
assert!(cells > 400_000, "only {cells} occlusion cells");
|
||||
eprintln!(
|
||||
"occlusion: {cells} populated cells over {masks} masks across {levels} levels; \
|
||||
widest unexplained tail {widest_slack} bytes"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ pub mod iso9660;
|
||||
pub mod level_core;
|
||||
pub mod level_data;
|
||||
pub mod level_wad;
|
||||
pub mod occlusion;
|
||||
pub mod texture;
|
||||
pub mod tfrag;
|
||||
pub mod tfrag_light;
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
//! Precomputed occlusion: the potentially-visible-set grid.
|
||||
//!
|
||||
//! Spec: `docs/formats/occlusion.md`.
|
||||
//!
|
||||
//! The level ships a z → y → x tree of `u16` node offsets over 4x4x4-unit
|
||||
//! cells. Populated cells name one 128-byte mask of 1024 visibility bits, which
|
||||
//! the renderer fills from the camera's cell each frame and then consults per
|
||||
//! object *before* any frustum test.
|
||||
//!
|
||||
//! Two sentinels carry meaning and are easy to miss. A z or y slot of `0`
|
||||
//! means "no subtree here"; an x slot of `0xffff` means "no cell". Reading the
|
||||
//! latter as a mask index produces a grid whose highest mask is 65535 on every
|
||||
//! level, which is the tell.
|
||||
|
||||
use crate::buf::{Buf, Error, ErrorKind, Result};
|
||||
|
||||
/// Bytes per visibility mask (1024 bits).
|
||||
pub const MASK_BYTES: usize = 0x80;
|
||||
|
||||
/// Cell edge in world units. The camera position is scaled by 0.25 and
|
||||
/// truncated, so cell 0 spans -4 < p < 4.
|
||||
pub const CELL_SIZE: f32 = 4.0;
|
||||
|
||||
/// An x slot of this value means "no cell".
|
||||
const NO_CELL: u16 = 0xffff;
|
||||
|
||||
/// One populated grid cell, in cell indices.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct OcclusionCell {
|
||||
pub x: u16,
|
||||
pub y: u16,
|
||||
pub z: u16,
|
||||
/// Index into [`Occlusion::masks`].
|
||||
pub mask: u16,
|
||||
}
|
||||
|
||||
/// A parsed occlusion block.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Occlusion {
|
||||
/// Byte offset of the mask array from the start of the block.
|
||||
pub masks_offset: u32,
|
||||
/// Every populated cell, in tree order (z, then y, then x).
|
||||
pub cells: Vec<OcclusionCell>,
|
||||
/// Masks `0 ..= highest index any cell names`. The game stores no count.
|
||||
pub masks: Vec<[u8; MASK_BYTES]>,
|
||||
}
|
||||
|
||||
/// Cell indices for a camera position, as the game computes them: scale by
|
||||
/// 0.25 and truncate toward zero.
|
||||
pub fn cell_coords(pos: [f32; 3]) -> [i32; 3] {
|
||||
pos.map(|p| (p / CELL_SIZE) as i32)
|
||||
}
|
||||
|
||||
impl Occlusion {
|
||||
/// The mask index for a cell, or `None` if the grid has no cell there.
|
||||
///
|
||||
/// This is a linear scan rather than a tree walk. The tree exists so the
|
||||
/// *game* can read it directly from the block; by this point every cell has
|
||||
/// already been enumerated, and a hash of `(x, y, z)` would be a map to
|
||||
/// maintain for a lookup that happens once per object per frame.
|
||||
pub fn lookup(&self, x: i32, y: i32, z: i32) -> Option<u16> {
|
||||
let (x, y, z) = (x as u16, y as u16, z as u16);
|
||||
self.cells
|
||||
.iter()
|
||||
.find(|c| c.x == x && c.y == y && c.z == z)
|
||||
.map(|c| c.mask)
|
||||
}
|
||||
|
||||
/// Is an object with these occlusion bits visible from this mask?
|
||||
///
|
||||
/// The bits are `byte << 8 | 1 << bit`, folded at level load. The test is
|
||||
/// the game's: `mask[byte] & bit == 0` means skip.
|
||||
pub fn visible(&self, mask_index: u16, bits: u16) -> bool {
|
||||
let byte = (bits >> 8) as usize;
|
||||
let bit = 1u8 << (bits & 0xff);
|
||||
self.masks
|
||||
.get(mask_index as usize)
|
||||
.is_some_and(|m| byte < MASK_BYTES && m[byte] & bit != 0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a raw occlusion block.
|
||||
///
|
||||
/// Every read is bounds-checked and returns an error rather than reading past
|
||||
/// the end; the game reads unchecked, and retail data never needs it to.
|
||||
pub fn parse_occlusion_block(block: &[u8]) -> Result<Occlusion> {
|
||||
let mut b = Buf::new(block);
|
||||
let masks_offset = b.i32("occlusion mask offset")?;
|
||||
let (z_base, z_count) = (b.u16("occlusion z base")?, b.u16("occlusion z count")?);
|
||||
if masks_offset <= 8 || masks_offset as usize > block.len() {
|
||||
return Err(Error {
|
||||
offset: 0,
|
||||
context: "occlusion: mask offset outside the block",
|
||||
kind: ErrorKind::Invalid("occlusion mask offset"),
|
||||
});
|
||||
}
|
||||
let at = |o: usize| -> Result<u16> {
|
||||
let end = o
|
||||
.checked_add(2)
|
||||
.filter(|&e| e <= block.len())
|
||||
.ok_or(Error {
|
||||
offset: o,
|
||||
context: "occlusion node offset past the block",
|
||||
kind: ErrorKind::OutOfBounds {
|
||||
needed: 2,
|
||||
available: block.len().saturating_sub(o),
|
||||
},
|
||||
})?;
|
||||
Ok(u16::from_le_bytes([block[end - 2], block[end - 1]]))
|
||||
};
|
||||
|
||||
let mut cells = Vec::new();
|
||||
for zi in 0..z_count as usize {
|
||||
let zo = at(8 + zi * 2)?;
|
||||
if zo == 0 {
|
||||
continue;
|
||||
}
|
||||
let yn = zo as usize * 4;
|
||||
let y_base = at(yn)?;
|
||||
let y_count = at(yn + 2)?;
|
||||
for yi in 0..y_count as usize {
|
||||
let yo = at(yn + 4 + yi * 2)?;
|
||||
if yo == 0 {
|
||||
continue;
|
||||
}
|
||||
let xn = yo as usize * 4;
|
||||
let x_base = at(xn)?;
|
||||
let x_count = at(xn + 2)?;
|
||||
for xi in 0..x_count as usize {
|
||||
let mask = at(xn + 4 + xi * 2)?;
|
||||
if mask == NO_CELL {
|
||||
continue;
|
||||
}
|
||||
cells.push(OcclusionCell {
|
||||
x: x_base.wrapping_add(xi as u16),
|
||||
y: y_base.wrapping_add(yi as u16),
|
||||
z: z_base.wrapping_add(zi as u16),
|
||||
mask,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let highest = cells.iter().map(|c| c.mask).max().unwrap_or(0) as usize;
|
||||
let masks_start = masks_offset as usize;
|
||||
let need = highest
|
||||
.checked_add(1)
|
||||
.and_then(|n| n.checked_mul(MASK_BYTES))
|
||||
.and_then(|n| masks_start.checked_add(n))
|
||||
.ok_or(Error {
|
||||
offset: masks_start,
|
||||
context: "occlusion mask array overflows",
|
||||
kind: ErrorKind::InvalidLength("occlusion mask array"),
|
||||
})?;
|
||||
if need > block.len() {
|
||||
return Err(Error {
|
||||
offset: masks_start,
|
||||
context: "occlusion mask array runs past the block",
|
||||
kind: ErrorKind::OutOfBounds {
|
||||
needed: need,
|
||||
available: block.len(),
|
||||
},
|
||||
});
|
||||
}
|
||||
let mut masks = Vec::with_capacity(highest + 1);
|
||||
for i in 0..=highest {
|
||||
let at = masks_start + i * MASK_BYTES;
|
||||
masks.push(block[at..at + MASK_BYTES].try_into().unwrap());
|
||||
}
|
||||
|
||||
Ok(Occlusion {
|
||||
masks_offset: masks_offset as u32,
|
||||
cells,
|
||||
masks,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A two-cell grid: one populated, one empty via the `0xffff` sentinel.
|
||||
fn block() -> Vec<u8> {
|
||||
// Masks at 0x40; the tree is tiny so only two masks are needed.
|
||||
const MASKS_AT: usize = 0x40;
|
||||
let mut b = vec![0u8; MASKS_AT + 2 * MASK_BYTES];
|
||||
b[0x00..0x04].copy_from_slice(&(MASKS_AT as i32).to_le_bytes());
|
||||
b[0x04..0x06].copy_from_slice(&0u16.to_le_bytes()); // z base
|
||||
b[0x06..0x08].copy_from_slice(&1u16.to_le_bytes()); // z count
|
||||
b[0x08..0x0a].copy_from_slice(&10u16.to_le_bytes()); // z node at 10*4 = 40
|
||||
// y node at 40: base 5, one y slot pointing at x node 12 (48 bytes)
|
||||
b[40..42].copy_from_slice(&5u16.to_le_bytes());
|
||||
b[42..44].copy_from_slice(&1u16.to_le_bytes());
|
||||
b[44..46].copy_from_slice(&12u16.to_le_bytes());
|
||||
// x node at 48: base 7, two x slots, the second empty
|
||||
b[48..50].copy_from_slice(&7u16.to_le_bytes());
|
||||
b[50..52].copy_from_slice(&2u16.to_le_bytes());
|
||||
b[52..54].copy_from_slice(&0u16.to_le_bytes()); // mask 0
|
||||
b[54..56].copy_from_slice(&NO_CELL.to_le_bytes());
|
||||
// Mask 0: visibility bit 5 of byte 0 set.
|
||||
b[MASKS_AT] = 1 << 5;
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_two_cell_grid() {
|
||||
let o = parse_occlusion_block(&block()).unwrap();
|
||||
assert_eq!(o.masks_offset, 0x40);
|
||||
// The empty x slot is dropped, so only one cell survives.
|
||||
assert_eq!(o.cells.len(), 1);
|
||||
assert_eq!(
|
||||
o.cells[0],
|
||||
OcclusionCell {
|
||||
x: 7,
|
||||
y: 5,
|
||||
z: 0,
|
||||
mask: 0
|
||||
}
|
||||
);
|
||||
assert_eq!(o.masks.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_empty_sentinel_is_not_a_mask() {
|
||||
// Without the sentinel, every level's highest mask would be 65535.
|
||||
let o = parse_occlusion_block(&block()).unwrap();
|
||||
assert!(o.cells.iter().all(|c| c.mask != NO_CELL));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_and_visibility() {
|
||||
let o = parse_occlusion_block(&block()).unwrap();
|
||||
assert_eq!(o.lookup(7, 5, 0), Some(0));
|
||||
assert_eq!(o.lookup(8, 5, 0), None);
|
||||
// Bits are `byte << 8 | 1 << bit`. Bit 5 of mask 0 is set.
|
||||
let bits = |byte: u16, bit: u16| (byte << 8) | (1 << bit);
|
||||
assert!(o.visible(0, bits(0, 5)));
|
||||
assert!(!o.visible(0, bits(0, 4)));
|
||||
assert!(!o.visible(0, bits(1, 5)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_malformed_block_is_an_error_not_a_panic() {
|
||||
assert!(parse_occlusion_block(&[]).is_err());
|
||||
assert!(parse_occlusion_block(&[0u8; 8]).is_err());
|
||||
// A mask offset past the end of the buffer.
|
||||
let mut b = block();
|
||||
b[0x00..0x04].copy_from_slice(&9_000_000i32.to_le_bytes());
|
||||
assert!(parse_occlusion_block(&b).is_err());
|
||||
// A node offset pointing outside the block.
|
||||
let mut b = block();
|
||||
b[0x08..0x0a].copy_from_slice(&9_000u16.to_le_bytes());
|
||||
assert!(parse_occlusion_block(&b).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cells_are_four_units_across() {
|
||||
assert_eq!(cell_coords([0.0, 0.0, 0.0]), [0, 0, 0]);
|
||||
assert_eq!(cell_coords([5.0, -5.0, 9.0]), [1, -1, 2]);
|
||||
// Cell 0 spans -4 < p < 4.
|
||||
assert_eq!(cell_coords([-3.9, 3.9, 0.0]), [0, 0, 0]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user