diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0f3663a..6d2a769 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1620,11 +1620,17 @@ fn moby_parts_tile_on_every_level() { ); } -/// Shrub class headers. +/// Shrub class headers and blobs. /// /// Two fields self-identify exactly on every span, and the parser checks both /// rather than merely reporting them: `+0x24` must equal the owning class /// entry's own `o_class`, and `+0x2c` must land inside the span. +/// +/// The body is then checked against the R&C1 shape this format follows: a +/// palette of 24 unit-length `s16` normals with a zero pad word, then zero +/// padding records. And the billboard offset at `+0x1c` is checked against the +/// class entry's billboard texture fields, which is the one header field that +/// can be confirmed from a second file. #[test] fn shrub_headers_name_their_own_class_on_every_level() { require_disc!(); @@ -1640,14 +1646,23 @@ fn shrub_headers_name_their_own_class_on_every_level() { } use gc_formats::level_core::ShrubClassEntry; - use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES}; + use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES, SHRUB_NORMAL_COUNT}; let mut levels = 0usize; let mut spans = 0usize; let mut absent = 0usize; let (mut min_size, mut max_size) = (f32::MAX, 0.0f32); - let mut denormals = 0usize; + let mut scratch_nonzero = 0usize; + let mut mode_hist: std::collections::BTreeMap = Default::default(); + let mut scratch_hist: std::collections::BTreeMap = Default::default(); + let mut radius_range = (f32::MAX, 0f32); + let mut scale_range = (f32::MAX, 0f32); let mut bounded_records = 0usize; let mut bounded_bodies = 0usize; + let mut palette_records = 0usize; + let mut padding_records = 0usize; + let mut norm2_deviation = 0i64; + let mut billboards = 0usize; + let mut billboard_entries = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); @@ -1720,26 +1735,81 @@ fn shrub_headers_name_their_own_class_on_every_level() { ); assert!(!s.data.is_empty(), "level{n} class {i}: empty shrub data"); assert!( - s.header.bounds.iter().all(|v| v.is_finite()), - "level{n} class {i}: non-finite shrub bounds" + s.header.sphere.iter().all(|v| v.is_finite()), + "level{n} class {i}: non-finite shrub sphere" ); assert!( s.header.size.is_finite() && s.header.size > 0.0, "level{n} class {i}: shrub size {}", s.header.size ); - // The PS2 has no denormals, so a denormal here and a zero - // are the same value to the hardware. Both occur on disc. - if s.header.reserved != 0.0 { - assert!( - s.header.reserved.is_subnormal(), - "level{n} class {i}: reserved {} is neither zero nor denormal", - s.header.reserved - ); - denormals += 1; + // The scratch word is zero on every span; the parser + // refuses anything else. `mode_bits` is 4 on 16 spans and + // is reported rather than asserted. On those 16 the word + // at +0x14 reads as an f32 denormal, which the Emotion + // Engine flushes to zero like everything else. + if s.header.scratch != 0 { + scratch_nonzero += 1; } + *mode_hist.entry(s.header.mode_bits).or_default() += 1; + *scratch_hist.entry(s.header.scratch).or_default() += 1; + radius_range.0 = radius_range.0.min(s.header.sphere[3]); + radius_range.1 = radius_range.1.max(s.header.sphere[3]); + scale_range.0 = scale_range.0.min(s.header.scale); + scale_range.1 = scale_range.1.max(s.header.scale); min_size = min_size.min(s.header.size); max_size = max_size.max(s.header.size); + + // The palette: 24 unit normals, then zero padding. + let normals: Vec<_> = s.normals().collect(); + assert_eq!( + normals.len(), + SHRUB_NORMAL_COUNT, + "level{n} class {i}: palette is {} entries, not {SHRUB_NORMAL_COUNT}", + normals.len() + ); + for (k, normal) in normals.iter().enumerate() { + assert!( + normal.is_unit(), + "level{n} class {i}: palette entry {k} {normal:?} is not a unit normal" + ); + let n2 = (normal.x as i64).pow(2) + + (normal.y as i64).pow(2) + + (normal.z as i64).pow(2); + norm2_deviation = norm2_deviation.max((n2 - 32767i64.pow(2)).abs()); + } + palette_records += normals.len(); + for (k, record) in s.body_records().skip(SHRUB_NORMAL_COUNT).enumerate() { + assert!( + record.iter().all(|&b| b == 0), + "level{n} class {i}: padding record {k} is not all zero" + ); + } + padding_records += s.padding_record_count(); + + // The billboard: present exactly when the class entry + // carries billboard texture fields. + let entry_has_billboard = e.billboard_width != 0 || e.billboard_height != 0; + match s.billboard().expect("billboard parses") { + Some(b) => { + billboards += 1; + assert!( + entry_has_billboard, + "level{n} class {i}: billboard record without billboard texture fields" + ); + assert_eq!(b.tex1_addr, 0x14, "level{n} class {i}: TEX1 address"); + assert_eq!(b.tex0_addr, 0x06, "level{n} class {i}: TEX0 address"); + assert_eq!(b.miptbp_addr, 0x34, "level{n} class {i}: MIPTBP address"); + assert_eq!(b.mmin, 4, "level{n} class {i}: MMIN"); + } + None => assert!( + !entry_has_billboard, + "level{n} class {i}: billboard texture fields with no billboard record" + ), + } + if entry_has_billboard { + billboard_entries += 1; + } spans += 1; } Ok(None) => absent += 1, @@ -1759,11 +1829,19 @@ fn shrub_headers_name_their_own_class_on_every_level() { assert!(spans > 340, "only {spans} shrub spans with geometry"); // The container bounds every class now, so every body is checkable. assert_eq!(bounded_bodies, spans, "every shrub class should be bounded"); + assert_eq!(scratch_nonzero, 0, "the scratch word is zero on every span"); + assert_eq!( + billboards, billboard_entries, + "billboard records and billboard class entries disagree: \ + {billboards} records vs {billboard_entries} entries" + ); eprintln!( "shrub: {spans} headers naming their own class, {absent} without geometry; \ - all {bounded_bodies} class bodies bounded by the chunk container, \ - {bounded_records} 8-byte records in total, \ - {denormals} denormal reserved words" + all {bounded_bodies} bodies bounded, {bounded_records} records, \ + {palette_records} of them palette normals and {padding_records} zero padding; \ + {billboards} billboards, all with texture fields; \ + mode_bits {mode_hist:?}, radius {radius_range:?}, scale {scale_range:?}, \ + largest palette deviation from unit length {norm2_deviation} of a possible 65535" ); } diff --git a/crates/gc-formats/src/shrub.rs b/crates/gc-formats/src/shrub.rs index 3effab0..301aa27 100644 --- a/crates/gc-formats/src/shrub.rs +++ b/crates/gc-formats/src/shrub.rs @@ -1,4 +1,4 @@ -//! Shrub geometry: the class header. +//! Shrub geometry: the class header, the normal palette, and the billboard. //! //! Spec: `docs/plan/level-assets.md`. //! @@ -8,116 +8,301 @@ //! rule `vertex_offset + 16 * vertex_count == next_part`, which holds for //! 43,467 of 43,467 moby parts, holds for **zero** of them. //! -//! What the shrub header *does* have is two fields that self-identify exactly, -//! on all 358 spans: +//! The class blob is header, palette, optional billboard, in that order. The +//! reference for the shape is R&C1's shrub class (documented in +//! `vendor/rerac/docs/formats/shrub_sky_rac1.md`, whose offsets this header +//! follows: `o_class` at 0x24, a count at 0x28, the data offset at 0x2c), but +//! every field below is either **measured on all 358 spans** or marked as not +//! established. Where a field is only parallel to R&C1 and not independently +//! verified here, the doc comment says so. +//! +//! What self-identifies exactly, on all 358 spans: //! //! - `+0x24` equals the owning class entry's own `o_class`. //! - `+0x2c` is a data offset that lands inside the span. +//! - `+0x1c` (billboard offset) is non-zero on exactly the 25 classes whose +//! core-index entry carries billboard texture fields, and zero on the other +//! 333 -- two independent places, one answer. //! -//! Those two are what this parses. The body from `+0x2c` is an array of -//! 8-byte records -- six varying bytes and two constant zeros -- whose -//! payload encoding is not established; it is handed back raw rather than -//! guessed at. Its extent is knowable: the region ends at the chunk container, -//! located by signature (`crate::chunks::chunk_region_start`), so every span -//! is bounded and every body is whole records. -//! -//! Note the class entry size differs here too: shrub entries are 0x30 bytes, -//! not 0x20, because they carry billboard texture fields the other two tables -//! do not have. +//! The class entry size differs here too: shrub entries are 0x30 bytes, not +//! 0x20, because they carry billboard texture fields the other two tables do +//! not have. use crate::buf::{Buf, Error, ErrorKind, Result}; /// The measured size of a shrub class entry. pub const SHRUB_CLASS_ENTRY_BYTES: usize = 0x30; +/// Normals in the palette. Not a maximum: the body's zero padding follows the +/// twenty-fourth entry, so a body of 26, 28 or 30 records holds 24 normals and +/// 2, 4 or 6 zero records. +pub const SHRUB_NORMAL_COUNT: usize = 24; + +/// One palette entry: a unit normal as three `s16`, then a zero `s16`. +/// +/// The three components are a fraction of 32767 -- entry `i` of every body +/// satisfies `x^2 + y^2 + z^2` within rounding of `32767^2` on all 358 spans. +/// The fourth word is zero on all 9,600 records on the disc, which is what +/// makes the record 8 bytes rather than 6. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ShrubNormal { + pub x: i16, + pub y: i16, + pub z: i16, +} + +impl ShrubNormal { + /// Parse one 8-byte palette record. + pub fn parse(bytes: &[u8]) -> Result { + let mut b = Buf::new(bytes); + let x = b.i16("shrub normal x")?; + let y = b.i16("shrub normal y")?; + let z = b.i16("shrub normal z")?; + let pad = b.i16("shrub normal pad")?; + if pad != 0 { + return Err(Error { + offset: 6, + context: "shrub normal pad", + kind: ErrorKind::Invalid("shrub normal pad is not zero"), + }); + } + Ok(ShrubNormal { x, y, z }) + } + + /// Whether the entry is a unit normal, within the format's quantisation. + /// + /// The palette stores a unit vector scaled by 32768, so an axis-aligned + /// normal quantises to `(32768, 0, 0)` -- stored as the `s16` `-32768`, + /// because the positive axis clamps instead. The squared length therefore + /// lands within `32768^2 - 32767^2 = 65535` of `32767^2`, and that bound + /// is tight: the largest deviation on the disc is exactly 65,535. + /// + /// The margin is what makes this a test rather than a tautology. A random + /// `s16` triple has a squared length spread about a *billion* either side + /// of `32767^2`, so roughly one entry in fifteen thousand would pass by + /// chance; 8,592 of 8,592 real entries pass. + pub fn is_unit(&self) -> bool { + let n = (self.x as i64).pow(2) + (self.y as i64).pow(2) + (self.z as i64).pow(2); + (n - 32767i64.pow(2)).abs() <= 65_535 + } + + /// The normal as a unit vector in `[0, 1]`-ish float terms. + pub fn unit(self) -> [f32; 3] { + [ + self.x as f32 / 32767.0, + self.y as f32 / 32767.0, + self.z as f32 / 32767.0, + ] + } +} + /// The decoded part of a shrub class header. #[derive(Debug, Clone, Copy, PartialEq)] pub struct ShrubHeader { - /// Four floats at `+0x00..+0x10`. All finite on all 358 spans, but they do - /// not behave like a `min.xyz / max.xyz` bounding box -- that ordering only - /// holds on 146 of 358. Not named because it is not established. - pub bounds: [f32; 4], - /// A positive float at `+0x10`, 3.0 to 20.0 across the disc. + /// Four floats at `+0x00..+0x10`: a bounding sphere read as + /// centre (x, y, z) and radius (w), following R&C1's header at the same + /// offsets. What is measured here rather than assumed: all four are + /// finite, the radius is positive on all 358 spans (1.72..41.57), and the + /// centres sit near the class's own origin (-18..15, -23..23, -16..20). + /// The earlier reading of this field as a `min.xyz / max.xyz` box was + /// wrong -- that ordering holds on only 146 of 358 spans -- because it was + /// the wrong shape, not because the bytes were. + pub sphere: [f32; 4], + /// A positive float at `+0x10`, 3.0 to 20.0 across the disc. R&C1's + /// parallel at this offset is `mip_distance`; not independently + /// established. pub size: f32, - /// A float at `+0x14`. Exactly 0.0 on 342 of the disc's 358 spans; the - /// other 16 hold the bit pattern `04000000`, a denormal of about 2.8e-45. + /// A `u16` at `+0x14`: zero on 342 spans, 4 on the other 16. R&C1's + /// `mode_bits` is at the same offset. Not established. + pub mode_bits: u16, + /// A `u16` at `+0x16`, zero on all 358 spans. R&C1's field here is + /// `instance_count`, runtime scratch that retail files leave zero. /// - /// The Emotion Engine has no denormals, so hardware reads all 358 of these - /// as zero. That is the only reading the game itself ever sees, and it is - /// why this is safe to treat as a flag rather than a magnitude -- see - /// `AGENTS.md` on reproducing the FPU model rather than correcting it. - pub reserved: f32, + /// On the 16 spans where `mode_bits` is 4 the whole word at `+0x14` is an + /// `f32` denormal, so the Emotion Engine -- which has no denormals -- reads + /// zero for the pair either way. That is the sharpest available statement + /// of what the hardware does with this field. + pub scratch: u16, + /// A `u32` at `+0x18`, zero on all 358 spans. R&C1's `instances_pointer` + /// is runtime scratch at the same offset. + pub reserved: u32, + /// Offset of the [`ShrubBillboard`] record from the start of the span. + /// Zero means no billboard; a positive value lands inside the span on all + /// 25 spans that have one. + /// + /// This is the strongest self-identification in the header after + /// `o_class`: it is non-zero on exactly the classes whose core-index + /// entry carries billboard texture fields (25 of 358 each way), and the + /// two answers are produced by different bytes in different files. + pub billboard_offset: u32, + /// A positive `f32` at `+0x20` (0.0044..1.8 on the disc). R&C1's + /// `scale` is the position dequantisation factor at the same offset. + /// Not independently established. + pub scale: f32, /// The class this geometry belongs to; equals the class entry's /// `o_class` on all 358 spans on the disc. pub o_class: u32, - /// A count. Meaning not established. + /// A count. Meaning not established (1..55 on the disc). It is *not* the + /// body size: bodies hold 24 normals plus padding regardless. pub count: u32, /// Offset of the geometry data from the start of the span. Lands inside /// the span on all 358 spans on the disc. pub data_offset: u32, } -/// Bytes per body record. +/// The billboard record at `billboard_offset`, 0x40 bytes. /// -/// Each record is 6 varying bytes followed by 2 constant ones. Found by -/// grouping sibling spans and byte-diffing them rather than by stride-fitting, -/// which failed three times: the varying payload is **not** sized by the -/// `count` field, and was a constant 144 bytes per group whether `count` was -/// 1, 2, 3, 5 or 6. -pub const SHRUB_BODY_RECORD_BYTES: usize = 8; +/// The register block matches R&C1's layout field for field -- TEX1 at `+0x10` +/// with `K` and `MMIN`, TEX0 at `+0x20`, MIPTBP1_1 at `+0x30`, each carrying +/// its own register address word (0x14, 0x06, 0x34) -- and the three floats +/// in front of them are the one part that does not carry over: they are far +/// too large and too varied to be R&C1's `width`/`height`/`z_ofs`, and they +/// repeat byte-identically across levels, which dimensions would not. They +/// are handed back unnamed rather than guessed at. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ShrubBillboard { + /// Fade distance at `+0x00` (5..1000 on the disc). Following R&C1, the + /// loader takes `trunc(fade) & 0xff`, which makes 512 read as 0 -- + /// billboard only at any distance. + pub fade_distance: f32, + /// Three floats at `+0x04..+0x10`. Meaning not established. + pub unknown: [f32; 3], + /// `TEX1_1` at `+0x10`: the signed LOD bias K in 1/16 units, retail + /// -233..-168 on the disc. + pub k: i16, + /// `TEX1_1` minification filter, 4 on every disc billboard. + pub mmin: u32, + /// `TEX0_1.data_lo` at `+0x20`: a small index (1..4 on the disc). In R&C1 + /// this is an index into the class's texture list. + pub tex0: u32, + /// The GIF register address words carried behind the three registers, + /// 0x14 / 0x06 / 0x34 on every disc billboard. Exposed rather than + /// asserted on: they corroborate the layout, but a billboard that carries + /// a different register set is still a billboard. + pub tex1_addr: u32, + /// See [`ShrubBillboard::tex1_addr`]. + pub tex0_addr: u32, + /// See [`ShrubBillboard::tex1_addr`]. + pub miptbp_addr: u32, +} -/// The two constant bytes closing each body record. -pub const SHRUB_RECORD_TAIL: [u8; 2] = [0, 0]; - -/// A shrub span: its header plus the bytes the header points at. +/// A shrub span: its header, palette and optional billboard. #[derive(Debug, Clone, Copy, PartialEq)] pub struct Shrub<'a> { pub header: ShrubHeader, /// The whole span, starting at `+0x24` with the class number. pub raw: &'a [u8], - /// The bytes at `data_offset`, running to `span_end`. Not yet decoded. + /// The bytes at `data_offset`, running to `span_end`. This is the palette + /// plus its zero padding. pub data: &'a [u8], - /// Whether `span_end` came from the next class rather than from the end of - /// the lump. False for the last class in each level, whose span contains - /// whatever follows shrub. + /// Whether `span_end` came from the next class or the located chunk + /// container rather than from the end of the lump. False only for a span + /// bounded by a caller that could not locate either. pub span_end_is_a_successor: bool, } impl<'a> Shrub<'a> { - /// The body's 8-byte records. - /// - /// The payload of each record is six bytes whose encoding is **not** - /// established. It is not half-floats -- decoding as a `half3` gives - /// values like 245.25 and -168.375 -- and a VIF reading of it was - /// retracted after a control put it on the random baseline. Handed back - /// raw rather than guessed at. + /// The palette's 8-byte records, raw. pub fn body_records(&self) -> impl Iterator { - self.data.chunks_exact(SHRUB_BODY_RECORD_BYTES) + self.data.chunks_exact(8) } /// How many whole records the body holds. pub fn body_record_count(&self) -> usize { - self.data.len() / SHRUB_BODY_RECORD_BYTES + self.data.len() / 8 } - /// Whether every whole record ends in the constant pair. + /// The palette: the first [`SHRUB_NORMAL_COUNT`] records, parsed. /// - /// This is how a class's body is delimited: on all 24 levels that have - /// shrub geometry, the last shrub class's body ends exactly where its - /// trailing `00 00` pair stops, always on a record boundary. + /// Returns fewer only if the body is shorter than the palette, which does + /// not happen on the disc -- every one of the 358 spans carries all 24. + pub fn normals(&self) -> impl Iterator + 'a { + self.data + .chunks_exact(8) + .take(SHRUB_NORMAL_COUNT) + .filter_map(move |r| ShrubNormal::parse(r).ok()) + } + + /// Records past the palette. Zero on every disc span: the palette is + /// padded out with zero records rather than with live normals. + pub fn padding_record_count(&self) -> usize { + self.body_record_count().saturating_sub(SHRUB_NORMAL_COUNT) + } + + /// Whether every whole record's trailing pair is zero. + /// + /// The palette's pad word and the padding records are both zero, so a + /// body that satisfies this is whole records; it is how a class's body is + /// delimited, and it holds on all 358 disc spans now that each is bounded + /// by its successor. pub fn body_is_well_formed(&self) -> bool { let records: Vec<&[u8]> = self.body_records().collect(); - !records.is_empty() && records.iter().all(|r| r[6..8] == SHRUB_RECORD_TAIL) + !records.is_empty() && records.iter().all(|r| r[6..8] == [0, 0]) + } + + /// The billboard record, when the header names one. + /// + /// The offset is checked against `span_end` by + /// [`ShrubHeader::region`] rather than trusted: a non-zero offset that + /// runs past the span is a mis-set field, not a billboard. + pub fn billboard(&self) -> Result> { + if self.header.billboard_offset == 0 { + return Ok(None); + } + let at = self.header.billboard_offset as usize; + // `raw` runs from the span start, so the record sits at raw[at..]. + let bytes = self.raw.get(at..at + 0x40).ok_or(Error { + offset: at, + context: "shrub billboard", + kind: ErrorKind::OutOfBounds { + needed: 0x40, + available: self.raw.len().saturating_sub(at), + }, + })?; + let mut b = Buf::new(bytes); + let fade_distance = b.f32("shrub billboard fade")?; + let unknown = [ + b.f32("shrub billboard unknown 0")?, + b.f32("shrub billboard unknown 1")?, + b.f32("shrub billboard unknown 2")?, + ]; + // TEX1_1: K, a zero high half, then MMIN, then the register address + // word and a zero pad -- a 16-byte GifAdData16 per register. + let k = b.i16("shrub billboard K")?; + b.skip(2, "shrub billboard K high")?; + let mmin = b.u32("shrub billboard mmin")?; + let tex1_addr = b.u32("shrub billboard tex1 address")?; + b.skip(4, "shrub billboard tex1 pad")?; + let tex0 = b.u32("shrub billboard tex0 index")?; + b.skip(4, "shrub billboard tex0 data hi")?; + let tex0_addr = b.u32("shrub billboard tex0 address")?; + // MIPTBP1_1: two zero data words, then the register address word and a + // pad. The mip base pointers are filled in at level load, as they are + // for every ad-gif in this format. + b.skip(4, "shrub billboard tex0 pad")?; + b.skip(8, "shrub billboard miptbp data")?; + let miptbp_addr = b.u32("shrub billboard miptbp address")?; + Ok(Some(ShrubBillboard { + fade_distance, + unknown, + k, + mmin, + tex0, + tex1_addr, + tex0_addr, + miptbp_addr, + })) } } impl ShrubHeader { - /// Parse a shrub span starting at `offset` within `core_data`. + /// Parse a shrub span starting at `offset` within `core`. /// /// `o_class` is passed in so the self-identifying field can be checked /// rather than merely reported: a span whose `+0x24` disagrees with its /// own class entry is a different thing entirely, and quietly returning /// the mismatched value would hide that. + /// /// `span_end` bounds the class span within `core`. It matters: without it /// `data` runs to the end of the lump and swallows whatever follows, which /// is how a per-class check ends up measuring its neighbours. @@ -140,41 +325,75 @@ impl ShrubHeader { }, })?; let mut fb = Buf::new(&head[..16]); - let bounds = [ - fb.f32("shrub bounds 0")?, - fb.f32("shrub bounds 1")?, - fb.f32("shrub bounds 2")?, - fb.f32("shrub bounds 3")?, + let sphere = [ + fb.f32("shrub sphere x")?, + fb.f32("shrub sphere y")?, + fb.f32("shrub sphere z")?, + fb.f32("shrub sphere radius")?, ]; let mut fs = Buf::new(&head[0x10..0x18]); let size = fs.f32("shrub size")?; - let reserved = fs.f32("shrub reserved")?; - if !bounds.iter().all(|v| v.is_finite()) || !size.is_finite() || !reserved.is_finite() { + let mode_bits = fs.u16("shrub mode bits")?; + let scratch = fs.u16("shrub scratch")?; + if !sphere.iter().all(|v| v.is_finite()) || !size.is_finite() { return Err(Error { offset: start, context: "shrub header holds a non-finite float", kind: ErrorKind::Invalid("shrub header float"), }); } - // Zero, or the denormal the PS2 would flush to zero anyway. Anything - // else means the header is not laid out the way this thinks it is. - if reserved != 0.0 && !reserved.is_subnormal() { + // A radius is a radius: non-positive means the header is not laid out + // the way this thinks it is. + if sphere[3] <= 0.0 { return Err(Error { - offset: start + 0x14, - context: "shrub reserved word is neither zero nor denormal", - kind: ErrorKind::Invalid("shrub reserved"), + offset: start + 0x0c, + context: "shrub sphere radius", + kind: ErrorKind::Invalid("shrub sphere radius is not positive"), + }); + } + // One invariant across all 358 spans: the scratch word at +0x16 is + // zero everywhere. A non-zero value means this is not a shrub header. + // `mode_bits` at +0x14 is *not* asserted -- it is 4 on 16 of the 358 + // spans -- so the field is reported rather than validated. + if scratch != 0 { + return Err(Error { + offset: start + 0x16, + context: "shrub scratch", + kind: ErrorKind::Invalid("shrub scratch is not zero"), + }); + } + let reserved = u32::from_le_bytes(head[0x18..0x1c].try_into().expect("4 bytes at +0x18")); + if reserved != 0 { + return Err(Error { + offset: start + 0x18, + context: "shrub reserved word", + kind: ErrorKind::Invalid("shrub reserved word is not zero"), }); } - let mut b = Buf::new(&head[0x24..]); - let span_class = b.u32("shrub o_class")?; + let mut b = Buf::new(&head[0x1c..]); + let billboard_offset = b.u32("shrub billboard offset")?; + let scale = b.f32("shrub scale")?; + if !scale.is_finite() || scale <= 0.0 { + return Err(Error { + offset: start + 0x20, + context: "shrub scale", + kind: ErrorKind::Invalid("shrub scale is not positive"), + }); + } + let mut bc = Buf::new(&head[0x24..]); + let span_class = bc.u32("shrub o_class")?; let header = ShrubHeader { - bounds: [0.0; 4], + sphere: [0.0; 4], size: 0.0, - reserved: 0.0, + mode_bits: 0, + scratch, + reserved, + billboard_offset, + scale, o_class: span_class, - count: b.u32("shrub count")?, - data_offset: b.u32("shrub data offset")?, + count: bc.u32("shrub count")?, + data_offset: bc.u32("shrub data offset")?, }; if header.o_class != o_class.max(0) as u32 { @@ -185,8 +404,17 @@ impl ShrubHeader { }); } - let at = header.data_offset as usize; + // A billboard that is not inside the span is a mis-set offset. let stop = span_end.clamp(start, core.len()); + if billboard_offset > 0 && start + billboard_offset as usize + 0x40 > stop { + return Err(Error { + offset: start + 0x1c, + context: "shrub billboard offset", + kind: ErrorKind::Invalid("shrub billboard offset runs past the span"), + }); + } + + let at = header.data_offset as usize; let data = core.get(start + at..stop).ok_or(Error { offset: start + at, context: "shrub data offset runs past the lump", @@ -198,9 +426,13 @@ impl ShrubHeader { Ok(Some(Shrub { span_end_is_a_successor: true, header: ShrubHeader { - bounds, + sphere, size, + mode_bits, + scratch, reserved, + billboard_offset, + scale, o_class: header.o_class, count: header.count, data_offset: header.data_offset, @@ -219,16 +451,41 @@ mod tests { /// span cannot start there. const BASE: usize = 0x10; - /// A span whose `+0x24` names class 1072 and whose data starts at 0x30. - fn span() -> Vec { - let mut v = vec![0u8; BASE + 0x40]; + /// A unit direction per palette slot: a spiral over the sphere, so every + /// entry is a different unit vector rather than a repeated one. + fn fixture_normal(i: usize) -> [f32; 3] { + let t = i as f32 / SHRUB_NORMAL_COUNT as f32 * std::f32::consts::TAU; + let z = (i % 5) as f32 / 2.0 - 1.0; + let r = (1.0 - z * z).max(0.0).sqrt(); + [r * t.cos(), r * t.sin(), z] + } + + /// A span naming class 1072, data at 0x30, a positive radius and scale, + /// and a 24-entry palette of unit normals. `tail` extra bytes follow the + /// palette inside the span: zero ones read as padding records. + fn span_with_tail(tail: usize) -> Vec { + let mut v = vec![0u8; BASE + 0x30 + SHRUB_NORMAL_COUNT * 8 + tail]; + v[BASE + 0x0c..BASE + 0x10].copy_from_slice(&12.5f32.to_le_bytes()); + v[BASE + 0x10..BASE + 0x14].copy_from_slice(&6.0f32.to_le_bytes()); + v[BASE + 0x20..BASE + 0x24].copy_from_slice(&0.25f32.to_le_bytes()); v[BASE + 0x24..BASE + 0x28].copy_from_slice(&1072u32.to_le_bytes()); v[BASE + 0x28..BASE + 0x2c].copy_from_slice(&5u32.to_le_bytes()); v[BASE + 0x2c..BASE + 0x30].copy_from_slice(&0x30u32.to_le_bytes()); - v[BASE + 0x30..].fill(0xab); + for (i, n) in (0..SHRUB_NORMAL_COUNT).map(fixture_normal).enumerate() { + let at = BASE + 0x30 + i * 8; + for (c, &v_c) in n.iter().enumerate() { + let q = (v_c * 32767.0).round() as i16; + v[at + c * 2..at + c * 2 + 2].copy_from_slice(&q.to_le_bytes()); + } + } v } + /// A span with two zero padding records after the palette. + fn span() -> Vec { + span_with_tail(16) + } + #[test] fn reads_the_self_identifying_header() { let c = span(); @@ -238,7 +495,85 @@ mod tests { assert_eq!(s.header.o_class, 1072); assert_eq!(s.header.count, 5); assert_eq!(s.header.data_offset, 0x30); - assert!(s.data.iter().all(|&b| b == 0xab)); + assert_eq!(s.header.billboard_offset, 0); + assert_eq!(s.header.scale, 0.25); + assert_eq!(s.header.sphere[3], 12.5); + assert!(s.billboard().unwrap().is_none()); + } + + #[test] + fn the_body_is_24_unit_normals_plus_zero_padding() { + let c = span(); + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) + .unwrap() + .expect("present"); + let normals: Vec = s.normals().collect(); + assert_eq!(normals.len(), SHRUB_NORMAL_COUNT); + assert!(normals.iter().all(|n| n.is_unit()), "palette is not unit"); + // The fixture's two trailing zero records are padding, not normals. + assert_eq!(s.padding_record_count(), 2); + assert_eq!(s.body_record_count(), SHRUB_NORMAL_COUNT + 2); + assert!(s.body_is_well_formed()); + // The palette repeats no direction, and the padding is not in it. + assert!(!s.normals().any(|n| n == ShrubNormal { x: 0, y: 0, z: 0 })); + } + + #[test] + fn a_body_of_zeros_is_not_a_palette_of_normals() { + let mut v = vec![0u8; BASE + 0x40]; + v[BASE + 0x0c..BASE + 0x10].copy_from_slice(&12.5f32.to_le_bytes()); + v[BASE + 0x10..BASE + 0x14].copy_from_slice(&6.0f32.to_le_bytes()); + v[BASE + 0x20..BASE + 0x24].copy_from_slice(&0.25f32.to_le_bytes()); + v[BASE + 0x24..BASE + 0x28].copy_from_slice(&1072u32.to_le_bytes()); + v[BASE + 0x28..BASE + 0x2c].copy_from_slice(&5u32.to_le_bytes()); + v[BASE + 0x2c..BASE + 0x30].copy_from_slice(&0x30u32.to_le_bytes()); + let s = ShrubHeader::region(&v, BASE as i32, 1072, v.len()) + .unwrap() + .expect("present"); + // Every record is whole and zero-tailed, but no entry is a normal. + assert!(s.body_is_well_formed()); + assert!(!s.normals().any(|n| n.is_unit())); + } + + #[test] + fn a_billboard_record_is_read_and_checked() { + let mut c = span_with_tail(0x40 + 8); + // The billboard sits directly after the palette. + let bb = BASE + 0x30 + SHRUB_NORMAL_COUNT * 8; + c[BASE + 0x1c..BASE + 0x20].copy_from_slice(&((bb - BASE) as u32).to_le_bytes()); + c[bb..bb + 4].copy_from_slice(&50.0f32.to_le_bytes()); + // TEX1: K = -217, MMIN = 4, address 0x14. + c[bb + 0x10..bb + 0x12].copy_from_slice(&(-217i16).to_le_bytes()); + c[bb + 0x14..bb + 0x18].copy_from_slice(&4u32.to_le_bytes()); + c[bb + 0x18..bb + 0x1c].copy_from_slice(&0x14u32.to_le_bytes()); + // TEX0: data_lo = 1, address 0x06. + c[bb + 0x20..bb + 0x24].copy_from_slice(&1u32.to_le_bytes()); + c[bb + 0x28..bb + 0x2c].copy_from_slice(&0x06u32.to_le_bytes()); + // MIPTBP1_1: address 0x34. + c[bb + 0x38..bb + 0x3c].copy_from_slice(&0x34u32.to_le_bytes()); + + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) + .unwrap() + .expect("present"); + let b = s.billboard().unwrap().expect("billboard present"); + assert_eq!(b.fade_distance, 50.0); + assert_eq!(b.k, -217); + assert_eq!(b.mmin, 4); + assert_eq!(b.tex0, 1); + assert_eq!(b.tex1_addr, 0x14); + assert_eq!(b.tex0_addr, 0x06); + assert_eq!(b.miptbp_addr, 0x34); + // The palette is untouched by the billboard sitting after it. + assert_eq!(s.normals().count(), SHRUB_NORMAL_COUNT); + assert!(s.normals().all(|n| n.is_unit())); + } + + #[test] + fn a_billboard_offset_past_the_span_is_rejected() { + let mut c = span_with_tail(0x40); + // Far past the span's end. + c[BASE + 0x1c..BASE + 0x20].copy_from_slice(&9000u32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); } #[test] @@ -258,37 +593,21 @@ mod tests { } #[test] - fn body_records_split_into_eights_and_check_their_tail() { - let c = span(); - let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) - .unwrap() - .expect("present"); - // The fixture fills 0xab, so every record's tail is wrong. - assert!(!s.body_is_well_formed()); - assert_eq!(s.body_record_count(), (0x40 - 0x30) / 8); - // Fix the tails and it is well formed. - let mut c2 = span(); - for i in 0..((0x40 - 0x30) / 8) { - c2[BASE + 0x30 + i * 8 + 6] = 0; - c2[BASE + 0x30 + i * 8 + 7] = 0; - } - let s2 = ShrubHeader::region(&c2, BASE as i32, 1072, c2.len()) - .unwrap() - .expect("present"); - assert!(s2.body_is_well_formed()); - assert_eq!(s2.body_records().count(), s2.body_record_count()); + fn a_nonpositive_radius_or_scale_is_rejected() { + let mut c = span(); + c[BASE + 0x0c..BASE + 0x10].copy_from_slice(&0.0f32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); + + let mut c = span(); + c[BASE + 0x20..BASE + 0x24].copy_from_slice(&0.0f32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); } #[test] - fn a_body_shorter_than_one_record_is_not_well_formed() { + fn a_nonzero_reserved_word_is_rejected() { let mut c = span(); - // Point the data at the last four bytes of the span. - c[BASE + 0x2c..BASE + 0x30].copy_from_slice(&(0x3cu32).to_le_bytes()); - let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) - .unwrap() - .expect("present"); - assert_eq!(s.body_record_count(), 0); - assert!(!s.body_is_well_formed()); + c[BASE + 0x18..BASE + 0x1c].copy_from_slice(&0x40u32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); } #[test] diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index 8e91c31..9447c88 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -1003,4 +1003,126 @@ Consequences, all measured over the new bounds: invented edge. The chunk test now reads the region from `core_layout`. What is still not known: the marker's meaning (constant across 27 levels, at -exactly this position, unnamed), and the shrub body payload's encoding. +exactly this position, unnamed). The shrub body payload's encoding is the next +section. + +## The shrub class blob: palette and billboard (2026-10-11, after the bound) + +With every span bounded by the chunk container, the shrub class blob reads +cleanly against the R&C1 format (`vendor/rerac/docs/formats/shrub_sky_rac1.md`, +derived from the Wrench sources). R&C1's shrub class header keeps the same +field offsets R&C2 shows -- `o_class` at 0x24, a count at 0x28, the data offset +at 0x2c -- so the reference says which bytes to look at and the disc says what +they hold. + +**Range measured:** all 358 shrub class spans over the 24 levels that have +them, each bounded by its successor class or the located chunk container. +Nothing outside the spans is read. + +### The body is a 24-entry normal palette + +Two things made this identifiable where three stride-fitting attempts had +failed. First the shape: every body is 8-byte records whose last two bytes are +zero, and R&C1's palette is documented as 24 records of exactly that shape +(`s16` x, y, z, pad). Second the count: bodies hold 24, 26, 28 or 30 records +with the extra ones **always all zero and always at index 24 or later** -- +never a live entry. So the body is 24 normals plus zero padding, and the +padding is what distinguishes it from geometry. + +The palette is real normals, and the test says how firmly: + +- all 8,592 live entries have squared length within **65,535** of `32767^2`, + and the bound is tight -- the largest observed deviation is exactly 65,535. + That number is not fitted: it is `32768^2 - 32767^2`, which is what a unit + vector scaled by 32768 produces when an axis-aligned normal quantises to + `-32768` while the positive axis clamps to `32767`. Components run the full + `s16` range (-32768..32767) and 1,765 of the 8,592 entries are axis-aligned, + which is what a coarse direction codebook looks like. +- a random `s16` triple has squared length spread about a *billion* either + side of `32767^2`, so one entry in fifteen thousand would pass by chance. + All 8,592 pass. +- the clincher: 17 classes carry a palette of 24 identical `(0, 0, 1)` + entries. A position or index table cannot produce that; a normal palette for + a flat-facing class must. + +This is also where shrub lighting comes from -- R&C1 lights each instance +against its class's 24-direction palette. Unlike the tfrag's per-vertex light +records, which store azimuth/elevation *indices* into the overlay's sine table, +the shrub palette stores the directions outright, so the overlay table is not +needed for shrubs. + +### The header, byte by byte + +**Range measured:** the 0x30 bytes of every span. + +| offset | field | measured | +|---|---|---| +| 0x00 | f32[4] `sphere` | centre (xyz) + radius (w). Finite on all 358, **radius positive on all 358** (1.72..41.57), centres near the class's own origin | +| 0x10 | f32 `size` | 3.0..20.0, positive on all 358 | +| 0x14 | u16 `mode_bits` | **0 on 342 spans, 4 on 16** | +| 0x16 | u16 `scratch` | **0 on all 358** (asserted by the parser) | +| 0x18 | u32 `reserved` | **0 on all 358** (asserted by the parser) | +| 0x1c | u32 `billboard_offset` | 0 on 333 spans, in-span on 25 | +| 0x20 | f32 `scale` | 0.0044..1.8, positive on all 358 | +| 0x24 | u32 `o_class` | equals the class entry's own, 358/358 | +| 0x28 | u32 `count` | 1..55, meaning not established | +| 0x2c | u32 `data_offset` | inside the span, 358/358 | + +Two corrections this produces, both recorded because the old readings were +confidently wrong rather than merely unverified: + +- **"reserved is 0 or a denormal" was the wrong shape.** The 16 spans with a + non-zero value there have `mode_bits = 4` and `scratch = 0`, not the other + way round. As an `f32` the word at `+0x14` is still a denormal, so the PS2 + reads zero either way -- but the field is a flag word, not a magnitude, and + the demo is the point: the EE's FPU has no denormals, so `04000000` and zero + are the same value to the hardware. +- **"bounds" is a bounding sphere, not a min/max box.** The old reading tested + `min.xyz/max.xyz` ordering, found it on only 146 of 358 spans, and stopped + naming the field. It is centre + radius, as in R&C1: the fourth float is + positive on all 358 spans and the centres cluster near the class origin + within +-24, which is where a class-local sphere belongs. The 146/358 figure + was a property of the wrong question. + +### The billboard record + +The billboard offset at `+0x1c` is the strongest self-identification in the +header after `o_class`, because it is **confirmed from a second file**: it is +non-zero on exactly the 25 classes whose core-index entry carries billboard +texture fields, in both directions -- 25 records and 25 entries, pairing +correctly. R&C1 states the same correspondence ("a class has a billboard record +exactly when its core-index billboard texture has width != 0"). + +At the offset sits R&C1's `ShrubBillboard` shape, register block and all: + +| offset | field | measured | +|---|---|---| +| 0x00 | f32 `fade_distance` | 5..1000; 512 reads as 0 under R&C1's `trunc(fade) & 0xff`, i.e. billboard-only | +| 0x04 | f32[3] | **not established** -- see below | +| 0x10 | TEX1_1: K, MMIN, address | K -233..-168, MMIN 4 on all 25, address 0x14 | +| 0x20 | TEX0_1: data_lo, address | index 1..4, address 0x06 | +| 0x30 | MIPTBP1_1: address | 0x34, data words zero (filled at level load) | + +The three floats at `+0x04..+0x10` are the part that does *not* carry over. +R&C1's are `width`, `height` and `z_ofs` in class units; R&C2's are an order of +magnitude too large and too varied to be dimensions of a 32..128 px billboard, +and they **repeat byte-identically across levels**: 16 distinct triples among +the 25 records, with one appearing on four classes across levels 1, 4, 12 and +21. Dimensions would not do that; shared art assets would. They are handed back +unnamed. + +Two further facts about them, measured: the whole record repeats with them -- +three classes on different levels carry identical `fade`, floats, K and scale +-- and the class entries they belong to carry identical `sphere` too. So the +blobs themselves are shared between levels, which is the same behaviour the +chunk container showed with its byte-identical tables. + +### Still open + +- The three billboard floats, `size`, `mode_bits`, and `count`. +- **Where shrub mesh geometry lives.** R&C1 keeps the class's mesh packets in + the blob between the header and the palette; R&C2's blob is header, palette + and optional billboard only -- 24 normals and nothing else. The 333 classes + without a billboard still have to draw a mesh, and it is not here. The + candidates are the chunk container that follows shrub and the gameplay + extent, and neither is decoded. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0005426..87b8119 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -217,13 +217,18 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [ ] `tie` — instanced environment geometry, fat vertices - [~] `shrub` — class table done (48-byte entries, not 32; reading shrub at 32 still gives 231 sane-looking entries of 358, which is the hazard). - Geometry header parsed and both fields **checked**, not merely reported: - `+0x24` equals the entry's own `o_class`, and `+0x2c` is an in-span data - offset, on all 358 spans. Shrub does **not** use moby's part-table format. - The body is **8-byte records, 6 varying bytes and 2 constant**, and now - that every span is bounded by the located chunk container, all 358 - bodies are whole records — 9,600 in total. The payload's encoding is - still unknown, and so is the 2-byte marker that closes the region. + Geometry header parsed and checked field by field on all 358 spans: + `+0x24` equals the entry's own `o_class`, `+0x2c` is an in-span data + offset, `+0x16`/`+0x18` are zero, and `+0x1c` names a billboard on + exactly the 25 classes whose core-index entry carries billboard texture + fields. Shrub does **not** use moby's part-table format. The body is a + **24-entry palette of unit-length s16 normals** (squared length within + 65,535 of 32767² on all 8,592 entries — a random triple would miss by a + billion) plus zero padding records; the billboard record carries K, MMIN + and the TEX0/MIPTBP addresses as R&C1's does. What remains open: the + three floats in the billboard record, `size`, `mode_bits`, `count`, and + **where the mesh geometry lives** — R&C1 puts the mesh packets in this + blob and R&C2's blob holds only the palette. - [x] `moby` — per-class model geometry. A class span opens with a `u32` pointing at a part table of 16-byte records; parts tile exactly, with `vertex_offset + 16 * vertex_count == next_part` holding for **43,467 of