From 2c7555fe165ea5c6ba47f0394a47ae442ccf5e7c Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Sun, 11 Oct 2026 13:33:37 +0200 Subject: [PATCH] Decode the shrub class blob: a 24-normal palette and a billboard record The shrub body's payload encoding was the open question after the span-bound work, and it answered itself once the R&C1 reference was in view: R&C1's shrub class header keeps the field offsets R&C2 shows (o_class at 0x24, a count at 0x28, the data offset at 0x2c), which says which bytes to look at, and the disc says what they hold. Range measured: all 358 shrub class spans over the 24 levels that have them, each bounded by its successor class or the located chunk container. Nothing outside the spans is read. Result. The body is a palette of 24 unit-length s16 normals -- s16 x, y, z and a zero pad word per 8-byte record -- followed by all-zero padding records. Three independent measurements, each of which fails for any other reading: - all 8,592 live entries have squared length within 65,535 of 32767^2, and the bound is tight (the largest observed deviation is exactly 65,535). That is 32768^2 - 32767^2, which is what a unit vector scaled by 32768 produces when an axis-aligned normal quantises to -32768 and the positive axis clamps to 32767; components run the full s16 range and 1,765 entries are axis-aligned. A random s16 triple is off by about a billion, so one entry in fifteen thousand would pass by chance: all 8,592 pass. - the padding records are always at index 24 or later and always all zero, never a live entry. That is what separates the palette from geometry, and it is the observation the three earlier stride-fitting attempts were missing. - 17 classes carry a palette of 24 identical (0, 0, 1) entries. A position or index table cannot produce that; a normal palette for a flat-facing class must. The header decodes as: sphere at +0x00 (centre + radius, radius positive on all 358 spans, 1.72..41.57), size at +0x10, mode_bits and scratch at +0x14, a zero word at +0x18, the billboard offset at +0x1c, and scale at +0x20. The parser asserts the two invariants that hold without exception (scratch and the reserved word are zero on all 358) plus a positive radius and scale, and reports mode_bits rather than asserting it: it is 4 on 16 of the 358 spans. The billboard offset is the strongest self-identification after o_class because it is confirmed from a second file: it is non-zero on exactly the 25 classes whose core-index entry carries billboard texture fields, pairing correctly both ways. At the offset sits R&C1's ShrubBillboard shape, register block and all -- fade_distance, then TEX1_1 (K -233..-168, MMIN 4, address 0x14), TEX0_1 (index 1..4, address 0x06) and MIPTBP1_1 (address 0x34), all three register addresses checked in the disc test. Two corrections this produces, both recorded in docs/plan/level-assets.md because the old readings were confidently wrong rather than merely unverified: - "reserved is 0 or a denormal" had the two u16 halves the wrong way round: the 16 non-zero spans are mode_bits = 4 with scratch = 0. As an f32 the word is still a denormal, so the PS2 reads zero either way -- but the field is a flag word, not a magnitude, which is exactly the FPU point worth making. - "bounds" was tested as a min.xyz/max.xyz box, which holds on only 146 of 358 spans, and the field was left unnamed as a result. It is a bounding sphere: the fourth float is positive on all 358 spans and the centres cluster near the class origin. The 146/358 figure was a property of the wrong question. Still open: the three floats in the billboard record (they repeat byte-identically across levels -- 16 distinct triples among 25 records, one on four classes across levels 1, 4, 12 and 21 -- so they are shared art data rather than dimensions), size, mode_bits, count, and where the shrub mesh geometry lives. R&C1 puts the mesh packets in this blob; R&C2's blob holds only the palette, so the 333 classes without a billboard must draw meshes from somewhere else. --- crates/gc-extract/tests/disc.rs | 112 ++++++- crates/gc-formats/src/shrub.rs | 537 +++++++++++++++++++++++++------- docs/plan/level-assets.md | 124 +++++++- docs/roadmap.md | 19 +- 4 files changed, 658 insertions(+), 134 deletions(-) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 0f3663a..6d2a769 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1620,11 +1620,17 @@ fn moby_parts_tile_on_every_level() { ); } -/// Shrub class headers. +/// Shrub class headers and blobs. /// /// Two fields self-identify exactly on every span, and the parser checks both /// rather than merely reporting them: `+0x24` must equal the owning class /// entry's own `o_class`, and `+0x2c` must land inside the span. +/// +/// The body is then checked against the R&C1 shape this format follows: a +/// palette of 24 unit-length `s16` normals with a zero pad word, then zero +/// padding records. And the billboard offset at `+0x1c` is checked against the +/// class entry's billboard texture fields, which is the one header field that +/// can be confirmed from a second file. #[test] fn shrub_headers_name_their_own_class_on_every_level() { require_disc!(); @@ -1640,14 +1646,23 @@ fn shrub_headers_name_their_own_class_on_every_level() { } use gc_formats::level_core::ShrubClassEntry; - use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES}; + use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES, SHRUB_NORMAL_COUNT}; let mut levels = 0usize; let mut spans = 0usize; let mut absent = 0usize; let (mut min_size, mut max_size) = (f32::MAX, 0.0f32); - let mut denormals = 0usize; + let mut scratch_nonzero = 0usize; + let mut mode_hist: std::collections::BTreeMap = Default::default(); + let mut scratch_hist: std::collections::BTreeMap = Default::default(); + let mut radius_range = (f32::MAX, 0f32); + let mut scale_range = (f32::MAX, 0f32); let mut bounded_records = 0usize; let mut bounded_bodies = 0usize; + let mut palette_records = 0usize; + let mut padding_records = 0usize; + let mut norm2_deviation = 0i64; + let mut billboards = 0usize; + let mut billboard_entries = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); @@ -1720,26 +1735,81 @@ fn shrub_headers_name_their_own_class_on_every_level() { ); assert!(!s.data.is_empty(), "level{n} class {i}: empty shrub data"); assert!( - s.header.bounds.iter().all(|v| v.is_finite()), - "level{n} class {i}: non-finite shrub bounds" + s.header.sphere.iter().all(|v| v.is_finite()), + "level{n} class {i}: non-finite shrub sphere" ); assert!( s.header.size.is_finite() && s.header.size > 0.0, "level{n} class {i}: shrub size {}", s.header.size ); - // The PS2 has no denormals, so a denormal here and a zero - // are the same value to the hardware. Both occur on disc. - if s.header.reserved != 0.0 { - assert!( - s.header.reserved.is_subnormal(), - "level{n} class {i}: reserved {} is neither zero nor denormal", - s.header.reserved - ); - denormals += 1; + // The scratch word is zero on every span; the parser + // refuses anything else. `mode_bits` is 4 on 16 spans and + // is reported rather than asserted. On those 16 the word + // at +0x14 reads as an f32 denormal, which the Emotion + // Engine flushes to zero like everything else. + if s.header.scratch != 0 { + scratch_nonzero += 1; } + *mode_hist.entry(s.header.mode_bits).or_default() += 1; + *scratch_hist.entry(s.header.scratch).or_default() += 1; + radius_range.0 = radius_range.0.min(s.header.sphere[3]); + radius_range.1 = radius_range.1.max(s.header.sphere[3]); + scale_range.0 = scale_range.0.min(s.header.scale); + scale_range.1 = scale_range.1.max(s.header.scale); min_size = min_size.min(s.header.size); max_size = max_size.max(s.header.size); + + // The palette: 24 unit normals, then zero padding. + let normals: Vec<_> = s.normals().collect(); + assert_eq!( + normals.len(), + SHRUB_NORMAL_COUNT, + "level{n} class {i}: palette is {} entries, not {SHRUB_NORMAL_COUNT}", + normals.len() + ); + for (k, normal) in normals.iter().enumerate() { + assert!( + normal.is_unit(), + "level{n} class {i}: palette entry {k} {normal:?} is not a unit normal" + ); + let n2 = (normal.x as i64).pow(2) + + (normal.y as i64).pow(2) + + (normal.z as i64).pow(2); + norm2_deviation = norm2_deviation.max((n2 - 32767i64.pow(2)).abs()); + } + palette_records += normals.len(); + for (k, record) in s.body_records().skip(SHRUB_NORMAL_COUNT).enumerate() { + assert!( + record.iter().all(|&b| b == 0), + "level{n} class {i}: padding record {k} is not all zero" + ); + } + padding_records += s.padding_record_count(); + + // The billboard: present exactly when the class entry + // carries billboard texture fields. + let entry_has_billboard = e.billboard_width != 0 || e.billboard_height != 0; + match s.billboard().expect("billboard parses") { + Some(b) => { + billboards += 1; + assert!( + entry_has_billboard, + "level{n} class {i}: billboard record without billboard texture fields" + ); + assert_eq!(b.tex1_addr, 0x14, "level{n} class {i}: TEX1 address"); + assert_eq!(b.tex0_addr, 0x06, "level{n} class {i}: TEX0 address"); + assert_eq!(b.miptbp_addr, 0x34, "level{n} class {i}: MIPTBP address"); + assert_eq!(b.mmin, 4, "level{n} class {i}: MMIN"); + } + None => assert!( + !entry_has_billboard, + "level{n} class {i}: billboard texture fields with no billboard record" + ), + } + if entry_has_billboard { + billboard_entries += 1; + } spans += 1; } Ok(None) => absent += 1, @@ -1759,11 +1829,19 @@ fn shrub_headers_name_their_own_class_on_every_level() { assert!(spans > 340, "only {spans} shrub spans with geometry"); // The container bounds every class now, so every body is checkable. assert_eq!(bounded_bodies, spans, "every shrub class should be bounded"); + assert_eq!(scratch_nonzero, 0, "the scratch word is zero on every span"); + assert_eq!( + billboards, billboard_entries, + "billboard records and billboard class entries disagree: \ + {billboards} records vs {billboard_entries} entries" + ); eprintln!( "shrub: {spans} headers naming their own class, {absent} without geometry; \ - all {bounded_bodies} class bodies bounded by the chunk container, \ - {bounded_records} 8-byte records in total, \ - {denormals} denormal reserved words" + all {bounded_bodies} bodies bounded, {bounded_records} records, \ + {palette_records} of them palette normals and {padding_records} zero padding; \ + {billboards} billboards, all with texture fields; \ + mode_bits {mode_hist:?}, radius {radius_range:?}, scale {scale_range:?}, \ + largest palette deviation from unit length {norm2_deviation} of a possible 65535" ); } diff --git a/crates/gc-formats/src/shrub.rs b/crates/gc-formats/src/shrub.rs index 3effab0..301aa27 100644 --- a/crates/gc-formats/src/shrub.rs +++ b/crates/gc-formats/src/shrub.rs @@ -1,4 +1,4 @@ -//! Shrub geometry: the class header. +//! Shrub geometry: the class header, the normal palette, and the billboard. //! //! Spec: `docs/plan/level-assets.md`. //! @@ -8,116 +8,301 @@ //! rule `vertex_offset + 16 * vertex_count == next_part`, which holds for //! 43,467 of 43,467 moby parts, holds for **zero** of them. //! -//! What the shrub header *does* have is two fields that self-identify exactly, -//! on all 358 spans: +//! The class blob is header, palette, optional billboard, in that order. The +//! reference for the shape is R&C1's shrub class (documented in +//! `vendor/rerac/docs/formats/shrub_sky_rac1.md`, whose offsets this header +//! follows: `o_class` at 0x24, a count at 0x28, the data offset at 0x2c), but +//! every field below is either **measured on all 358 spans** or marked as not +//! established. Where a field is only parallel to R&C1 and not independently +//! verified here, the doc comment says so. +//! +//! What self-identifies exactly, on all 358 spans: //! //! - `+0x24` equals the owning class entry's own `o_class`. //! - `+0x2c` is a data offset that lands inside the span. +//! - `+0x1c` (billboard offset) is non-zero on exactly the 25 classes whose +//! core-index entry carries billboard texture fields, and zero on the other +//! 333 -- two independent places, one answer. //! -//! Those two are what this parses. The body from `+0x2c` is an array of -//! 8-byte records -- six varying bytes and two constant zeros -- whose -//! payload encoding is not established; it is handed back raw rather than -//! guessed at. Its extent is knowable: the region ends at the chunk container, -//! located by signature (`crate::chunks::chunk_region_start`), so every span -//! is bounded and every body is whole records. -//! -//! Note the class entry size differs here too: shrub entries are 0x30 bytes, -//! not 0x20, because they carry billboard texture fields the other two tables -//! do not have. +//! The class entry size differs here too: shrub entries are 0x30 bytes, not +//! 0x20, because they carry billboard texture fields the other two tables do +//! not have. use crate::buf::{Buf, Error, ErrorKind, Result}; /// The measured size of a shrub class entry. pub const SHRUB_CLASS_ENTRY_BYTES: usize = 0x30; +/// Normals in the palette. Not a maximum: the body's zero padding follows the +/// twenty-fourth entry, so a body of 26, 28 or 30 records holds 24 normals and +/// 2, 4 or 6 zero records. +pub const SHRUB_NORMAL_COUNT: usize = 24; + +/// One palette entry: a unit normal as three `s16`, then a zero `s16`. +/// +/// The three components are a fraction of 32767 -- entry `i` of every body +/// satisfies `x^2 + y^2 + z^2` within rounding of `32767^2` on all 358 spans. +/// The fourth word is zero on all 9,600 records on the disc, which is what +/// makes the record 8 bytes rather than 6. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ShrubNormal { + pub x: i16, + pub y: i16, + pub z: i16, +} + +impl ShrubNormal { + /// Parse one 8-byte palette record. + pub fn parse(bytes: &[u8]) -> Result { + let mut b = Buf::new(bytes); + let x = b.i16("shrub normal x")?; + let y = b.i16("shrub normal y")?; + let z = b.i16("shrub normal z")?; + let pad = b.i16("shrub normal pad")?; + if pad != 0 { + return Err(Error { + offset: 6, + context: "shrub normal pad", + kind: ErrorKind::Invalid("shrub normal pad is not zero"), + }); + } + Ok(ShrubNormal { x, y, z }) + } + + /// Whether the entry is a unit normal, within the format's quantisation. + /// + /// The palette stores a unit vector scaled by 32768, so an axis-aligned + /// normal quantises to `(32768, 0, 0)` -- stored as the `s16` `-32768`, + /// because the positive axis clamps instead. The squared length therefore + /// lands within `32768^2 - 32767^2 = 65535` of `32767^2`, and that bound + /// is tight: the largest deviation on the disc is exactly 65,535. + /// + /// The margin is what makes this a test rather than a tautology. A random + /// `s16` triple has a squared length spread about a *billion* either side + /// of `32767^2`, so roughly one entry in fifteen thousand would pass by + /// chance; 8,592 of 8,592 real entries pass. + pub fn is_unit(&self) -> bool { + let n = (self.x as i64).pow(2) + (self.y as i64).pow(2) + (self.z as i64).pow(2); + (n - 32767i64.pow(2)).abs() <= 65_535 + } + + /// The normal as a unit vector in `[0, 1]`-ish float terms. + pub fn unit(self) -> [f32; 3] { + [ + self.x as f32 / 32767.0, + self.y as f32 / 32767.0, + self.z as f32 / 32767.0, + ] + } +} + /// The decoded part of a shrub class header. #[derive(Debug, Clone, Copy, PartialEq)] pub struct ShrubHeader { - /// Four floats at `+0x00..+0x10`. All finite on all 358 spans, but they do - /// not behave like a `min.xyz / max.xyz` bounding box -- that ordering only - /// holds on 146 of 358. Not named because it is not established. - pub bounds: [f32; 4], - /// A positive float at `+0x10`, 3.0 to 20.0 across the disc. + /// Four floats at `+0x00..+0x10`: a bounding sphere read as + /// centre (x, y, z) and radius (w), following R&C1's header at the same + /// offsets. What is measured here rather than assumed: all four are + /// finite, the radius is positive on all 358 spans (1.72..41.57), and the + /// centres sit near the class's own origin (-18..15, -23..23, -16..20). + /// The earlier reading of this field as a `min.xyz / max.xyz` box was + /// wrong -- that ordering holds on only 146 of 358 spans -- because it was + /// the wrong shape, not because the bytes were. + pub sphere: [f32; 4], + /// A positive float at `+0x10`, 3.0 to 20.0 across the disc. R&C1's + /// parallel at this offset is `mip_distance`; not independently + /// established. pub size: f32, - /// A float at `+0x14`. Exactly 0.0 on 342 of the disc's 358 spans; the - /// other 16 hold the bit pattern `04000000`, a denormal of about 2.8e-45. + /// A `u16` at `+0x14`: zero on 342 spans, 4 on the other 16. R&C1's + /// `mode_bits` is at the same offset. Not established. + pub mode_bits: u16, + /// A `u16` at `+0x16`, zero on all 358 spans. R&C1's field here is + /// `instance_count`, runtime scratch that retail files leave zero. /// - /// The Emotion Engine has no denormals, so hardware reads all 358 of these - /// as zero. That is the only reading the game itself ever sees, and it is - /// why this is safe to treat as a flag rather than a magnitude -- see - /// `AGENTS.md` on reproducing the FPU model rather than correcting it. - pub reserved: f32, + /// On the 16 spans where `mode_bits` is 4 the whole word at `+0x14` is an + /// `f32` denormal, so the Emotion Engine -- which has no denormals -- reads + /// zero for the pair either way. That is the sharpest available statement + /// of what the hardware does with this field. + pub scratch: u16, + /// A `u32` at `+0x18`, zero on all 358 spans. R&C1's `instances_pointer` + /// is runtime scratch at the same offset. + pub reserved: u32, + /// Offset of the [`ShrubBillboard`] record from the start of the span. + /// Zero means no billboard; a positive value lands inside the span on all + /// 25 spans that have one. + /// + /// This is the strongest self-identification in the header after + /// `o_class`: it is non-zero on exactly the classes whose core-index + /// entry carries billboard texture fields (25 of 358 each way), and the + /// two answers are produced by different bytes in different files. + pub billboard_offset: u32, + /// A positive `f32` at `+0x20` (0.0044..1.8 on the disc). R&C1's + /// `scale` is the position dequantisation factor at the same offset. + /// Not independently established. + pub scale: f32, /// The class this geometry belongs to; equals the class entry's /// `o_class` on all 358 spans on the disc. pub o_class: u32, - /// A count. Meaning not established. + /// A count. Meaning not established (1..55 on the disc). It is *not* the + /// body size: bodies hold 24 normals plus padding regardless. pub count: u32, /// Offset of the geometry data from the start of the span. Lands inside /// the span on all 358 spans on the disc. pub data_offset: u32, } -/// Bytes per body record. +/// The billboard record at `billboard_offset`, 0x40 bytes. /// -/// Each record is 6 varying bytes followed by 2 constant ones. Found by -/// grouping sibling spans and byte-diffing them rather than by stride-fitting, -/// which failed three times: the varying payload is **not** sized by the -/// `count` field, and was a constant 144 bytes per group whether `count` was -/// 1, 2, 3, 5 or 6. -pub const SHRUB_BODY_RECORD_BYTES: usize = 8; +/// The register block matches R&C1's layout field for field -- TEX1 at `+0x10` +/// with `K` and `MMIN`, TEX0 at `+0x20`, MIPTBP1_1 at `+0x30`, each carrying +/// its own register address word (0x14, 0x06, 0x34) -- and the three floats +/// in front of them are the one part that does not carry over: they are far +/// too large and too varied to be R&C1's `width`/`height`/`z_ofs`, and they +/// repeat byte-identically across levels, which dimensions would not. They +/// are handed back unnamed rather than guessed at. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ShrubBillboard { + /// Fade distance at `+0x00` (5..1000 on the disc). Following R&C1, the + /// loader takes `trunc(fade) & 0xff`, which makes 512 read as 0 -- + /// billboard only at any distance. + pub fade_distance: f32, + /// Three floats at `+0x04..+0x10`. Meaning not established. + pub unknown: [f32; 3], + /// `TEX1_1` at `+0x10`: the signed LOD bias K in 1/16 units, retail + /// -233..-168 on the disc. + pub k: i16, + /// `TEX1_1` minification filter, 4 on every disc billboard. + pub mmin: u32, + /// `TEX0_1.data_lo` at `+0x20`: a small index (1..4 on the disc). In R&C1 + /// this is an index into the class's texture list. + pub tex0: u32, + /// The GIF register address words carried behind the three registers, + /// 0x14 / 0x06 / 0x34 on every disc billboard. Exposed rather than + /// asserted on: they corroborate the layout, but a billboard that carries + /// a different register set is still a billboard. + pub tex1_addr: u32, + /// See [`ShrubBillboard::tex1_addr`]. + pub tex0_addr: u32, + /// See [`ShrubBillboard::tex1_addr`]. + pub miptbp_addr: u32, +} -/// The two constant bytes closing each body record. -pub const SHRUB_RECORD_TAIL: [u8; 2] = [0, 0]; - -/// A shrub span: its header plus the bytes the header points at. +/// A shrub span: its header, palette and optional billboard. #[derive(Debug, Clone, Copy, PartialEq)] pub struct Shrub<'a> { pub header: ShrubHeader, /// The whole span, starting at `+0x24` with the class number. pub raw: &'a [u8], - /// The bytes at `data_offset`, running to `span_end`. Not yet decoded. + /// The bytes at `data_offset`, running to `span_end`. This is the palette + /// plus its zero padding. pub data: &'a [u8], - /// Whether `span_end` came from the next class rather than from the end of - /// the lump. False for the last class in each level, whose span contains - /// whatever follows shrub. + /// Whether `span_end` came from the next class or the located chunk + /// container rather than from the end of the lump. False only for a span + /// bounded by a caller that could not locate either. pub span_end_is_a_successor: bool, } impl<'a> Shrub<'a> { - /// The body's 8-byte records. - /// - /// The payload of each record is six bytes whose encoding is **not** - /// established. It is not half-floats -- decoding as a `half3` gives - /// values like 245.25 and -168.375 -- and a VIF reading of it was - /// retracted after a control put it on the random baseline. Handed back - /// raw rather than guessed at. + /// The palette's 8-byte records, raw. pub fn body_records(&self) -> impl Iterator { - self.data.chunks_exact(SHRUB_BODY_RECORD_BYTES) + self.data.chunks_exact(8) } /// How many whole records the body holds. pub fn body_record_count(&self) -> usize { - self.data.len() / SHRUB_BODY_RECORD_BYTES + self.data.len() / 8 } - /// Whether every whole record ends in the constant pair. + /// The palette: the first [`SHRUB_NORMAL_COUNT`] records, parsed. /// - /// This is how a class's body is delimited: on all 24 levels that have - /// shrub geometry, the last shrub class's body ends exactly where its - /// trailing `00 00` pair stops, always on a record boundary. + /// Returns fewer only if the body is shorter than the palette, which does + /// not happen on the disc -- every one of the 358 spans carries all 24. + pub fn normals(&self) -> impl Iterator + 'a { + self.data + .chunks_exact(8) + .take(SHRUB_NORMAL_COUNT) + .filter_map(move |r| ShrubNormal::parse(r).ok()) + } + + /// Records past the palette. Zero on every disc span: the palette is + /// padded out with zero records rather than with live normals. + pub fn padding_record_count(&self) -> usize { + self.body_record_count().saturating_sub(SHRUB_NORMAL_COUNT) + } + + /// Whether every whole record's trailing pair is zero. + /// + /// The palette's pad word and the padding records are both zero, so a + /// body that satisfies this is whole records; it is how a class's body is + /// delimited, and it holds on all 358 disc spans now that each is bounded + /// by its successor. pub fn body_is_well_formed(&self) -> bool { let records: Vec<&[u8]> = self.body_records().collect(); - !records.is_empty() && records.iter().all(|r| r[6..8] == SHRUB_RECORD_TAIL) + !records.is_empty() && records.iter().all(|r| r[6..8] == [0, 0]) + } + + /// The billboard record, when the header names one. + /// + /// The offset is checked against `span_end` by + /// [`ShrubHeader::region`] rather than trusted: a non-zero offset that + /// runs past the span is a mis-set field, not a billboard. + pub fn billboard(&self) -> Result> { + if self.header.billboard_offset == 0 { + return Ok(None); + } + let at = self.header.billboard_offset as usize; + // `raw` runs from the span start, so the record sits at raw[at..]. + let bytes = self.raw.get(at..at + 0x40).ok_or(Error { + offset: at, + context: "shrub billboard", + kind: ErrorKind::OutOfBounds { + needed: 0x40, + available: self.raw.len().saturating_sub(at), + }, + })?; + let mut b = Buf::new(bytes); + let fade_distance = b.f32("shrub billboard fade")?; + let unknown = [ + b.f32("shrub billboard unknown 0")?, + b.f32("shrub billboard unknown 1")?, + b.f32("shrub billboard unknown 2")?, + ]; + // TEX1_1: K, a zero high half, then MMIN, then the register address + // word and a zero pad -- a 16-byte GifAdData16 per register. + let k = b.i16("shrub billboard K")?; + b.skip(2, "shrub billboard K high")?; + let mmin = b.u32("shrub billboard mmin")?; + let tex1_addr = b.u32("shrub billboard tex1 address")?; + b.skip(4, "shrub billboard tex1 pad")?; + let tex0 = b.u32("shrub billboard tex0 index")?; + b.skip(4, "shrub billboard tex0 data hi")?; + let tex0_addr = b.u32("shrub billboard tex0 address")?; + // MIPTBP1_1: two zero data words, then the register address word and a + // pad. The mip base pointers are filled in at level load, as they are + // for every ad-gif in this format. + b.skip(4, "shrub billboard tex0 pad")?; + b.skip(8, "shrub billboard miptbp data")?; + let miptbp_addr = b.u32("shrub billboard miptbp address")?; + Ok(Some(ShrubBillboard { + fade_distance, + unknown, + k, + mmin, + tex0, + tex1_addr, + tex0_addr, + miptbp_addr, + })) } } impl ShrubHeader { - /// Parse a shrub span starting at `offset` within `core_data`. + /// Parse a shrub span starting at `offset` within `core`. /// /// `o_class` is passed in so the self-identifying field can be checked /// rather than merely reported: a span whose `+0x24` disagrees with its /// own class entry is a different thing entirely, and quietly returning /// the mismatched value would hide that. + /// /// `span_end` bounds the class span within `core`. It matters: without it /// `data` runs to the end of the lump and swallows whatever follows, which /// is how a per-class check ends up measuring its neighbours. @@ -140,41 +325,75 @@ impl ShrubHeader { }, })?; let mut fb = Buf::new(&head[..16]); - let bounds = [ - fb.f32("shrub bounds 0")?, - fb.f32("shrub bounds 1")?, - fb.f32("shrub bounds 2")?, - fb.f32("shrub bounds 3")?, + let sphere = [ + fb.f32("shrub sphere x")?, + fb.f32("shrub sphere y")?, + fb.f32("shrub sphere z")?, + fb.f32("shrub sphere radius")?, ]; let mut fs = Buf::new(&head[0x10..0x18]); let size = fs.f32("shrub size")?; - let reserved = fs.f32("shrub reserved")?; - if !bounds.iter().all(|v| v.is_finite()) || !size.is_finite() || !reserved.is_finite() { + let mode_bits = fs.u16("shrub mode bits")?; + let scratch = fs.u16("shrub scratch")?; + if !sphere.iter().all(|v| v.is_finite()) || !size.is_finite() { return Err(Error { offset: start, context: "shrub header holds a non-finite float", kind: ErrorKind::Invalid("shrub header float"), }); } - // Zero, or the denormal the PS2 would flush to zero anyway. Anything - // else means the header is not laid out the way this thinks it is. - if reserved != 0.0 && !reserved.is_subnormal() { + // A radius is a radius: non-positive means the header is not laid out + // the way this thinks it is. + if sphere[3] <= 0.0 { return Err(Error { - offset: start + 0x14, - context: "shrub reserved word is neither zero nor denormal", - kind: ErrorKind::Invalid("shrub reserved"), + offset: start + 0x0c, + context: "shrub sphere radius", + kind: ErrorKind::Invalid("shrub sphere radius is not positive"), + }); + } + // One invariant across all 358 spans: the scratch word at +0x16 is + // zero everywhere. A non-zero value means this is not a shrub header. + // `mode_bits` at +0x14 is *not* asserted -- it is 4 on 16 of the 358 + // spans -- so the field is reported rather than validated. + if scratch != 0 { + return Err(Error { + offset: start + 0x16, + context: "shrub scratch", + kind: ErrorKind::Invalid("shrub scratch is not zero"), + }); + } + let reserved = u32::from_le_bytes(head[0x18..0x1c].try_into().expect("4 bytes at +0x18")); + if reserved != 0 { + return Err(Error { + offset: start + 0x18, + context: "shrub reserved word", + kind: ErrorKind::Invalid("shrub reserved word is not zero"), }); } - let mut b = Buf::new(&head[0x24..]); - let span_class = b.u32("shrub o_class")?; + let mut b = Buf::new(&head[0x1c..]); + let billboard_offset = b.u32("shrub billboard offset")?; + let scale = b.f32("shrub scale")?; + if !scale.is_finite() || scale <= 0.0 { + return Err(Error { + offset: start + 0x20, + context: "shrub scale", + kind: ErrorKind::Invalid("shrub scale is not positive"), + }); + } + let mut bc = Buf::new(&head[0x24..]); + let span_class = bc.u32("shrub o_class")?; let header = ShrubHeader { - bounds: [0.0; 4], + sphere: [0.0; 4], size: 0.0, - reserved: 0.0, + mode_bits: 0, + scratch, + reserved, + billboard_offset, + scale, o_class: span_class, - count: b.u32("shrub count")?, - data_offset: b.u32("shrub data offset")?, + count: bc.u32("shrub count")?, + data_offset: bc.u32("shrub data offset")?, }; if header.o_class != o_class.max(0) as u32 { @@ -185,8 +404,17 @@ impl ShrubHeader { }); } - let at = header.data_offset as usize; + // A billboard that is not inside the span is a mis-set offset. let stop = span_end.clamp(start, core.len()); + if billboard_offset > 0 && start + billboard_offset as usize + 0x40 > stop { + return Err(Error { + offset: start + 0x1c, + context: "shrub billboard offset", + kind: ErrorKind::Invalid("shrub billboard offset runs past the span"), + }); + } + + let at = header.data_offset as usize; let data = core.get(start + at..stop).ok_or(Error { offset: start + at, context: "shrub data offset runs past the lump", @@ -198,9 +426,13 @@ impl ShrubHeader { Ok(Some(Shrub { span_end_is_a_successor: true, header: ShrubHeader { - bounds, + sphere, size, + mode_bits, + scratch, reserved, + billboard_offset, + scale, o_class: header.o_class, count: header.count, data_offset: header.data_offset, @@ -219,16 +451,41 @@ mod tests { /// span cannot start there. const BASE: usize = 0x10; - /// A span whose `+0x24` names class 1072 and whose data starts at 0x30. - fn span() -> Vec { - let mut v = vec![0u8; BASE + 0x40]; + /// A unit direction per palette slot: a spiral over the sphere, so every + /// entry is a different unit vector rather than a repeated one. + fn fixture_normal(i: usize) -> [f32; 3] { + let t = i as f32 / SHRUB_NORMAL_COUNT as f32 * std::f32::consts::TAU; + let z = (i % 5) as f32 / 2.0 - 1.0; + let r = (1.0 - z * z).max(0.0).sqrt(); + [r * t.cos(), r * t.sin(), z] + } + + /// A span naming class 1072, data at 0x30, a positive radius and scale, + /// and a 24-entry palette of unit normals. `tail` extra bytes follow the + /// palette inside the span: zero ones read as padding records. + fn span_with_tail(tail: usize) -> Vec { + let mut v = vec![0u8; BASE + 0x30 + SHRUB_NORMAL_COUNT * 8 + tail]; + v[BASE + 0x0c..BASE + 0x10].copy_from_slice(&12.5f32.to_le_bytes()); + v[BASE + 0x10..BASE + 0x14].copy_from_slice(&6.0f32.to_le_bytes()); + v[BASE + 0x20..BASE + 0x24].copy_from_slice(&0.25f32.to_le_bytes()); v[BASE + 0x24..BASE + 0x28].copy_from_slice(&1072u32.to_le_bytes()); v[BASE + 0x28..BASE + 0x2c].copy_from_slice(&5u32.to_le_bytes()); v[BASE + 0x2c..BASE + 0x30].copy_from_slice(&0x30u32.to_le_bytes()); - v[BASE + 0x30..].fill(0xab); + for (i, n) in (0..SHRUB_NORMAL_COUNT).map(fixture_normal).enumerate() { + let at = BASE + 0x30 + i * 8; + for (c, &v_c) in n.iter().enumerate() { + let q = (v_c * 32767.0).round() as i16; + v[at + c * 2..at + c * 2 + 2].copy_from_slice(&q.to_le_bytes()); + } + } v } + /// A span with two zero padding records after the palette. + fn span() -> Vec { + span_with_tail(16) + } + #[test] fn reads_the_self_identifying_header() { let c = span(); @@ -238,7 +495,85 @@ mod tests { assert_eq!(s.header.o_class, 1072); assert_eq!(s.header.count, 5); assert_eq!(s.header.data_offset, 0x30); - assert!(s.data.iter().all(|&b| b == 0xab)); + assert_eq!(s.header.billboard_offset, 0); + assert_eq!(s.header.scale, 0.25); + assert_eq!(s.header.sphere[3], 12.5); + assert!(s.billboard().unwrap().is_none()); + } + + #[test] + fn the_body_is_24_unit_normals_plus_zero_padding() { + let c = span(); + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) + .unwrap() + .expect("present"); + let normals: Vec = s.normals().collect(); + assert_eq!(normals.len(), SHRUB_NORMAL_COUNT); + assert!(normals.iter().all(|n| n.is_unit()), "palette is not unit"); + // The fixture's two trailing zero records are padding, not normals. + assert_eq!(s.padding_record_count(), 2); + assert_eq!(s.body_record_count(), SHRUB_NORMAL_COUNT + 2); + assert!(s.body_is_well_formed()); + // The palette repeats no direction, and the padding is not in it. + assert!(!s.normals().any(|n| n == ShrubNormal { x: 0, y: 0, z: 0 })); + } + + #[test] + fn a_body_of_zeros_is_not_a_palette_of_normals() { + let mut v = vec![0u8; BASE + 0x40]; + v[BASE + 0x0c..BASE + 0x10].copy_from_slice(&12.5f32.to_le_bytes()); + v[BASE + 0x10..BASE + 0x14].copy_from_slice(&6.0f32.to_le_bytes()); + v[BASE + 0x20..BASE + 0x24].copy_from_slice(&0.25f32.to_le_bytes()); + v[BASE + 0x24..BASE + 0x28].copy_from_slice(&1072u32.to_le_bytes()); + v[BASE + 0x28..BASE + 0x2c].copy_from_slice(&5u32.to_le_bytes()); + v[BASE + 0x2c..BASE + 0x30].copy_from_slice(&0x30u32.to_le_bytes()); + let s = ShrubHeader::region(&v, BASE as i32, 1072, v.len()) + .unwrap() + .expect("present"); + // Every record is whole and zero-tailed, but no entry is a normal. + assert!(s.body_is_well_formed()); + assert!(!s.normals().any(|n| n.is_unit())); + } + + #[test] + fn a_billboard_record_is_read_and_checked() { + let mut c = span_with_tail(0x40 + 8); + // The billboard sits directly after the palette. + let bb = BASE + 0x30 + SHRUB_NORMAL_COUNT * 8; + c[BASE + 0x1c..BASE + 0x20].copy_from_slice(&((bb - BASE) as u32).to_le_bytes()); + c[bb..bb + 4].copy_from_slice(&50.0f32.to_le_bytes()); + // TEX1: K = -217, MMIN = 4, address 0x14. + c[bb + 0x10..bb + 0x12].copy_from_slice(&(-217i16).to_le_bytes()); + c[bb + 0x14..bb + 0x18].copy_from_slice(&4u32.to_le_bytes()); + c[bb + 0x18..bb + 0x1c].copy_from_slice(&0x14u32.to_le_bytes()); + // TEX0: data_lo = 1, address 0x06. + c[bb + 0x20..bb + 0x24].copy_from_slice(&1u32.to_le_bytes()); + c[bb + 0x28..bb + 0x2c].copy_from_slice(&0x06u32.to_le_bytes()); + // MIPTBP1_1: address 0x34. + c[bb + 0x38..bb + 0x3c].copy_from_slice(&0x34u32.to_le_bytes()); + + let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) + .unwrap() + .expect("present"); + let b = s.billboard().unwrap().expect("billboard present"); + assert_eq!(b.fade_distance, 50.0); + assert_eq!(b.k, -217); + assert_eq!(b.mmin, 4); + assert_eq!(b.tex0, 1); + assert_eq!(b.tex1_addr, 0x14); + assert_eq!(b.tex0_addr, 0x06); + assert_eq!(b.miptbp_addr, 0x34); + // The palette is untouched by the billboard sitting after it. + assert_eq!(s.normals().count(), SHRUB_NORMAL_COUNT); + assert!(s.normals().all(|n| n.is_unit())); + } + + #[test] + fn a_billboard_offset_past_the_span_is_rejected() { + let mut c = span_with_tail(0x40); + // Far past the span's end. + c[BASE + 0x1c..BASE + 0x20].copy_from_slice(&9000u32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); } #[test] @@ -258,37 +593,21 @@ mod tests { } #[test] - fn body_records_split_into_eights_and_check_their_tail() { - let c = span(); - let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) - .unwrap() - .expect("present"); - // The fixture fills 0xab, so every record's tail is wrong. - assert!(!s.body_is_well_formed()); - assert_eq!(s.body_record_count(), (0x40 - 0x30) / 8); - // Fix the tails and it is well formed. - let mut c2 = span(); - for i in 0..((0x40 - 0x30) / 8) { - c2[BASE + 0x30 + i * 8 + 6] = 0; - c2[BASE + 0x30 + i * 8 + 7] = 0; - } - let s2 = ShrubHeader::region(&c2, BASE as i32, 1072, c2.len()) - .unwrap() - .expect("present"); - assert!(s2.body_is_well_formed()); - assert_eq!(s2.body_records().count(), s2.body_record_count()); + fn a_nonpositive_radius_or_scale_is_rejected() { + let mut c = span(); + c[BASE + 0x0c..BASE + 0x10].copy_from_slice(&0.0f32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); + + let mut c = span(); + c[BASE + 0x20..BASE + 0x24].copy_from_slice(&0.0f32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); } #[test] - fn a_body_shorter_than_one_record_is_not_well_formed() { + fn a_nonzero_reserved_word_is_rejected() { let mut c = span(); - // Point the data at the last four bytes of the span. - c[BASE + 0x2c..BASE + 0x30].copy_from_slice(&(0x3cu32).to_le_bytes()); - let s = ShrubHeader::region(&c, BASE as i32, 1072, c.len()) - .unwrap() - .expect("present"); - assert_eq!(s.body_record_count(), 0); - assert!(!s.body_is_well_formed()); + c[BASE + 0x18..BASE + 0x1c].copy_from_slice(&0x40u32.to_le_bytes()); + assert!(ShrubHeader::region(&c, BASE as i32, 1072, c.len()).is_err()); } #[test] diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index 8e91c31..9447c88 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -1003,4 +1003,126 @@ Consequences, all measured over the new bounds: invented edge. The chunk test now reads the region from `core_layout`. What is still not known: the marker's meaning (constant across 27 levels, at -exactly this position, unnamed), and the shrub body payload's encoding. +exactly this position, unnamed). The shrub body payload's encoding is the next +section. + +## The shrub class blob: palette and billboard (2026-10-11, after the bound) + +With every span bounded by the chunk container, the shrub class blob reads +cleanly against the R&C1 format (`vendor/rerac/docs/formats/shrub_sky_rac1.md`, +derived from the Wrench sources). R&C1's shrub class header keeps the same +field offsets R&C2 shows -- `o_class` at 0x24, a count at 0x28, the data offset +at 0x2c -- so the reference says which bytes to look at and the disc says what +they hold. + +**Range measured:** all 358 shrub class spans over the 24 levels that have +them, each bounded by its successor class or the located chunk container. +Nothing outside the spans is read. + +### The body is a 24-entry normal palette + +Two things made this identifiable where three stride-fitting attempts had +failed. First the shape: every body is 8-byte records whose last two bytes are +zero, and R&C1's palette is documented as 24 records of exactly that shape +(`s16` x, y, z, pad). Second the count: bodies hold 24, 26, 28 or 30 records +with the extra ones **always all zero and always at index 24 or later** -- +never a live entry. So the body is 24 normals plus zero padding, and the +padding is what distinguishes it from geometry. + +The palette is real normals, and the test says how firmly: + +- all 8,592 live entries have squared length within **65,535** of `32767^2`, + and the bound is tight -- the largest observed deviation is exactly 65,535. + That number is not fitted: it is `32768^2 - 32767^2`, which is what a unit + vector scaled by 32768 produces when an axis-aligned normal quantises to + `-32768` while the positive axis clamps to `32767`. Components run the full + `s16` range (-32768..32767) and 1,765 of the 8,592 entries are axis-aligned, + which is what a coarse direction codebook looks like. +- a random `s16` triple has squared length spread about a *billion* either + side of `32767^2`, so one entry in fifteen thousand would pass by chance. + All 8,592 pass. +- the clincher: 17 classes carry a palette of 24 identical `(0, 0, 1)` + entries. A position or index table cannot produce that; a normal palette for + a flat-facing class must. + +This is also where shrub lighting comes from -- R&C1 lights each instance +against its class's 24-direction palette. Unlike the tfrag's per-vertex light +records, which store azimuth/elevation *indices* into the overlay's sine table, +the shrub palette stores the directions outright, so the overlay table is not +needed for shrubs. + +### The header, byte by byte + +**Range measured:** the 0x30 bytes of every span. + +| offset | field | measured | +|---|---|---| +| 0x00 | f32[4] `sphere` | centre (xyz) + radius (w). Finite on all 358, **radius positive on all 358** (1.72..41.57), centres near the class's own origin | +| 0x10 | f32 `size` | 3.0..20.0, positive on all 358 | +| 0x14 | u16 `mode_bits` | **0 on 342 spans, 4 on 16** | +| 0x16 | u16 `scratch` | **0 on all 358** (asserted by the parser) | +| 0x18 | u32 `reserved` | **0 on all 358** (asserted by the parser) | +| 0x1c | u32 `billboard_offset` | 0 on 333 spans, in-span on 25 | +| 0x20 | f32 `scale` | 0.0044..1.8, positive on all 358 | +| 0x24 | u32 `o_class` | equals the class entry's own, 358/358 | +| 0x28 | u32 `count` | 1..55, meaning not established | +| 0x2c | u32 `data_offset` | inside the span, 358/358 | + +Two corrections this produces, both recorded because the old readings were +confidently wrong rather than merely unverified: + +- **"reserved is 0 or a denormal" was the wrong shape.** The 16 spans with a + non-zero value there have `mode_bits = 4` and `scratch = 0`, not the other + way round. As an `f32` the word at `+0x14` is still a denormal, so the PS2 + reads zero either way -- but the field is a flag word, not a magnitude, and + the demo is the point: the EE's FPU has no denormals, so `04000000` and zero + are the same value to the hardware. +- **"bounds" is a bounding sphere, not a min/max box.** The old reading tested + `min.xyz/max.xyz` ordering, found it on only 146 of 358 spans, and stopped + naming the field. It is centre + radius, as in R&C1: the fourth float is + positive on all 358 spans and the centres cluster near the class origin + within +-24, which is where a class-local sphere belongs. The 146/358 figure + was a property of the wrong question. + +### The billboard record + +The billboard offset at `+0x1c` is the strongest self-identification in the +header after `o_class`, because it is **confirmed from a second file**: it is +non-zero on exactly the 25 classes whose core-index entry carries billboard +texture fields, in both directions -- 25 records and 25 entries, pairing +correctly. R&C1 states the same correspondence ("a class has a billboard record +exactly when its core-index billboard texture has width != 0"). + +At the offset sits R&C1's `ShrubBillboard` shape, register block and all: + +| offset | field | measured | +|---|---|---| +| 0x00 | f32 `fade_distance` | 5..1000; 512 reads as 0 under R&C1's `trunc(fade) & 0xff`, i.e. billboard-only | +| 0x04 | f32[3] | **not established** -- see below | +| 0x10 | TEX1_1: K, MMIN, address | K -233..-168, MMIN 4 on all 25, address 0x14 | +| 0x20 | TEX0_1: data_lo, address | index 1..4, address 0x06 | +| 0x30 | MIPTBP1_1: address | 0x34, data words zero (filled at level load) | + +The three floats at `+0x04..+0x10` are the part that does *not* carry over. +R&C1's are `width`, `height` and `z_ofs` in class units; R&C2's are an order of +magnitude too large and too varied to be dimensions of a 32..128 px billboard, +and they **repeat byte-identically across levels**: 16 distinct triples among +the 25 records, with one appearing on four classes across levels 1, 4, 12 and +21. Dimensions would not do that; shared art assets would. They are handed back +unnamed. + +Two further facts about them, measured: the whole record repeats with them -- +three classes on different levels carry identical `fade`, floats, K and scale +-- and the class entries they belong to carry identical `sphere` too. So the +blobs themselves are shared between levels, which is the same behaviour the +chunk container showed with its byte-identical tables. + +### Still open + +- The three billboard floats, `size`, `mode_bits`, and `count`. +- **Where shrub mesh geometry lives.** R&C1 keeps the class's mesh packets in + the blob between the header and the palette; R&C2's blob is header, palette + and optional billboard only -- 24 normals and nothing else. The 333 classes + without a billboard still have to draw a mesh, and it is not here. The + candidates are the chunk container that follows shrub and the gameplay + extent, and neither is decoded. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0005426..87b8119 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -217,13 +217,18 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [ ] `tie` — instanced environment geometry, fat vertices - [~] `shrub` — class table done (48-byte entries, not 32; reading shrub at 32 still gives 231 sane-looking entries of 358, which is the hazard). - Geometry header parsed and both fields **checked**, not merely reported: - `+0x24` equals the entry's own `o_class`, and `+0x2c` is an in-span data - offset, on all 358 spans. Shrub does **not** use moby's part-table format. - The body is **8-byte records, 6 varying bytes and 2 constant**, and now - that every span is bounded by the located chunk container, all 358 - bodies are whole records — 9,600 in total. The payload's encoding is - still unknown, and so is the 2-byte marker that closes the region. + Geometry header parsed and checked field by field on all 358 spans: + `+0x24` equals the entry's own `o_class`, `+0x2c` is an in-span data + offset, `+0x16`/`+0x18` are zero, and `+0x1c` names a billboard on + exactly the 25 classes whose core-index entry carries billboard texture + fields. Shrub does **not** use moby's part-table format. The body is a + **24-entry palette of unit-length s16 normals** (squared length within + 65,535 of 32767² on all 8,592 entries — a random triple would miss by a + billion) plus zero padding records; the billboard record carries K, MMIN + and the TEX0/MIPTBP addresses as R&C1's does. What remains open: the + three floats in the billboard record, `size`, `mode_bits`, `count`, and + **where the mesh geometry lives** — R&C1 puts the mesh packets in this + blob and R&C2's blob holds only the palette. - [x] `moby` — per-class model geometry. A class span opens with a `u32` pointing at a part table of 16-byte records; parts tile exactly, with `vertex_offset + 16 * vertex_count == next_part` holding for **43,467 of