Going-Commando/crates/gc-formats/src/chunks.rs

//! The post-shrub chunk container.
//!
//! Spec: `docs/formats/core-data-regions.md`.
//!
//! This is the container that fills the region after the last shrub class body
//! and before the constant trailer. It is *not* established to be moby_anim;
//! a 49 x 368-byte table inside it turned out to be byte-identical on all 27
//! levels, which is evidence against that identification rather than for it.
//!
//! The framing is established. Each chunk is:
//!
//! ```text
//! +0x00  16 bytes, last byte 0xff     (opaque, differs per chunk)
//! +0x10  10 zero bytes
//! +0x1a  N tagged offsets, top byte 0xf0
//! ```
//!
//! and each tagged offset is `0xF0000000 | low24`. Resolving a record at
//! `chunk_start + low` yields records that share a field layout: 16.7% to 20.0%
//! of byte positions are constant across a chunk's own records, on every level.
//! That is what distinguishes this from a coincidental byte pattern. Records in
//! one chunk are **not** all the same length -- each runs to its successor's
//! offset -- so the statistic is computed over the span every record shares.
//!
//! The stride between records is a property of the chunk, not of the region.
//! Level 0 alone contains chunks at 148, 340, 368, 288, 320, 236, 272, 384,
//! 116, 160, 250, 580 and 576. Because the strides differ, a region-wide
//! autocorrelation averages them to the noise floor -- which is why an
//! earlier claim of a "368-byte period" for the region as a whole was wrong.
//! 368 is the stride of one chunk.
//!
//! Fields are *not* named. The 16-byte header is opaque, the `0xf0` tag is
//! unexplained, and it is not known which fields are an index, a length and
//! an offset. What this parser provides is bounds-checked chunk and record
//! extraction, which is what the format analysis actually established.

use crate::buf::{Buf, ErrorKind, Result};

/// Bytes in a chunk's opaque leading header.
pub const CHUNK_HEADER_BYTES: usize = 16;

/// Zero bytes between the header and the offset table.
pub const CHUNK_GAP_BYTES: usize = 10;

/// The value every tagged offset's top byte carries, on all 27 levels.
pub const CHUNK_TAG: u8 = 0xf0;

/// Bytes between a chunk record's leading word and the payload the table
/// points at.
///
/// A record is `[leading u16][two index bytes][payload]`, and the disc's
/// offset table points at the *payload*. The loaded table in RAM points two
/// bytes earlier, at the leading word -- the payload's index bytes are then at
/// `+2` from the pointer -- so the game's record starts here and the disc's
/// table is two bytes late. See [`Chunk::prefix`].
pub const RECORD_PREFIX_BYTES: usize = 2;

/// The two bytes immediately preceding every chunk region on the disc.
///
/// The last class body in a level -- shrub where the level has shrub, tie
/// where it does not -- ends with this constant pair before the chunk
/// container begins. It is *not* a unique pattern: the pair occurs 25 to 103
/// times in a 5 to 17 MB lump, so on its own it means nothing. What locates
/// the boundary is the pair sitting exactly in front of a chunk signature
/// (see [`chunk_region_start`]).
///
/// Not interpreted. Whether it is a terminator, an end count or a tag is not
/// established; what is established is that it is constant across all 27
/// levels and sits at this one place.
/// Alignment of the first byte of a chunk record.
///
/// The leading byte is not decoded -- what it indexes is not established, and
/// naming it would be a guess. What *is* established is that it is a small
/// bucket index, not data: within a chunk type it takes 16 discrete values
/// (0x00, 0x10, ... 0xf0) with near-uniform counts, so it reads as an index into
/// groups rather than a running offset.
///
/// **The alignment step is per type, not global.** Across all 27 levels, over
/// 1,509 chunks and 34,491 records:
///
/// - the leading byte is a multiple of **8** on 34,432 of 34,491;
/// - a multiple of **16** on only 33,349 -- the 1,142-record difference falls in
///   **seven** types, four of which step their buckets by 8 (0x00, 0x08, 0x10,
///   ...) instead of 16;
/// - and 59 records in three types break even the 8-byte rule, which is where the
///   variable-length type lives (0x00f2ef4275, records 76 to 104 bytes).
///
/// So [`record_is_aligned`] is checked at 8, not 16, and a type that steps by 16
/// is not distinguishable from one that steps by 8 without looking at the type.
pub const CHUNK_RECORD_ALIGNMENT: usize = 8;

/// True when a record's leading byte is a multiple of [`CHUNK_RECORD_ALIGNMENT`].
///
/// Measured over all 27 levels: 34,432 of 34,491 records, with the exceptions
/// confined to the seven types above.
pub fn record_is_aligned(record: &[u8]) -> bool {
    match record.first() {
        Some(&b) => (b as usize) % CHUNK_RECORD_ALIGNMENT == 0,
        None => false,
    }
}

pub const CHUNK_REGION_MARKER: [u8; 2] = [0xf6, 0x7c];

/// A chunk header plus the records its offsets address.
///
/// `stride` is the distance between the first two resolved offsets, and nothing
/// more. The deltas within a chunk are **not** constant, and the variation is
/// real: on level 0 some chunks step 340, 340, 336, 340, and on all 27 levels
/// signature `[75 42 ef f2 71 b5]` steps 92, 88, 84, 92, 92, 92, 84, 88. Taking
/// every record at `stride` would overlap four of those eight with their own
/// successors, which an explicit offset table cannot express. Record *lengths*
/// come from the deltas; see [`resolve`].
pub struct Chunk {
    /// The 16 opaque header bytes.
    pub header: [u8; CHUNK_HEADER_BYTES],
    /// Offset of this chunk within the region it was parsed from.
    pub offset: usize,
    /// Record count, from header byte 14.
    ///
    /// Measured: equals `offsets.len()` on **1,123 of 1,123** chunks across
    /// all 27 levels, zero mismatches. This is the one header field that is
    /// decoded rather than merely measured.
    pub count: u8,
    /// The low 24 bits of each tagged offset, in file order.
    pub offsets: Vec<u32>,
    /// Record stride, from the first two offsets. Zero if fewer than two.
    ///
    pub stride: usize,
    /// The `u16` immediately before the first record's payload -- the record's
    /// own leading word, which the loaded table points at.
    ///
    /// **Measured on the disc:** the same at every record of a chunk on
    /// **1,439 of 1,509 chunks** -- `None` on the 70 that vary -- and fixed
    /// within every chunk type, over the 71 types. Its values are 1, 2, 4, 6,
    /// 8 and 10; 554 chunks carry 2 and 520 carry 4.
    ///
    /// **Measured against the running game:** in PCSX2's RAM the rewritten
    /// offset table points at this word rather than at the payload, two bytes
    /// earlier than the disc's table, so the game's record starts here.
    ///
    /// Not named. What the value counts is not established: it is *not* the
    /// payload's entry count (the payload length divided by it is a whole
    /// number on 1,344 chunks and an odd fraction on the other 162, leaving 57
    /// distinct "sizes"), and it is not the number of distinct leading bytes
    /// (agrees on 0 of 1,506). An earlier note in this file read the
    /// `02 00` seen in RAM as a prefix the loader had inserted; those bytes
    /// are on the disc already, at every record, which is what this field is.
    pub prefix: Option<u16>,
}

/// One chunk, with its records already bounds-checked.
pub struct ResolvedChunk<'a> {
    /// The offset table.
    pub chunk: Chunk,
    /// Record bytes, one per entry in `chunk.offsets`.
    pub records: Vec<&'a [u8]>,
}

impl<'a> ResolvedChunk<'a> {
    /// The per-chunk phase and step that explain every record's leading byte, or
    /// `None` when no `(step, phase)` in that form fits.
    ///
    /// **Measured: 1,262 of 1,509 chunks** across all 27 levels are fully explained
    /// by `record[i].leading == step * ((i + phase) mod (256 / step))`, with
    /// `step` taking 8 on 51 chunks, 16 on 554, 32 on 520 and 64 on 137. The other
    /// 247 chunks, across 11 chunk types, are not explained by it.
    ///
    /// Where this returns `Some`, the leading byte carries **no per-record
    /// information**: it is a deterministic function of the record's index and the
    /// chunk's phase, so it is a position marker rather than data. The 247 chunks
    /// where it returns `None` are the ones where that has *not* been shown, and
    /// nothing should skip their leading byte on the strength of this.
    pub fn bucket_rule(&self) -> Option<(u8, usize)> {
        use std::collections::BTreeSet;
        if self.records.is_empty() {
            return None;
        }
        let n = self.records.len();
        for step in [8u16, 16, 32, 64] {
            let m = (256 / step) as usize;
            for phase in 0..m {
                let mut ok = true;
                for (i, r) in self.records.iter().enumerate() {
                    let Some(&b) = r.first() else {
                        ok = false;
                        break;
                    };
                    if b != ((((i + phase) % m) * step as usize % 256) as u8) {
                        ok = false;
                        break;
                    }
                }
                if ok {
                    return Some((step as u8, phase));
                }
            }
        }
        let _ = n;
        let _: Option<BTreeSet<u8>> = None;
        None
    }

    /// True when every record in this chunk has the same length.
    ///
    /// **Measured: true for 257 of 1,509 chunks** across all 27 levels. Record
    /// length varies *between* chunks of the same type -- one type runs 76 to 104
    /// bytes -- and, on 1,252 chunks, *within* one as well.
    ///
    /// This method used to report 1,509 of 1,509, and that was an artefact of
    /// `resolve`: it sliced every record at the chunk's first offset delta, so
    /// every record came out the same length by construction, and this predicate
    /// was confirming the parser back to itself. `resolve` now bounds each record
    /// by its successor's offset, which is the only coherent reading -- a record
    /// of the nominal stride would overlap the next one whenever the delta is
    /// smaller, and an explicit offset table cannot express an overlap.
    ///
    /// So the earlier withdrawal, "an earlier claim that some type had
    /// variable-length records was pooling across chunks", was also wrong. The
    /// records are variable-length and were never anything else.
    pub fn has_uniform_record_length(&self) -> bool {
        let mut it = self.records.iter().filter(|r| !r.is_empty());
        let Some(first) = it.next().map(|r| r.len()) else {
            return true;
        };
        it.all(|r| r.len() == first)
    }

    /// The per-chunk `(step, phase) that makes the first **two** bytes of every
    /// record encode that record's own index, or `None` when no such pair fits.
    ///
    /// With `g = 256 / step` and `j = index + phase`:
    ///
    /// ```text
    /// record[0] == step * (j mod g)
    /// record[1] == (j / g) mod 256
    /// ```
    ///
    /// so the pair is the record index in mixed radix, and the leading bytes carry
    /// **no per-record information** -- a record knows where it is.
    ///
    /// **Measured: 1,235 of 1,509 chunks** across all 27 levels fit this exactly.
    /// [`Self::bucket_rule`] -- byte 0 alone -- fits 1,262; the 27-chunk difference
    /// are chunks whose first byte follows the rule but whose second does not.
    ///
    /// The remaining 274 chunks are not explained by it.
    /// Nothing should skip a leading byte on the strength of this where it returns
    /// `None`.
    pub fn index_rule(&self) -> Option<(u8, usize)> {
        for step in [8u16, 16, 32, 64] {
            let g = (256 / step) as usize;
            for phase in 0..g {
                let mut ok = true;
                for (i, r) in self.records.iter().enumerate() {
                    if r.len() < 2 {
                        continue;
                    }
                    let j = i + phase;
                    if r[0] != ((j % g) * step as usize % 256) as u8
                        || r[1] != ((j / g) % 256) as u8
                    {
                        ok = false;
                        break;
                    }
                }
                if ok && self.records.iter().any(|r| r.len() >= 2) {
                    return Some((step as u8, phase));
                }
            }
        }
        None
    }
}

/// True if `offset` looks like the start of a chunk.
///
/// The signature is the header's last byte being `0xff`, the 10-byte gap
/// being zero, and at least `min_offsets` following words carrying the `0xf0`
/// tag.
pub fn is_chunk_start(region: &[u8], offset: usize, min_offsets: usize) -> bool {
    let hdr_end = offset + CHUNK_HEADER_BYTES;
    if hdr_end + CHUNK_GAP_BYTES + min_offsets * 4 > region.len() {
        return false;
    }
    if region[hdr_end - 1] != 0xff {
        return false;
    }
    if region[hdr_end..hdr_end + CHUNK_GAP_BYTES] != [0u8; CHUNK_GAP_BYTES] {
        return false;
    }
    (0..min_offsets).all(|k| {
        let at = hdr_end + CHUNK_GAP_BYTES + k * 4;
        read_u32(region, at) >> 24 == CHUNK_TAG as u32
    })
}

/// The start of the chunk container: the first offset at or after `from`
/// carrying a chunk signature immediately preceded by [`CHUNK_REGION_MARKER`].
///
/// This is what closes the last class region (shrub, or tie on the levels
/// without shrub). The two parts of the signature are each weak alone -- the
/// marker pair occurs by chance dozens of times per lump, and the chunk shape
/// matches random data often enough to need four tagged offsets to be believed
/// -- and together they are exact: on all 27 levels the marker sits exactly
/// two bytes before the first chunk header, with no further marker-paired
/// signature after the region start.
///
/// Returns `None` when the signature is not found, so an unlocated boundary
/// stays visible rather than being closed against a guess.
pub fn chunk_region_start(core: &[u8], from: usize) -> Option<usize> {
    if from < 2 {
        return None;
    }
    // The marker's first byte is the cheapest rejection, so test it before
    // comparing the pair or the signature behind it.
    let mut at = from - 2;
    while at + 2 <= core.len() {
        if core[at] == CHUNK_REGION_MARKER[0]
            && core[at + 1] == CHUNK_REGION_MARKER[1]
            && is_chunk_start(core, at + 2, 4)
        {
            return Some(at + 2);
        }
        at += 1;
    }
    None
}

/// Little-endian, matching `Buf::u32` and the PS2.
fn read_u32(bytes: &[u8], at: usize) -> u32 {
    u32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]])
}

/// Parse the chunk at `offset`, without resolving records.
///
/// Returns `None` if `offset` does not carry the chunk signature.
pub fn parse_chunk(region: &[u8], offset: usize) -> Result<Option<Chunk>> {
    if !is_chunk_start(region, offset, 4) {
        return Ok(None);
    }
    let mut b = Buf::new(region);
    b.seek(offset);

    let header_bytes = b.read_bytes(CHUNK_HEADER_BYTES, "chunk header")?;
    let mut header = [0u8; CHUNK_HEADER_BYTES];
    header.copy_from_slice(header_bytes);

    b.skip(CHUNK_GAP_BYTES, "chunk gap")?;

    // The offset table runs until the first word without the tag.
    let table_start = b.pos();
    let mut offsets = Vec::new();
    while b.pos() + 4 <= region.len() {
        let v = b.u32("chunk offset")?;
        if v >> 24 != CHUNK_TAG as u32 {
            b.seek(table_start);
            break;
        }
        offsets.push(v & 0x00ff_ffff);
    }

    // The offset table runs to the first untagged word, but the header carries
    // its own count. Truncating to that count stops a stray tagged word in the
    // record data from extending the table.
    offsets.truncate(header[14] as usize);

    let stride = if offsets.len() >= 2 {
        (offsets[1] - offsets[0]) as usize
    } else {
        0
    };

    // The leading word sits two bytes before the payload the table points at.
    // Reading it and requiring it to be the same at every record is a
    // signature check like the header's last byte: it holds on every chunk on
    // The leading word sits two bytes before the payload the table points at.
    // It is the same at every record of a chunk on 1,439 of 1,509 chunks; the
    // other 70 vary, so it stays an Option rather than a value a caller could
    // mistake for a field that is always there.
    let read_prefix = |low: u32| -> Option<u16> {
        let at = offset
            .checked_add(low as usize)?
            .checked_sub(RECORD_PREFIX_BYTES)?;
        region
            .get(at..at + RECORD_PREFIX_BYTES)
            .map(|b| u16::from_le_bytes([b[0], b[1]]))
    };
    let prefix = offsets.first().copied().and_then(read_prefix).filter(|&p| {
        offsets
            .iter()
            .skip(1)
            .all(|&low| read_prefix(low) == Some(p))
    });

    Ok(Some(Chunk {
        count: header[14],
        header,
        offset,
        offsets,
        stride,
        prefix,
    }))
}

/// Resolve every record a chunk addresses, bounds-checked against `region`.
///
/// A record runs from its own offset to the **next** record's offset, not to
/// `chunk.offset + stride`.
///
/// This is not a stylistic choice. `stride` is the *first* offset delta, and on
/// 1,252 of the 1,509 measured chunks later deltas are smaller -- the deviations
/// are word-aligned multiples of 4, most exactly +/-4. Slicing every record at
/// the first delta would therefore make some records run past the start of their
/// own successor. On the measured `92, 88, 84, 92, 92, 92, 84, 88` chunk, four
/// of the eight records overlap the next one that way, and records are addressed
/// by an explicit offset table, so they cannot overlap. The only coherent
/// reading is that a record ends where the next begins.
///
/// Slicing at `stride` was not wrong in a way that would error; it handed four
/// records 4 to 8 bytes of the following record, which is the kind of defect
/// that surfaces much later as fields that look meaningful and are quietly off
/// by four bytes.
///
/// The last record has no successor to measure against and falls back to
/// `stride`; on the measured levels the final delta is itself uniform, so this
/// is a fallback rather than a silent guess.
///
/// A record that would run past the end of the region is dropped rather than
/// clamped, so the returned length may be shorter than `chunk.offsets` -- on the
/// measured levels it is not, but a parser that silently clamps would hide a
/// malformed chunk rather than report it.
pub fn resolve<'r>(chunk: &Chunk, region: &'r [u8]) -> Result<ResolvedChunk<'r>> {
    let mut records = Vec::with_capacity(chunk.offsets.len());
    for (i, &low) in chunk.offsets.iter().enumerate() {
        let start = chunk
            .offset
            .checked_add(low as usize)
            .ok_or(crate::buf::Error {
                offset: chunk.offset,
                context: "chunk record",
                kind: ErrorKind::Invalid("chunk offset overflows usize"),
            })?;
        let len = match chunk.offsets.get(i + 1) {
            Some(&next) => (next - low) as usize,
            None => chunk.stride,
        };
        if start + len > region.len() {
            continue;
        }
        records.push(&region[start..start + len]);
    }
    Ok(ResolvedChunk {
        chunk: Chunk {
            count: chunk.count,
            header: chunk.header,
            prefix: chunk.prefix,
            offset: chunk.offset,
            offsets: chunk.offsets.clone(),
            stride: chunk.stride,
        },
        records,
    })
}

/// Walk every chunk in `region`, resolving records.
///
/// Returns chunks that carried the signature but whose records did not all
/// resolve, alongside the ones that did. Callers that care about coverage
/// should check both.
pub fn parse_all_chunks(region: &[u8]) -> Result<Vec<ResolvedChunk<'_>>> {
    let mut out = Vec::new();
    let mut at = 0usize;
    while at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + 16 <= region.len() {
        let Some(chunk) = parse_chunk(region, at)? else {
            at += 1;
            continue;
        };
        // Resume past this chunk's header and offset table so a chunk's own
        // record data is not rescanned as further chunk headers.
        let consumed = CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + chunk.offsets.len() * 4;
        let resolved = resolve(&chunk, region)?;
        out.push(resolved);
        at += consumed.max(1);
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A chunk signature at `at`, preceded by the marker: the marker pair, the
    /// 16-byte header whose last byte is `0xff`, the 10 zero bytes, then four
    /// tagged offsets.
    fn plant_chunk(core: &mut [u8], at: usize) {
        let base = at - 2;
        core[base..base + 2].copy_from_slice(&CHUNK_REGION_MARKER);
        // Header: byte 14 is the record count, matching the four tagged words.
        core[at + 14] = 4;
        core[at + 15] = 0xff;
        // The 10 zero bytes of the gap at +0x10. Their low bits resolve to a
        // zero stride, which is fine: only the signature is under test.
        core[at + CHUNK_HEADER_BYTES..at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES].fill(0);
        for k in 0..4 {
            let word = (CHUNK_TAG as u32) << 24;
            core[at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + k * 4
                ..at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + k * 4 + 4]
                .copy_from_slice(&word.to_le_bytes());
        }
    }

    #[test]
    fn finds_the_markered_signature() {
        let mut core = vec![0x5au8; 256];
        plant_chunk(&mut core, 128);
        assert_eq!(chunk_region_start(&core, 64), Some(128));
        // Scanning from earlier still finds it: the signature is what locates
        // the boundary, not the search start.
        assert_eq!(chunk_region_start(&core, 2), Some(128));
    }

    #[test]
    fn a_marker_without_a_signature_is_not_a_boundary() {
        let mut core = vec![0x5au8; 256];
        core[10..12].copy_from_slice(&CHUNK_REGION_MARKER);
        assert_eq!(chunk_region_start(&core, 2), None);
    }

    #[test]
    fn a_signature_without_the_marker_is_not_a_boundary() {
        let mut core = vec![0x5au8; 256];
        // Plant the signature, then break the marker in front of it.
        plant_chunk(&mut core, 128);
        core[126] = 0x5a;
        assert_eq!(chunk_region_start(&core, 2), None);
    }

    #[test]
    fn an_unlocated_boundary_stays_none() {
        assert_eq!(chunk_region_start(&[], 2), None);
        // `from` below the marker's own minimum is refused rather than
        // underflowing.
        assert_eq!(chunk_region_start(&[0xf6], 0), None);
    }
}