From 291153a0d9228685d264c8ce1f853a0b7c4bba46 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Mon, 12 Oct 2026 04:13:52 +0200 Subject: [PATCH] Find the tie LOD block terminator; packets == 0 means no block at that LOD parse_tie_block walks to a 0xf6 terminator that every real block carries, on 2177 of 2177 across all 27 levels, followed by zero padding. This is what makes the blocks parseable: packets[1] bounds one on only 461 of 1864 classes, so the terminator is the only general way to find one. The terminator was worth a control, because 0xf6 could have been byte-frequency coincidence. It occurs in payloads at 0.83x the uniform rate -- slightly LESS often than chance -- while the last non-zero byte is 0xf6 on 909 of 922 blocks in the first subset, against 17 of 922 for the same blocks cut at a random point. So it is a terminator, not a common byte. packets == 0 means the class has NO block at that LOD: of 3,728 (class, LOD) slots, 2177 point at a block and 1551 are exactly zero. Corrected header, over all 2177: +0x00 count multiple of 16 on all; 42 distinct; never zero +0x02 zero on all +0x04 exactly four values: 0x501 (1069), 0xa02 (720), 0xf03 (350), 0x1404 (38) +0x06 16 distinct, never zero +0x08 180 distinct, never zero +0x0a 61 distinct, never zero +0x0c 125 distinct, never zero, clustered 0x043f..0x0443 +0x0e 11 distinct, zero on 2162 -- the only non-universal field +0x0c clusters around 1087-1091, close enough to a block length to be worth stating, but it is NOT the length: the difference against the true payload length scatters from -12 to -172. A correction to the previous commit. It reported 3,728 blocks all terminating and packed_texture taking 842 distinct values, and on that basis retracted the four-value claim. Both numbers were wrong the same way: the scan did not skip packets == 0, so it computed class_base + 0 and walked forward from the CLASS BODY, finding a terminator there by accident. That contaminated 1551 of the 3728. With the zeros excluded the slot takes exactly four values on all 2177 blocks. The four-value claim was right; my retraction of it was wrong; and the retraction was made on a number from the same contaminated set. A retraction is not automatically better than the claim it retracts -- it needs its own bounds. Still open: the payload's records. Established this round: blocks are self-delimiting, absent LODs are marked by a zero packets word, and the header is fully characterised. Not established: any record boundary inside the payload, which stays variable-length. --- crates/gc-extract/tests/disc.rs | 81 ++++++++++++++++-------- crates/gc-formats/src/tie.rs | 100 +++++++++++++++++++++++------- docs/formats/core-data-regions.md | 51 +++++++++++++++ 3 files changed, 186 insertions(+), 46 deletions(-) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 9734b93..c2e7294 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2702,15 +2702,19 @@ fn tie_classes_resolve_through_both_indirections() { } } - // Tie LOD block headers: measured invariants across all 27 levels. - use gc_formats::tie::{TieBlockHeader, TIE_BLOCK_HEADER_SIZE, TIE_PACKED_TEXTURE_VALUES}; + // Tie LOD blocks are self-delimiting, which is what makes them parseable + // without trusting packets[1] to bound the block. + use gc_formats::tie::{ + parse_tie_block, TIE_BLOCK_HEADER_SIZE, TIE_BLOCK_TERMINATOR, TIE_PACKED_TEXTURE_VALUES, + }; let mut blocks = 0usize; + let mut absent = 0usize; let mut hdr_ok = 0usize; let mut count_mult16 = 0usize; let mut reserved_02_zero = 0usize; let mut packed_ok = 0usize; - let mut reserved_0e_zero = 0usize; + let mut padding_all_zero = 0usize; let mut packed_seen: Vec = Vec::new(); for n in 0..27 { @@ -2720,43 +2724,58 @@ fn tie_classes_resolve_through_both_indirections() { let t = h.tie_classes; for c in 0..t.count as usize { let r = resolve_tie_class(&idx, &core, t, c).expect("tie class"); - let p = r.header.packets; - if !(p[0] < p[1] && p[1] < p[2]) { - continue; - } let base = r.entry.offset_in_asset_wad as usize; - for i in 0..2 { - let s = base + p[i] as usize; - let e = base + p[i + 1] as usize; - if e > core.len() || e <= s + TIE_BLOCK_HEADER_SIZE { + for lod in [1usize, 2] { + let off = r.header.packets[lod] as i64; + if off == 0 { + // packets == 0 means this class has no block at this LOD. + absent += 1; continue; } - let bh = TieBlockHeader::parse(&core[s..e]).expect("block header"); + let s = (base as i64 + off) as usize; + if s + TIE_BLOCK_HEADER_SIZE > core.len() { + continue; + } + let Some(blk) = parse_tie_block(&core, s) else { + continue; + }; + blocks += 1; - if bh.count % 16 == 0 { + if blk.header.count % 16 == 0 { count_mult16 += 1 } - if bh.reserved_02 == 0 { + if blk.header.reserved_02 == 0 { reserved_02_zero += 1 } - if TIE_PACKED_TEXTURE_VALUES.contains(&bh.packed_texture) { + if TIE_PACKED_TEXTURE_VALUES.contains(&blk.header.packed_texture) { packed_ok += 1 } - if bh.reserved_0e == 0 { - reserved_0e_zero += 1 - } - if bh.invariants_hold() { + if blk.header.invariants_hold() { hdr_ok += 1 } - if !packed_seen.contains(&bh.packed_texture) { - packed_seen.push(bh.packed_texture); + if blk.padding.iter().all(|&b| b == 0) { + padding_all_zero += 1 + } + assert_eq!( + blk.payload.last(), + Some(&TIE_BLOCK_TERMINATOR), + "level{n} class{c} lod{lod}: payload does not end on the terminator" + ); + if !packed_seen.contains(&blk.header.packed_texture) { + packed_seen.push(blk.header.packed_texture); } } } } packed_seen.sort_unstable(); - assert!(blocks >= 900, "only {blocks} measurable tie blocks"); + assert!(blocks >= 2100, "only {blocks} tie blocks located"); + assert!( + absent > 1000, + "only {absent} absent (packets == 0) LOD blocks" + ); + assert_eq!(blocks + absent, 2 * classes, + "expected two LOD slots per class, got {blocks} present + {absent} absent for {classes} classes"); assert_eq!( count_mult16, blocks, @@ -2778,13 +2797,25 @@ fn tie_classes_resolve_through_both_indirections() { assert_eq!( hdr_ok, blocks, - "invariants failed on {} of {blocks}", + "header invariants failed on {} of {blocks}", blocks - hdr_ok ); + assert_eq!( + padding_all_zero, + blocks, + "non-zero padding on {} of {blocks}", + blocks - padding_all_zero + ); + assert_eq!( + packed_seen.len(), + TIE_PACKED_TEXTURE_VALUES.len(), + "packed_texture took {} distinct values, expected the four {packed_seen:?}", + packed_seen.len() + ); eprintln!( - "tie blocks: {blocks}; count %16 on all; reserved_02 == 0 on all; \ - reserved_0e == 0 on {reserved_0e_zero}; packed_texture in {packed_seen:?} on all" + "tie blocks: {blocks} present, {absent} absent (packets == 0); all end on 0xf6 with zero padding; \ + count %16 on all; reserved_02 == 0 on all; packed_texture in {packed_seen:?} on all" ); assert!(classes >= 1800, "only {classes} tie classes resolved"); diff --git a/crates/gc-formats/src/tie.rs b/crates/gc-formats/src/tie.rs index 73dceb3..2e3cf31 100644 --- a/crates/gc-formats/src/tie.rs +++ b/crates/gc-formats/src/tie.rs @@ -46,13 +46,13 @@ pub struct TieClassHeader { /// `0x00`: packet data offsets for LOD 0 (highest detail), 1 and 2. /// /// [`TIE_PACKETS0`] is constant across all 1,864 classes and is reported - /// separately; only the second and third words behave like offsets. Those - /// two are **strictly increasing on 461 of 1,864 classes** -- on the other - /// 1,403 they are not ordered, so they cannot both be block starts into a - /// single ascending region and the block layout is not decoded. + /// separately. `packets[1]` and `packets[2]` are the LOD block offsets, + /// **signed**, relative to the class header: **zero means that class has no + /// block at that LOD**, which is 1,551 of the 3,728 slots; the other 2,177 + /// point at a [`TieBlock`]. /// - /// Where they do increase, both resulting block sizes are divisible by 16 - /// on **922 of 922**, which is the evidence for a 16-byte record. + /// They are strictly increasing on only 461 of 1,864 classes, so neither one + /// bounds the other -- the `0xf6` terminator does that instead. pub packets: [i32; 3], /// `0x10`, `0x14`, `0x18`: LOD switch distances in world units. pub near_dist: f32, @@ -110,11 +110,11 @@ impl TieClassHeader { /// Size of a tie LOD block's own header. pub const TIE_BLOCK_HEADER_SIZE: usize = 0x10; -/// The four values [`TieBlockHeader::packed_texture`] takes across the disc. +/// The four most common values of [`TieBlockHeader::packed_texture`]. /// /// Low byte 1-4, high byte 5/10/15/20, so the word is two small fields rather -/// than a magnitude. Which is the texture index and which the scale is not -/// established; that the word is packed is. +/// than a magnitude. These are the *only* values the slot takes: 0x501 on 1,069 +/// blocks, 0xa02 on 720, 0xf03 on 350, 0x1404 on 38. pub const TIE_PACKED_TEXTURE_VALUES: [u16; 4] = [0x0501, 0x0a02, 0x0f03, 0x1404]; /// A tie LOD block's fixed 16-byte header. @@ -124,26 +124,29 @@ pub const TIE_PACKED_TEXTURE_VALUES: [u16; 4] = [0x0501, 0x0a02, 0x0f03, 0x1404] /// 1,403 classes are unmeasured, not measured-and-different. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TieBlockHeader { - /// `+0x00`: element count. A multiple of 16 on 922 of 922 blocks, range - /// 16 to 2,096, with 72 distinct values across the disc. + /// `+0x00`: element count. A multiple of 16 on all 2,177 blocks, 42 distinct + /// values, never zero. /// /// It is **not** a record count over the block: `block_size - header` is /// divisible by `count` on 0 of 922 blocks, for any record size and any /// header up to 512 bytes. pub count: u16, - /// `+0x02`: zero on 922 of 922. + /// `+0x02`: zero on all 2,177 blocks. pub reserved_02: u16, - /// `+0x04`: one of [`TIE_PACKED_TEXTURE_VALUES`] on every block measured. + /// `+0x04`: one of exactly [`TIE_PACKED_TEXTURE_VALUES`] on all 2,177 blocks. pub packed_texture: u16, - /// `+0x06`: 15 distinct values, 0x0206 to 0x0512. + /// `+0x06`: 16 distinct values, never zero. pub field_06: u16, - /// `+0x08`: 135 distinct values, 0x0b05 to 0x4ee4. + /// `+0x08`: 180 distinct values, never zero. pub field_08: u16, - /// `+0x0a`: 45 distinct values. + /// `+0x0a`: 61 distinct values, never zero. pub field_0a: u16, - /// `+0x0c`: 94 distinct values, 0x010e to 0x0553. + /// `+0x0c`: 125 distinct values, never zero, clustered at 0x043f to 0x0443 + /// (~1,087 to 1,091). **Not** the payload length: the difference against the + /// true length scatters from -12 to -172. pub field_0c: u16, - /// `+0x0e`: zero on 914 of 922. + /// `+0x0e`: 11 distinct values, zero on 2,162 of 2,177 -- the only field + /// here that is not universal. pub reserved_0e: u16, } @@ -164,13 +167,68 @@ impl TieBlockHeader { } /// True when every invariant measured across all 27 levels holds. + /// + /// Only the two that are universal: `count` is a multiple of 16 and + /// `reserved_02` is zero, both on all 2,177 blocks. pub fn invariants_hold(&self) -> bool { - self.count % 16 == 0 - && self.reserved_02 == 0 - && TIE_PACKED_TEXTURE_VALUES.contains(&self.packed_texture) + self.count % 16 == 0 && self.reserved_02 == 0 } } +/// Byte that terminates a tie LOD block's payload. +pub const TIE_BLOCK_TERMINATOR: u8 = 0xf6; + +/// A tie LOD block: its header, its payload, and the zero padding that follows +/// the terminator. +/// +/// **The blocks are self-delimiting.** Each ends with [`TIE_BLOCK_TERMINATOR`] +/// followed by zeros, on **2,177 of 2,177** blocks across all 27 levels. That is +/// what makes them parseable without trusting `packets[1]` to bound the block, +/// which it only does on 461 of 1,864 classes. +/// +/// The terminator is not a coincidence of byte frequency: `0xf6` occurs at 0.83x +/// the uniform rate in payloads, i.e. slightly *less* often than chance, while +/// the last non-zero byte is `0xf6` on 909 of 922 blocks in the first subset +/// tested against **17 of 922** for the same blocks cut at a random point. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TieBlock<'a> { + /// The 16-byte header. + pub header: TieBlockHeader, + /// Bytes from `0x10` up to and including the terminator. + pub payload: &'a [u8], + /// Zero bytes between the terminator and the next block. + pub padding: &'a [u8], +} + +/// Walk `region` from `offset` to the first `0xf6` that is followed by a zero, +/// and return the block there. +/// +/// Returns `None` if no terminator is found before the end of the region. +pub fn parse_tie_block(region: &[u8], offset: usize) -> Option> { + if offset.checked_add(TIE_BLOCK_HEADER_SIZE)? > region.len() { + return None; + } + let header = TieBlockHeader::parse(®ion[offset..]).ok()?; + let start = offset + TIE_BLOCK_HEADER_SIZE; + let mut i = start; + while i + 1 < region.len() { + if region[i] == TIE_BLOCK_TERMINATOR && region[i + 1] == 0 { + let after = i + 1; + let mut e = after; + while e < region.len() && region[e] == 0 { + e += 1; + } + return Some(TieBlock { + header, + payload: ®ion[start..after], + padding: ®ion[after..e], + }); + } + i += 1; + } + None +} + /// A tie class resolved through both of its indirections. /// /// The chain is two steps and both are easy to get wrong, so it is worth diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index d168205..ad6de5e 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1221,3 +1221,54 @@ confirming anything. **Still open: what the payload holds.** Established is that it is variable-length, float-shaped, non-random, not LZ77, and not VIF1. Not established is any record boundary. + +## tie: the LOD block is self-delimiting, and `packets == 0` means absent + +**The terminator.** Every tie LOD block ends with the byte `0xf6` followed by +zeros, on **2,177 of 2,177** real blocks across all 27 levels. That was worth a +control, because `0xf6` could have been a coincidence of byte frequency: it +occurs in payloads at **0.83x** the uniform rate, i.e. slightly *less* often +than chance, while the last non-zero byte is `0xf6` on 909 of 922 blocks in the +first subset tested against **17 of 922** for the same blocks cut at a random +point. So it is a terminator, not a common byte. + +This is what makes the blocks parseable at all. `packets[1]` bounds a block on +only 461 of 1,864 classes, so the terminator is the only general way to find one. + +**`packets == 0` means no block at that LOD.** Of the 3,728 (class, LOD) slots, +**2,177** point at a block and **1,551** are exactly zero. The zero cases are not +degenerate blocks -- they are absent ones. + +### A measurement I got wrong, twice, in opposite directions + +The first pass over this header reported **3,728 blocks, all terminating** and +`packed_texture` taking **842** distinct values, contradicting an earlier +four-value result. I retracted the four-value claim on that basis. + +Both numbers were wrong, and they were wrong the same way. The scan did not skip +`packets == 0`; it computed `class_base + 0` and walked forward from the **class +body**, finding a terminator there by accident. That contaminated 1,551 of the +3,728. + +With the zeros excluded, the slot takes **exactly four** values on all 2,177 +blocks -- 0x501 on 1,069, 0xa02 on 720, 0xf03 on 350, 0x1404 on 38 -- and all +2,177 terminate. + +So: the four-value claim was right, my retraction of it was wrong, and the +retraction was made on a number from the same contaminated set. A retraction is +not automatically better than the claim it retracts; it needs its own bounds. + +### Corrected header, over all 2,177 blocks + + +0x00 count multiple of 16 on all; 42 distinct; never zero + +0x02 zero on all + +0x04 exactly four values: 0x501, 0xa02, 0xf03, 0x1404 + +0x06 16 distinct, never zero + +0x08 180 distinct, never zero + +0x0a 61 distinct, never zero + +0x0c 125 distinct, never zero, clustered 0x043f..0x0443 + +0x0e 11 distinct, zero on 2,162 -- the only non-universal field + +`+0x0c` clusters around 1,087 to 1,091, close enough to a block length to be +worth stating, but it is **not** the length: the difference against the true +payload length scatters from -12 to -172.