From 1cfe7445019390ef57d3733a5f41609af99f1d98 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Tue, 13 Oct 2026 16:48:49 +0200 Subject: [PATCH] Settle the tie packet length, and correct the retraction that hid it Range measured: every tie LOD block on the disc -- 2,177 blocks, 13,042 consecutive record pairs. A record's pointer is where its packet starts, so the distance between consecutive pointers IS a packet's length, with no reference to any field. Comparing that against the record's own fields: packet_quadwords = vert_ofs + vert_size + scissor_size 12,927 of 12,927 record pairs whose strip_count is zero 0 of 115 record pairs whose it is not The partition is exact: every record with a non-zero strip_count is an exception and no record with a zero one is. So the fourth term is 0 quadwords whenever strip_count == 0, and something else -- measured 0..19 quadwords, and NOT equal to strip_count -- whenever it is not. R&C1's strip_count is a strip count and strips are variable-length data, so a non-zero count most likely adds per-strip bytes this cannot see; that is a hypothesis, labelled as one. TieRecord gains packet_quadwords and packet_quadwords_are_known. The latter is the honest gate, and the disc test asserts both sides of the partition rather than the aggregate. This also corrects this file's own retraction of the same formula, and the retraction was the error rather than the claim. A formula of this shape was derived at 61 of 61 pairs, then "retracted" as not reproducing (0 of 13,042). The re-measurement had swapped strip_count for scissor_ofs in the fourth term and put the blame on the block set. Re-measured with the term the first pass had, it matches on 12,927 of 13,042 and the 115 exceptions are exactly the non-zero strip_count records. Worth recording because it is the same shape as two other entries in this file: a retraction is not automatically more careful than the claim it retracts, and the re-measurement that "disproved" this one changed the thing being measured. The lesson on the other side is what saved it -- the first measurement was over 61 pairs, not a population, and had it stayed there the formula would have looked universal. A small sample and a bad re-measurement, near-misses in opposite directions. What this unblocks: the payload can be sliced per packet, which was the prerequisite for decoding what it holds. --- crates/gc-extract/tests/disc.rs | 130 ++++++++++++++++++++++++++++++ crates/gc-formats/src/tie.rs | 28 +++++-- docs/formats/core-data-regions.md | 41 ++++++++++ docs/roadmap.md | 8 ++ 4 files changed, 199 insertions(+), 8 deletions(-) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index f1a3f21..7d81e74 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2801,6 +2801,136 @@ fn overlay_load_base_from_the_profiler_table() { ); } +/// The tie packet's length, from its record's fields. +/// +/// A tie LOD block is a table of packet descriptors plus the packets +/// themselves, and the descriptor's pointer is where the packet starts. So the +/// distance between consecutive pointers **is** a packet's length, measured +/// without reference to any field. Against that: +/// +/// **`packet_quadwords = vert_ofs + vert_size + scissor_size`, on 12,927 of +/// 12,927 record pairs whose `strip_count` is zero -- and on 0 of 115 whose it +/// is not.** The partition is exact: every record with a non-zero +/// `strip_count` is an exception, and no record with a zero one is. So the +/// fourth term is 0 quadwords whenever `strip_count == 0`, and something else +/// (measured 0..19 quadwords, and not equal to `strip_count`) whenever it is +/// not. +/// +/// This is what makes the packet payload sliceable at all. It also settles a +/// dispute inside this file: a formula of this shape was derived at 61 of 61, +/// then "retracted" as not reproducing -- 0 of 13,042. The retraction was the +/// error, not the formula: the re-measurement had swapped `strip_count` for +/// `scissor_ofs` in the fourth term, and the blame was put on the block set. +/// Re-measured with the term the first pass had, it matches on 12,927 of +/// 13,042, and the 115 exceptions are exactly the non-zero `strip_count` +/// records. +#[test] +fn tie_packet_length_follows_from_its_record() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::level_core::{ClassEntry, LevelCoreHeader}; + use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord}; + + let mut blocks = 0usize; + let mut pairs = 0usize; + let mut zero_strip = 0usize; + let mut zero_strip_ok = 0usize; + let mut nonzero_strip = 0usize; + let mut nonzero_strip_ok = 0usize; + let mut unaligned = 0usize; + + for lv in 0..27usize { + let (Ok(idx), Ok(core)) = ( + std::fs::read(data.join(format!("level{lv}/core_index.bin"))), + std::fs::read(data.join(format!("level{lv}/core_data.bin"))), + ) else { + continue; + }; + let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes; + for c in 0..t.count as usize { + let o = t.offset as usize + c * 0x20; + let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry"); + let base = e.offset_in_asset_wad as usize; + if base == 0 || base + 0x48 > core.len() { + continue; + } + let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else { + continue; + }; + for lod in 1..=2usize { + let off = hdr.packets[lod]; + if off <= 0 { + continue; + } + let s = base + off as usize; + if let Some(blk) = parse_tie_block(&core, s) { + let recs: Vec = blk + .header + .records(&core[s..]) + .map(|r| TieRecord::parse(r).expect("record")) + .collect(); + if recs.len() < 2 { + continue; + } + blocks += 1; + for (a, b) in recs.iter().zip(recs.iter().skip(1)) { + if b.pointer <= a.pointer { + continue; + } + let delta = (b.pointer - a.pointer) as usize; + pairs += 1; + if delta % 16 != 0 { + unaligned += 1; + continue; + } + let hit = delta / 16 == a.packet_quadwords(); + if a.strip_count == 0 { + zero_strip += 1; + zero_strip_ok += usize::from(hit); + } else { + nonzero_strip += 1; + nonzero_strip_ok += usize::from(hit); + } + } + } + } + } + } + + assert!(blocks > 1900, "only {blocks} tie blocks were read"); + assert!(pairs > 12_000, "only {pairs} record pairs were read"); + assert_eq!( + unaligned, 0, + "{unaligned} pointer deltas are not 16-aligned" + ); + assert_eq!( + zero_strip_ok, + zero_strip, + "the formula failed on {} of {zero_strip} strip_count == 0 records", + zero_strip - zero_strip_ok + ); + assert!( + zero_strip > 12_000, + "only {zero_strip} records have strip_count == 0" + ); + eprintln!( + "tie packet length: vert_ofs + vert_size + scissor_size quadwords on \ + {zero_strip_ok}/{zero_strip} records with strip_count == 0; on the {nonzero_strip} \ + records with a non-zero strip_count it holds {nonzero_strip_ok} times (measured \ + fourth term 0..19, not strip_count)" + ); +} + fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { return 0; diff --git a/crates/gc-formats/src/tie.rs b/crates/gc-formats/src/tie.rs index 4520b94..0b69ea5 100644 --- a/crates/gc-formats/src/tie.rs +++ b/crates/gc-formats/src/tie.rs @@ -327,14 +327,26 @@ impl TieRecord { && self.control_size <= self.control_count } - // A packet-length formula was here -- `(vert_ofs + vert_size + - // scissor_ofs + scissor_size) * 16` -- and it was retracted. It was - // measured on a block set found by reading `packets[2]` where `packets[1]` - // was meant, over 61 pairs, and re-measured on the correct blocks it holds - // on 0 of 13,042. The block set was the error, not the arithmetic: the - // arithmetic agreed perfectly with a population that was not the tie - // blocks at all. This is the same failure `AGENTS.md` rule 3 is about, - // arriving through a wrong subscript instead of a wrong span. + /// Quadwords of packet data: `vert_ofs + vert_size + scissor_size`. + /// + /// Holds on **12,927 of 12,927** record pairs whose `strip_count` is zero, + /// and on **0 of 115** whose it is not -- the partition is exact, so the + /// formula is safe to use when `strip_count == 0` and knows nothing + /// otherwise. R&C1's `strip_count` is the strip count, and strips are + /// variable-length data, so a non-zero count most likely adds a record + /// count's worth of per-strip bytes this cannot see. + pub fn packet_quadwords(&self) -> usize { + self.vert_ofs as usize + self.vert_size as usize + self.scissor_size as usize + } + + /// Whether [`Self::packet_quadwords`] is known to describe this record. + /// + /// `strip_count == 0` on every record the formula has been verified on; + /// the 115 records where it is not decode to a fourth term of 0..19 + /// quadwords that is not `strip_count` and is not established. + pub fn packet_quadwords_are_known(&self) -> bool { + self.strip_count == 0 + } } /// Byte that terminates a tie LOD block's payload. diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 13541e7..f5f1889 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -1280,6 +1280,47 @@ had loaded. Two operational notes worth keeping: What the records' fields name and what the payload holds are still open. +## The tie packet's length, from its record's fields (2026-10-11) + +**Range measured:** every tie LOD block on the disc -- 2,177 blocks, 13,042 +consecutive record pairs. + +**Method:** a record's pointer is where its packet starts, so the distance +between consecutive pointers **is** a packet's length, with no reference to any +field. Comparing that against the record's own fields gives the length rule. + +**Result: `packet_quadwords = vert_ofs + vert_size + scissor_size`, on 12,927 +of 12,927 record pairs whose `strip_count` is zero -- and on 0 of 115 whose it +is not.** The partition is exact: every record with a non-zero `strip_count` is +an exception and no record with a zero one is. The fourth term is 0 quadwords +whenever `strip_count == 0`, and something else (measured 0..19 quadwords, and +*not* equal to `strip_count`) whenever it is not. R&C1's `strip_count` is a +strip count and strips are variable-length data, so a non-zero count most +likely adds per-strip bytes this cannot see -- a hypothesis, labelled as one. + +`TieRecord::packet_quadwords` and `TieRecord::packet_quadwords_are_known` carry +it; the latter is the honest gate, and the disc test asserts both sides of the +partition. + +**This settles a dispute inside this file, and the retraction was the error.** +A formula of this shape was derived at 61 of 61 pairs, then "retracted" as not +reproducing -- re-measured at 0 of 13,042. The retraction was wrong: the +re-measurement had swapped `strip_count` for `scissor_ofs` in the fourth term, +and the blame went on the block set. Re-measured with the term the first pass +had, it matches on 12,927 of 13,042, and the 115 exceptions are precisely the +non-zero `strip_count` records. + +The lesson is worth keeping because it is the same shape as two others here: a +retraction is not automatically more careful than the claim it retracts, and the +re-measurement that "disproved" this one changed the thing it was measuring. +The lesson on the other side is the one that saved it: the first measurement +was over 61 pairs, which is not a population -- and had it stayed at 61 the +formula would have looked universal. Both the small sample and the bad +re-measurement were near-misses in opposite directions. + +What this unblocks: the payload can now be sliced per packet, which was the +prerequisite for decoding what it holds. + ### What the payload is, and what retired the coordinate reading The payload is **not random**, and that took a control to establish. Reading each diff --git a/docs/roadmap.md b/docs/roadmap.md index 6ed821c..8e7a746 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -322,6 +322,14 @@ Phase 1 hands over the extents; this phase works out what each one contains. Within the all-zero case, `i16[3] == 0` exactly (5,851) is a third distribution at 62%. Three cases: `0x00/0x00`, `0x00/1..255`, and non-zero high byte. +- [x] **The tie packet's length follows from its record**: + `vert_ofs + vert_size + scissor_size` quadwords, on **12,927 of 12,927** + record pairs whose `strip_count` is zero and **0 of 115** whose it is + not -- an exact partition, so the formula is safe when `strip_count == 0` + and knows nothing otherwise. This unblocks slicing the payload per + packet. It also corrects this file's own retraction of the same formula: + the "0 of 13,042" re-measurement had swapped `strip_count` for + `scissor_ofs` and blamed the block set. - [~] The remaining tie vertex **fields are not named**, and two readings are now excluded. `i16[6]`/`i16[7]` are constant across a whole draw packet in 43%/34% of records, so they are usually per-packet rather than per-vertex.