From 572cd1898dd29e0a3ee9a92d545dc46cadc7f4a6 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Tue, 13 Oct 2026 17:40:31 +0200 Subject: [PATCH] Record a defect packet slicing found: the tie block terminator is early Range measured: every tie LOD block on the disc, 2,177 blocks. The last packet's end is known two ways. Its length comes from its record's packet_quadwords, and every other record's length is confirmed by the next record's pointer -- so only the last packet of each table is unconfirmed. Comparing its computed end against where parse_tie_block stopped: the last packet's computed end lands 2 to 16 bytes past the located 0xf6, on all 2,150 blocks measured. Two readings fit. Either the terminator was located too early -- inside the last packet's data -- or the last packet's length over-predicts by a small amount. The first is likelier: a length error on the last record of every table would have to be a coincidence. Neither is settled here, and the test asserts the measurement rather than the reading. Why the existing test missed it: "2,177 of 2,177 blocks end on 0xf6" is satisfied by a false terminator exactly as well as a true one -- it checks that the byte found IS a 0xf6, not that it is the block's end. A test that confirms the shape it walked to will pass on the wrong byte. The cost: the block extent is short by that much, so payload and padding under-report and anything computed from them is correspondingly short. The block starts are unaffected and RAM-confirmed. Recorded as a limitation with the measurement attached, not fixed. A terminator packet data cannot imitate is its own task, and guessing one would be the plausible-sounding alternative this project has been bitten by. A packets() accessor was tried and removed for the same reason: with the extent short it silently yields the wrong answer, and an API that is quietly wrong is worse than no API. --- crates/gc-extract/tests/disc.rs | 107 ++++++++++++++++++++++++++++++++ crates/gc-formats/src/tie.rs | 6 ++ docs/roadmap.md | 6 ++ 3 files changed, 119 insertions(+) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 7d81e74..11cee75 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -2931,6 +2931,113 @@ fn tie_packet_length_follows_from_its_record() { ); } +/// The located `0xf6` terminator lies inside the last packet's data. +/// +/// `TieBlock::payload` ends just past the `0xf6` that `parse_tie_block` walks +/// to. Each record's `packet_quadwords` gives its packet's length, and for +/// every record but the last of a table that length is confirmed by the *next* +/// record's pointer. So the last packet's end is known independently, and +/// comparing it against the located terminator: +/// +/// **the last packet's computed end lands 2 to 16 bytes past the terminator, +/// on every block measured.** The last packet's length is independent of the +/// terminator (it comes from its record's quadword count, confirmed for every +/// record but the last by the next record's pointer), so either the terminator +/// was located too early -- inside the last packet's data -- or the last +/// packet's length over-predicts by a small amount. Both readings fit; the +/// first is the likelier, because a length error only on the last record of +/// every table would have to be a coincidence. +/// +/// This is a real limitation in `parse_tie_block`, and the existing +/// `0xf6`-terminator test could not see it because a false terminator +/// satisfies that test just as well as a true one. Not fixed here: a +/// terminator that packet data cannot imitate is its own task, and guessing +/// one would be the "plausible-sounding alternative" this project has been +/// bitten by before. +/// +/// What still holds: the lengths are right for every record but the last +/// (12,927 of 12,927 record pairs) and the block *starts* are confirmed +/// against RAM. The block *extent* is what is short. +#[test] +fn tie_block_terminator_is_inside_the_last_packet() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::level_core::{ClassEntry, LevelCoreHeader}; + use gc_formats::tie::{parse_tie_block, TieClassHeader, TieRecord, TIE_RECORD_SIZE}; + + let mut blocks = 0usize; + let mut overruns: std::collections::BTreeMap = Default::default(); + + for lv in 0..27usize { + let (Ok(idx), Ok(core)) = ( + std::fs::read(data.join(format!("level{lv}/core_index.bin"))), + std::fs::read(data.join(format!("level{lv}/core_data.bin"))), + ) else { + continue; + }; + let t = LevelCoreHeader::parse(&idx).expect("hdr").tie_classes; + for c in 0..t.count as usize { + let o = t.offset as usize + c * 0x20; + let e = ClassEntry::parse(&idx[o..o + 0x20]).expect("entry"); + let base = e.offset_in_asset_wad as usize; + if base == 0 || base + 0x48 > core.len() { + continue; + } + let Ok(hdr) = TieClassHeader::parse(&core[base..base + 0x48]) else { + continue; + }; + for lod in 1..=2usize { + let off = hdr.packets[lod]; + if off <= 0 { + continue; + } + let s = base + off as usize; + if let Some(blk) = parse_tie_block(&core, s) { + let recs: Vec = blk + .header + .records(&core[s..]) + .map(|r| TieRecord::parse(r).expect("record")) + .collect(); + let Some(last) = recs.last() else { + continue; + }; + if !last.packet_quadwords_are_known() { + continue; + } + blocks += 1; + let packet_end = + s + last.pointer as usize + last.packet_quadwords() * TIE_RECORD_SIZE; + let payload_end = s + blk.payload.len(); + if packet_end > payload_end { + *overruns.entry(packet_end - payload_end).or_default() += 1; + } + } + } + } + } + + assert!(blocks > 1900, "only {blocks} tie blocks measured"); + let measured: usize = overruns.values().sum(); + assert_eq!(measured, blocks, "{measured} of {blocks} blocks overran"); + let mut top: Vec<(usize, usize)> = overruns.into_iter().collect(); + top.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); + top.truncate(6); + eprintln!( + "tie block terminator: the last packet overruns it on all {measured} blocks, \ + by a small byte count (top: {top:?})" + ); +} + fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { return 0; diff --git a/crates/gc-formats/src/tie.rs b/crates/gc-formats/src/tie.rs index 0b69ea5..0373d0d 100644 --- a/crates/gc-formats/src/tie.rs +++ b/crates/gc-formats/src/tie.rs @@ -372,6 +372,12 @@ pub struct TieBlock<'a> { /// The 16-byte header. pub header: TieBlockHeader, /// Bytes from `0x10` up to and including the terminator. + /// + /// **This is short.** The `0xf6` the walk stops at lies inside the last + /// packet's data, an even 2 to 16 bytes before that packet's computed end, + /// so the block's real extent runs past it. `tie_block_payload_divides_ + /// into_contiguous_packets` measures this; finding a terminator packet + /// data cannot imitate is its own task. pub payload: &'a [u8], /// Zero bytes between the terminator and the next block. pub padding: &'a [u8], diff --git a/docs/roadmap.md b/docs/roadmap.md index 8e7a746..2f11603 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -283,6 +283,12 @@ Phase 1 hands over the extents; this phase works out what each one contains. R&C1 carries the same quadruple at `+0x20` and it is not an offset on R&C2. The fourth byte is unidentified and is *not* the shader total (disagrees on 1,827 of 1,864). +- [~] The tie block's **located `0xf6` terminator is inside the last packet's + data** -- the last packet's computed end lands 2 to 16 bytes past it on + all 2,150 blocks measured. So the block's *extent* is short by that much + (the `0xf6` test could not see it: a false terminator passes that test). + The block *starts* are unaffected and RAM-confirmed. The packet lengths + themselves are verified for every record but the last. - [x] `tie` — **fully chained.** `resolve_tie_class` walks both indirections: the class table is in `core_index.bin`, and the entry's `offset_in_asset_wad` addresses `core_data.bin` (not `core_data.wad`). All **1,864** tie classes