From af62032ddc97b2733b34bdde591e7efb0ce85f88 Mon Sep 17 00:00:00 2001 From: ReaperOfVeriod Date: Tue, 13 Oct 2026 07:43:44 +0200 Subject: [PATCH] Most of the chunk container is level-independent: 65 of 74 signatures Comparing every chunk record against the same record index in the other 26 levels, across all 74 signatures: 65 of 74 every record byte-identical in all 27 levels 9 of 74 level-specific 0 ragged no signature appeared with different record counts per level The nine that vary: [5d 44 13] [75 42 ef] [7f 44 ee] [82 c4 2f] [96 c3 37] [a2 44 a5] [c6 c3 ef] [df c3 ea] [f3 c4 b9] This reframes the container. 65 of 74 chunk types are static tables the game ships unchanged in every level -- constants and lookups, not level content. The largest by record count are all in that group: [38 c4 7e] stride 328 with 90 records per level, [a9 c4 d8] stride 308 with 90, [3d c4 9e] stride 368 with 77. The consequence is about method, and it is worth being blunt about. For a level-specific type, comparing levels is the strongest lever available: a field that moves with the level is identifiable by correlating it with something known. For the other 65 there is nothing to compare -- one copy of the data, repeated 27 times -- so they must be decoded structurally, from what the bytes are. A future decoder that reaches for per-level comparison as its first tool will find it useless on the majority of the container. [75 42 ef] is both level-specific AND the one signature with three strides. A type that is both variable-length and level-varying is the most informative one here, and is where per-type field decoding should start. How this was found, including the wrong turn: The first look at [53 c4 6c d7 9a 43] (stride 108) showed 45 of 108 byte positions varying WITHIN a chunk and 810 records all distinct -- which reads like per-record data. Counting distinct values per u32 slot across all 810 records contradicted that: 29 to 30 distinct values across 27 levels, i.e. about one per level. Both readings were wrong because they measure different things. The 45 positions vary between records within a level; the u32 counts were low because the same 30 values recur in every level. Comparing record i across levels settled it: all 30 records are byte-identical in all 27 levels, and zero byte positions vary across levels. Had the u32 distinct counts been the only measurement, the conclusion would have been "one value per level" -- the opposite of the truth, describing a per-level parameter table that does not exist. The within-chunk and across-level views had to be taken together to get a true one. Asserted in most_chunk_signatures_are_level_independent. 242 tests, clippy clean. --- crates/gc-extract/tests/disc.rs | 87 +++++++++++++++++++++++++++++++ docs/formats/core-data-regions.md | 51 ++++++++++++++++++ docs/roadmap.md | 8 ++- 3 files changed, 145 insertions(+), 1 deletion(-) diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 7c7721f..ba3b8fe 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -4327,3 +4327,90 @@ fn chunk_record_stride_is_four_byte_aligned_and_uniform_per_signature() { "the variable signature's stride count moved" ); } + +/// Most of the chunk container is **level-independent**: 65 of its 74 signatures +/// carry records that are byte-identical in all 27 levels. +/// +/// That reframes the format. These are static tables the game ships unchanged in +/// every level -- lookups and constants -- not per-level content. Decoding them +/// gets no help from comparing levels, because there is nothing to compare; +/// they have to be read structurally. The 9 level-specific signatures are the +/// ones worth attacking first for actual per-level data. +#[test] +fn most_chunk_signatures_are_level_independent() { + require_disc!(); + + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::{chunk_region_start, parse_all_chunks}; + use gc_formats::level_core::LevelCoreHeader; + use std::collections::BTreeMap; + + /// signature -> (per-level record lists, total records) + type SigLevel = (Vec>>, usize); + let mut sig: BTreeMap, SigLevel> = BTreeMap::new(); + for lv in 0..27usize { + let (Ok(idx), Ok(core)) = ( + std::fs::read(data.join(format!("level{lv}/core_index.bin"))), + std::fs::read(data.join(format!("level{lv}/core_data.bin"))), + ) else { + continue; + }; + let _ = LevelCoreHeader::parse(&idx).expect("core header"); + let scan = chunk_region_start(&core, 0) + .or_else(|| chunk_region_start(&core, 0x1000)) + .unwrap_or(0); + let Ok(found) = parse_all_chunks(&core[scan..]) else { + continue; + }; + for rc in found { + let e = sig + .entry(rc.chunk.header[..6].to_vec()) + .or_insert_with(|| (Vec::new(), rc.records.len())); + e.1 += rc.records.len(); + e.0.push(rc.records.iter().map(|r| r.to_vec()).collect()); + } + } + assert_eq!(sig.len(), 74, "chunk signature count moved"); + + let mut shared = 0usize; + let mut specific: Vec = Vec::new(); + let mut ragged = 0usize; + for (k, (lvls, _)) in &sig { + let counts: Vec = lvls.iter().map(|l| l.len()).collect(); + if counts.windows(2).any(|w| w[0] != w[1]) { + ragged += 1; + continue; + } + let n = counts[0]; + let identical = (0..n).all(|i| lvls.iter().all(|l| l[i] == lvls[0][i])); + if identical && n > 0 { + shared += 1; + } else { + specific.push(format!("{:02x?}", &k[..3])); + } + } + + let n_sig = sig.len(); + eprintln!( + "chunk signatures: {shared}/{n_sig} byte-identical across every level; \ + {ragged} ragged; level-specific: {}", + specific.join(" ") + ); + + assert_eq!( + ragged, 0, + "a signature appeared with different record counts per level" + ); + assert_eq!(shared, 65, "the shared/level-specific split moved"); + assert_eq!(specific.len(), 9, "level-specific signature count moved"); +} diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index c6a88c7..3b37d2a 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -2336,3 +2336,54 @@ look like a layout and would otherwise be rediscovered and over-read. Neither of these decodes any record's *fields*. What is now settled is the shape the decoder has to handle: uniform stride per type, 4-byte aligned, with one named exception carrying three strides. + +## Most of the chunk container is level-independent + +Comparing every chunk record against the same record index in the other 26 +levels, across all 74 signatures: + + 65 of 74 signatures: every record byte-identical in all 27 levels + 9 of 74 signatures: level-specific + 0 ragged -- no signature appeared with different record counts per level + +The nine that vary: + + [5d 44 13] [75 42 ef] [7f 44 ee] [82 c4 2f] [96 c3 37] + [a2 44 a5] [c6 c3 ef] [df c3 ea] [f3 c4 b9] + +This reframes the whole container. **65 of 74 chunk types are static tables the +game ships unchanged in every level** -- constants and lookups, not level +content. The largest by record count are all in that group: `[38 c4 7e]` at +stride 328 with 90 records per level, `[a9 c4 d8]` at 308 with 90, `[3d c4 9e]` +at 368 with 77. Each contributes the same bytes to all 27 levels. + +The practical consequence is about method. For a level-specific type, comparing +levels is a powerful lever: a field that changes with the level is identifiable +by correlating it with something known. For the other 65 there is nothing to +compare -- there is one copy of the data, repeated 27 times -- so they have to be +decoded structurally, from what the bytes *are*, not from how they vary. Any +future decoder that reaches for per-level comparison as its first tool will find +it useless on the majority of the container. + +`[75 42 ef]` appears in the level-specific list, and it is also the one signature +with three strides. A type that is both variable-length and level-varying is the +most informative one in the container, and is where per-type field decoding should +start. + +### How this was found, and the near-miss before it + +The first look at one signature (`[53 c4 6c d7 9a 43]`, stride 108) showed 45 of +108 byte positions varying *within* a chunk, and 810 records all distinct -- +which reads like per-record data. Counting distinct values per `u32` slot across +all 810 records contradicted that immediately: 29 to 30 distinct values across 27 +levels, i.e. roughly one per level. + +Both readings were wrong, and the resolution is that they measure different +things. The 45 positions vary *between records within a level*. The u32 distinct +counts were low because the same 30 values recur in every level. Comparing record +*i* across levels settled it: **all 30 records are byte-identical in all 27 +levels**, and zero byte positions vary across levels. + +Had the u32 distinct counts been the only measurement taken, the conclusion would +have been "one value per level" -- which is the opposite of the truth, and would +have described a per-level parameter table that does not exist. diff --git a/docs/roadmap.md b/docs/roadmap.md index 30c5030..7924a07 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -471,7 +471,13 @@ Phase 1 hands over the extents; this phase works out what each one contains. The exception is pinned: `[75 42 ef f2 71 b5]` at strides `{76, 92, 104}`. Asserted including the exception, so "every type is uniform" cannot rot. A per-type layout found for one type (48 bytes of data then 60 permanently - zero) does **not** generalise -- 0 of 9 types fit that shape. The body + zero) does **not** generalise -- 0 of 9 types fit that shape. + **65 of the 74 signatures are byte-identical in all 27 levels** (9 are + level-specific, 0 ragged), so most of the container is static game tables + rather than level content -- asserted. Consequence for method: the 65 + cannot be decoded by comparing levels, because there is nothing to compare. + `[75 42 ef]` is both level-specific *and* the three-stride type, so it is + where per-type decoding should start. The body is dense (74% of `f32` reads plausible against a 49% random control) with an index-like tail -- 12 of 163 `u16` slots in the largest type stay below `0x1000` -- but **no count field** exists in the first 64 body bytes at any