diff --git a/crates/gc-extract/src/probe.rs b/crates/gc-extract/src/probe.rs index af6172a..835ab05 100644 --- a/crates/gc-extract/src/probe.rs +++ b/crates/gc-extract/src/probe.rs @@ -95,7 +95,7 @@ pub fn probe_level(dir: &Path, level: u32) -> Result { header.push((name.into(), range(v))); } - let regions = gc_formats::regions::core_layout(&index, &h, core.len()); + let regions = gc_formats::regions::core_layout(&index, &core, &h); let mut classes = Vec::new(); for (name, r, stride) in [ diff --git a/crates/gc-extract/tests/disc.rs b/crates/gc-extract/tests/disc.rs index 220d7b8..0f3663a 100644 --- a/crates/gc-extract/tests/disc.rs +++ b/crates/gc-extract/tests/disc.rs @@ -1648,7 +1648,6 @@ fn shrub_headers_name_their_own_class_on_every_level() { let mut denormals = 0usize; let mut bounded_records = 0usize; let mut bounded_bodies = 0usize; - let mut last_bodies = 0usize; for n in 0..27 { let dir = data.join(format!("level{n}")); @@ -1661,10 +1660,10 @@ fn shrub_headers_name_their_own_class_on_every_level() { let h = gc_formats::level_core::LevelCoreHeader::parse(&index).expect("core header"); let range = h.shrub_classes; // Class spans tile between consecutive geometry offsets, with the last - // one bounded by the start of core_data's constant trailer. Without - // this the last class absorbs the trailer and the body check measures - // its neighbours. - let trailer = gc_formats::regions::CORE_DATA_TRAILER_BYTES; + // one bounded by the chunk container. It is found by signature, since + // the core index carries no successor offset for shrub; without it the + // last class absorbs the container and the body check measures its + // neighbours. let mut offsets: Vec = Vec::new(); for i in 0..range.count.max(0) as usize { let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES; @@ -1675,48 +1674,45 @@ fn shrub_headers_name_their_own_class_on_every_level() { } } offsets.sort_unstable(); + // Levels without shrub geometry have no span to bound. The container + // that follows tie on those levels is checked by + // `chunk_container_closes_the_last_class_region`. + let container = match offsets.last() { + Some(&last) if !offsets.is_empty() => { + gc_formats::chunks::chunk_region_start(&core, last).unwrap_or_else(|| { + panic!("level{n}: no chunk container after the shrub classes") + }) + } + _ => core.len(), + }; let mut span_end_of: std::collections::HashMap = std::collections::HashMap::new(); for (i, &o) in offsets.iter().enumerate() { - let end = offsets.get(i + 1).copied().unwrap_or(core.len() - trailer); + let end = offsets.get(i + 1).copied().unwrap_or(container); span_end_of.insert(o, end); } for i in 0..range.count.max(0) as usize { let at = range.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES; let e = ShrubClassEntry::parse(&index[at..at + SHRUB_CLASS_ENTRY_BYTES]) .unwrap_or_else(|e| panic!("level{n}: shrub class {i}: {e}")); - let (span_end, has_successor) = match span_end_of - .get(&(e.base.offset_in_asset_wad.max(0) as usize)) - { - Some(&e2) - if offsets.last() != Some(&(e.base.offset_in_asset_wad.max(0) as usize)) => - { - (e2, true) - } - _ => (core.len(), false), - }; + let (span_end, has_successor) = + match span_end_of.get(&(e.base.offset_in_asset_wad.max(0) as usize)) { + Some(&e2) => (e2, true), + None => (core.len(), false), + }; match ShrubHeader::region(&core, e.base.offset_in_asset_wad, e.base.o_class, span_end) { Ok(Some(mut s)) => { s.span_end_is_a_successor = has_successor; - // Only a class whose span ends at the *next* class has a - // bounded body. The last class's span runs to the trailer - // and contains the 690 KB region that follows shrub, so its - // body is not wholly records -- that is the open question, - // not a defect, and asserting either way would be wrong. - if s.span_end_is_a_successor { - assert!( - s.body_is_well_formed(), - "level{n} class {i}: bounded shrub body is not whole 8-byte records" - ); - bounded_bodies += 1; - } - last_bodies += 1; - // Count records only where the span is actually bounded. - // Summing the unbounded last class too reproduces the - // retracted "3.5 million records" figure. - if s.span_end_is_a_successor { - bounded_records += s.body_record_count(); - } + // Every class is bounded now, so every body must be whole + // 8-byte records. That was not assertable while the last + // class ran to the lump's end, and it is the check the + // boundary earns its keep for. + assert!( + s.body_is_well_formed(), + "level{n} class {i}: shrub body is not whole 8-byte records" + ); + bounded_bodies += 1; + bounded_records += s.body_record_count(); assert_eq!( s.header.o_class, e.base.o_class.max(0) as u32, @@ -1761,27 +1757,29 @@ fn shrub_headers_name_their_own_class_on_every_level() { spans + absent ); assert!(spans > 340, "only {spans} shrub spans with geometry"); + // The container bounds every class now, so every body is checkable. + assert_eq!(bounded_bodies, spans, "every shrub class should be bounded"); eprintln!( "shrub: {spans} headers naming their own class, {absent} without geometry; \ - {bounded_bodies}/{last_bodies} class bodies bounded by a successor, \ - {bounded_records} 8-byte records in bounded bodies, \ + all {bounded_bodies} class bodies bounded by the chunk container, \ + {bounded_records} 8-byte records in total, \ {denormals} denormal reserved words" ); } /// Region bounds derived from successor boundaries, on every level. /// -/// The point is not that these are correct -- shrub's end is still unknown -- -/// but that the unknown is visible. Every region here is closed by the next -/// region's start rather than by `core_data.len()`, so no span silently -/// absorbs the tail of the lump the way the last shrub class was doing. +/// Every region is closed by the next region's start, or -- for the last class +/// region -- by the located chunk container, never by `core_data.len()`, so no +/// span silently absorbs the tail of the lump the way the last shrub class was +/// doing. #[test] fn core_regions_chain_on_every_level() { require_disc!(); let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { - let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { @@ -1790,23 +1788,26 @@ fn core_regions_chain_on_every_level() { } use gc_formats::level_core::LevelCoreHeader; - use gc_formats::regions::core_layout; + use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES}; let mut levels = 0usize; let mut unresolved_shrub = 0usize; + let mut unresolved_chunks = 0usize; let mut bounded = 0usize; let mut shrub_bytes = 0usize; + let mut chunk_bytes = 0usize; let (mut first, mut last) = (usize::MAX, 0usize); for n in 0..27 { let dir = data.join(format!("level{n}")); - let Ok(index) = std::fs::read(dir.join("core_index.bin")) else { + let (Ok(core), Ok(index)) = ( + std::fs::read(dir.join("core_data.bin")), + std::fs::read(dir.join("core_index.bin")), + ) else { continue; }; - let core_len = std::fs::metadata(dir.join("core_data.bin")) - .map(|m| m.len() as usize) - .unwrap_or(0); + let core_len = core.len(); let h = LevelCoreHeader::parse(&index).expect("core header"); - let regions = core_layout(&index, &h, core_len); + let regions = core_layout(&index, &core, &h); // Regions appear in order, never overlap, and never run past the lump. for pair in regions.windows(2) { @@ -1838,7 +1839,7 @@ fn core_regions_chain_on_every_level() { // Regions appear in the canonical order, but not every level has every // region -- level 10 has no occlusion and no shrub at all. - const ORDER: [&str; 7] = [ + const ORDER: [&str; 8] = [ "tfrags", "occlusion", "sky", @@ -1846,6 +1847,7 @@ fn core_regions_chain_on_every_level() { "moby", "tie", "shrub", + "chunks", ]; let names: Vec<_> = regions.iter().map(|r| r.name).collect(); let ranks: Vec = names @@ -1862,11 +1864,11 @@ fn core_regions_chain_on_every_level() { "level{n}: regions out of order: {names:?}" ); if let Some(s) = regions.iter().find(|r| r.name == "shrub") { - // The constant trailer bounds it; assert it is actually inside - // the lump rather than at its end. match s.end { Some(e) => { - assert!(e < core_len, "level{n}: shrub ends at the lump's end"); + // The container's start, not the trailer and not the lump's + // end: the trailer is the chunk region's business. + assert!(e < core_len - CORE_DATA_TRAILER_BYTES); shrub_bytes += s.len().unwrap_or(0); bounded += 1; } @@ -1875,20 +1877,33 @@ fn core_regions_chain_on_every_level() { first = first.min(s.start); last = last.max(s.start); } + if let Some(c) = regions.iter().find(|r| r.name == "chunks") { + match c.end { + Some(e) => { + // The trailer is the only thing past the chunk container. + assert_eq!(e, core_len - CORE_DATA_TRAILER_BYTES); + chunk_bytes += c.len().unwrap_or(0); + } + None => unresolved_chunks += 1, + } + } levels += 1; } assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); - // Known gap, not a bug: the offset ending the shrub region is not in the - // core index and textures sit before moby, so they cannot bound it. assert_eq!( unresolved_shrub, 0, - "every level with shrub should bound it" + "every level with shrub should bound it at the chunk container" ); assert_eq!(bounded, 24, "expected 24 levels with a shrub region"); + assert_eq!( + unresolved_chunks, 0, + "every chunk region should end at the trailer" + ); eprintln!( - "regions: shrub starts {first}..{last}; {shrub_bytes} bytes total, \ - bounded by the constant trailer on all {bounded} levels" + "regions: shrub starts {first}..{last}; {shrub_bytes} bytes of shrub and \ + {chunk_bytes} bytes of chunk data, shrub closed by the located container on \ + all {bounded} levels" ); } @@ -1896,6 +1911,15 @@ fn core_regions_chain_on_every_level() { /// signature on every level, and their offsets resolve to records with a /// shared field layout. /// +/// The region is bounded properly rather than by the fixed window this test +/// first used: it runs from the marker-paired chunk signature that closes the +/// last class region to the constant trailer. The window approach worked and +/// then over-worked -- a fixed 900,000 bytes from the end reaches past the +/// container on some levels and stops inside it on others, which is how 27 +/// chunks came to be truncated at a "window edge" that is not a format +/// boundary. The locator is asserted by +/// `chunk_container_closes_the_last_class_region`. +/// /// The layout facts asserted here are measured, not assumed: /// /// - every tagged offset has top byte `0xf0`, on all 27 levels; @@ -1909,7 +1933,7 @@ fn chunk_containers_resolve_on_every_level() { let data = std::env::var_os("GC_EXTRACTED") .map(PathBuf::from) .unwrap_or_else(|| { - let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); PathBuf::from(home).join("Documents/gc-extracted") }); if !data.exists() { @@ -1917,6 +1941,9 @@ fn chunk_containers_resolve_on_every_level() { return; } + use gc_formats::level_core::LevelCoreHeader; + use gc_formats::regions::{core_layout, CORE_DATA_TRAILER_BYTES}; + let mut levels = 0usize; let mut chunks = 0usize; let mut records = 0usize; @@ -1928,20 +1955,30 @@ fn chunk_containers_resolve_on_every_level() { for n in 0..27 { let dir = data.join(format!("level{n}")); - let core = match std::fs::read(dir.join("core_data.bin")) { - Ok(c) => c, - Err(_) => continue, + let (Ok(core), Ok(index)) = ( + std::fs::read(dir.join("core_data.bin")), + std::fs::read(dir.join("core_index.bin")), + ) else { + continue; }; - // The region runs from after the last shrub geometry to the constant - // trailer, whose size is measured and identical on all 27 levels. - const TRAILER: usize = 0x38a10; - let region_start = core.len().saturating_sub(TRAILER + 900_000); - let region_end = core.len() - TRAILER; - if region_end <= region_start { + // The region runs from the chunk signature that closes the last class + // region to the constant trailer, both measured and identical in size + // on all 27 levels. core_layout locates the start through the same + // signature the chunk parser uses. + let h = LevelCoreHeader::parse(&index).expect("core header"); + let Some(region) = core_layout(&index, &core, &h) + .into_iter() + .find(|r| r.name == "chunks") + else { continue; - } - let region = &core[region_start..region_end]; + }; + let Some(end) = region.end else { + panic!("level{n}: chunk region is unbounded"); + }; + assert_eq!(end, core.len() - CORE_DATA_TRAILER_BYTES); + assert!(!region.is_empty(), "level{n}: empty chunk region"); + let region = &core[region.start..end]; let found = gc_formats::chunks::parse_all_chunks(region).expect("chunks should parse"); if found.is_empty() { @@ -1970,9 +2007,9 @@ fn chunk_containers_resolve_on_every_level() { strides.insert(chunk.chunk.stride); // Records resolve unless the last one would run past the region - // end -- which happens for chunks sitting at the window edge, since - // the window is a fixed 900000 bytes and does not align to chunk - // boundaries. Truncation is counted rather than asserted away. + // end, which happens for the chunk sitting at the trailer end: + // the container's records stop there, so its final record or two + // fall outside. Truncation is counted rather than asserted away. assert!( chunk.records.len() <= chunk.chunk.offsets.len(), "level{n}: chunk at {} resolved MORE records than offsets", @@ -1993,7 +2030,7 @@ fn chunk_containers_resolve_on_every_level() { } } - assert!(levels >= 20, "only {levels} levels had chunks"); + assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); assert!(chunks > 500, "only {chunks} chunks resolved"); assert!(records > 5_000, "only {records} records resolved"); assert!( @@ -2011,7 +2048,7 @@ fn chunk_containers_resolve_on_every_level() { eprintln!( "chunks: {chunks} over {levels} levels, {records} records, {} distinct strides, \ constant {constant_total}/{stride_total} ({:.1}%), per-chunk min {:.1}% max {:.1}%, \ - {truncated} chunks truncated at the window edge", + {truncated} chunks truncated at the trailer end", strides.len(), mean_frac * 100.0, frac.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0, @@ -2019,6 +2056,185 @@ fn chunk_containers_resolve_on_every_level() { ); } +/// The chunk container closes the last class region, on every level. +/// +/// The core index carries no successor offset for shrub -- unlike moby and +/// tie, whose boundaries fall out of the class tables -- so the boundary is +/// located in `core_data`: the constant marker pair sitting immediately in +/// front of a chunk signature. Both halves of that signature are weak alone, +/// which is why the test measures them separately: +/// +/// - the marker pair `f6 7c` occurs 25 to 103 times in a 5 to 17 MB lump, so +/// on its own it means nothing; +/// - the chunk shape (16-byte header ending `0xff`, 10 zeros, tagged offsets) +/// matches random data often enough to need four tagged offsets before it +/// means anything. +/// +/// Together they are exact: on all 27 levels the marker sits exactly two bytes +/// before the first chunk header. Three checks pin that down per level: +/// +/// 1. the located boundary carries the marker; +/// 2. the last shrub body walks to 8-byte records that end exactly where the +/// signature begins, with no chunk signature inside the body itself; +/// 3. the container's records fill the region up to the constant trailer. +/// +/// Levels without shrub are the control for the locator: there the container +/// closes tie instead, and the same marker precedes it. +#[test] +fn chunk_container_closes_the_last_class_region() { + require_disc!(); + let data = std::env::var_os("GC_EXTRACTED") + .map(PathBuf::from) + .unwrap_or_else(|| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".into()); + PathBuf::from(home).join("Documents/gc-extracted") + }); + if !data.exists() { + eprintln!("skipped: unpack the disc first"); + return; + } + + use gc_formats::chunks::{self, CHUNK_REGION_MARKER}; + use gc_formats::level_core::{LevelCoreHeader, ShrubClassEntry}; + use gc_formats::regions::{core_layout, last_geometry, CORE_DATA_TRAILER_BYTES}; + use gc_formats::shrub::{ShrubHeader, SHRUB_CLASS_ENTRY_BYTES}; + + let mut levels = 0usize; + let mut with_shrub = 0usize; + let mut marker_wide_max = 0usize; + let mut body_records = 0usize; + let mut container_bytes = 0usize; + let mut fill_gap_max = 0usize; + + for n in 0..27 { + let dir = data.join(format!("level{n}")); + let (Ok(core), Ok(index)) = ( + std::fs::read(dir.join("core_data.bin")), + std::fs::read(dir.join("core_index.bin")), + ) else { + panic!("level{n}: missing core_data or core_index"); + }; + let trailer = core.len() - CORE_DATA_TRAILER_BYTES; + let h = LevelCoreHeader::parse(&index).expect("core header"); + + let last_shrub = last_geometry(&index, h.shrub_classes, SHRUB_CLASS_ENTRY_BYTES); + let last_tie = last_geometry(&index, h.tie_classes, 0x20); + // The container follows the last of the shrub and tie classes, so the + // signature search starts after the class bodies rather than inside + // them. + let scan_from = last_shrub + .or(last_tie) + .unwrap_or_else(|| panic!("level{n}: no class geometry to follow")); + let container = chunks::chunk_region_start(&core, scan_from) + .unwrap_or_else(|| panic!("level{n}: no chunk container after {scan_from}")); + + // 1. The marker, and how common it is without the signature behind it. + assert_eq!( + &core[container - 2..container], + &CHUNK_REGION_MARKER, + "level{n}: container at {container} is not preceded by the marker" + ); + marker_wide_max = marker_wide_max.max( + core.windows(2) + .filter(|w| *w == CHUNK_REGION_MARKER) + .count(), + ); + + // 2. The last shrub body ends exactly at the marker. This is the check + // that the boundary is the successor of shrub rather than a coincident + // pair inside it. + if let Some(last) = last_shrub { + let o_class = (0..h.shrub_classes.count.max(0) as usize) + .filter_map(|i| { + let at = h.shrub_classes.offset.max(0) as usize + i * SHRUB_CLASS_ENTRY_BYTES; + let e = ShrubClassEntry::parse( + index + .get(at..at + SHRUB_CLASS_ENTRY_BYTES) + .unwrap_or_default(), + ) + .ok()?; + (e.base.offset_in_asset_wad == last as i32).then_some(e.base.o_class) + }) + .next() + .unwrap_or_else(|| panic!("level{n}: no shrub class at {last}")); + let span = ShrubHeader::region(&core, last as i32, o_class, trailer) + .expect("last shrub span parses") + .expect("last shrub span present"); + let data_start = last + span.header.data_offset as usize; + + // Walk 8-byte records, bounded by the container so the walk can + // never continue into the chunk region itself. + let mut at = data_start; + while at + 8 <= container && core[at + 6] == 0 && core[at + 7] == 0 { + at += 8; + } + assert_eq!( + at + 2, + container, + "level{n}: shrub body at {data_start} does not end at the marker" + ); + assert!(at > data_start, "level{n}: last shrub body is empty"); + body_records += (at - data_start) / 8; + + // No chunk signature inside the body: the search for the boundary + // must not match one it should have skipped. + let sig_inside = (last..container - 2).any(|off| chunks::is_chunk_start(&core, off, 4)); + assert!( + !sig_inside, + "level{n}: chunk signature inside the last shrub body" + ); + with_shrub += 1; + } else { + // Without shrub the container closes tie, and the marker precedes + // it the same way -- the locator is not shrub-specific. + let tie = core_layout(&index, &core, &h) + .into_iter() + .find(|r| r.name == "tie") + .unwrap_or_else(|| panic!("level{n}: no tie region to close")); + assert_eq!( + tie.end, + Some(container), + "level{n}: tie must end at the container" + ); + } + + // 3. The container fills its region up to the constant trailer: the + // furthest record should reach within a few KB of it, never inside it. + let region = &core[container..trailer]; + let found = chunks::parse_all_chunks(region).expect("chunks should parse"); + let region_ptr = region.as_ptr() as usize; + let furthest = found + .iter() + .flat_map(|c| { + c.records + .iter() + .map(move |r| r.as_ptr() as usize + r.len() - region_ptr) + }) + .max() + .unwrap_or(0); + assert!( + region.len() - furthest < 4096, + "level{n}: chunk records stop {} bytes short of the trailer", + region.len() - furthest + ); + container_bytes += region.len(); + fill_gap_max = fill_gap_max.max(region.len() - furthest); + levels += 1; + } + + assert_eq!(levels, 27, "expected all 27 levels, got {levels}"); + assert_eq!( + with_shrub, 24, + "expected 24 levels with shrub, got {with_shrub}" + ); + eprintln!( + "container: {levels} levels ({with_shrub} closing shrub), \ + {body_records} 8-byte records in last-shrub bodies, \ + {container_bytes} bytes of chunk data, largest trailer gap {fill_gap_max}, \ + marker pair occurs at most {marker_wide_max} times per lump" + ); +} + /// Byte positions that hold the same value in every supplied record. fn constant_positions(records: &[&[u8]]) -> usize { let Some(first) = records.first() else { diff --git a/crates/gc-formats/src/chunks.rs b/crates/gc-formats/src/chunks.rs index dd65724..23c4d6a 100644 --- a/crates/gc-formats/src/chunks.rs +++ b/crates/gc-formats/src/chunks.rs @@ -43,6 +43,20 @@ pub const CHUNK_GAP_BYTES: usize = 10; /// The value every tagged offset's top byte carries, on all 27 levels. pub const CHUNK_TAG: u8 = 0xf0; +/// The two bytes immediately preceding every chunk region on the disc. +/// +/// The last class body in a level -- shrub where the level has shrub, tie +/// where it does not -- ends with this constant pair before the chunk +/// container begins. It is *not* a unique pattern: the pair occurs 25 to 103 +/// times in a 5 to 17 MB lump, so on its own it means nothing. What locates +/// the boundary is the pair sitting exactly in front of a chunk signature +/// (see [`chunk_region_start`]). +/// +/// Not interpreted. Whether it is a terminator, an end count or a tag is not +/// established; what is established is that it is constant across all 27 +/// levels and sits at this one place. +pub const CHUNK_REGION_MARKER: [u8; 2] = [0xf6, 0x7c]; + /// A chunk header plus the records its offsets address. /// /// `stride` is the distance between the first two resolved offsets. Every @@ -97,6 +111,38 @@ pub fn is_chunk_start(region: &[u8], offset: usize, min_offsets: usize) -> bool }) } +/// The start of the chunk container: the first offset at or after `from` +/// carrying a chunk signature immediately preceded by [`CHUNK_REGION_MARKER`]. +/// +/// This is what closes the last class region (shrub, or tie on the levels +/// without shrub). The two parts of the signature are each weak alone -- the +/// marker pair occurs by chance dozens of times per lump, and the chunk shape +/// matches random data often enough to need four tagged offsets to be believed +/// -- and together they are exact: on all 27 levels the marker sits exactly +/// two bytes before the first chunk header, with no further marker-paired +/// signature after the region start. +/// +/// Returns `None` when the signature is not found, so an unlocated boundary +/// stays visible rather than being closed against a guess. +pub fn chunk_region_start(core: &[u8], from: usize) -> Option { + if from < 2 { + return None; + } + // The marker's first byte is the cheapest rejection, so test it before + // comparing the pair or the signature behind it. + let mut at = from - 2; + while at + 2 <= core.len() { + if core[at] == CHUNK_REGION_MARKER[0] + && core[at + 1] == CHUNK_REGION_MARKER[1] + && is_chunk_start(core, at + 2, 4) + { + return Some(at + 2); + } + at += 1; + } + None +} + /// Little-endian, matching `Buf::u32` and the PS2. fn read_u32(bytes: &[u8], at: usize) -> u32 { u32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) @@ -207,3 +253,62 @@ pub fn parse_all_chunks(region: &[u8]) -> Result>> { } Ok(out) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A chunk signature at `at`, preceded by the marker: the marker pair, the + /// 16-byte header whose last byte is `0xff`, the 10 zero bytes, then four + /// tagged offsets. + fn plant_chunk(core: &mut [u8], at: usize) { + let base = at - 2; + core[base..base + 2].copy_from_slice(&CHUNK_REGION_MARKER); + // Header: byte 14 is the record count, matching the four tagged words. + core[at + 14] = 4; + core[at + 15] = 0xff; + // The 10 zero bytes of the gap at +0x10. Their low bits resolve to a + // zero stride, which is fine: only the signature is under test. + core[at + CHUNK_HEADER_BYTES..at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES].fill(0); + for k in 0..4 { + let word = (CHUNK_TAG as u32) << 24; + core[at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + k * 4 + ..at + CHUNK_HEADER_BYTES + CHUNK_GAP_BYTES + k * 4 + 4] + .copy_from_slice(&word.to_le_bytes()); + } + } + + #[test] + fn finds_the_markered_signature() { + let mut core = vec![0x5au8; 256]; + plant_chunk(&mut core, 128); + assert_eq!(chunk_region_start(&core, 64), Some(128)); + // Scanning from earlier still finds it: the signature is what locates + // the boundary, not the search start. + assert_eq!(chunk_region_start(&core, 2), Some(128)); + } + + #[test] + fn a_marker_without_a_signature_is_not_a_boundary() { + let mut core = vec![0x5au8; 256]; + core[10..12].copy_from_slice(&CHUNK_REGION_MARKER); + assert_eq!(chunk_region_start(&core, 2), None); + } + + #[test] + fn a_signature_without_the_marker_is_not_a_boundary() { + let mut core = vec![0x5au8; 256]; + // Plant the signature, then break the marker in front of it. + plant_chunk(&mut core, 128); + core[126] = 0x5a; + assert_eq!(chunk_region_start(&core, 2), None); + } + + #[test] + fn an_unlocated_boundary_stays_none() { + assert_eq!(chunk_region_start(&[], 2), None); + // `from` below the marker's own minimum is refused rather than + // underflowing. + assert_eq!(chunk_region_start(&[0xf6], 0), None); + } +} diff --git a/crates/gc-formats/src/regions.rs b/crates/gc-formats/src/regions.rs index 4cf4db9..8837a16 100644 --- a/crates/gc-formats/src/regions.rs +++ b/crates/gc-formats/src/regions.rs @@ -10,7 +10,7 @@ //! distributions -- were silently wrong, and nothing errored. //! //! So bounds are computed once, here, from successor boundaries, and every -//! caller uses these rather than inventing its own. Where a bound is not yet +//! caller uses these rather than inventing their own. Where a bound is not yet //! known it is `None`, not a guess: [`Region::end`] being `None` means //! "unresolved", and [`core_layout`] refuses to close a region whose successor //! it cannot find. @@ -24,14 +24,24 @@ //! | sky | sky | collision | //! | collision | collision | first moby class | //! | moby | first moby | first tie class | -//! | tie | first tie | first shrub class | -//! | shrub | first shrub | **unresolved** | +//! | tie | first tie | first shrub class, | +//! | | | else the chunk container | +//! | shrub | first shrub | the chunk container | +//! | chunks | the chunk | lump length − 0x38A10 | +//! | | container | | //! -//! The shrub end is the open item. Texture texels sit *before* moby, not after -//! shrub, so they cannot bound it: on level 0 the texel range is 3539328 to -//! 5326208 while the last shrub class starts at 16044288. The ~925 KB after -//! the last shrub class is real, non-zero data that is not accounted for yet. +//! The chunk container's start is located by signature rather than read from +//! the core index: the class table that precedes it carries no successor +//! offset, so the boundary is found in `core_data` itself -- see +//! [`crate::chunks::chunk_region_start`]. Every level has one: on the 24 levels +//! with shrub it sits exactly two bytes after the last shrub body, and on the +//! three levels without shrub it closes tie instead. +//! +//! Texture texels do **not** bound anything: they sit before moby, not after +//! shrub. On level 0 the texel range is 3539328 to 5326208 while the last +//! shrub class starts at 16044288. +use crate::chunks::chunk_region_start; use crate::level_core::{ArrayRange, LevelCoreHeader}; /// Bytes at the end of `core_data` that are identical across all 27 levels. @@ -85,12 +95,40 @@ pub fn first_geometry(index: &[u8], range: ArrayRange, stride: usize) -> Option< best } +/// The highest geometry offset in a class table, ignoring absent entries. +/// +/// This is the *last* class in the region, which is where the search for the +/// successor region starts. Starting there rather than at the region's head +/// matters: the search is for the first chunk signature after this offset, so +/// beginning it inside the region's own class bodies would risk matching one. +pub fn last_geometry(index: &[u8], range: ArrayRange, stride: usize) -> Option { + let count = range.count.max(0) as usize; + let base = range.offset.max(0) as usize; + let mut best: Option = None; + for i in 0..count { + let at = base + i * stride; + let chunk = index.get(at..at + 4)?; + let off = i32::from_le_bytes(chunk.try_into().expect("4 bytes")); + if off > 0 { + best = Some(best.map_or(off as usize, |b: usize| b.max(off as usize))); + } + } + best +} + /// Compute the region's bounds for one level. -pub fn core_layout(index: &[u8], h: &LevelCoreHeader, core_len: usize) -> Vec { +/// +/// `core` is the level's `core_data` lump. It is needed for one thing only: the +/// class regions end at the chunk container, whose start is located by +/// signature in `core` because the class table that precedes it carries no +/// successor offset. +pub fn core_layout(index: &[u8], core: &[u8], h: &LevelCoreHeader) -> Vec { let at = |v: i32| -> Option { (v > 0).then_some(v as usize) }; let first_moby = first_geometry(index, h.moby_classes, 0x20); let first_tie = first_geometry(index, h.tie_classes, 0x20); + let last_tie = last_geometry(index, h.tie_classes, 0x20); let first_shrub = first_geometry(index, h.shrub_classes, 0x30); + let last_shrub = last_geometry(index, h.shrub_classes, 0x30); // Each region ends where the next one begins. `chained` keeps `None` // propagating rather than silently closing a region against a gap. @@ -103,10 +141,20 @@ pub fn core_layout(index: &[u8], h: &LevelCoreHeader, core_len: usize) -> Vec chunk_region_start(core, shrub), + (None, Some(tie)) => chunk_region_start(core, tie), + (None, None) => None, + }; + // The final region is bounded by the constant trailer rather than by // `core_data.len()`, which would let it absorb the trailer and whatever // tail follows the last named region. - let tail_end = core_len + let tail_end = core + .len() .checked_sub(CORE_DATA_TRAILER_BYTES) .filter(|&e| e > 0); @@ -115,9 +163,12 @@ pub fn core_layout(index: &[u8], h: &LevelCoreHeader, core_len: usize) -> Vec Vec { let h = crate::level_core::LevelCoreHeader::parse(v).unwrap(); - core_layout(v, &h, 1 << 24) + core_layout(v, &core(v), &h) + } + + /// A zeroed lump, optionally carrying a real chunk-container signature just + /// past the last class -- the layout locates the container in `core`, so a + /// core of plain zeros locates nothing. + fn core(v: &[u8]) -> Vec { + let h = crate::level_core::LevelCoreHeader::parse(v).unwrap(); + let mut c = vec![0u8; 1 << 24]; + let last_class = [ + last_geometry(v, h.tie_classes, 0x20), + last_geometry(v, h.shrub_classes, 0x30), + ] + .into_iter() + .flatten() + .max(); + if let Some(last) = last_class { + plant_chunk(&mut c, last + 100); + } + c + } + + /// The marker pair plus a chunk signature at `at`, as the disc carries it. + fn plant_chunk(core: &mut [u8], at: usize) { + core[at - 2..at].copy_from_slice(&crate::chunks::CHUNK_REGION_MARKER); + core[at + 14] = 4; + core[at + 15] = 0xff; + for k in 0..4 { + let word = (crate::chunks::CHUNK_TAG as u32) << 24; + let at = + at + crate::chunks::CHUNK_HEADER_BYTES + crate::chunks::CHUNK_GAP_BYTES + k * 4; + core[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } } #[test] @@ -170,7 +253,8 @@ mod tests { "collision", "moby", "tie", - "shrub" + "shrub", + "chunks" ] ); assert_eq!(regions[0].start, 0); @@ -203,7 +287,7 @@ mod tests { } #[test] - fn the_last_region_ends_at_the_constant_trailer() { + fn the_class_regions_end_at_the_chunk_container() { let core_len = 1 << 24; let v = index( 100, @@ -212,29 +296,74 @@ mod tests { &[(200, &[1000]), (400, &[2000]), (600, &[3000])], ); let regions = layout(&v); + // The container sits 100 bytes past the last shrub class, which is + // where the search for it starts. + let container = 3000 + 100; let shrub = regions.iter().find(|r| r.name == "shrub").unwrap(); assert_eq!(shrub.start, 3000); + assert_eq!(shrub.end, Some(container)); + let chunks = regions.iter().find(|r| r.name == "chunks").unwrap(); + assert_eq!(chunks.start, container); // Not core_data.len(): the trailer is constant across levels and does // not belong to any region. - assert_eq!(shrub.end, Some(core_len - CORE_DATA_TRAILER_BYTES)); - assert!(shrub.end.unwrap() < core_len); + assert_eq!(chunks.end, Some(core_len - CORE_DATA_TRAILER_BYTES)); + assert!(chunks.end.unwrap() < core_len); + // Tie still ends at shrub, not at the container. let tie = regions.iter().find(|r| r.name == "tie").unwrap(); assert_eq!(tie.end, Some(3000)); } #[test] - fn a_lump_shorter_than_the_trailer_leaves_the_end_unresolved() { - let mut v = index( + fn tie_ends_at_the_container_when_there_is_no_shrub() { + // Third class table absent: tie is the last class region, so it is tie + // the container closes. + let v = index(100, 200, 300, &[(200, &[1000]), (400, &[2000])]); + let h = crate::level_core::LevelCoreHeader::parse(&v).unwrap(); + let core = core(&v); + let container = 2000 + 100; + let regions = core_layout(&v, &core, &h); + let tie = regions.iter().find(|r| r.name == "tie").unwrap(); + assert_eq!(tie.end, Some(container)); + assert!(regions.iter().all(|r| r.name != "shrub")); + let chunks = regions.iter().find(|r| r.name == "chunks").unwrap(); + assert_eq!(chunks.start, container); + } + + #[test] + fn an_unlocated_container_leaves_the_class_end_unresolved() { + // A core of plain zeros carries no signature, so shrub cannot be + // closed -- the end stays visible rather than clamped to the lump's + // tail, and the chunk region does not exist at all. + let v = index( 100, 200, 300, &[(200, &[1000]), (400, &[2000]), (600, &[3000])], ); let h = crate::level_core::LevelCoreHeader::parse(&v).unwrap(); - let regions = core_layout(&v, &h, CORE_DATA_TRAILER_BYTES - 1); + let regions = core_layout(&v, &[0u8; 4 << 20], &h); let shrub = regions.iter().find(|r| r.name == "shrub").unwrap(); - assert_eq!(shrub.end, None, "must not clamp a backwards bound"); - let _ = &mut v; + assert_eq!(shrub.end, None); + assert!(regions.iter().all(|r| r.name != "chunks")); + } + + #[test] + fn a_lump_shorter_than_the_trailer_leaves_the_end_unresolved() { + let v = index( + 100, + 200, + 300, + &[(200, &[1000]), (400, &[2000]), (600, &[3000])], + ); + let h = crate::level_core::LevelCoreHeader::parse(&v).unwrap(); + // Shorter than the trailer, so the container starts but cannot end. + let mut core = vec![0u8; CORE_DATA_TRAILER_BYTES - 1]; + plant_chunk(&mut core, 3000 + 100); + let regions = core_layout(&v, &core, &h); + let shrub = regions.iter().find(|r| r.name == "shrub").unwrap(); + assert_eq!(shrub.end, Some(3100)); + let chunks = regions.iter().find(|r| r.name == "chunks").unwrap(); + assert_eq!(chunks.end, None, "must not clamp a backwards bound"); } #[test] diff --git a/crates/gc-formats/src/shrub.rs b/crates/gc-formats/src/shrub.rs index 4e8def9..3effab0 100644 --- a/crates/gc-formats/src/shrub.rs +++ b/crates/gc-formats/src/shrub.rs @@ -14,9 +14,12 @@ //! - `+0x24` equals the owning class entry's own `o_class`. //! - `+0x2c` is a data offset that lands inside the span. //! -//! Those two are what this parses. The body past the data offset is not -//! decoded: the run of bytes there does not tile at any stride that divides the -//! tail consistently with the count at `+0x28`. +//! Those two are what this parses. The body from `+0x2c` is an array of +//! 8-byte records -- six varying bytes and two constant zeros -- whose +//! payload encoding is not established; it is handed back raw rather than +//! guessed at. Its extent is knowable: the region ends at the chunk container, +//! located by signature (`crate::chunks::chunk_region_start`), so every span +//! is bounded and every body is whole records. //! //! Note the class entry size differs here too: shrub entries are 0x30 bytes, //! not 0x20, because they carry billboard texture fields the other two tables diff --git a/docs/formats/core-data-regions.md b/docs/formats/core-data-regions.md index 48cd4ef..b29119f 100644 --- a/docs/formats/core-data-regions.md +++ b/docs/formats/core-data-regions.md @@ -27,8 +27,10 @@ than inventing their own. | `sky` | `sky` | `collision` | | `collision`| `collision` | first moby class | | `moby` | first moby | first tie class | -| `tie` | first tie | first shrub class | -| `shrub` | first shrub | lump length − 0x38A10 | +| `tie` | first tie | first shrub class, | +| | | else chunk container | +| `shrub` | first shrub | chunk container | +| `chunks` | chunk container| lump length − 0x38A10| Class regions start at the lowest non-zero geometry offset in their class table. A zero entry means *absent*, not position zero -- the same trap the @@ -43,26 +45,79 @@ Comparing all 27 levels byte-for-byte from the end: the last **231,952 bytes therefore ends at `len - 0x38A10`, and the final region is bounded there rather than at `core_data.len()`. -This is what closes shrub. Across the 24 levels that have a shrub region it -gives **24,442,928 bytes** total, and it is the same bound the disc test -asserts on every level. +This is what closes the chunk container. The regions it separates from level +content are measured on the disc: **2,033,776 bytes of shrub region across the +24 levels that have one, and 25,775,850 bytes of chunk container** across all +27. The disc test asserts the bound on every level. What the trailer *is* is not established. It may be fixed data, or padding from a fixed-size decompression buffer. Either way it is constant, which is all that is needed to bound the tail. -## Open: what was in the region before it +## Found: the chunk container closes the class region -The offset ending the shrub region is not in the core index. +The offset ending the shrub region is not in the core index, and the class +table that precedes the container carries no successor offset for it -- unlike +moby and tie, whose boundaries fall out of the tables. Texture texels do not +bound it either: they sit before moby, not after shrub. So the boundary is +located in `core_data` itself. -Texture texels do **not** bound it. They sit before moby, not after shrub: on -level 0 the texel range is 3539328..5326208 while the last shrub class starts -at 16044288, leaving ~925 KB of real, non-zero data after the last shrub class -that nothing yet accounts for. +**Range measured:** for each level, from the last shrub class's geometry +offset to the constant trailer. Nothing outside that range is read. -`core_layout` reports `end: None` for shrub on the 24 levels that have one, and -the disc test asserts it, so the gap stays visible rather than being papered -over by clamping to the end of the lump. +**Method:** scan forward from the last shrub class for the pair (a) the +constant two bytes `f6 7c`, sitting immediately in front of (b) a chunk +signature -- a 16-byte header ending `0xff`, ten zero bytes, then four or more +`0xf0`-tagged offsets. `gc_formats::chunks::chunk_region_start`. + +Each half of that signature is weak on its own, which is why the halves are +measured separately: + +- the pair `f6 7c` occurs **25 to 103 times** in a 5 to 17 MB lump, so + matching it alone picks dozens of wrong offsets; +- the chunk shape matches random data often enough that `parse_chunk` demands + four tagged offsets before a signature is believed. + +**Result: on all 27 levels the pair sits exactly two bytes before the first +chunk header.** The two are never found apart. This is the boundary, and +`core_layout` closes shrub with it -- `shrub`'s end is the container's start, +and the container's own end is the trailer, so every byte of level-specific +`core_data` now belongs to a named region. + +Two controls, both asserted by the disc test `chunk_container_closes_the_last_class_region`: + +- **No signature inside the body.** Scanning the last shrub body for a bare + chunk signature finds none, on any level -- so the boundary search was not + matching something it should have skipped, and the marker's job is + corroboration rather than load-bearing disambiguation. +- **The three levels without shrub** (10, 25, 26) carry the same marker two + bytes before the container. There the container closes *tie* instead, which + is what makes the locator a property of the region rather than a shrub + rule. + +On level 0: + +| what | range | bytes | +|---|---|---| +| last shrub class data | 16046800 .. 16047040 | 240 | +| marker `f6 7c` | 16047040 .. 16047042 | 2 | +| **chunk container** | 16047042 .. 16737344 | **690,302** | +| constant trailer | 16737344 .. 16969296 | 231,952 | + +The marker's meaning is not established. It is constant across all 27 levels +and it appears at exactly this position, but whether it is a terminator, an end +count or a tag is unknown, so it is not named. + +**Correction this produces.** The chunk-container measurements recorded below +were taken over "the 900,000 bytes before the constant trailer" -- a window +wide enough to contain the container on every level but not aligned to its +true extent. Chunks sitting at the window's edge were truncated (27 of 1,123), +and the region's size was never correctly stated. Both are now bounded +properly: on level 0 the container is 690,302 bytes, not ~900,000, and the +25,775,850 bytes of chunk data across 27 levels are the measured total rather +than a windowed sample. The `chunk_containers_resolve_on_every_level` test now +reads the region from `core_layout` instead of measuring backwards from the +end of the lump. ## Correction: it is mostly texture data, not an unnamed region diff --git a/docs/plan/level-assets.md b/docs/plan/level-assets.md index bfe1ff2..8e91c31 100644 --- a/docs/plan/level-assets.md +++ b/docs/plan/level-assets.md @@ -961,3 +961,46 @@ This is the second time in this format that a confident aggregate hid a bad bound -- the first was the VIF reading. The pattern is worth noting: aggregates across many spans are only as trustworthy as the span bounds underneath them, and those are the easy thing to get wrong because nothing errors. + +## Shrub span bounds: resolved (2026-10-11, later still) + +The offset that ends the shrub region is the **start of the chunk container**, +and it is found by signature rather than read from the core index, which has no +successor offset for shrub. + +**Range measured:** for each level, from the last shrub class's geometry offset +to the constant trailer. Nothing outside that range is read. + +**Method:** scan forward from the last shrub class for two bytes `f6 7c` +sitting immediately in front of a chunk signature (16-byte header ending `0xff`, +ten zero bytes, four or more `0xf0`-tagged offsets). Each half of that +signature is weak alone -- the marker pair occurs 25 to 103 times per lump -- +and together they are exact. + +**Result: on all 27 levels the marker sits exactly two bytes before the first +chunk header.** The two parts of the signature are never found apart, and no +chunk signature occurs inside any last shrub body. The three levels without +shrub (10, 25, 26) carry the same marker before the container that closes +*tie*, so the locator is a property of the region, not a shrub rule. +`gc_formats::chunks::chunk_region_start` implements it and +`regions::core_layout` closes shrub with it. + +Consequences, all measured over the new bounds: + +- **All 358 shrub class bodies are bounded and well formed** -- every one is + whole 8-byte records. That was not assertable while the last class in each + level ran out to the lump's end, and it is the check the boundary earns. +- **9,600 records in total**, against the 8,938 previously measurable in + successor-bounded spans; 662 of them sit in the 24 last-class bodies, 24 to + 30 per level. The retracted "3.5 million records / 28 MB of shrub body" + figure stays retracted: the correct total for the shrub region is + **2,033,776 bytes** across the 24 levels that have one. +- The chunk container is **25,775,850 bytes across 27 levels** (1,236 chunks, + 22,562 records), not an ~690-900 KB afterthought per level. The earlier + container measurements were taken over "the 900,000 bytes before the + trailer", a window that did not align to the container's real extent: on + level 0 it is 690,302 bytes, and 27 of 1,123 chunks were truncated at that + invented edge. The chunk test now reads the region from `core_layout`. + +What is still not known: the marker's meaning (constant across 27 levels, at +exactly this position, unnamed), and the shrub body payload's encoding. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0b825a0..0005426 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # Roadmap -Status as of **2026-10-09**. +Status as of **2026-10-11**. A from-scratch native Rust reimplementation of Ratchet & Clank: Going Commando (PS2). Each phase ends in a **gate** — something checkable, not a feeling that @@ -57,6 +57,14 @@ early, while they are still cheap to retire. through `LightTfrags`. The renderer still counts and reports how many vertices it lit, because a silent fallback to stored colours would look plausible and be wrong. +- [x] **The shrub region is bounded, and with it the whole lump.** Shrub's end + was the last unlocated region bound: it is the start of the post-class + chunk container, found by a marker-paired signature in `core_data` (the + index carries no successor offset). The container closes shrub on the + 24 shrub levels and tie on the three without. All 358 shrub class bodies + are now bounded and are whole 8-byte records, so shrub measurements are + trustworthy for the first time -- the "3.5M records" figure from the + unbounded final span is replaced by a measured 9,600. - [ ] Lighting *magnitudes* are unverified: they need comparison against reference frames, which needs the PCSX2 comparison loop. - [ ] Then the GS register set and the guard-band clip test. @@ -153,6 +161,9 @@ Phase 1 hands over the extents; this phase works out what each one contains. - [x] Region layout of `core_data` established: tfrag, occlusion, sky, collision and `textures_base` are offsets, strictly increasing on all 27 levels. Terrain is `[0, occlusion)` and carries inline texels. +- [x] Region layout **closed**: the chunk container's marker-paired signature + ends the last class region, and the constant trailer ends the container. + Every byte of level-specific `core_data` now belongs to a named region. - [x] **tfrag vertex format: recovered.** The tfrags block header gives a table offset (`0x40`) and a patch count; each patch is a 0x40-byte header whose `data` offset is relative to the table, not to the block. Positions are @@ -209,8 +220,10 @@ Phase 1 hands over the extents; this phase works out what each one contains. Geometry header parsed and both fields **checked**, not merely reported: `+0x24` equals the entry's own `o_class`, and `+0x2c` is an in-span data offset, on all 358 spans. Shrub does **not** use moby's part-table format. - The body past `+0x2c` is not decoded — it tiles at no stride consistent - with the count at `+0x28`. + The body is **8-byte records, 6 varying bytes and 2 constant**, and now + that every span is bounded by the located chunk container, all 358 + bodies are whole records — 9,600 in total. The payload's encoding is + still unknown, and so is the 2-byte marker that closes the region. - [x] `moby` — per-class model geometry. A class span opens with a `u32` pointing at a part table of 16-byte records; parts tile exactly, with `vertex_offset + 16 * vertex_count == next_part` holding for **43,467 of