#!/usr/bin/env python3 """region.py: build another region's version of the game from the PAL sources. python tools/region.py us DECOMP_REPO OUT_DIR The decomp, the host sources and every tool here work in the PAL executable's addresses (SCES-50916), and they name functions and data by address (func_0012F4A8, D_0013A548). region/us.json (made by tools/region_map.py from both executables; addresses only) maps every PAL address to the US one (SCUS-97199). This writes, under OUT_DIR: repo/ the decomp as the tools read it, in US addresses: its catalogues (progress/report.json, config/*, config/overlays/functions.tsv), headers and sources with every func_/D_ name and address literal rewritten host/ the host sources (host/src, host/include, host/overrides) the same way region_report.txt every literal rewritten, and what could not be mapped A PAL function the US build does not have is renamed func_palonly_X, so nothing is ever placed at its PAL address; a US function the PAL build does not have is added to the catalogue (and translated from the bytes like the rest). The build then runs the usual tools on these trees (build.py --region us).""" import bisect, json, os, re, shutil, sys HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) REGIONS = {"us": {"map": os.path.join(HERE, "region", "us.json"), "exe": "SCUS_971.99", "pal_exe": "SCES_509.16", "video": "ntsc"}} RESIDENT_PAL = 0x0015F000 CORE_TEXT = (0x00112380, 0x0012F580) # PAL core.text: the Sony libraries class RegionMap: def __init__(self, path): m = json.load(open(path)) self.m = m self.exe = {int(k, 16): v for k, v in m["exe_funcs"].items()} self.exe_starts = sorted(self.exe) self.data = sorted(m["data"]) self.lv = {lv: {int(k, 16): v for k, v in d["funcs"].items()} for lv, d in m["levels"].items()} self.lv_starts = {lv: sorted(d) for lv, d in self.lv.items()} self.lv_data = {lv: sorted(r) for lv, r in m["level_data"].items()} self.sections = m["sections"] self.us_only = m["exe_us_only"] self.lv_us_only = {lv: d["us_only"] for lv, d in m["levels"].items()} @staticmethod def _func(table, starts, a, interior=True): if a in table: return table[a][0] if not interior: return None k = bisect.bisect_right(starts, a) - 1 if k >= 0: s = starts[k]; ua, sz, method = table[s] if s < a < s + sz and method == "fingerprint": return ua + (a - s) # same instructions: same offsets return None @staticmethod def _runs(runs, a): for r in runs: if r[0] <= a < r[1]: return a + r[2] return None def section_edge(self, a): """a PAL section's first address or its end: the US section's""" if a == 0x0023E740: # the end of PAL's game program as the tools give it (consts) t = self.sections[".text"]; return (t[2] + t[3] + 15) & ~15 for n, (pa, ps, ua, us_) in self.sections.items(): if not pa: continue if a == pa: return ua if a == pa + ps: return ua + us_ if a == (pa + ps + 15) & ~15: return (ua + us_ + 15) & ~15 # (an end rounded up to 16) return None def exe_addr(self, a, interior=True): """the US address of a PAL executable address, or None""" if 0x00100000 <= a < 0x00112380: return a # .reginfo, .vutext: the same in both f = self._func(self.exe, self.exe_starts, a, interior) if f is not None: return f return self._runs(self.data, a) def level_addr(self, lv, a, interior=True): if a < RESIDENT_PAL: return self.exe_addr(a, interior) # the core: shared by every program f = self._func(self.lv.get(lv, {}), self.lv_starts.get(lv, []), a, interior) if f is not None: return f return self._runs(self.lv_data.get(lv, []), a) def exe_func(self, a): return self.exe[a][0] if a in self.exe else None def level_func(self, lv, a): return self.lv[lv][a][0] if lv in self.lv and a in self.lv[lv] else None def consts(region): """what the tools need to know about the executable, for this region""" pal = {"exe": "SCES_509.16", "main_text": (0x001E9080, 0x0023E740), "resident": 0x0015F000, "core_text": CORE_TEXT, "extra_ranges": [(0x001252A0, 0x00125630), (0x0011DF10, 0x001207B8)]} if region in (None, "", "pal"): return pal rm = RegionMap(REGIONS[region]["map"]) t = rm.sections[".text"]; lit = rm.sections[".lit"]; ct = rm.sections["core.text"] return {"exe": REGIONS[region]["exe"], "main_text": (t[2], (t[2] + t[3] + 15) & ~15), "resident": lit[2], "core_text": (ct[2], ct[2] + ct[3]), "extra_ranges": [(rm.exe_addr(lo), rm.exe_addr(hi) if rm.exe_addr(hi) else rm.exe_addr(hi - 4) + 4) for lo, hi in pal["extra_ranges"]]} # ---- rewriting --------------------------------------------------------------------------------------- NAME = re.compile(r"\b(func|D)_(?:L(\d\d)_)?([0-9A-Fa-f]{6,8})(?![0-9A-Fa-f])") LIT = re.compile(r"\b0x(00[1-9][0-9A-Fa-f]{5}|0[1-9][0-9A-Fa-f]{5}|[1-9][0-9A-Fa-f]{5})(?=[uUlL]*\b)") # a line marked region:keep holds values that only look like addresses (IOP addresses, handles, lui values) class Rewriter: def __init__(self, rm): self.rm = rm self.log, self.unmapped = [], [] self.taken = {} def name(self, kind, lv_name, hexpart, lv_ctx, where): a = int(hexpart, 16) up = hexpart.isupper() or not any(c.isalpha() for c in hexpart) fmt = "%08X" if len(hexpart) == 8 else ("%06X" if up else "%06x") if kind == "func": if lv_name is not None: # a level catalogue name: its address in that level ua = self.rm.level_func(lv_name, a) if ua is None: self.unmapped.append((where, "func_L%s_%s" % (lv_name, hexpart))); return "func_palonly_L%s_%s" % (lv_name, hexpart) return "func_L%s_%s" % (lv_name, fmt % ua) ua = self.rm.exe_func(a) if ua is None and CORE_TEXT[0] <= a < CORE_TEXT[1]: ua = self.rm.exe_addr(a) # a library entry inside a split unit if ua is None: # the executable's main-segment functions are also catalogue names of level copies self.unmapped.append((where, "func_" + hexpart)); return "func_palonly_" + hexpart return "func_" + (fmt % ua) ctx = lv_name or lv_ctx # D_LNN_X: level NN's own data pre = "D_L%s_" % lv_name if lv_name else "D_" ua = self.rm.level_addr(ctx, a) if ctx else self.rm.exe_addr(a) if ua is None: self.unmapped.append((where, pre + hexpart)); return pre.replace("D_", "D_palonly_") + hexpart # one to one: a second PAL address landing on a US address already taken (two neighbours either # side of a shift boundary) is not trusted; its uses then fail to build instead of reading other data key = (ctx, bool(lv_name), ua) if self.taken.setdefault(key, a) != a: self.unmapped.append((where, "%s%s (clashes with PAL %08X)" % (pre, hexpart, self.taken[key]))); return pre.replace("D_", "D_palonly_") + hexpart return pre + (fmt % ua) def text(self, s, where, lv_ctx=None, literals=True): def nm(q): r = self.name(q.group(1), q.group(2), q.group(3), lv_ctx, where) return r s = NAME.sub(nm, s) if literals: def lit(q): h = q.group(1); a = int(h, 16) if not (0x00100000 <= a < 0x02000000): return q.group(0) # only what is surely an address: a section's edge, a function's first instruction, data # (a constant inside a function's range, such as a size of 0x200000, stays as it is) ua = self.rm.section_edge(a) if ua is None: core = CORE_TEXT[0] <= a < CORE_TEXT[1] # the Sony libraries: entries inside split units, same code in both ua = self.rm.level_addr(lv_ctx, a, interior=core) if lv_ctx else self.rm.exe_addr(a, interior=core) if ua is None: self.unmapped.append((where, "0x" + h)); return q.group(0) if ua != a: self.log.append((where, "0x%s -> 0x%08X" % (h, ua))) return "0x%0*X" % (len(h), ua) s = "".join(l if "region:keep" in l else LIT.sub(lit, l) for l in s.splitlines(True)) return s def level_of(rel): q = re.search(r"overlays[/\\]l(\d\d)_", rel) return q.group(1) if q else None def write_repo(rm, rw, repo, out): """the decomp, as the tools read it, in US addresses""" for sub in ("include", "src", "nonmatching", "tools"): top = os.path.join(repo, sub) for root, _, files in os.walk(top): for f in files: p = os.path.join(root, f); rel = os.path.relpath(p, repo); d = os.path.join(out, rel) os.makedirs(os.path.dirname(d), exist_ok=True) if sub == "tools" or not f.endswith((".c", ".h", ".inc", ".s")): shutil.copyfile(p, d); continue d_name = os.path.join(os.path.dirname(d), rw.text(f, rel, literals=False)) open(d_name, "w").write(rw.text(open(p, errors="replace").read(), rel, level_of(rel))) os.makedirs(os.path.join(out, "config", "overlays"), exist_ok=True) os.makedirs(os.path.join(out, "progress"), exist_ok=True) # symbols for fn in ("config/symbol_addrs.txt", "config/port_symbols.txt"): p = os.path.join(repo, fn) if not os.path.exists(p): continue lines = [] for l in open(p, errors="replace"): q = re.match(r"^(\s*)(\w+)(\s*=\s*)0x([0-9A-Fa-f]+)(.*)$", l.rstrip("\n")) if q: a = int(q.group(4), 16); ua = rm.exe_addr(a) if ua is None: rw.unmapped.append((fn, q.group(2))); continue l = "%s%s%s0x%08X%s" % (q.group(1), rw.text(q.group(2), fn, literals=False), q.group(3), ua, q.group(5)) else: l = rw.text(l.rstrip("\n"), fn) lines.append(l) open(os.path.join(out, fn), "w").write("\n".join(lines) + "\n") for fn in ("config/names.tsv", "config/symbol_names.txt", "config/file_cflags.txt", "config/linker_remnants.txt", "config/handwritten_asm.txt"): p = os.path.join(repo, fn) if os.path.exists(p): open(os.path.join(out, fn), "w").write(rw.text(open(p, errors="replace").read(), fn, literals=False)) # the executable's functions (progress/report.json): US addresses and sizes rep = json.load(open(os.path.join(repo, "progress", "report.json"))) seen = set() for u in rep.get("units", []): fl = [] for f in u.get("functions", []): md = f.get("metadata") or {} if "virtual_address" not in md: fl.append(f); continue a = int(md["virtual_address"]) if a in rm.exe: ua, sz, _ = rm.exe[a] f = dict(f); f["name"] = rw.text(f["name"], "report.json", literals=False); f["size"] = str(sz) md = dict(md); md["virtual_address"] = str(ua); f["metadata"] = md fl.append(f); seen.add(ua) # (PAL-only: dropped) u["functions"] = fl extra = [{"name": "func_%08X" % ua, "size": str(sz), "metadata": {"virtual_address": str(ua)}} for ua, sz in rm.us_only if ua not in seen and sz >= 8] rep.setdefault("units", []).append({"name": "us_only", "functions": extra}) json.dump(rep, open(os.path.join(out, "progress", "report.json"), "w")) # the US-only functions: assembly entries in a source file of their own with open(os.path.join(out, "src", "game", "us_only.c"), "w") as f: f.write('/* region.py: functions the US executable has and PAL does not (translated from the bytes) */\n#include "common.h"\n\n') for e in extra: f.write('INCLUDE_ASM("asm/us_only", %s);\n' % e["name"]) # the level catalogue: each function's places in every level, in US addresses rows = [] for l in open(os.path.join(repo, "config", "overlays", "functions.tsv")): if l.startswith("#") or not l.strip(): rows.append(l.rstrip("\n")); continue f = l.rstrip("\n").split("\t") pl, size = [], None for x in f[5].split(","): if ":" not in x: continue lv, a = x.split(":"); a = int(a, 16) ua = rm.level_func(lv, a) if ua is None: continue pl.append("%s:%08X" % (lv, ua)) if size is None: size = rm.lv[lv][a][1] if not pl: continue name = rw.text(f[0], "functions.tsv", literals=False) if name.startswith(("func_palonly", "D_palonly")): # (named after a level copy the US build lacks: name it after one it has) lv0, a0 = pl[0].split(":"); name = "func_L%s_%s" % (lv0, a0) f[0], f[2], f[4], f[5] = name, str(size or f[2]), str(len({p.split(':')[0] for p in pl})), ",".join(pl) rows.append("\t".join(f)) open(os.path.join(out, "config", "overlays", "functions.tsv"), "w").write("\n".join(rows) + "\n") for fn in ("config/overlays/names.tsv",): p = os.path.join(repo, fn) if os.path.exists(p): open(os.path.join(out, fn), "w").write(rw.text(open(p, errors="replace").read(), fn, literals=False)) os.makedirs(os.path.join(out, "baserom"), exist_ok=True) # host sources that use the game's addresses (the rest: the platform, the GS, VU and movie code) HOST_GAME = ("src/rac1/", "src/sdk/", "overrides/", "src/main.c", "src/runtime.c", "include/") def write_host(rm, rw, src_host, out): for root, dirs, files in os.walk(src_host): dirs[:] = [d for d in dirs if d not in ("uya_only", "drafts")] for f in files: p = os.path.join(root, f); rel = os.path.relpath(p, src_host).replace(os.sep, "/") if rel.startswith("region/"): # host/region//: code only that region has, already in its addresses if rel.startswith("region/%s/" % REGION): d = os.path.join(out, "src", "region", rel[len("region/%s/" % REGION):]) os.makedirs(os.path.dirname(d), exist_ok=True); shutil.copyfile(p, d) continue d = os.path.join(out, os.path.dirname(rel), rw.text(f, "host/" + rel, literals=False)) os.makedirs(os.path.dirname(d), exist_ok=True) if not f.endswith((".c", ".h")): shutil.copyfile(p, d); continue s = open(p, errors="replace").read() s = s.replace("SCES_509.16", REGIONS[REGION]["exe"]) # the executable's file name on the disc open(d, "w").write(rw.text(s, "host/" + rel, literals=rel.startswith(HOST_GAME)) if rel.startswith(HOST_GAME) else s) REGION = None def main(): global REGION region, repo, out = sys.argv[1], sys.argv[2], sys.argv[3] REGION = region cfg = REGIONS[region] rm = RegionMap(cfg["map"]) rw = Rewriter(rm) shutil.rmtree(os.path.join(out, "repo"), ignore_errors=True); shutil.rmtree(os.path.join(out, "host"), ignore_errors=True) write_repo(rm, rw, repo, os.path.join(out, "repo")) write_host(rm, rw, os.path.join(HERE, "host"), os.path.join(out, "host")) with open(os.path.join(out, "region_report.txt"), "w") as f: f.write("region %s: %d literals rewritten, %d names and literals not mapped\n\n== not mapped\n" % (region, len(rw.log), len(rw.unmapped))) for w, x in sorted(set(rw.unmapped)): f.write("%s\t%s\n" % (w, x)) f.write("\n== literals rewritten\n") for w, x in rw.log: f.write("%s\t%s\n" % (w, x)) hosts = [(w, x) for w, x in set(rw.unmapped) if w.startswith("host/")] print("region %s: %d literals rewritten; not mapped: %d (host sources: %d) -> %s" % ( region, len(rw.log), len(set(rw.unmapped)), len(hosts), os.path.join(out, "region_report.txt"))) if __name__ == "__main__": main()