294 lines
16 KiB
Python
294 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""build.py: generate host sources from OpenRAC rac1-decomp and build the RAC1 Wrapper.
|
|
|
|
usage: python build.py [--repo PATH_TO_DECOMP] [--platform linux|windows] [-j N]
|
|
[--iso PATH_TO_DISC_IMAGE] [--levels 00,01,...|none]
|
|
|
|
Needs Python 3 and zig (pip install ziglang, or zig on PATH). The decomp repo
|
|
is only read. Output: build/<platform>/rac1_wrapper[.exe]
|
|
|
|
Level programs (docs/LEVELS.md): each level listed in --levels (default 00,
|
|
Veldin 1) is compiled from src/overlays and its own bytes. Those bytes come
|
|
from your disc image (--iso, else RAC1_ISO, else iso= in
|
|
build/<platform>/rac1_wrapper.cfg), or from an OpenRAC dump in the decomp
|
|
repo (baserom/overlays/level_NN); they are extracted to build/levels/ and
|
|
never leave your machine.
|
|
"""
|
|
import argparse, glob, json, os, shutil, subprocess, sys
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--repo", default=os.environ.get("UYA_DECOMP", os.path.join(HERE, "..", "rac1-decomp-main")))
|
|
ap.add_argument("--platform", default="windows" if os.name == "nt" else "linux")
|
|
ap.add_argument("-j", type=int, default=os.cpu_count() or 2)
|
|
ap.add_argument("-O", default="2")
|
|
ap.add_argument("--iso", default=None)
|
|
ap.add_argument("--levels", default="00")
|
|
ap.add_argument("--region", default="pal", choices=("pal", "us"),
|
|
help="pal: SCES-50916 (the decomp's own); us: SCUS-97199, made from the same sources by tools/region.py")
|
|
a = ap.parse_args()
|
|
sys.path.insert(0, os.path.join(HERE, "tools"))
|
|
import region as _region
|
|
RC = _region.consts(a.region)
|
|
EXE_NAME = RC["exe"]
|
|
|
|
zig = [shutil.which("zig")] if shutil.which("zig") else [sys.executable, "-m", "ziglang"]
|
|
# Linux: 32-bit glibc 2.17 or newer (any distribution since 2013), linked dynamically so it can load
|
|
# the system's SDL2 and OpenGL driver (32-bit: i386 libraries)
|
|
target = {"linux": "x86-linux-gnu.2.17", "windows": "x86-windows-gnu"}[a.platform]
|
|
out = os.path.join(HERE, "build", a.platform if a.region == "pal" else "%s_%s" % (a.platform, a.region))
|
|
gen = os.path.join(out, "gen")
|
|
stage = os.path.join(out, "gen.new")
|
|
obj = os.path.join(out, "obj")
|
|
os.makedirs(obj, exist_ok=True)
|
|
levels = [] if a.levels in ("", "none") else sorted({int(x) for x in a.levels.split(",")})
|
|
|
|
|
|
def find_iso():
|
|
if a.iso: return a.iso
|
|
if os.environ.get("RAC1_ISO"): return os.environ["RAC1_ISO"]
|
|
cfg = os.path.join(out, "rac1_wrapper.cfg")
|
|
if os.path.exists(cfg):
|
|
for l in open(cfg, errors="replace"):
|
|
if l.strip().startswith("iso="): return l.strip()[4:].strip()
|
|
return None
|
|
|
|
|
|
def sync(src, dst):
|
|
"""copy the freshly generated tree over gen/, touching only files that
|
|
changed, so unchanged sources keep their objects"""
|
|
os.makedirs(dst, exist_ok=True)
|
|
keep = set()
|
|
for root, _, fs in os.walk(src):
|
|
for f in fs:
|
|
s = os.path.join(root, f)
|
|
rel = os.path.relpath(s, src)
|
|
d = os.path.join(dst, rel)
|
|
keep.add(rel)
|
|
os.makedirs(os.path.dirname(d), exist_ok=True)
|
|
if not os.path.exists(d) or open(s, "rb").read() != open(d, "rb").read():
|
|
shutil.copyfile(s, d)
|
|
for root, _, fs in os.walk(dst):
|
|
for f in fs:
|
|
rel = os.path.relpath(os.path.join(root, f), dst)
|
|
if rel not in keep: os.remove(os.path.join(root, f))
|
|
shutil.rmtree(src)
|
|
|
|
|
|
def iso_file(iso, name):
|
|
"""a file in the disc image's root directory (ISO 9660)"""
|
|
import struct
|
|
with open(iso, "rb") as f:
|
|
f.seek(16 * 2048); pvd = f.read(2048)
|
|
if pvd[1:6] != b"CD001": return None
|
|
dlsn, dlen = struct.unpack_from("<I", pvd, 158)[0], struct.unpack_from("<I", pvd, 166)[0]
|
|
f.seek(dlsn * 2048); d = f.read(dlen)
|
|
o = 0
|
|
while o < len(d):
|
|
n = d[o]
|
|
if n == 0: o = (o // 2048 + 1) * 2048; continue
|
|
fn = d[o + 33:o + 33 + d[o + 32]].decode("ascii", "replace").split(";")[0]
|
|
if fn == name:
|
|
lsn, size = struct.unpack_from("<I", d, o + 2)[0], struct.unpack_from("<I", d, o + 10)[0]
|
|
f.seek(lsn * 2048); return f.read(size)
|
|
o += n
|
|
return None
|
|
|
|
|
|
env = dict(os.environ, UYA_PLATFORM=a.platform, RAC_PLATFORM=a.platform, UYA_REGION=a.region)
|
|
# another region: the decomp and the host sources rewritten in its addresses (tools/region.py)
|
|
HOST = os.path.join(HERE, "host")
|
|
if a.region != "pal":
|
|
rdir = os.path.join(out, "region")
|
|
subprocess.run([sys.executable, os.path.join(HERE, "tools", "region.py"), a.region, a.repo, rdir], check=True)
|
|
a.repo = os.path.join(rdir, "repo")
|
|
HOST = os.path.join(rdir, "host")
|
|
env["UYA_HOST"] = HOST
|
|
# the static recompilation is built from the game itself: the executable comes
|
|
# from the disc image (the decomp repo only lends its symbol and function lists)
|
|
if find_iso():
|
|
_exe = iso_file(find_iso(), EXE_NAME)
|
|
if _exe:
|
|
os.makedirs(os.path.join(HERE, "build", "disc"), exist_ok=True)
|
|
_p = os.path.join(HERE, "build", "disc", EXE_NAME)
|
|
if not os.path.exists(_p) or open(_p, "rb").read() != _exe: open(_p, "wb").write(_exe)
|
|
env["UYA_EXE"] = _p
|
|
print("executable read from the disc image:", find_iso())
|
|
shutil.rmtree(stage, ignore_errors=True)
|
|
py = sys.executable
|
|
def progress(phase, i, n):
|
|
"""a line the Project Solana launcher reads for its progress bar"""
|
|
print("[progress] %s %d %d" % (phase, i, n), flush=True)
|
|
progress("prepare", 0, 3)
|
|
subprocess.run([py, os.path.join(HERE, "tools", "gen_host.py"), a.repo, stage], check=True, env=env)
|
|
dumps = os.path.join(HERE, "build", "levels" if a.region == "pal" else "levels_" + a.region)
|
|
# the level programs first: the executable's translation looks at the core
|
|
# addresses their code calls (tools/recomp.py)
|
|
for lv in levels:
|
|
cmd = [py, os.path.join(HERE, "tools", "level_dump.py"), dumps, str(lv), "--repo", a.repo]
|
|
if find_iso(): cmd += ["--iso", find_iso()]
|
|
subprocess.run(cmd, check=True)
|
|
progress("prepare", 1, 3)
|
|
subprocess.run([py, os.path.join(HERE, "tools", "recomp.py"), stage, a.repo], check=True, env=env)
|
|
progress("prepare", 2, 3)
|
|
# VU0/VU1 microprograms (.vutext) to C
|
|
subprocess.run([py, os.path.join(HERE, "tools", "vu_recomp.py"), stage, a.repo], check=True, env=env)
|
|
level_info = []
|
|
for k, lv in enumerate(levels):
|
|
progress("levels", k, len(levels))
|
|
LV = "%02d" % lv
|
|
lg = os.path.join(stage, "l" + LV)
|
|
subprocess.run([py, os.path.join(HERE, "tools", "gen_host.py"), a.repo, lg, "--level", LV, "--exe-gen", stage], check=True, env=env)
|
|
subprocess.run([py, os.path.join(HERE, "tools", "recomp.py"), lg, a.repo, "--level", LV, "--dump", dumps, "--exe-gen", stage],
|
|
check=True, env=env)
|
|
man = json.load(open(os.path.join(dumps, "level_" + LV, "manifest.json")))
|
|
text = [r for r in man["records"] if r["name"] == "text"][0]
|
|
level_info.append((LV, man["entry_point"], text["address"], text["bytes"]))
|
|
# the levels built in: ParseBin looks a loaded program up here (host/src/rac1/levels.c)
|
|
with open(os.path.join(stage, "level_table.c"), "w") as f:
|
|
f.write('/* generated by build.py: the level programs in this build */\n#include "rac1_levels.h"\n')
|
|
for LV, *_ in level_info:
|
|
f.write("extern const uya_tramp_t uya_tramp_table_l%s[]; extern const int uya_tramp_count_l%s;\n" % (LV, LV))
|
|
f.write("extern const rac1_rc_row_t rc%s_table[]; extern const int rc%s_table_count;\n" % (LV, LV))
|
|
f.write("const rac1_level_t rac1_levels[] = {\n")
|
|
for LV, entry, ta, tb in level_info:
|
|
f.write(" {%d, 0x%08X, 0x%08X, 0x%X, uya_tramp_table_l%s, &uya_tramp_count_l%s, rc%s_table, &rc%s_table_count},\n"
|
|
% (int(LV), entry, ta, tb, LV, LV, LV, LV))
|
|
if not level_info: f.write(" {-1, 0, 0, 0, 0, 0, 0, 0},\n")
|
|
f.write("};\nconst int rac1_level_count = %d;\n" % len(level_info))
|
|
sync(stage, gen)
|
|
|
|
common = ["-target", target, "-O" + a.O, "-g", "-march=nehalem", "-mfpmath=sse", "-malign-double", "-fno-strict-aliasing", "-fwrapv",
|
|
"-fno-PIE", "-fno-stack-protector", "-I", os.path.join(gen, "include"), "-I", os.path.join(HOST, "include")]
|
|
if a.platform == "linux": common += ["-D_FILE_OFFSET_BITS=64"] # (32-bit glibc: disc images are over 2 GB)
|
|
# frontbin code: keep it permissive, it was written for gcc 2.95
|
|
menu_flags = ["-w", "-Wno-error=incompatible-pointer-types", "-Wno-error=int-conversion",
|
|
"-Wno-error=implicit-function-declaration", "-Wno-error=implicit-int",
|
|
"-Wno-error=incompatible-function-pointer-types", "-Wno-error=return-type",
|
|
"-fno-builtin",
|
|
# no inlining across the game's functions: the decomp names overlapping PS2 data
|
|
# with separate symbols (D_001997D0 is a struct whose +0x2C is also D_001997FC),
|
|
# which the compiler takes for distinct objects. A call keeps such stores and loads
|
|
# in order; an inlined body does not (func_001EB300 inlined into func_001EABE8 kept
|
|
# the help-text count in a register, so the text tables were never relocated)
|
|
"-fno-inline-functions",
|
|
# the game's C keeps 16-byte vectors (u128, qcopy) in locals; the x86 stack is
|
|
# not 16-byte aligned on every path into it (translated code, thunks), and
|
|
# an aligned load of a misaligned local faults (func_L00_001FE940 on Veldin's
|
|
# first frame): every function realigns its own frame
|
|
"-mstackrealign"]
|
|
jobs = [] # (source, extra flags, object name, forced header)
|
|
for c in sorted(glob.glob(os.path.join(gen, "*.c"))):
|
|
jobs.append((c, menu_flags if os.path.basename(c) not in ("tramp_table.c", "level_table.c") else [], None, None))
|
|
jobs.append((os.path.join(gen, "abs_syms.s"), [], None, None))
|
|
for c in sorted(glob.glob(os.path.join(HOST, "overrides", "*.c"))):
|
|
jobs.append((c, menu_flags, None, None))
|
|
for c in sorted(glob.glob(os.path.join(HOST, "src", "**", "*.c"), recursive=True)):
|
|
jobs.append((c, ["-Wall", "-Wno-unused"], None, None))
|
|
for LV, *_ in level_info:
|
|
lg = os.path.join(gen, "l" + LV)
|
|
hdr = os.path.join(lg, "l%s_names.h" % LV)
|
|
os.makedirs(os.path.join(obj, "l" + LV), exist_ok=True)
|
|
for c in sorted(glob.glob(os.path.join(lg, "*.c"))):
|
|
jobs.append((c, menu_flags + ["-include", hdr], os.path.join("l" + LV, os.path.basename(c) + ".o"), hdr))
|
|
jobs.append((os.path.join(lg, "abs_syms.s"), [], os.path.join("l" + LV, "abs_syms.s.o"), None))
|
|
# the executable's main-segment overrides that the level has: the level's copies
|
|
for nm in open(os.path.join(lg, "overrides.txt")).read().split():
|
|
c = os.path.join(HOST, "overrides", nm + ".c")
|
|
jobs.append((c, menu_flags + ["-include", hdr], os.path.join("l" + LV, "override_" + nm + ".c.o"), hdr))
|
|
|
|
|
|
# compiler flags: a change rebuilds everything
|
|
_stamp = os.path.join(obj, "flags.txt")
|
|
_flags = " ".join(common + menu_flags)
|
|
if not os.path.exists(_stamp) or open(_stamp).read() != _flags:
|
|
os.makedirs(obj, exist_ok=True)
|
|
open(_stamp, "w").write(_flags)
|
|
flags_time = os.path.getmtime(_stamp)
|
|
|
|
# headers every generated file sees: a change rebuilds them all
|
|
headers_time = max([flags_time] + [os.path.getmtime(h) for h in glob.glob(os.path.join(gen, "include", "*.h")) + glob.glob(os.path.join(HOST, "include", "*.h"))])
|
|
|
|
|
|
def cc(job):
|
|
src, extra, oname, hdr = job
|
|
o = os.path.join(obj, oname or (os.path.basename(src) + ".o"))
|
|
if os.path.exists(o) and src.endswith(".c") and HOST not in src:
|
|
t = os.path.getmtime(o)
|
|
if t > os.path.getmtime(src) and t > headers_time and (not hdr or t > os.path.getmtime(hdr)):
|
|
return o, None
|
|
r = subprocess.run(zig + ["cc"] + common + extra + ["-c", src, "-o", o], capture_output=True, text=True)
|
|
return o, (r.stderr if r.returncode else None)
|
|
|
|
try: # Linux: the link opens every object at once (~5,700); the default soft limit is often 1,024
|
|
import resource
|
|
_s, _h = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
if _s != resource.RLIM_INFINITY and (_h == resource.RLIM_INFINITY or _s < _h):
|
|
resource.setrlimit(resource.RLIMIT_NOFILE, (_h if _h != resource.RLIM_INFINITY else 1 << 20, _h))
|
|
except (ImportError, ValueError, OSError):
|
|
pass
|
|
progress("compile", 0, len(jobs))
|
|
with ThreadPoolExecutor(a.j) as ex:
|
|
futs = [ex.submit(cc, j) for j in jobs]
|
|
done_n = 0
|
|
for _ in as_completed(futs):
|
|
done_n += 1
|
|
if done_n % 8 == 0 or done_n == len(jobs): progress("compile", done_n, len(jobs))
|
|
res = [f.result() for f in futs]
|
|
progress("link", 0, 1)
|
|
|
|
|
|
def objs_rsp(name):
|
|
"""the object files as a response file (@file): Windows limits a command line to
|
|
32767 characters, and the ~5700 objects of a full build are far more than that"""
|
|
p = os.path.join(obj, name + ".rsp")
|
|
with open(p, "w") as f:
|
|
for o, _ in res: f.write('"%s"\n' % o.replace("\\", "/"))
|
|
return "@" + p.replace("\\", "/")
|
|
errs = [e for _, e in res if e]
|
|
for e in errs: print(e[-3000:])
|
|
if errs:
|
|
sys.exit("%d files failed to compile" % len(errs))
|
|
if a.platform == "linux":
|
|
exe = os.path.join(out, "rac1_wrapper")
|
|
# at 0x40000000 like the Windows DLL (lower addresses are the PS2's); its symbols exported
|
|
# (-rdynamic) so the crash handler can name functions
|
|
link = zig + ["cc", "-target", target, "-o", exe, objs_rsp("link")] + ["-no-pie", "-rdynamic", "-Wl,--image-base=0x40000000",
|
|
"-ldl", "-lpthread", "-lm"]
|
|
r = subprocess.run(link, capture_output=True, text=True)
|
|
if r.returncode:
|
|
print(r.stderr[-6000:]); sys.exit("link failed")
|
|
else:
|
|
# the menu and runtime as a DLL at 0x40000000, plus the small launcher that owns PS2 RAM
|
|
dll = os.path.join(out, "rac1_game.dll")
|
|
link = zig + ["cc", "-shared", "-target", target, "-o", dll, objs_rsp("link")] + ["-Wl,--image-base=0x40000000", "-Wl,--no-dynamicbase", "-lshell32", "-lwinmm", "-lgdi32", "-luser32", "-ladvapi32"]
|
|
r = subprocess.run(link, capture_output=True, text=True)
|
|
if r.returncode:
|
|
print(r.stderr[-6000:]); sys.exit("link failed (dll)")
|
|
# the function names for crash reports (the .pdb is deleted after a launcher build)
|
|
try:
|
|
sys.path.insert(0, os.path.join(HERE, "tools"))
|
|
import pdb_syms
|
|
print("symbols:", pdb_syms.write_sym(dll, os.path.join(out, "rac1_game.pdb"), os.path.join(out, "rac1_game.sym")))
|
|
except Exception as e:
|
|
print("no symbol map (%s): crash reports will show offsets only" % e)
|
|
exe =os.path.join(out, "rac1_wrapper.exe")
|
|
L = os.path.join(HERE, "launcher")
|
|
link = zig + ["cc", "-target", target, "-O2", "-nostdlib", "-o", exe, os.path.join(L, "launcher_win32.c"),
|
|
os.path.join(L, "ps2ram_win32.c"), "-lkernel32", "-luser32",
|
|
"-Wl,--image-base=0x10000", "-Wl,--entry=launcher_start@0", "-Wl,--no-dynamicbase"]
|
|
r = subprocess.run(link, capture_output=True, text=True)
|
|
if r.returncode:
|
|
print(r.stderr[-6000:]); sys.exit("link failed (launcher)")
|
|
# the library app (frontend/solana.c): build/Project Solana.exe, which finds the game in build/windows
|
|
F = os.path.join(HERE, "frontend")
|
|
if not os.path.isdir(F): F = None # (the game kit players download has no launcher source)
|
|
link = None if not F else zig + ["cc", "-target", target, "-O2", "-municode", "-Wl,--subsystem,windows", "-o", os.path.join(HERE, "build", "Project Solana.exe"),
|
|
os.path.join(F, "solana.c"), os.path.join(F, "solana.rc"), "-lgdi32", "-luser32", "-lshell32", "-lcomdlg32", "-lole32", "-lmsimg32", "-lwinhttp", "-lbcrypt", "-lwindowscodecs", "-luuid"]
|
|
r = subprocess.run(link, capture_output=True, text=True, cwd=F) if link else None
|
|
if r and r.returncode:
|
|
print(r.stderr[-6000:]); sys.exit("link failed (Project Solana.exe)")
|
|
progress("done", 1, 1)
|
|
print("built", exe)
|