Run the pipeline on Linux

README.md

@@ -67,6 +67,24 @@ python recomp\web\serve.py --game "C:\Program Files (x86)\Nostale" --user <accou
 `serve.py` mints the launcher's auth code from the NosCore WebApi for that account, or you can pass one with
 `?auth=<code>`.
 
+### Linux
+
+The same pipeline runs on Linux (Python 3 with `capstone` and `pefile`, Emscripten's `emcc` on `PATH`, Ghidra 12 with
+its JDK, Node with Playwright). Generated code and builds go to `$NOSRECOMP_WORK` (default `~/re/work`).
+
+```bash
+GHIDRA_HOME=/opt/ghidra tools/decompile.sh ~/Nostale/NosCore.exe        # Ghidra only: IDR needs Windows
+W=${NOSRECOMP_WORK:-~/re/work}
+python3 recomp/lift.py $W/input/NosCore.exe $W/functions.txt $W/recomp/gen --seeds $W/recomp/seeds.txt
+python3 recomp/wasm/compile_all.py --web
+python3 recomp/web/serve.py --game ~/Nostale --user <account> --password <password>
+python3 recomp/web/iterate.py      # seeds each function the client stops on, re-lifts and rebuilds
+```
+
+Without IDR's seeds the function list is shorter; the lifter takes unit initialization and finalization routines
+from the client's init table, and `iterate.py` is the browser counterpart of `iterate.ps1` for whatever is still
+only reached indirectly.
+
 ### Scripted runs
 
 `node recomp/web/drive.mjs [--maps 2,20,145] [--trace] [--dump-frame]` (Playwright with the installed Chrome) logs in,

recomp/lift.py

@@ -880,6 +880,9 @@ def memval(op):
         "fwait", "wait", "sete", "setne"} | set(JCC)
 
 
+PROLOGUE_REGS = {"ebp", "ebx", "esi", "edi"}
+
+
 def reloc_code_seeds(L, fns):
     """Relocated pointers into the code section that no function covers and that decode as plausible code
     (interface thunks, method pointers only referenced from data)."""
@@ -937,10 +940,24 @@ def entry_shape(t):
     for t in sorted(L.reloc_targets):
         if t in stored or covered(t) or not entry_shape(t):
             continue
+        # An aligned pointer to a Delphi prologue is an entry however long the routine is before its ret.
+        head = L.disasm_range(t, t + 2)
+        if t % 4 == 0 and len(head) == 2 and head[0].mnemonic == "push" and head[0].op_str in PROLOGUE_REGS and (
+                head[1].mnemonic == "push" and head[1].op_str in PROLOGUE_REGS
+                or (head[1].mnemonic, head[1].op_str) == ("mov", "ebp, esp")
+                or head[1].mnemonic in ("add", "sub") and head[1].op_str.startswith("esp, ")):
+            out.append(t)
+            continue
         pos, ok = t, False
-        for _ in range(24):
+        for _ in range(256):
             dec = L.disasm_range(pos, pos + 1)
-            if not dec or dec[0].mnemonic not in SANE:
+            # 'add byte ptr [eax], al' is two zero bytes: the walk has run into data (a VMT or RTTI field).
+            if not dec or dec[0].mnemonic not in SANE or dec[0].bytes[:2] == b"\0\0":
+                break
+            # An aligned 'push ebp; mov ebp, esp' ahead is the next routine: the walk started in a string literal or
+            # other data Delphi places before a routine.
+            if any(a != t and a % 4 == 0 and L.code[a - L.code_lo: a - L.code_lo + 3] == b"\x55\x8b\xec"
+                   for a in range(pos, pos + dec[0].size)):
                 break
             if dec[0].mnemonic in ("jmp", "ret"):
                 ok = True
@@ -952,6 +969,29 @@ def entry_shape(t):
     return out
 
 
+def init_table_seeds(L):
+    """Unit initialization and finalization routines, listed in the PackageInfo the entry point hands to @InitExe
+    ('mov eax, PackageInfo; call @InitExe'). They are only reached through that table, and a long one never reaches
+    the ret that reloc_code_seeds' sanity walk looks for."""
+    entry = L.base + L.pe.OPTIONAL_HEADER.AddressOfEntryPoint
+    dec = L.disasm_range(entry, entry + 128)
+    for a, b in zip(dec, dec[1:]):
+        if b.mnemonic != "call":
+            continue
+        if a.mnemonic == "mov" and a.op_str.startswith("eax, 0x") and a.operands[1].type == X.X86_OP_IMM:
+            info = a.operands[1].imm & 0xFFFFFFFF
+            count, table = (struct.unpack("<I", L.pe.get_data(info - L.base + k, 4))[0] for k in (0, 4))
+            out = set()
+            for k in range(min(count, 65536) * 2):
+                v = struct.unpack("<I", L.pe.get_data(table - L.base + 4 * k, 4))[0]
+                if L.code_lo <= v < L.code_hi:
+                    out.add(v)
+            print(f"{len(out)} unit init/finalization routines in the init table")
+            return sorted(out)
+        break
+    return []
+
+
 def call_targets(L, fns):
     """Direct call targets that land mid-function become extra entry labels of the owning function;
     targets no function covers become new functions."""
@@ -1140,6 +1180,7 @@ def main():
         if rs and L.code_lo <= e < L.code_hi:
             fns.append((e, name, rs))
     fns = extend_functions(L, fns)
+    fns.extend(discover(L, fns, init_table_seeds(L)))
     fns.extend(discover(L, fns, reloc_code_seeds(L, fns)))
     if a.seeds and os.path.exists(a.seeds):
         fns.extend(discover(L, fns, [int(x.split()[0], 16) for x in open(a.seeds) if x.strip() and not x.startswith("#")]))

recomp/port/hle_misc.c

@@ -442,7 +442,7 @@ static void ws_recv(CPU* c) {
         memmove(k->rx, k->rx + n, k->rx_len - n);
         k->rx_len -= n;
         static int logged;
-        if (logged++ < 20) {
+        if (logged++ < 20 || getenv("PORT_LOG_RECV")) {
             char hex[3 * 64 + 1] = {0};
             for (uint32_t i = 0; i < n && i < 64; i++) snprintf(hex + 3 * i, 4, "%02x ", M8(buf + i));
             port_log("recv %08x: %u bytes %s", ARG(0), n, hex);

recomp/port/hle_user32.c

@@ -2066,8 +2066,8 @@ EM_JS(int, js_input_poll, (int* out), {
     if (!ring) return 0;
     const head = Atomics.load(ring, 0), tail = Atomics.load(ring, 1);
     if (head === tail) return 0;
-    const cap = (ring.length - 2) >> 2, at = 2 + (tail % cap) * 4;
-    for (let i = 0; i < 4; i++) HEAP32[(out >> 2) + i] = ring[at + i];
+    const cap = ((ring.length - 2) / 5) | 0, at = 2 + (tail % cap) * 5;
+    for (let i = 0; i < 5; i++) HEAP32[(out >> 2) + i] = ring[at + i];
     Atomics.store(ring, 1, tail + 1);
     return 1;
 });
@@ -2076,7 +2076,7 @@ EM_JS(int, js_input_peek_change, (void), {
     if (!ring) return 0;
     const head = Atomics.load(ring, 0), tail = Atomics.load(ring, 1);
     if (head === tail) return 0;
-    const type = ring[2 + (tail % ((ring.length - 2) >> 2)) * 4];
+    const type = ring[2 + (tail % (((ring.length - 2) / 5) | 0)) * 5];
     return type === 2 || type === 3 || type === 5 || type === 6 ? 1 : 0;
 });
 /* Blocks until the page writes an input event or the timeout passes; returns 0 when there is no ring to wait on. */
@@ -2112,7 +2112,7 @@ void hle_screen_to_page(int* x, int* y) {
 static struct { uint32_t hwnd, msg, time; int x, y; } last_click;
 
 /* Mouse input goes to the capture window or the one under the cursor, after the hit test and cursor messages. */
-static void mouse_event(CPU* c, int kind, int button, int sx, int sy, int wheel) {
+static void mouse_event(CPU* c, int kind, int button, int sx, int sy, int wheel, uint32_t time) {
     static const uint32_t vk[3] = {1, 4, 2}, down_msg[3] = {0x201, 0x207, 0x204};
     cursor_x = sx;
     cursor_y = sy;
@@ -2171,13 +2171,13 @@ static void mouse_event(CPU* c, int kind, int button, int sx, int sy, int wheel)
     if (hit == HTCLIENT) {
         int ox, oy;
         client_origin(t, &ox, &oy);
-        uint32_t now = (uint32_t)port_now_ms();
         if (kind == 2 && (t->cls->style & 8) && last_click.hwnd == target && last_click.msg == msg &&
-            now - last_click.time <= 500 && abs(sx - last_click.x) <= 2 && abs(sy - last_click.y) <= 2) {
+            time - last_click.time <= 500 && abs(sx - last_click.x) <= 2 && abs(sy - last_click.y) <= 2) {
             msg += 2;
             last_click.hwnd = 0;
+            if (getenv("PORT_LOG_INPUT")) port_log("double click -> %08x", target);
         } else if (kind == 2) {
-            last_click.hwnd = target; last_click.msg = msg; last_click.time = now; last_click.x = sx; last_click.y = sy;
+            last_click.hwnd = target; last_click.msg = msg; last_click.time = time; last_click.x = sx; last_click.y = sy;
         }
         post(target, msg, mouse_keys(), ((uint32_t)(sy - oy) & 0xffff) << 16 | ((uint32_t)(sx - ox) & 0xffff));
     } else if (hit != HTNOWHERE && hit != HTTRANSPARENT) {
@@ -2192,21 +2192,21 @@ enum { IN_MOVE = 1, IN_DOWN, IN_UP, IN_WHEEL, IN_KEYDOWN, IN_KEYUP, IN_CHAR, IN_
 static uint32_t last_change_frame = 0xffffffffu;
 
 static void pump_input(CPU* c) {
-    int e[4];
+    int e[5];
     while (display_hwnd) {
         int change = js_input_peek_change();
         if (change && last_change_frame == port_frames) break;
         if (!js_input_poll(e)) break;
         if (change) last_change_frame = port_frames;
         static int logged;
-        if (e[0] != IN_MOVE && e[0] != IN_CHAR && logged++ < 10) port_log("input %d %d (%d,%d) -> %08x", e[0], e[1], e[2], e[3], display_hwnd);
+        if (e[0] != IN_MOVE && e[0] != IN_CHAR && (logged++ < 10 || getenv("PORT_LOG_INPUT"))) port_log("input %d %d (%d,%d) -> %08x", e[0], e[1], e[2], e[3], display_hwnd);
         int sx, sy;
         page_to_screen(e[2], e[3], &sx, &sy);
         switch (e[0]) {
-        case IN_MOVE: case IN_UP: case IN_WHEEL: mouse_event(c, e[0], e[1], sx, sy, e[1]); break;
+        case IN_MOVE: case IN_UP: case IN_WHEEL: mouse_event(c, e[0], e[1], sx, sy, e[1], (uint32_t)e[4]); break;
         case IN_DOWN:
             if (getenv("PORT_TRACE_CLICK")) { port_trace_budget = 1500; port_fn_trace_start(c); }
-            mouse_event(c, e[0], e[1], sx, sy, 0);
+            mouse_event(c, e[0], e[1], sx, sy, 0, (uint32_t)e[4]);
             break;
         case IN_KEYDOWN: key_event(c, (uint32_t)e[1] & 0xff, 1, 0); break;
         case IN_KEYUP: key_event(c, (uint32_t)e[1] & 0xff, 0, 0); break;

recomp/wasm/compile_all.py

@@ -5,16 +5,18 @@
 import concurrent.futures as cf
 import glob
 import os
+import shutil
 import subprocess
 import sys
 import time
 
 HERE = os.path.dirname(os.path.abspath(__file__))
-GEN = r"C:\tools\re\work\recomp\gen"
-OUT = r"C:\tools\re\work\wasm"
+WORK = os.environ.get("NOSRECOMP_WORK", r"C:\tools\re\work" if os.name == "nt" else os.path.expanduser("~/re/work"))
+GEN = os.path.join(WORK, "recomp", "gen")
+OUT = os.path.join(WORK, "wasm")
 RT = os.path.join(HERE, "..", "runtime")
 PORT = os.path.join(HERE, "..", "port")
-EMCC = r"C:\tools\emsdk\upstream\emscripten\emcc.exe"
+EMCC = os.environ.get("EMCC") or shutil.which("emcc") or r"C:\tools\emsdk\upstream\emscripten\emcc.exe"
 FLAGS = ["-O2", "-w", "-c", "-sSUPPORT_LONGJMP=0"]
 LINK = ["-O2", "-sGLOBAL_BASE=536870912", "-sINITIAL_MEMORY=1024MB", "-sALLOW_MEMORY_GROWTH=1",
         "-sMAXIMUM_MEMORY=2048MB", "-sSTACK_SIZE=64MB", "-sNODERAWFS=1", "-sEXIT_RUNTIME=1",

recomp/web/drive.mjs

@@ -5,11 +5,13 @@
 //                       [--maps 2,145] [--cpuprofile]
 import { createRequire } from 'node:module';
 import { mkdirSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
 import { join } from 'node:path';
 
 const require = createRequire(import.meta.url);
+const windows = process.platform === 'win32';
 const playwrightPath = process.env.PLAYWRIGHT_PATH
-  || 'C:/Program Files/nodejs/node_modules/@playwright/cli/node_modules/playwright';
+  || (windows ? 'C:/Program Files/nodejs/node_modules/@playwright/cli/node_modules/playwright' : 'playwright');
 const { chromium } = require(playwrightPath);
 
 const arg = (name, fallback) => {
@@ -18,7 +20,8 @@ const arg = (name, fallback) => {
 };
 const flag = (name) => process.argv.includes(name);
 
-const out = arg('--out', 'C:/tools/re/work/drive');
+const work = process.env.NOSRECOMP_WORK || (windows ? 'C:/tools/re/work' : join(homedir(), 're/work'));
+const out = arg('--out', join(work, 'drive'));
 // Message boxes answer themselves with their default button so nothing waits on a human.
 const url = arg('--url', 'http://127.0.0.1:8077/') + '?autodismiss=1' + (flag('--trace') ? '&trace=1' : '') + (arg('--params', '') ? '&' + arg('--params', '') : '');
 mkdirSync(out, { recursive: true });
@@ -32,8 +35,8 @@ const GROUND = [1000, 520];
 
 const browser = await chromium.launch({
   headless: !flag('--headed'),
-  channel: process.env.DRIVE_CHANNEL || 'chrome',
-  args: ['--use-angle=d3d11', '--enable-gpu', '--ignore-gpu-blocklist', '--enable-unsafe-swiftshader'],
+  channel: process.env.DRIVE_CHANNEL || (windows ? 'chrome' : undefined),
+  args: [...(windows ? ['--use-angle=d3d11'] : []), '--enable-gpu', '--ignore-gpu-blocklist', '--enable-unsafe-swiftshader'],
 });
 let page, profiler;
 // --cpuprofile samples the worker from entering the world to the end of the run into worker.cpuprofile (open it in
@@ -214,6 +217,7 @@ try {
   }
 } catch (e) {
   console.log('error:', e.message);
+  process.exitCode = 1;
 } finally {
   if (profiler) {
     const { profile } = await profiler('Profiler.stop');

recomp/web/index.html

@@ -363,22 +363,22 @@
   // Same event codes as pump_input in hle_user32.c.
   const IN = { move: 1, down: 2, up: 3, wheel: 4, keydown: 5, keyup: 6, char: 7, dumpFrame: 8 };
   let ring = null;
-  function send(type, a, x = 0, y = 0) {
+  function send(type, a, x = 0, y = 0, time = performance.now()) {
     if (!ring) return;
-    const head = Atomics.load(ring, 0), cap = (ring.length - 2) >> 2;
+    const head = Atomics.load(ring, 0), cap = ((ring.length - 2) / 5) | 0;
     if (head - Atomics.load(ring, 1) >= cap) return;
-    const at = 2 + (head % cap) * 4;
-    ring[at] = type; ring[at + 1] = a; ring[at + 2] = x; ring[at + 3] = y;
+    const at = 2 + (head % cap) * 5;
+    ring[at] = type; ring[at + 1] = a; ring[at + 2] = x; ring[at + 3] = y; ring[at + 4] = Math.round(time);
     Atomics.store(ring, 0, head + 1);
     Atomics.notify(ring, 0);
   }
   function pos(e) {
     const r = canvas.getBoundingClientRect();
     return [Math.round((e.clientX - r.left) * canvas.width / r.width), Math.round((e.clientY - r.top) * canvas.height / r.height)];
   }
-  canvas.addEventListener('mousemove', (e) => send(IN.move, 0, ...pos(e)));
-  canvas.addEventListener('mousedown', (e) => { canvas.focus(); send(IN.down, e.button, ...pos(e)); e.preventDefault(); });
-  canvas.addEventListener('mouseup', (e) => send(IN.up, e.button, ...pos(e)));
+  canvas.addEventListener('mousemove', (e) => send(IN.move, 0, ...pos(e), e.timeStamp));
+  canvas.addEventListener('mousedown', (e) => { canvas.focus(); send(IN.down, e.button, ...pos(e), e.timeStamp); e.preventDefault(); });
+  canvas.addEventListener('mouseup', (e) => send(IN.up, e.button, ...pos(e), e.timeStamp));
   canvas.addEventListener('wheel', (e) => { send(IN.wheel, e.deltaY < 0 ? 120 : -120, ...pos(e)); e.preventDefault(); }, { passive: false });
   canvas.addEventListener('contextmenu', (e) => e.preventDefault());
   const charKeys = { Enter: 13, Backspace: 8, Tab: 9, Escape: 27 };

recomp/web/iterate.py

@@ -0,0 +1,70 @@
+"""Seed loop for the browser build: run drive.mjs, seed the first function nobody discovered, re-lift, rebuild, repeat.
+
+The browser counterpart of iterate.ps1, for machines without the native MSVC build. serve.py must be running.
+
+usage: iterate.py [--rounds 25] [--url http://127.0.0.1:8077/] [--before "<command>"] [-- extra drive.mjs arguments]
+
+--before runs a shell command ahead of each round, for instance restarting the NosCore servers so the world server
+does not still hold the previous round's session ("account name is currently in use").
+"""
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+RECOMP = os.path.dirname(HERE)
+WORK = os.environ.get("NOSRECOMP_WORK", r"C:\tools\re\work" if os.name == "nt" else os.path.expanduser("~/re/work"))
+MISSING = re.compile(r"no recompiled code for (?:guest address|indirect jump target) ([0-9a-f]{8})")
+
+
+def main():
+    argv = sys.argv[1:]
+    extra = argv[argv.index("--") + 1:] if "--" in argv else []
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--rounds", type=int, default=25)
+    ap.add_argument("--url", default="http://127.0.0.1:8077/")
+    ap.add_argument("--before")
+    a = ap.parse_args(argv[:argv.index("--")] if "--" in argv else argv)
+
+    gen = os.path.join(WORK, "recomp", "gen")
+    seeds = os.path.join(WORK, "recomp", "seeds.txt")
+    out = os.path.join(WORK, "drive", "iterate")
+    open(seeds, "a").close()
+    for rnd in range(1, a.rounds + 1):
+        if a.before:
+            subprocess.run(a.before, shell=True, check=True)
+        log_path = os.path.join(out, "full.log")
+        if os.path.exists(log_path):
+            os.remove(log_path)
+        drive = subprocess.run(["node", os.path.join(HERE, "drive.mjs"), "--url", a.url, "--out", out, *extra])
+        if not os.path.exists(log_path):
+            print(f"round {rnd}: drive.mjs wrote no log (exit {drive.returncode})")
+            sys.exit(1)
+        with open(log_path, encoding="utf-8", errors="replace") as f:
+            log = f.read()
+        m = MISSING.search(log)
+        if not m and drive.returncode:
+            print(f"round {rnd}: drive.mjs failed (exit {drive.returncode}) before any missing-code stop")
+            print("\n".join(l for l in log.splitlines() if l.startswith("FATAL"))[:2000])
+            sys.exit(1)
+        if not m:
+            print(f"round {rnd}: no missing-code stop")
+            print("\n".join(l for l in log.splitlines() if l.startswith("FATAL"))[:2000])
+            return
+        addr = m.group(1)
+        print(f"round {rnd}: missing {addr}")
+        with open(seeds) as f:
+            if addr in f.read().split():
+                print(f"already seeded {addr}, giving up")
+                sys.exit(1)
+        with open(seeds, "a") as f:
+            f.write(addr + "\n")
+        subprocess.run([sys.executable, os.path.join(RECOMP, "lift.py"), os.path.join(WORK, "input", "NosCore.exe"),
+                        os.path.join(WORK, "functions.txt"), gen, "--seeds", seeds], check=True)
+        subprocess.run([sys.executable, os.path.join(RECOMP, "wasm", "compile_all.py"), "--web"], check=True)
+
+
+if __name__ == "__main__":
+    main()

recomp/web/serve.py

@@ -257,7 +257,8 @@ def do_HEAD(self):
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--game", required=True)
-    ap.add_argument("--build", default=r"C:\tools\re\work\wasm\web")
+    work = os.environ.get("NOSRECOMP_WORK", r"C:\tools\re\work" if os.name == "nt" else os.path.expanduser("~/re/work"))
+    ap.add_argument("--build", default=os.path.join(work, "wasm", "web"))
     ap.add_argument("--port", type=int, default=8077)
     ap.add_argument("--auth-url", default="https://127.0.0.1:7001")
     ap.add_argument("--user", help="local NosCore account used to mint _NC_AUTH_CODE (dev only)")

recomp/web/worker.js

@@ -2,9 +2,9 @@
 // /game/ (mountGameFile), so nothing of the client is bundled with the page.
 const params = new URLSearchParams(self.location.search);
 
-// Input ring shared with the page: [0] = events written, [1] = events read, then 4 ints per event. The worker never
+// Input ring shared with the page: [0] = events written, [1] = events read, then 5 ints per event (type, argument, x, y, page time in ms). The worker never
 // yields, so postMessage could not deliver input; the host drains this from PeekMessage.
-self.inputRing = new Int32Array(new SharedArrayBuffer((2 + 4 * 1024) * 4));
+self.inputRing = new Int32Array(new SharedArrayBuffer((2 + 5 * 1024) * 4));
 // audioDone[sample] = generation of the last sound on that sample handle that finished playing (hle_audio.c).
 self.audioDone = new Int32Array(new SharedArrayBuffer(128 * 4));
 // Text pasted in the page: [0] = paste sequence, [1] = length, then UTF-16 code units (hle_menu.c).