From 52c771d555918f1fa715758d7483352b935e4894 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:25:25 +0000 Subject: [PATCH 1/8] Add cpp_permuter: source permuter for matching C++ functions A C++17 tool in the spirit of decomp-permuter for MSVC 6 matching projects such as gta2_re. It locates one function in a .cpp, parses its body with a tolerant statement/expression parser (no preprocessing, so the rest of the file stays byte-identical), applies mutation passes, compiles each candidate with a user command and scores the function's disassembly (llvm-objdump) against a target object. Passes: reorder_saves (every dependency-respecting order of runs of local saves), move_stmt, inline_local, sink_decl, hoist_decl, merge_decl, swap_operands, flip_compare, invert_if, ternary, compound_assign. Exhaustive and random search, parallel compiles, dry runs, and a VC6-under-wine compile script for gta2_re. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + CMakeLists.txt | 34 + README.md | 124 ++++ examples/gta2/PromoteMemberToLeader.conf | 10 + examples/gta2/README.md | 75 ++ examples/gta2/compile.sh | 30 + src/analysis.cpp | 522 ++++++++++++++ src/analysis.hpp | 49 ++ src/lexer.cpp | 201 ++++++ src/lexer.hpp | 30 + src/main.cpp | 766 ++++++++++++++++++++ src/parser.cpp | 611 ++++++++++++++++ src/parser.hpp | 89 +++ src/passes.cpp | 875 +++++++++++++++++++++++ src/passes.hpp | 49 ++ src/runner.cpp | 99 +++ src/runner.hpp | 22 + src/scorer.cpp | 359 ++++++++++ src/scorer.hpp | 45 ++ tests/e2e/ops_base.cpp | 16 + tests/e2e/ops_target.cpp | 16 + tests/e2e/run.sh | 31 + tests/e2e/saves_base.cpp | 18 + tests/e2e/saves_target.cpp | 18 + tests/tests.cpp | 325 +++++++++ 25 files changed, 4417 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 examples/gta2/PromoteMemberToLeader.conf create mode 100644 examples/gta2/README.md create mode 100755 examples/gta2/compile.sh create mode 100644 src/analysis.cpp create mode 100644 src/analysis.hpp create mode 100644 src/lexer.cpp create mode 100644 src/lexer.hpp create mode 100644 src/main.cpp create mode 100644 src/parser.cpp create mode 100644 src/parser.hpp create mode 100644 src/passes.cpp create mode 100644 src/passes.hpp create mode 100644 src/runner.cpp create mode 100644 src/runner.hpp create mode 100644 src/scorer.cpp create mode 100644 src/scorer.hpp create mode 100644 tests/e2e/ops_base.cpp create mode 100644 tests/e2e/ops_target.cpp create mode 100755 tests/e2e/run.sh create mode 100644 tests/e2e/saves_base.cpp create mode 100644 tests/e2e/saves_target.cpp create mode 100644 tests/tests.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..44f58f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +build/ +permuter_out/ +*.permuter[0-9]*.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..08dec5e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.16) +project(cpp_permuter CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +find_package(Threads REQUIRED) + +add_library(permuter_core STATIC + src/lexer.cpp + src/parser.cpp + src/analysis.cpp + src/passes.cpp + src/scorer.cpp + src/runner.cpp +) +target_include_directories(permuter_core PUBLIC src) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(permuter_core PRIVATE -Wall -Wextra) +endif() + +add_executable(cpp_permuter src/main.cpp) +target_link_libraries(cpp_permuter PRIVATE permuter_core Threads::Threads) + +enable_testing() +add_executable(permuter_tests tests/tests.cpp) +target_link_libraries(permuter_tests PRIVATE permuter_core Threads::Threads) +add_test(NAME unit COMMAND permuter_tests) +add_test(NAME end_to_end + COMMAND ${CMAKE_COMMAND} -E env PERMUTER=$ + sh ${CMAKE_SOURCE_DIR}/tests/e2e/run.sh) diff --git a/README.md b/README.md new file mode 100644 index 0000000..e947a33 --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +# cpp_permuter + +Brute-forces source permutations of a single C++ function until its compiled code matches a +target object, or gets closer to it. It is built for matching decompilation projects such as +[gta2_re](https://github.com/CriminalRETeam/gta2_re) (MSVC 6), and works like +[decomp-permuter](https://github.com/simonlindholm/decomp-permuter), but for C++ and written +in C++. + +How it works: + +1. Find the function in the `.cpp` and parse its body into statements, declarations and + expressions. +2. Make candidates by applying mutation passes (reorder local saves, inline a local, move + declarations in and out of blocks, swap operands, invert an `if`, ...). +3. Splice each candidate back into the file, compile it with your compile command, and + disassemble the function from the object with `llvm-objdump`. +4. Score it against the target: 0 means identical, lower is closer. Improvements are written + to the output directory. + +## Building + +Needs CMake 3.16+ and a C++17 compiler. Scoring also needs `llvm-objdump` on `PATH`. + +```sh +cmake -S . -B build -G Ninja # or leave out -G for make +cmake --build build +ctest --test-dir build # unit tests, plus end-to-end tests with clang +``` + +## Usage + +```sh +cpp_permuter -s Source/PedGroup.cpp -f PedGroup::PromoteMemberToLeader_4C9680 \ + -c 'examples/gta2/compile.sh {src} {obj}' \ + -t Scripts/asm/PedGroup.cpp.obj \ + -m exhaustive -p reorder_saves -j 4 +``` + +| option | meaning | +|---|---| +| `-s/--source` | the `.cpp` holding the function | +| `-f/--function` | `Class::Method`, `Method`, `Class::Class`, `Class::~Class` | +| `-c/--compile` | compile command. `{src}` is the candidate source, `{obj}` the object it must write, `{dir}` the original source's directory | +| `-t/--target-obj` | object file holding the target code | +| `--symbol`, `--target-symbol` | symbol names, if the guess from `--function` is wrong (MSVC `?Name@Class@@...`, Itanium, or C) | +| `--score-cmd` | score with your own command instead (`{obj}`, `{src}`). It must print the score (0 = match) as the last number in its output | +| `--ignore-reloc-names` | compare relocated operands without their symbol names, and treat absolute addresses as symbols. Use this when the target came from raw exe asm | +| `-m exhaustive` | try every candidate the selected passes produce. `--depth N` chains N mutations, `--max-candidates` caps the count | +| `-m random` | (default) apply 1..`--max-mutations` random mutations, starting from the base or from the best so far. Runs until a match, `-n` compiles, or Ctrl-C | +| `-p a,b` | passes to use (default: all) | +| `-j N` | parallel compiles | +| `--dry-run` | print the candidates as diffs without compiling. A quick check of what a pass would do | +| `--show-ast` | print how the function was parsed | +| `--show-base-diff` | print the target/base asm alignment before starting | +| `--config FILE` | `key = value` lines, same keys as the long options. Handy for one config per function | +| `--keep-going` | keep searching after an exact match | + +Each improvement goes to `permuter_out/output--/`: + +- `function.cpp`: the permuted function. +- `source.cpp`: the whole file. +- `diff.txt`: what changed. +- `asm_diff.txt`: target and candidate asm aligned side by side. + +The exit status is 0 if a match was found and 3 if not. + +Candidate sources are written next to the original (`PedGroup.permuter0.cpp`, ...) so relative +`#include`s still resolve. They are deleted on exit. `--candidate-dir` puts them somewhere +else. + +## Passes + +`cpp_permuter --list-passes` prints these: + +| pass | what it does | +|---|---| +| `reorder_saves` | Runs of consecutive "local saves" (declarations, or assignments to locals, that don't call or store) in every order that keeps dependencies. Four independent saves give 23 new orders (the `PedGroup::PromoteMemberToLeader_4C9680` quirk) | +| `move_stmt` | Move one statement up or down past neighbours it doesn't depend on. Field loads and stores are assumed not to alias; locals and calls are respected | +| `inline_local` | Remove a local by putting its initializer into its uses, plainly and with a cast to the local's type. A local that is never used is dropped | +| `sink_decl` | Move a declaration into the innermost block that holds all its uses, or to just before its first use | +| `hoist_decl` | Move a declaration to the top of its block or of the function, splitting `T x = e;` into `T x;` plus `x = e;`. Also offers one candidate that hoists every declaration at once (C89 style) | +| `merge_decl` | `T x; ... x = e;` becomes `T x = e;` at the assignment | +| `swap_operands` | `a + b` becomes `b + a`, for `+ * & \| ^ == !=` | +| `flip_compare` | `a < b` becomes `b > a` | +| `invert_if` | `if (c) A else B` becomes `if (!c) B else A`, negating comparisons directly | +| `ternary` | `x = c ? a : b` and `return c ? a : b` become `if`/`else` (and an early-return form), and back | +| `compound_assign` | `x = x op y` and `x op= y`, both ways | + +The passes aim to keep behaviour the same, but they don't prove it. For example they assume +that two different fields don't alias. That is fine here, because a candidate that matches the +target byte for byte is correct by construction. A rewrite that changes behaviour just won't +match. + +## The parser + +This is not a full C++ front end, and that is deliberate. It works on the unpreprocessed +source and only rewrites the body of the target function. Everything else in the file stays +byte-identical, so candidates go through cl.exe with the original includes and macros. Inside +the body it recognises statements, local declarations and expressions. Anything it can't +classify (`__asm`, unknown macros, local types) becomes a fixed barrier that is never moved +across. If it misreads something, that candidate fails to compile and is skipped. + +Running clang on the whole file instead would mean the code has to compile under clang. MSVC 6 +code with its own headers, `__asm` and naked marker functions doesn't, and clang's error +recovery drops the very declarations we want to move. + +## Scoring + +The function is disassembled with `llvm-objdump -d -r --disassemble-symbols=`. Before +comparing: + +- relocated operands are replaced with their symbol; +- branch targets become instruction indices, so a size change upstream doesn't change every + jump; +- `<...>` annotations are stripped. + +Target and candidate are then aligned with an edit distance. Penalties: 10 for a register +difference, 50 for other operand differences, 60 for a moved instruction, 100 for an insertion +or a deletion. The weights are the same idea as decomp-permuter's. + +## gta2_re + +`examples/gta2/` has a VC6-under-wine compile script and a walkthrough. See +[examples/gta2/README.md](examples/gta2/README.md). diff --git a/examples/gta2/PromoteMemberToLeader.conf b/examples/gta2/PromoteMemberToLeader.conf new file mode 100644 index 0000000..c0336ff --- /dev/null +++ b/examples/gta2/PromoteMemberToLeader.conf @@ -0,0 +1,10 @@ +# cpp_permuter --config PromoteMemberToLeader.conf +# Paths are relative to where cpp_permuter runs (here: the gta2_re checkout). +source = Source/PedGroup.cpp +function = PedGroup::PromoteMemberToLeader_4C9680 +compile = /path/to/cpp_permuter/examples/gta2/compile.sh {src} {obj} +target-obj = Scripts/asm/PedGroup.cpp.obj +mode = exhaustive +passes = reorder_saves +jobs = 4 +output-dir = permuter_out/PromoteMemberToLeader diff --git a/examples/gta2/README.md b/examples/gta2/README.md new file mode 100644 index 0000000..b586d15 --- /dev/null +++ b/examples/gta2/README.md @@ -0,0 +1,75 @@ +# Using cpp_permuter with gta2_re + +## Setup (Linux) + +Set up wine and the gta2_re VC6 toolchain as gta2_re's `CLAUDE.md` describes, so the +`3rdParty/gta2_re_compile_tools` submodule is checked out. Then: + +```sh +export GTA2_RE=/path/to/gta2_re +export WINEDEBUG=-all +wineserver -p # keep wineserver running: compiles drop from seconds to well under one +``` + +Check that a single TU builds: + +```sh +examples/gta2/compile.sh $GTA2_RE/Source/PedGroup.cpp /tmp/PedGroup.obj +``` + +`compile.sh` uses the `gta2_lib` flags (`/O2 /GX /ML ...`). Files that `cmake/vc6.cmake` gives +extra flags need `EXTRA_CFLAGS`, for example `EXTRA_CFLAGS=/Gz` for `Network_20324.cpp`. + +## A target object + +The permuter compares against an object file that holds the original function. It needs the +original `10.5.exe` in `Scripts/bin_comp/`, as for objdiff: + +```sh +cd $GTA2_RE +python3 Scripts/generate_target_asm_for_objs.py PedGroup.cpp # writes Scripts/asm/PedGroup.cpp.asm + make_objs.sh +(cd Scripts/asm && ./make_objs.sh) # -> Scripts/asm/PedGroup.cpp.obj +``` + +Calls in that asm are resolved to mangled names, so they compare with the symbols in our +object. If some operands are still raw addresses from the exe, add `--ignore-reloc-names`. + +## Running + +Remove `WIP_IMPLEMENTED` / `NOT_IMPLEMENTED` from the function first, since they add code. + +```sh +cpp_permuter -s $GTA2_RE/Source/PedGroup.cpp -f PedGroup::SomeFunction_4C9xxx \ + -c "$PWD/examples/gta2/compile.sh {src} {obj}" \ + -t $GTA2_RE/Scripts/asm/PedGroup.cpp.obj \ + --show-base-diff -j 4 +``` + +Or put the options in a file (see `PromoteMemberToLeader.conf`) and run +`cpp_permuter --config PromoteMemberToLeader.conf`. + +Suggestions: + +- When you already suspect the kind of fix, use exhaustive mode with just that pass: + `-m exhaustive -p reorder_saves` tries every order of every run of local saves. +- `--dry-run` shows what a pass would produce without compiling anything. +- Random mode (the default) mixes all passes and is good for leaving running. Add + `-p reorder_saves,move_stmt` and similar to focus it. +- Results go to `permuter_out/output--/`. Copy the function back from + `function.cpp`, then run `build.py` and `compare_builds.py` as usual. + +## Checking the setup without the original exe + +A `MATCH_FUNC` is its own target. Compile the unmodified TU, scramble the function, and the +permuter should find the way back: + +```sh +examples/gta2/compile.sh $GTA2_RE/Source/PedGroup.cpp /tmp/target.obj +# swap a few lines of PedGroup::PromoteMemberToLeader_4C9680 in a copy of the file, then +cpp_permuter -s $GTA2_RE/Source/PedGroup_scrambled.cpp -f PedGroup::PromoteMemberToLeader_4C9680 \ + -c "$PWD/examples/gta2/compile.sh {src} {obj}" -t /tmp/target.obj \ + -m exhaustive -p reorder_saves -j 4 +``` + +With the four weapon saves scrambled (base score 520), this tried the other 23 orders and found +the matching one in about 4 seconds. diff --git a/examples/gta2/compile.sh b/examples/gta2/compile.sh new file mode 100755 index 0000000..4242edf --- /dev/null +++ b/examples/gta2/compile.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Compiles one gta2_re translation unit with VC6 under wine, using the same +# flags as gta2_lib in gta2_re's CMakeLists.txt / build.py --single_cpp. +# +# compile.sh +# +# GTA2_RE must point at a gta2_re checkout with the 3rdParty/gta2_re_compile_tools +# submodule. EXTRA_CFLAGS adds per-file flags (cmake/vc6.cmake), e.g. "/Gz" for +# Network_20324.cpp. +# +# Use it as: cpp_permuter -c 'examples/gta2/compile.sh {src} {obj}' ... +set -e +SRC=$1 +OBJ=$2 +: "${GTA2_RE:?set GTA2_RE to your gta2_re checkout}" +TOOLS="$GTA2_RE/3rdParty/gta2_re_compile_tools" + +winpath() { printf 'Z:%s' "$(realpath -m "$1")" | tr / '\\'; } + +export WINEDEBUG=-all +export WINEPATH="$(winpath "$TOOLS/VC98/Bin");$(winpath "$TOOLS/Common/MSDev98/Bin")" +export INCLUDE="$(winpath "$TOOLS/VC98/ATL/Include");$(winpath "$TOOLS/VC98/Include");$(winpath "$TOOLS/VC98/MFC/Include")" +export LIB="$(winpath "$TOOLS/VC98/Lib")" + +# cl.exe exits 0 on warnings and 2 on errors; its output goes to stdout. +wine cl.exe /nologo /TP /c \ + /I"$(winpath "$GTA2_RE/Source")" /I"$(winpath "$GTA2_RE")" /I"$(winpath "$TOOLS")" \ + /DWIN32 /D_WINDOWS /D_CRT_SECURE_NO_WARNINGS /D_CRT_NON_CONFORMING_SWPRINTFS /DIMGUI_DLL \ + /W3 /Zm1000 /EHsc /GX /ML /O2 /DNDEBUG $EXTRA_CFLAGS \ + /Fo"$(winpath "$OBJ")" "$(winpath "$SRC")" diff --git a/src/analysis.cpp b/src/analysis.cpp new file mode 100644 index 0000000..5995f1c --- /dev/null +++ b/src/analysis.cpp @@ -0,0 +1,522 @@ +#include "analysis.hpp" + +namespace perm { + +int binPrec(const std::string& op) { + if (op == ",") return 1; + if (op == "||") return 4; + if (op == "&&") return 5; + if (op == "|") return 6; + if (op == "^") return 7; + if (op == "&") return 8; + if (op == "==" || op == "!=") return 9; + if (op == "<" || op == ">" || op == "<=" || op == ">=") return 10; + if (op == "<<" || op == ">>") return 11; + if (op == "+" || op == "-") return 12; + if (op == "*" || op == "/" || op == "%") return 13; + if (op == ".*" || op == "->*") return 14; + return 0; +} + +void forEachExpr(Expr& x, const std::function& fn) { + fn(x); + for (auto& k : x.kids) forEachExpr(*k, fn); +} + +namespace { + +// True if token i can end an operand, so a following '*', '&', '-' is binary. +bool operandEnd(const Func& f, int i) { + if (i < 0) return false; + const Token& t = f.toks[i]; + switch (t.kind) { + case TokKind::Number: + case TokKind::String: + case TokKind::Char: return true; + case TokKind::Ident: + return !isKeyword(t.text) || t.text == "this" || t.text == "true" || t.text == "false" || + t.text == "nullptr"; + case TokKind::Punct: return t.text == ")" || t.text == "]" || t.text == "++" || t.text == "--"; + default: return false; + } +} + +struct ExprParser { + const Func& f; + const std::vector& T; + int pos, end; + bool fail = false; + + ExprParser(const Func& fn, int b, int e) : f(fn), T(fn.toks), pos(b), end(e) {} + + bool at(const char* s) const { + return pos < end && T[pos].kind == TokKind::Punct && T[pos].text == s; + } + + std::unique_ptr node(Expr::K k, int b) { + auto x = std::make_unique(); + x->k = k; + x->b = b; + return x; + } + + bool isTypeTok(int i) const { + const Token& t = T[i]; + if (t.kind == TokKind::Ident) { + if (isBuiltinType(t.text)) return true; + if (t.text == "const" || t.text == "volatile" || t.text == "struct" || + t.text == "class" || t.text == "union" || t.text == "enum" || + t.text == "unsigned" || t.text == "signed") + return true; + return !isKeyword(t.text) && !f.locals.count(t.text); + } + if (t.kind == TokKind::Punct) + return t.text == "::" || t.text == "*" || t.text == "&" || t.text == "<" || + t.text == ">" || t.text == ","; + return false; + } + + // '(' at pos starts a C-style cast? + bool isCast() const { + int close = matchBracket(T, pos); + if (close < 0 || close >= end || close == pos + 1) return false; + int idents = 0; + bool ptr = false, builtin = false; + for (int i = pos + 1; i < close; ++i) { + if (!isTypeTok(i)) return false; + if (T[i].kind == TokKind::Ident) { + idents++; + if (isBuiltinType(T[i].text)) builtin = true; + } else if (T[i].text == "*" || T[i].text == "&") { + ptr = true; + } + } + if (idents == 0) return false; + if (close + 1 >= end) return false; + const Token& n = T[close + 1]; + if (n.kind == TokKind::Punct) { + const std::string& s = n.text; + bool ambiguous = s == "-" || s == "+" || s == "*" || s == "&"; + if (ambiguous && !ptr && !builtin && idents == 1) return false; + return s == "(" || s == "!" || s == "~" || s == "-" || s == "+" || s == "*" || + s == "&" || s == "++" || s == "--" || s == "::"; + } + if (n.kind == TokKind::Ident) return !isKeyword(n.text) || n.text == "this" || + n.text == "sizeof" || n.text == "true" || + n.text == "false" || isBuiltinType(n.text); + return n.kind != TokKind::PP && n.kind != TokKind::End; + } + + std::unique_ptr parse(int minPrec) { + auto lhs = unary(); + if (!lhs) return nullptr; + while (pos < end && !fail) { + const Token& t = T[pos]; + if (t.kind != TokKind::Punct) break; + const std::string& op = t.text; + int b = lhs->b; + if (isAssignOp(op)) { + if (minPrec > 2) break; + int o = pos++; + auto rhs = parse(2); + if (!rhs) return nullptr; + auto x = node(Expr::Assign, b); + x->op = o; + x->e = rhs->e; + x->kids.push_back(std::move(lhs)); + x->kids.push_back(std::move(rhs)); + lhs = std::move(x); + } else if (op == "?") { + if (minPrec > 3) break; + int q = pos++; + auto mid = parse(1); + if (!mid || !at(":")) return fail = true, nullptr; + int c = pos++; + auto rhs = parse(2); + if (!rhs) return nullptr; + auto x = node(Expr::Ternary, b); + x->op = q; + x->colon = c; + x->e = rhs->e; + x->kids.push_back(std::move(lhs)); + x->kids.push_back(std::move(mid)); + x->kids.push_back(std::move(rhs)); + lhs = std::move(x); + } else { + int p = binPrec(op); + if (p == 0 || p < minPrec) break; + int o = pos++; + auto rhs = parse(p + 1); + if (!rhs) return nullptr; + auto x = node(op == "," ? Expr::Comma : Expr::Binary, b); + x->op = o; + x->e = rhs->e; + x->kids.push_back(std::move(lhs)); + x->kids.push_back(std::move(rhs)); + lhs = std::move(x); + } + } + return fail ? nullptr : std::move(lhs); + } + + std::unique_ptr unary() { + if (pos >= end) return fail = true, nullptr; + const Token& t = T[pos]; + int b = pos; + if (t.kind == TokKind::Punct) { + const std::string& s = t.text; + if (s == "+" || s == "-" || s == "!" || s == "~" || s == "*" || s == "&" || + s == "++" || s == "--") { + pos++; + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Unary, b); + x->op = b; + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + if (s == "(" && isCast()) { + int close = matchBracket(T, pos); + pos = close + 1; + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Cast, b); + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + } else if (t.kind == TokKind::Ident) { + if (t.text == "sizeof") { + pos++; + if (at("(")) { + // sizeof(type) or sizeof(expr): treat the group as opaque + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + pos = close + 1; + auto x = node(Expr::Primary, b); + x->e = pos; + return postfix(std::move(x)); + } + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Unary, b); + x->op = b; + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + if (t.text == "new" || t.text == "delete" || t.text == "throw") + return fail = true, nullptr; + } + return postfix(primary()); + } + + std::unique_ptr primary() { + if (pos >= end) return fail = true, nullptr; + const Token& t = T[pos]; + int b = pos; + if (t.kind == TokKind::Number || t.kind == TokKind::Char) { + pos++; + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::String) { + while (pos < end && T[pos].kind == TokKind::String) pos++; + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::Ident || (t.kind == TokKind::Punct && t.text == "::")) { + const std::string& s = t.text; + if (s == "static_cast" || s == "reinterpret_cast" || s == "const_cast" || + s == "dynamic_cast") { + pos++; + if (!at("<")) return fail = true, nullptr; + int depth = 0; + while (pos < end) { + if (T[pos].text == "<") depth++; + if (T[pos].text == ">" && --depth == 0) break; + pos++; + } + pos++; + if (!at("(")) return fail = true, nullptr; + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + ExprParser inner(f, pos + 1, close); + auto in = inner.parse(1); + if (!in || inner.pos != close) return fail = true, nullptr; + pos = close + 1; + auto x = node(Expr::Cast, b); + x->e = pos; + x->kids.push_back(std::move(in)); + return x; + } + if (t.kind == TokKind::Ident && isKeyword(s) && s != "this" && s != "true" && + s != "false" && s != "nullptr" && !isBuiltinType(s)) + return fail = true, nullptr; + if (t.kind == TokKind::Punct) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + pos++; + while (at("::") && pos + 1 < end) { + pos++; + if (at("~")) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + pos++; + } + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::Punct && t.text == "(") { + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + ExprParser inner(f, pos + 1, close); + auto in = inner.parse(1); + if (!in || inner.pos != close) return fail = true, nullptr; + pos = close + 1; + auto x = node(Expr::Paren, b); + x->e = pos; + x->kids.push_back(std::move(in)); + return x; + } + return fail = true, nullptr; + } + + std::unique_ptr postfix(std::unique_ptr x) { + if (!x) return nullptr; + while (pos < end && !fail) { + int b = x->b; + if (at("(")) { + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + auto c = node(Expr::Call, b); + c->kids.push_back(std::move(x)); + int p = pos + 1; + while (p < close) { + ExprParser a(f, p, close); + auto arg = a.parse(2); + if (!arg) return fail = true, nullptr; + c->kids.push_back(std::move(arg)); + p = a.pos; + if (p < close) { + if (T[p].text != ",") return fail = true, nullptr; + p++; + } + } + pos = close + 1; + c->e = pos; + x = std::move(c); + } else if (at("[")) { + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + ExprParser a(f, pos + 1, close); + auto idx = a.parse(1); + if (!idx || a.pos != close) return fail = true, nullptr; + auto c = node(Expr::Index, b); + c->kids.push_back(std::move(x)); + c->kids.push_back(std::move(idx)); + pos = close + 1; + c->e = pos; + x = std::move(c); + } else if (at(".") || at("->")) { + pos++; + if (at("~")) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + pos++; + while (at("::") && pos + 1 < end && T[pos + 1].kind == TokKind::Ident) pos += 2; + auto c = node(Expr::Member, b); + c->kids.push_back(std::move(x)); + c->e = pos; + x = std::move(c); + } else if (at("++") || at("--")) { + auto c = node(Expr::Postfix, b); + c->op = pos++; + c->kids.push_back(std::move(x)); + c->e = pos; + x = std::move(c); + } else { + break; + } + } + return fail ? nullptr : std::move(x); + } +}; + +} // namespace + +std::unique_ptr parseExpr(const Func& f, int b, int e) { + if (b >= e) return nullptr; + ExprParser p(f, b, e); + auto x = p.parse(1); + if (!x || p.pos != e) return nullptr; + return x; +} + +std::vector exprSites(const Func& f) { + std::vector out; + auto addDecl = [&](const Stmt& s) { + for (auto& d : s.decls) + if (d.init == Declarator::Assign && f.t(d.initBegin) != "{") + out.push_back({d.initBegin, d.initEnd, &s}); + }; + forEachStmt(*f.body, [&](const Stmt& s) { + switch (s.kind) { + case SK::Expr: + if (s.end - 1 > s.begin && f.t(s.end - 1) == ";") out.push_back({s.begin, s.end - 1, &s}); + break; + case SK::Return: + if (f.t(s.begin) == "return" && f.t(s.end - 1) == ";" && s.end - 1 > s.begin + 1) + out.push_back({s.begin + 1, s.end - 1, &s}); + break; + case SK::Decl: addDecl(s); break; + case SK::If: + case SK::While: + case SK::Switch: + case SK::Do: + if (s.condOpen >= 0) out.push_back({s.condOpen + 1, s.condClose, &s}); + break; + case SK::For: { + int depth = 0, segStart = s.condOpen + 1, seg = 0; + for (int i = s.condOpen + 1; i <= s.condClose; ++i) { + const std::string& x = f.t(i); + if (i < s.condClose && (x == "(" || x == "[" || x == "{")) depth++; + else if (i < s.condClose && (x == ")" || x == "]" || x == "}")) depth--; + if ((x == ";" && depth == 0) || i == s.condClose) { + if (seg == 0 && s.forInit) addDecl(*s.forInit); + else if (i > segStart) out.push_back({segStart, i, &s}); + segStart = i + 1; + seg++; + } + } + break; + } + default: break; + } + }); + return out; +} + +bool isWriteUse(const Func& f, int i) { + int n = (int)f.toks.size(); + const std::string& next = i + 1 < n ? f.t(i + 1) : ""; + const std::string& prev = i > 0 ? f.t(i - 1) : ""; + bool prevUnary = i < 2 || !operandEnd(f, i - 2); + if (isAssignOp(next) && !(prev == "*" && prevUnary)) return true; + if (next == "++" || next == "--" || prev == "++" || prev == "--") return true; + if (prev == "&" && prevUnary) return true; // address taken + return false; +} + +Effects effectsOf(const Func& f, int b, int e) { + Effects r; + for (int i = b; i < e; ++i) { + const Token& t = f.toks[i]; + if (t.kind == TokKind::PP) { + r.barrier = true; + continue; + } + if (t.kind == TokKind::Ident) { + const std::string& x = t.text; + if (x == "return" || x == "break" || x == "continue" || x == "goto" || x == "throw" || + x == "__asm" || x == "_asm" || x == "asm" || x == "case" || x == "default" || + x == "__leave") { + r.barrier = true; + continue; + } + if (x == "new" || x == "delete") { + r.call = true; + continue; + } + if (f.isLocal(i)) { + if (isWriteUse(f, i)) { + r.writes.insert(x); + if (f.t(i + 1) != "=") r.reads.insert(x); + } else { + r.reads.insert(x); + } + continue; + } + if (isKeyword(x)) continue; + if (i + 1 < (int)f.toks.size() && f.t(i + 1) == "(") { + r.call = true; + continue; + } + r.memRead = true; // member, global or something we can't see + continue; + } + if (t.kind != TokKind::Punct) continue; + const std::string& x = t.text; + if (x == "->" || x == "[") r.memRead = true; + else if (x == "*" && !operandEnd(f, i - 1)) r.memRead = true; + else if (isAssignOp(x)) { + bool plainLocal = f.isLocal(i - 1) && + !(i >= 2 && (f.t(i - 2) == "*" || f.t(i - 2) == "&") && + !operandEnd(f, i - 3)); + if (!plainLocal) r.memWrite = true; + } else if (x == "++" || x == "--") { + bool local = f.isLocal(i + 1) || (f.isLocal(i - 1)); + if (!local) r.memWrite = true; + } + } + return r; +} + +Effects effectsOfStmt(const Func& f, const Stmt& s) { + Effects r = effectsOf(f, s.begin, s.end); + switch (s.kind) { + case SK::Return: + case SK::Break: + case SK::Continue: + case SK::Goto: + case SK::Label: + case SK::Case: + case SK::Default: + case SK::Asm: + case SK::PP: + case SK::Other: + case SK::Try: r.barrier = true; break; + case SK::Decl: + if (s.isStatic) r.barrier = true; + for (auto& d : s.decls) { + r.writes.insert(f.t(d.nameTok)); + r.reads.erase(f.t(d.nameTok)); + } + break; + default: break; + } + return r; +} + +bool conflicts(const Effects& a, const Effects& b) { + if (a.barrier || b.barrier) return true; + for (auto& w : a.writes) + if (b.reads.count(w) || b.writes.count(w)) return true; + for (auto& w : b.writes) + if (a.reads.count(w)) return true; + bool aw = a.call || a.memWrite, bw = b.call || b.memWrite; + if (aw && (bw || b.memRead)) return true; + if (bw && a.memRead) return true; + return false; +} + +bool conflictsRelaxed(const Effects& a, const Effects& b) { + if (a.barrier || b.barrier) return true; + for (auto& w : a.writes) + if (b.reads.count(w) || b.writes.count(w)) return true; + for (auto& w : b.writes) + if (a.reads.count(w)) return true; + if (a.call && (b.call || b.memWrite || b.memRead)) return true; + if (b.call && (a.memWrite || a.memRead)) return true; + return false; +} + +std::vector usesOf(const Func& f, const std::string& name, int b, int e) { + std::vector r; + for (int i = b; i < e; ++i) + if (f.toks[i].kind == TokKind::Ident && f.toks[i].text == name && f.isLocal(i)) r.push_back(i); + return r; +} + +} // namespace perm diff --git a/src/analysis.hpp b/src/analysis.hpp new file mode 100644 index 0000000..d7838e8 --- /dev/null +++ b/src/analysis.hpp @@ -0,0 +1,49 @@ +// Expression parsing and read/write effects over token ranges. +#pragma once + +#include "parser.hpp" + +#include +#include +#include + +namespace perm { + +struct Expr { + enum K { Primary, Paren, Call, Index, Member, Postfix, Unary, Cast, Binary, Assign, Ternary, Comma } k; + int b = 0, e = 0; // token range [b, e) + int op = -1; // operator token (Binary/Assign/Unary/Postfix/Ternary '?') + int colon = -1; // Ternary ':' + std::vector> kids; +}; + +// Parses tokens [b, e) as one expression. Returns null if the whole range +// doesn't parse. +std::unique_ptr parseExpr(const Func& f, int b, int e); +int binPrec(const std::string& op); // 0 = not a binary operator +void forEachExpr(Expr& x, const std::function& fn); + +// Token ranges in the function that hold a complete expression. +struct ExprSite { + int b, e; + const Stmt* stmt; +}; +std::vector exprSites(const Func& f); + +struct Effects { + std::set reads, writes; + bool memRead = false, memWrite = false, call = false; + bool barrier = false; // control flow, labels, asm, preprocessor +}; +Effects effectsOf(const Func& f, int b, int e); +Effects effectsOfStmt(const Func& f, const Stmt& s); +bool conflicts(const Effects& a, const Effects& b); +// Like conflicts(), but two statements that only read or write memory (no +// calls) may swap: field stores and loads are assumed not to alias. +bool conflictsRelaxed(const Effects& a, const Effects& b); + +// Number of token uses of a local name in [b, e). +std::vector usesOf(const Func& f, const std::string& name, int b, int e); +bool isWriteUse(const Func& f, int tok); + +} // namespace perm diff --git a/src/lexer.cpp b/src/lexer.cpp new file mode 100644 index 0000000..362bc72 --- /dev/null +++ b/src/lexer.cpp @@ -0,0 +1,201 @@ +#include "lexer.hpp" + +#include +#include +#include + +namespace perm { + +namespace { + +const char* kPuncts[] = { + ">>=", "<<=", "->*", "...", "::", "->", "++", "--", "<<", ">>", "<=", ">=", "==", "!=", + "&&", "||", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", ".*", "##", +}; + +bool identStart(char c) { return std::isalpha((unsigned char)c) || c == '_' || c == '$'; } +bool identChar(char c) { return std::isalnum((unsigned char)c) || c == '_' || c == '$'; } + +} // namespace + +std::vector lex(const std::string& s) { + std::vector out; + size_t i = 0, n = s.size(); + int line = 1; + bool lineStart = true; // only whitespace seen on this line so far + std::string lead; + + auto advance = [&](size_t to) { + for (size_t k = i; k < to && k < n; ++k) + if (s[k] == '\n') line++; + i = to; + }; + + while (i < n) { + char c = s[i]; + // whitespace + if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v') { + if (c == '\n') lineStart = true; + lead += c; + advance(i + 1); + continue; + } + // line continuation outside of PP lines + if (c == '\\' && i + 1 < n && (s[i + 1] == '\n' || s[i + 1] == '\r')) { + lead += c; + advance(i + 1); + continue; + } + // comments + if (c == '/' && i + 1 < n && s[i + 1] == '/') { + size_t e = s.find('\n', i); + if (e == std::string::npos) e = n; + lead.append(s, i, e - i); + advance(e); + continue; + } + if (c == '/' && i + 1 < n && s[i + 1] == '*') { + size_t e = s.find("*/", i + 2); + e = (e == std::string::npos) ? n : e + 2; + lead.append(s, i, e - i); + advance(e); + continue; + } + + Token t; + t.line = line; + t.offset = i; + t.lead = std::move(lead); + lead.clear(); + size_t start = i, end = i; + + if (c == '#' && lineStart) { + // preprocessor line, with backslash continuations + end = i; + while (end < n) { + if (s[end] == '\n') { + size_t b = end; + if (b > start && s[b - 1] == '\r') b--; + if (b > start && s[b - 1] == '\\') { + end++; + continue; + } + break; + } + // a block comment can span lines inside a directive + if (s[end] == '/' && end + 1 < n && s[end + 1] == '*') { + size_t e = s.find("*/", end + 2); + end = (e == std::string::npos) ? n : e + 2; + continue; + } + end++; + } + // keep the trailing \r out of the token text + while (end > start && (s[end - 1] == '\r')) end--; + t.kind = TokKind::PP; + } else if (identStart(c)) { + end = i; + while (end < n && identChar(s[end])) end++; + std::string id = s.substr(start, end - start); + if (end < n && (s[end] == '"' || s[end] == '\'') && + (id == "L" || id == "u" || id == "U" || id == "u8")) { + char q = s[end]; + end++; + while (end < n && s[end] != q) { + if (s[end] == '\\') end++; + end++; + } + if (end < n) end++; + t.kind = q == '"' ? TokKind::String : TokKind::Char; + } else { + t.kind = TokKind::Ident; + } + } else if (std::isdigit((unsigned char)c) || + (c == '.' && i + 1 < n && std::isdigit((unsigned char)s[i + 1]))) { + end = i; + bool hex = c == '0' && i + 1 < n && (s[i + 1] == 'x' || s[i + 1] == 'X'); + while (end < n) { + char d = s[end]; + if (identChar(d) || d == '.') { + end++; + continue; + } + if ((d == '+' || d == '-') && !hex && end > start && + (s[end - 1] == 'e' || s[end - 1] == 'E')) { + end++; + continue; + } + break; + } + t.kind = TokKind::Number; + } else if (c == '"' || c == '\'') { + end = i + 1; + while (end < n && s[end] != c && s[end] != '\n') { + if (s[end] == '\\') end++; + end++; + } + if (end < n && s[end] == c) end++; + t.kind = c == '"' ? TokKind::String : TokKind::Char; + } else { + t.kind = TokKind::Punct; + end = i + 1; + for (const char* p : kPuncts) { + size_t len = std::strlen(p); + if (s.compare(i, len, p) == 0) { + end = i + len; + break; + } + } + } + t.text = s.substr(start, end - start); + advance(end); + lineStart = false; + out.push_back(std::move(t)); + } + Token e; + e.kind = TokKind::End; + e.lead = std::move(lead); + e.line = line; + e.offset = n; + out.push_back(std::move(e)); + return out; +} + +std::string joinTokens(const std::vector& toks, int b, int e) { + std::string r; + for (int i = b; i < e; ++i) { + if (i != b) r += toks[i].lead; + r += toks[i].text; + } + return r; +} + +bool isKeyword(const std::string& s) { + static const std::unordered_set kw = { + "if", "else", "for", "while", "do", "switch", "case", "default", "return", "break", + "continue", "goto", "sizeof", "new", "delete", "this", "true", "false", "static", + "const", "volatile", "register", "extern", "mutable", "inline", "typedef", "struct", + "class", "union", "enum", "template", "typename", "using", "namespace", "try", + "catch", "throw", "operator", "static_cast", "reinterpret_cast", "const_cast", + "dynamic_cast", "__asm", "_asm", "asm", "__try", "__except", "__finally", "__leave", + "__forceinline", "__inline", "__declspec", "__stdcall", "__cdecl", "__fastcall", + "__thiscall", "virtual", "public", "private", "protected", "friend", "explicit", + "auto", "typeid", "nullptr", + }; + return kw.count(s) || isBuiltinType(s); +} + +bool isBuiltinType(const std::string& s) { + static const std::unordered_set t = { + "void", "char", "short", "int", "long", "float", "double", "signed", "unsigned", + "bool", "wchar_t", "__int8", "__int16", "__int32", "__int64", + }; + return t.count(s) > 0; +} + +bool isAssignOp(const std::string& s) { + return s == "=" || s == "+=" || s == "-=" || s == "*=" || s == "/=" || s == "%=" || + s == "&=" || s == "|=" || s == "^=" || s == "<<=" || s == ">>="; +} + +} // namespace perm diff --git a/src/lexer.hpp b/src/lexer.hpp new file mode 100644 index 0000000..247b3e1 --- /dev/null +++ b/src/lexer.hpp @@ -0,0 +1,30 @@ +// Tolerant C++ tokenizer. Every token keeps the whitespace and comments that +// precede it ("lead"), so joining lead + text over all tokens reproduces the +// input exactly. Preprocessor lines become single PP tokens. +#pragma once + +#include +#include + +namespace perm { + +enum class TokKind { Ident, Number, String, Char, Punct, PP, End }; + +struct Token { + TokKind kind = TokKind::End; + std::string text; + std::string lead; // whitespace/comments before the token + int line = 0; // 1-based line of the token text + size_t offset = 0; // byte offset of text in the source +}; + +std::vector lex(const std::string& src); + +// Joins tokens [b, e): lead of b is omitted, leads of the rest are kept. +std::string joinTokens(const std::vector& toks, int b, int e); + +bool isKeyword(const std::string& s); +bool isBuiltinType(const std::string& s); +bool isAssignOp(const std::string& s); + +} // namespace perm diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..0f57068 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,766 @@ +// cpp_permuter: brute-forces source permutations of one C++ function until +// its compiled code matches (or gets closer to) a target object. + +#include "analysis.hpp" +#include "parser.hpp" +#include "passes.hpp" +#include "runner.hpp" +#include "scorer.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace perm; + +namespace { + +const char* kUsage = R"(usage: cpp_permuter [options] + +Finds a source permutation of one function that compiles to the target code. + +Required: + -s, --source FILE .cpp file holding the function + -f, --function NAME function to permute, e.g. PedGroup::PromoteMemberToLeader_4C9680 + -c, --compile CMD compile command. {src} is the candidate source, {obj} the + object to write, {dir} the original source's directory + -t, --target-obj FILE object file with the target code (not needed with --score-cmd) + +Scoring: + --symbol SYM symbol of the function in the compiled object + (default: guessed from --function) + --target-symbol SYM symbol in the target object (default: same as --symbol) + --score-cmd CMD score with this command instead ({obj}, {src}); it must + print the score (lower is better, 0 = match) as its last number + --objdump PATH llvm-objdump to use (default: llvm-objdump) + --ignore-reloc-names compare relocated operands without their symbol names + (for targets built from raw executable asm) + +Search: + -m, --mode MODE random (default) or exhaustive + -p, --passes LIST comma separated passes (default: all). See --list-passes + --depth N exhaustive: apply up to N mutations in a row (default 1) + --max-candidates N exhaustive: stop after N candidates (default 200000) + -n, --iterations N random: stop after N compiles (default: run until match/Ctrl-C) + --max-mutations N random: up to N mutations per candidate (default 3) + --seed N random seed + --keep-going don't stop at the first exact match + -j, --jobs N parallel compiles (default 1) + --timeout SEC compile timeout (default 120) + +Output: + -o, --output-dir DIR where improvements go (default: permuter_out) + --candidate-dir DIR where candidate sources are written (default: next to + the source, so relative #includes keep working) + --config FILE read "key = value" lines as if they were --key value + --list-passes list the passes and exit + --show-ast print the parsed statement tree and exit + --dry-run print the candidates' diffs instead of compiling + --show-base-diff print the asm diff of the unmodified function first + -v, --verbose print compile errors +)"; + +struct Options { + std::string source, function, compile, targetObj, symbol, targetSymbol, scoreCmd; + std::string objdump = "llvm-objdump"; + bool ignoreRelocNames = false; + std::string mode = "random"; + std::vector passes; + int depth = 1; + long maxCandidates = 200000; + long iterations = 0; + int maxMutations = 3; + unsigned long long seed = 0; + bool seedSet = false; + bool keepGoing = false; + int jobs = 1; + int timeout = 120; + std::string outputDir = "permuter_out"; + std::string candidateDir; + bool listPasses = false, showAst = false, dryRun = false, showBaseDiff = false, verbose = false; +}; + +std::atomic gStop{false}; + +void onSignal(int) { gStop = true; } + +std::vector split(const std::string& s, char sep) { + std::vector r; + std::string cur; + for (char c : s) { + if (c == sep) { + if (!cur.empty()) r.push_back(cur); + cur.clear(); + } else if (c != ' ') { + cur += c; + } + } + if (!cur.empty()) r.push_back(cur); + return r; +} + +bool readFile(const std::string& path, std::string& out) { + std::ifstream in(path, std::ios::binary); + if (!in) return false; + std::ostringstream ss; + ss << in.rdbuf(); + out = ss.str(); + return true; +} + +bool writeFile(const std::string& path, const std::string& data) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) return false; + out << data; + return (bool)out; +} + +bool applyOption(Options& o, const std::string& key, const std::string& val, bool hasVal, + std::string& err) { + auto need = [&]() { + if (!hasVal) err = "--" + key + " needs a value"; + return hasVal; + }; + auto num = [&](auto& dst) { + if (!need()) return false; + try { + dst = (std::remove_reference_t)std::stoll(val); + } catch (...) { + err = "--" + key + ": not a number: " + val; + return false; + } + return true; + }; + if (key == "source" || key == "s") return need() && (o.source = val, true); + if (key == "function" || key == "f") return need() && (o.function = val, true); + if (key == "compile" || key == "c") return need() && (o.compile = val, true); + if (key == "target-obj" || key == "t") return need() && (o.targetObj = val, true); + if (key == "symbol") return need() && (o.symbol = val, true); + if (key == "target-symbol") return need() && (o.targetSymbol = val, true); + if (key == "score-cmd") return need() && (o.scoreCmd = val, true); + if (key == "objdump") return need() && (o.objdump = val, true); + if (key == "ignore-reloc-names") return o.ignoreRelocNames = true; + if (key == "mode" || key == "m") return need() && (o.mode = val, true); + if (key == "passes" || key == "p") return need() && (o.passes = split(val, ','), true); + if (key == "depth") return num(o.depth); + if (key == "max-candidates") return num(o.maxCandidates); + if (key == "iterations" || key == "n") return num(o.iterations); + if (key == "max-mutations") return num(o.maxMutations); + if (key == "seed") return num(o.seed) && (o.seedSet = true); + if (key == "keep-going") return o.keepGoing = true; + if (key == "jobs" || key == "j") return num(o.jobs); + if (key == "timeout") return num(o.timeout); + if (key == "output-dir" || key == "o") return need() && (o.outputDir = val, true); + if (key == "candidate-dir") return need() && (o.candidateDir = val, true); + if (key == "list-passes") return o.listPasses = true; + if (key == "show-ast") return o.showAst = true; + if (key == "dry-run") return o.dryRun = true; + if (key == "show-base-diff") return o.showBaseDiff = true; + if (key == "verbose" || key == "v") return o.verbose = true; + err = "unknown option --" + key; + return false; +} + +bool isFlag(const std::string& k) { + static const std::set flags = { + "ignore-reloc-names", "keep-going", "list-passes", "show-ast", "dry-run", + "show-base-diff", "verbose", "v", + }; + return flags.count(k) > 0; +} + +bool loadConfig(Options& o, const std::string& path, std::string& err) { + std::string text; + if (!readFile(path, text)) { + err = "can't read config " + path; + return false; + } + std::istringstream in(text); + std::string line; + while (std::getline(in, line)) { + size_t b = line.find_first_not_of(" \t"); + if (b == std::string::npos || line[b] == '#') continue; + line = line.substr(b); + while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) line.pop_back(); + size_t eq = line.find('='); + std::string key = line.substr(0, eq), val; + while (!key.empty() && key.back() == ' ') key.pop_back(); + if (eq != std::string::npos) { + val = line.substr(eq + 1); + size_t vb = val.find_first_not_of(" \t"); + val = vb == std::string::npos ? "" : val.substr(vb); + if (val.size() >= 2 && val.front() == '"' && val.back() == '"') val = val.substr(1, val.size() - 2); + } + if (!applyOption(o, key, val, eq != std::string::npos, err)) return false; + } + return true; +} + +bool parseArgs(int argc, char** argv, Options& o, std::string& err) { + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "-h" || a == "--help") { + std::cout << kUsage; + std::exit(0); + } + std::string key, val; + bool hasVal = false; + if (a.rfind("--", 0) == 0) { + key = a.substr(2); + size_t eq = key.find('='); + if (eq != std::string::npos) { + val = key.substr(eq + 1); + key = key.substr(0, eq); + hasVal = true; + } + } else if (a.size() == 2 && a[0] == '-') { + key = a.substr(1); + } else { + err = "unexpected argument: " + a; + return false; + } + if (!hasVal && !isFlag(key) && i + 1 < argc) { + val = argv[++i]; + hasVal = true; + } + if (key == "config") { + if (!hasVal || !loadConfig(o, val, err)) { + if (err.empty()) err = "--config needs a value"; + return false; + } + continue; + } + if (!applyOption(o, key, val, hasVal, err)) return false; + } + return true; +} + +// Line diff (LCS) with a little context, for the output files. +std::string lineDiff(const std::string& a, const std::string& b) { + auto lines = [](const std::string& s) { + std::vector r; + std::istringstream in(s); + std::string l; + while (std::getline(in, l)) r.push_back(l); + return r; + }; + auto A = lines(a), B = lines(b); + size_t n = A.size(), m = B.size(); + std::vector> L(n + 1, std::vector(m + 1, 0)); + for (size_t i = n; i-- > 0;) + for (size_t j = m; j-- > 0;) + L[i][j] = A[i] == B[j] ? L[i + 1][j + 1] + 1 : std::max(L[i + 1][j], L[i][j + 1]); + std::vector> ops; + size_t i = 0, j = 0; + while (i < n || j < m) { + if (i < n && j < m && A[i] == B[j]) ops.push_back({' ', A[i++]}), j++; + else if (j < m && (i == n || L[i][j + 1] >= L[i + 1][j])) ops.push_back({'+', B[j++]}); + else ops.push_back({'-', A[i++]}); + } + std::string out; + const int ctx = 2; + std::vector show(ops.size(), false); + for (size_t k = 0; k < ops.size(); ++k) + if (ops[k].first != ' ') + for (long d = -ctx; d <= ctx; ++d) + if ((long)k + d >= 0 && (long)k + d < (long)ops.size()) show[k + d] = true; + bool gap = false; + for (size_t k = 0; k < ops.size(); ++k) { + if (!show[k]) { + if (!gap && !out.empty()) out += "...\n"; + gap = true; + continue; + } + gap = false; + out += ops[k].first; + out += ops[k].second + "\n"; + } + return out; +} + +double lastNumber(const std::string& s, bool& ok) { + ok = false; + double v = 0; + for (size_t i = 0; i < s.size(); ++i) { + if (std::isdigit((unsigned char)s[i]) || (s[i] == '-' && i + 1 < s.size() && std::isdigit((unsigned char)s[i + 1]))) { + size_t used = 0; + try { + v = std::stod(s.substr(i), &used); + ok = true; + i += used - 1; + } catch (...) { + } + } + } + return v; +} + +struct Evaluator { + const Options& o; + ScoreConfig sc; + std::string prefix, suffix, srcDir, stem, ext, symbol, targetSymbol; + std::vector target; + std::vector candPaths; + + std::string srcPath(int w) const { + std::string dir = o.candidateDir.empty() ? srcDir : o.candidateDir; + return (fs::path(dir) / (stem + ".permuter" + std::to_string(w) + ext)).string(); + } + std::string objPath(int w) const { + return (fs::path(o.outputDir) / ".work" / ("w" + std::to_string(w) + ".obj")).string(); + } + + bool compile(int w, const std::string& funcText, std::string& log) const { + std::string src = srcPath(w), obj = objPath(w); + if (!writeFile(src, prefix + funcText + suffix)) { + log = "can't write " + src; + return false; + } + std::error_code ec; + fs::remove(obj, ec); + std::string cmd = expandTemplate(o.compile, {{"src", src}, {"obj", obj}, {"dir", srcDir}}); + CmdResult r = runCommand(cmd, o.timeout); + log = r.output; + if (r.timedOut) log += "\n(timed out)"; + return r.status == 0 && fs::exists(obj); + } + + // Score of a compiled candidate; -1 if it couldn't be scored. + long score(int w, std::vector* insns, std::string& log) const { + std::string obj = objPath(w); + if (!o.scoreCmd.empty()) { + CmdResult r = runCommand(expandTemplate(o.scoreCmd, {{"obj", obj}, {"src", srcPath(w)}}), o.timeout); + bool ok; + double v = lastNumber(r.output, ok); + if (r.status != 0 || !ok) { + log = r.output; + return -1; + } + return (long)(v + 0.5); + } + std::vector mine; + if (!disassemble(sc, obj, symbol, mine, log)) return -1; + if (insns) *insns = mine; + return scoreInsns(sc, target, mine); + } + + long eval(int w, const std::string& funcText, std::string& log, std::vector* insns = nullptr) const { + if (!compile(w, funcText, log)) return -1; + return score(w, insns, log); + } +}; + +struct State { + std::mutex mu; + std::string baseText, bestText; + long baseScore = 0, bestScore = 0; + long compiles = 0, failures = 0, improvements = 0; + std::unordered_set seen; + int outputs = 0; +}; + +void writeOutput(const Options& o, const Evaluator& ev, State& st, const std::string& text, + long score, const std::vector& insns) { + std::string dir = (fs::path(o.outputDir) / ("output-" + std::to_string(score) + "-" + + std::to_string(++st.outputs))) + .string(); + fs::create_directories(dir); + writeFile(dir + "/function.cpp", text + "\n"); + writeFile(dir + "/source.cpp", ev.prefix + text + ev.suffix); + writeFile(dir + "/diff.txt", lineDiff(st.baseText, text)); + if (!insns.empty()) writeFile(dir + "/asm_diff.txt", diffInsns(ev.sc, ev.target, insns)); + writeFile(dir + "/score.txt", std::to_string(score) + "\n"); + std::cout << "\n[" << score << "] new best (was " << st.bestScore << "), written to " << dir + << "\n" + << lineDiff(st.baseText, text) << std::flush; +} + +// Records a scored candidate. Returns true if it was a match and we should stop. +bool report(const Options& o, const Evaluator& ev, State& st, const std::string& text, + long score, const std::vector& insns, const std::string& log) { + std::lock_guard lk(st.mu); + st.compiles++; + if (score < 0) { + st.failures++; + if (o.verbose) std::cout << "\ncandidate failed:\n" << log << "\n"; + return false; + } + if (score < st.bestScore) { + writeOutput(o, ev, st, text, score, insns); + st.bestScore = score; + st.bestText = text; + st.improvements++; + } + return score == 0 && !o.keepGoing; +} + +size_t hashText(const std::string& s) { return std::hash()(s); } + +void printStatus(State& st) { + std::lock_guard lk(st.mu); + std::cout << "\riterations: " << st.compiles << ", failed: " << st.failures + << ", base: " << st.baseScore << ", best: " << st.bestScore << " " + << std::flush; +} + +std::vector pickPasses(const Options& o, std::string& err) { + std::vector r; + if (o.passes.empty()) { + for (auto& p : allPasses()) r.push_back(&p); + return r; + } + for (auto& n : o.passes) { + const Pass* p = findPass(n); + if (!p) { + err = "unknown pass '" + n + "' (see --list-passes)"; + return {}; + } + r.push_back(p); + } + return r; +} + +// One random candidate derived from start; empty if nothing applied. +std::string randomCandidate(const Options& o, const std::vector& passes, + const std::string& start, Rng& rng) { + int total = 0; + for (auto* p : passes) total += p->weight; + int k = std::uniform_int_distribution(1, std::max(1, o.maxMutations))(rng); + std::string text = start; + for (int m = 0; m < k; ++m) { + std::string err; + auto f = parseFunc(text, err); + if (!f) break; + for (int attempt = 0; attempt < 10; ++attempt) { + int r = std::uniform_int_distribution(0, total - 1)(rng); + const Pass* p = passes.back(); + for (auto* q : passes) { + if (r < q->weight) { + p = q; + break; + } + r -= q->weight; + } + std::string out; + if (randomMutation(*p, *f, rng, out) && out != text) { + text = out; + break; + } + } + } + return text == start ? "" : text; +} + +int runRandom(const Options& o, const Evaluator& ev, State& st, + const std::vector& passes) { + unsigned long long seed = o.seedSet ? o.seed : (unsigned long long)std::random_device{}(); + std::vector workers; + std::atomic idle{0}; + for (int w = 0; w < o.jobs; ++w) { + workers.emplace_back([&, w]() { + Rng rng(seed + (unsigned long long)w * 7919); + int misses = 0; + while (!gStop) { + std::string start; + { + std::lock_guard lk(st.mu); + if (o.iterations && st.compiles >= o.iterations) break; + bool fromBest = st.bestText != st.baseText && + std::uniform_int_distribution(0, 1)(rng) == 0; + start = fromBest ? st.bestText : st.baseText; + } + std::string cand = randomCandidate(o, passes, start, rng); + bool fresh = false; + if (!cand.empty()) { + std::lock_guard lk(st.mu); + fresh = st.seen.insert(hashText(cand)).second; + } + if (!fresh) { + if (++misses > 2000) break; // nothing new left to try + continue; + } + misses = 0; + std::string log; + std::vector insns; + long s = ev.eval(w, cand, log, &insns); + if (report(o, ev, st, cand, s, insns, log)) gStop = true; + } + idle++; + }); + } + while (idle < o.jobs) { + for (int i = 0; i < 20 && idle < o.jobs; ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + printStatus(st); + } + for (auto& t : workers) t.join(); + return 0; +} + +int runExhaustive(const Options& o, const Evaluator& ev, State& st, + const std::vector& passes) { + std::mutex qmu; + std::condition_variable qcv; + std::deque queue; + bool done = false; + long produced = 0; + + auto worker = [&](int w) { + while (true) { + std::string cand; + { + std::unique_lock lk(qmu); + qcv.wait(lk, [&] { return !queue.empty() || done || gStop; }); + if (gStop || (queue.empty() && done)) return; + cand = std::move(queue.front()); + queue.pop_front(); + } + qcv.notify_all(); + std::string log; + std::vector insns; + long s = ev.eval(w, cand, log, &insns); + if (report(o, ev, st, cand, s, insns, log)) { + gStop = true; + qcv.notify_all(); + } + } + }; + std::vector workers; + for (int w = 0; w < o.jobs; ++w) workers.emplace_back(worker, w); + + std::thread producer([&]() { + std::vector level = {st.baseText}; + for (int d = 1; d <= o.depth && !gStop; ++d) { + std::vector next; + for (auto& text : level) { + std::string err; + auto f = parseFunc(text, err); + if (!f) continue; + for (auto* p : passes) { + p->enumerate(*f, [&](Mutation m) { + if (gStop || produced >= o.maxCandidates) return false; + std::string s = m(); + if (s.empty()) return true; + { + std::lock_guard lk(st.mu); + if (!st.seen.insert(hashText(s)).second) return true; + } + produced++; + if (d < o.depth) next.push_back(s); + std::unique_lock lk(qmu); + qcv.wait(lk, [&] { return queue.size() < 256 || gStop; }); + queue.push_back(std::move(s)); + qcv.notify_all(); + return true; + }); + } + } + level = std::move(next); + } + std::lock_guard lk(qmu); + done = true; + qcv.notify_all(); + }); + + while (true) { + { + std::lock_guard lk(qmu); + if ((done && queue.empty()) || gStop) break; + } + for (int i = 0; i < 20; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::lock_guard lk(qmu); + if ((done && queue.empty()) || gStop) break; + } + printStatus(st); + } + producer.join(); + { + std::lock_guard lk(qmu); + done = true; + } + qcv.notify_all(); + for (auto& t : workers) t.join(); + printStatus(st); + std::cout << "\n" << produced << " candidates generated\n"; + return 0; +} + +int dryRun(const Options& o, const std::string& base, const std::vector& passes) { + std::string err; + auto f = parseFunc(base, err); + if (o.mode == "exhaustive") { + long count = 0; + std::unordered_set seen; + for (auto* p : passes) { + long mine = 0; + p->enumerate(*f, [&](Mutation m) { + std::string s = m(); + if (s.empty() || !seen.insert(hashText(s)).second) return true; + mine++; + if (++count <= 50) std::cout << "=== " << p->name << " #" << mine << "\n" << lineDiff(base, s); + return count < o.maxCandidates; + }); + std::cerr << p->name << ": " << mine << " candidates\n"; + } + std::cerr << count << " candidates in total\n"; + return 0; + } + Rng rng(o.seedSet ? o.seed : 1); + long n = o.iterations ? o.iterations : 10; + for (long i = 0; i < n; ++i) { + std::string s = randomCandidate(o, passes, base, rng); + std::cout << "=== random #" << i + 1 << "\n" << (s.empty() ? "(no change)\n" : lineDiff(base, s)); + } + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + Options o; + std::string err; + if (!parseArgs(argc, argv, o, err)) { + std::cerr << "error: " << err << "\n\n" << kUsage; + return 2; + } + if (o.listPasses) { + for (auto& p : allPasses()) std::cout << " " << p.name << "\n " << p.description << "\n"; + return 0; + } + if (o.source.empty() || o.function.empty()) { + std::cerr << "error: --source and --function are required\n\n" << kUsage; + return 2; + } + std::string src; + if (!readFile(o.source, src)) { + std::cerr << "error: can't read " << o.source << "\n"; + return 1; + } + size_t start, end; + if (!locateFunction(src, o.function, start, end, err)) { + std::cerr << "error: " << err << "\n"; + return 1; + } + if (!err.empty()) std::cerr << err << "\n"; + std::string funcText = src.substr(start, end - start); + auto func = parseFunc(funcText, err); + if (!func) { + std::cerr << "error: can't parse " << o.function << ": " << err << "\n"; + return 1; + } + if (o.showAst) { + std::cout << dumpStmt(*func, *func->body); + std::cout << "params:"; + for (auto& p : func->params) std::cout << " " << p; + std::cout << "\nlocals:"; + for (auto& l : func->locals) std::cout << " " << l; + std::cout << "\n"; + return 0; + } + auto passes = pickPasses(o, err); + if (passes.empty()) { + std::cerr << "error: " << err << "\n"; + return 2; + } + if (o.mode != "random" && o.mode != "exhaustive") { + std::cerr << "error: --mode must be random or exhaustive\n"; + return 2; + } + if (o.dryRun) return dryRun(o, funcText, passes); + + if (o.compile.empty() || (o.targetObj.empty() && o.scoreCmd.empty())) { + std::cerr << "error: --compile and --target-obj (or --score-cmd) are required\n"; + return 2; + } + if (o.jobs < 1) o.jobs = 1; + + Evaluator ev{o}; + ev.sc.objdump = o.objdump; + ev.sc.ignoreRelocNames = o.ignoreRelocNames; + ev.prefix = src.substr(0, start); + ev.suffix = src.substr(end); + fs::path sp = fs::absolute(o.source); + ev.srcDir = sp.parent_path().string(); + ev.stem = sp.stem().string(); + ev.ext = sp.extension().string(); + fs::create_directories(fs::path(o.outputDir) / ".work"); + + std::signal(SIGINT, onSignal); + std::signal(SIGTERM, onSignal); + auto cleanup = [&]() { + std::error_code ec; + for (int w = 0; w < o.jobs; ++w) fs::remove(ev.srcPath(w), ec); + }; + + // the unmodified function + std::string log; + if (!ev.compile(0, funcText, log)) { + std::cerr << "error: the unmodified source doesn't compile:\n" << log << "\n"; + cleanup(); + return 1; + } + if (o.scoreCmd.empty()) { + if (o.symbol.empty()) { + auto syms = listSymbols(ev.sc, ev.objPath(0), err); + ev.symbol = guessSymbol(syms, o.function); + if (ev.symbol.empty()) { + std::cerr << "error: can't find the symbol for " << o.function + << " in the compiled object; pass --symbol\n"; + cleanup(); + return 1; + } + } else { + ev.symbol = o.symbol; + } + ev.targetSymbol = o.targetSymbol.empty() ? ev.symbol : o.targetSymbol; + if (!disassemble(ev.sc, o.targetObj, ev.targetSymbol, ev.target, err)) { + std::cerr << "error: target: " << err << "\n"; + cleanup(); + return 1; + } + std::cout << "symbol: " << ev.symbol << " (" << ev.target.size() << " target instructions)\n"; + } + std::vector baseInsns; + long base = ev.score(0, &baseInsns, log); + if (base < 0) { + std::cerr << "error: can't score the unmodified source:\n" << log << "\n"; + cleanup(); + return 1; + } + if (o.showBaseDiff && !baseInsns.empty()) std::cout << diffInsns(ev.sc, ev.target, baseInsns); + std::cout << "base score: " << base << "\n"; + + State st; + st.baseText = st.bestText = funcText; + st.baseScore = st.bestScore = base; + st.seen.insert(hashText(funcText)); + if (base == 0) { + std::cout << "already matching\n"; + cleanup(); + return 0; + } + if (o.mode == "random") runRandom(o, ev, st, passes); + else runExhaustive(o, ev, st, passes); + cleanup(); + + std::cout << "\n"; + if (st.bestScore == 0) std::cout << "MATCH found"; + else if (st.bestScore < base) std::cout << "best score " << st.bestScore << " (base " << base << ")"; + else std::cout << "no improvement over base score " << base; + std::cout << " after " << st.compiles << " compiles (" << st.failures << " failed)\n"; + return st.bestScore == 0 ? 0 : 3; +} diff --git a/src/parser.cpp b/src/parser.cpp new file mode 100644 index 0000000..02243ad --- /dev/null +++ b/src/parser.cpp @@ -0,0 +1,611 @@ +#include "parser.hpp" + +#include + +namespace perm { + +const char* skName(SK k) { + switch (k) { + case SK::Block: return "Block"; + case SK::Decl: return "Decl"; + case SK::Expr: return "Expr"; + case SK::If: return "If"; + case SK::For: return "For"; + case SK::While: return "While"; + case SK::Do: return "Do"; + case SK::Switch: return "Switch"; + case SK::Case: return "Case"; + case SK::Default: return "Default"; + case SK::Label: return "Label"; + case SK::Return: return "Return"; + case SK::Break: return "Break"; + case SK::Continue: return "Continue"; + case SK::Goto: return "Goto"; + case SK::Asm: return "Asm"; + case SK::PP: return "PP"; + case SK::Empty: return "Empty"; + case SK::Try: return "Try"; + case SK::Other: return "Other"; + } + return "?"; +} + +int matchBracket(const std::vector& toks, int i) { + const std::string& o = toks[i].text; + std::string c = o == "(" ? ")" : o == "[" ? "]" : o == "{" ? "}" : ""; + if (c.empty()) return -1; + int depth = 0; + for (int j = i; j < (int)toks.size(); ++j) { + if (toks[j].kind != TokKind::Punct) continue; + const std::string& t = toks[j].text; + if (t == "(" || t == "[" || t == "{") depth++; + else if (t == ")" || t == "]" || t == "}") { + depth--; + if (depth == 0) return t == c ? j : -1; + } + } + return -1; +} + +bool Func::isLocal(int i) const { + if (i < 0 || i >= (int)toks.size() || toks[i].kind != TokKind::Ident) return false; + if (!locals.count(toks[i].text)) return false; + if (i > 0) { + const std::string& p = toks[i - 1].text; + if (p == "." || p == "->" || p == "::") return false; + } + if (i + 1 < (int)toks.size() && toks[i + 1].text == "::") return false; + return true; +} + +std::string Func::indentOf(int i) const { + for (int k = i; k >= 0; --k) { + const std::string& lead = toks[k].lead; + size_t nl = lead.rfind('\n'); + if (nl == std::string::npos) continue; + std::string ind = "\n"; + for (size_t j = nl + 1; j < lead.size() && (lead[j] == ' ' || lead[j] == '\t'); ++j) + ind += lead[j]; + return ind; + } + return "\n"; +} + +void forEachStmt(const Stmt& s, const std::function& fn) { + fn(s); + for (auto& k : s.kids) forEachStmt(*k, fn); +} + +// --------------------------------------------------------------------------- +// Locating the function + +namespace { + +std::vector nameTokens(const std::string& qual) { + std::vector out; + size_t i = 0; + while (i <= qual.size()) { + size_t j = qual.find("::", i); + std::string comp = qual.substr(i, j == std::string::npos ? std::string::npos : j - i); + if (!out.empty()) out.push_back("::"); + if (!comp.empty() && comp[0] == '~') { + out.push_back("~"); + comp = comp.substr(1); + } + out.push_back(comp); + if (j == std::string::npos) break; + i = j + 2; + } + return out; +} + +} // namespace + +bool locateFunction(const std::string& src, const std::string& qualName, size_t& start, + size_t& end, std::string& err) { + std::vector toks = lex(src); + std::vector pat = nameTokens(qualName); + int n = (int)toks.size(); + int found = 0; + for (int k = 0; k + (int)pat.size() < n; ++k) { + bool ok = true; + for (size_t p = 0; p < pat.size() && ok; ++p) ok = toks[k + p].text == pat[p]; + if (!ok) continue; + int open = k + (int)pat.size(); + if (toks[open].text != "(") continue; + // extend backwards over qualifiers the caller left out + int s = k; + while (s >= 2 && toks[s - 1].text == "::" && toks[s - 2].kind == TokKind::Ident) s -= 2; + if (s >= 1 && (toks[s - 1].text == "." || toks[s - 1].text == "->")) continue; + int close = matchBracket(toks, open); + if (close < 0) continue; + int j = close + 1, body = -1; + while (j < n) { + const std::string& t = toks[j].text; + if (toks[j].kind == TokKind::Punct) { + if (t == "{") { + body = j; + break; + } + if (t == "(" || t == "[") { + j = matchBracket(toks, j); + if (j < 0) break; + j++; + continue; + } + if (t == ":" || t == "," || t == "::" || t == "*" || t == "&" || t == "<" || + t == ">") { + j++; + continue; + } + break; + } + if (toks[j].kind != TokKind::Ident) break; + j++; + } + if (body < 0) continue; + int bodyEnd = matchBracket(toks, body); + if (bodyEnd < 0) continue; + if (found++ == 0) { + start = toks[s].offset; + end = toks[bodyEnd].offset + 1; + } + } + if (found == 0) { + err = "definition of '" + qualName + "' not found"; + return false; + } + if (found > 1) + err = "warning: " + std::to_string(found) + " definitions of '" + qualName + + "' found, using the first"; + return true; +} + +// --------------------------------------------------------------------------- +// Statement parser + +namespace { + +struct Parser { + Func& f; + const std::vector& T; + + explicit Parser(Func& fn) : f(fn), T(fn.toks) {} + + const std::string& t(int i) const { + static const std::string empty; + return i >= 0 && i < (int)T.size() ? T[i].text : empty; + } + bool punct(int i, const char* p) const { + return i >= 0 && i < (int)T.size() && T[i].kind == TokKind::Punct && T[i].text == p; + } + + // Index of the ';' ending the statement starting at i, or -1 if a '}' at + // depth 0 (or limit) comes first. + int findSemi(int i, int limit) const { + int depth = 0; + for (int j = i; j < limit; ++j) { + if (T[j].kind != TokKind::Punct) continue; + const std::string& s = T[j].text; + if (s == "(" || s == "[" || s == "{") depth++; + else if (s == ")" || s == "]" || s == "}") { + if (depth == 0) return -1; + depth--; + } else if (s == ";" && depth == 0) return j; + } + return -1; + } + + // End (exclusive) of an unterminated statement: the next '}' at depth 0. + int findRecover(int i, int limit) const { + int depth = 0; + for (int j = i; j < limit; ++j) { + if (T[j].kind != TokKind::Punct) continue; + const std::string& s = T[j].text; + if (s == "(" || s == "[" || s == "{") depth++; + else if (s == ")" || s == "]" || s == "}") { + if (depth == 0) return j > i ? j : i + 1; + depth--; + } + } + return limit > i ? limit : i + 1; + } + + std::unique_ptr make(SK k, int b, int e) { + auto s = std::make_unique(); + s->kind = k; + s->begin = b; + s->end = e; + return s; + } + + std::unique_ptr toSemi(SK k, int i, int limit) { + int semi = findSemi(i, limit); + if (semi < 0) return make(SK::Other, i, findRecover(i, limit)); + return make(k, i, semi + 1); + } + + void adopt(Stmt& p, std::unique_ptr c) { + c->parent = &p; + p.kids.push_back(std::move(c)); + } + + std::unique_ptr parseBlock(int open) { + int close = matchBracket(T, open); + if (close < 0) close = (int)T.size() - 1; + auto s = make(SK::Block, open, close + 1); + int i = open + 1; + while (i < close) { + auto c = parseStmt(i, close); + if (c->end <= i) c->end = i + 1; + i = c->end; + adopt(*s, std::move(c)); + } + return s; + } + + int matchAngle(int i, int limit) const { + int depth = 0; + for (int j = i; j < limit; ++j) { + const std::string& s = T[j].text; + if (s == "<") depth++; + else if (s == ">") { + if (--depth == 0) return j; + } else if (s == ">>") { + depth -= 2; + if (depth <= 0) return depth == 0 ? j : -1; + } else if (s == ";" || s == "{" || s == "}" || s == "&&" || s == "||") + return -1; + } + return -1; + } + + // Tries to read tokens [b, e) (e is the ';' or the end of a for-init) as a + // declaration. + bool tryDecl(Stmt& s, int b, int e) { + int i = b; + bool isStatic = false, isConst = false; + auto isStorage = [&](const std::string& x) { + return x == "static" || x == "const" || x == "volatile" || x == "register" || + x == "extern" || x == "mutable" || x == "typename"; + }; + while (i < e && isStorage(t(i))) { + if (t(i) == "static") isStatic = true; + if (t(i) == "const") isConst = true; + i++; + } + if (t(i) == "struct" || t(i) == "class" || t(i) == "union" || t(i) == "enum") i++; + if (punct(i, "::")) i++; + if (i >= e) return false; + if (isBuiltinType(t(i))) { + while (i < e && isBuiltinType(t(i))) i++; + } else if (T[i].kind == TokKind::Ident && (t(i) == "auto" || !isKeyword(t(i))) && + !f.locals.count(t(i))) { + i++; + while (i < e) { + if (punct(i, "<")) { + int j = matchAngle(i, e); + if (j < 0) return false; + i = j + 1; + continue; + } + if (punct(i, "::") && i + 1 < e && T[i + 1].kind == TokKind::Ident) { + i += 2; + continue; + } + break; + } + } else { + return false; + } + while (i < e && (t(i) == "const" || t(i) == "volatile")) { + if (t(i) == "const") isConst = true; + i++; + } + int typeEnd = i; + std::vector decls; + while (true) { + Declarator d; + d.begin = i; + while (i < e && (punct(i, "*") || punct(i, "&") || t(i) == "const" || + t(i) == "volatile" || t(i) == "__stdcall" || t(i) == "__cdecl" || + t(i) == "__fastcall" || t(i) == "__thiscall")) { + if (punct(i, "&")) d.isRef = true; + i++; + } + if (punct(i, "(") && (punct(i + 1, "*") || punct(i + 1, "&"))) { + // function pointer: (*name)(params) + int close = matchBracket(T, i); + if (close < 0 || close >= e) return false; + int j = i + 1; + while (j < close && (punct(j, "*") || punct(j, "&") || t(j) == "const")) j++; + if (j >= close || T[j].kind != TokKind::Ident) return false; + d.nameTok = j; + i = close + 1; + if (!punct(i, "(")) return false; + int pc = matchBracket(T, i); + if (pc < 0 || pc >= e) return false; + i = pc + 1; + } else if (i < e && T[i].kind == TokKind::Ident && !isKeyword(t(i))) { + d.nameTok = i++; + } else { + return false; + } + while (punct(i, "[")) { + int close = matchBracket(T, i); + if (close < 0 || close >= e) return false; + d.isArray = true; + i = close + 1; + } + d.declEnd = i; + if (punct(i, "=")) { + i++; + d.initBegin = i; + int depth = 0; + while (i < e) { + if (T[i].kind == TokKind::Punct) { + const std::string& x = T[i].text; + if (x == "(" || x == "[" || x == "{") depth++; + else if (x == ")" || x == "]" || x == "}") depth--; + else if (x == "," && depth == 0) break; + } + i++; + } + d.initEnd = i; + d.init = Declarator::Assign; + if (d.initBegin == d.initEnd) return false; + } else if (punct(i, "(") || punct(i, "{")) { + int close = matchBracket(T, i); + if (close < 0 || close >= e) return false; + d.init = punct(i, "(") ? Declarator::Paren : Declarator::Brace; + d.initBegin = i + 1; + d.initEnd = close; + i = close + 1; + } + d.end = i; + decls.push_back(d); + if (i == e) break; + if (!punct(i, ",")) return false; + i++; + } + s.kind = SK::Decl; + s.typeBegin = b; + s.typeEnd = typeEnd; + s.decls = std::move(decls); + s.isStatic = isStatic; + s.isConst = isConst; + for (auto& d : s.decls) f.locals.insert(t(d.nameTok)); + return true; + } + + std::unique_ptr parseCond(SK k, int i, int limit) { + // keyword ( cond ) stmt + if (!punct(i + 1, "(")) return toSemi(SK::Other, i, limit); + int close = matchBracket(T, i + 1); + if (close < 0 || close >= limit) return make(SK::Other, i, findRecover(i, limit)); + auto s = make(k, i, close + 1); + s->condOpen = i + 1; + s->condClose = close; + if (close + 1 >= limit) return s; + auto body = parseStmt(close + 1, limit); + s->end = body->end; + adopt(*s, std::move(body)); + return s; + } + + std::unique_ptr parseStmt(int i, int limit) { + const Token& tk = T[i]; + const std::string& x = tk.text; + if (tk.kind == TokKind::PP) return make(SK::PP, i, i + 1); + if (tk.kind == TokKind::Punct) { + if (x == "{") return parseBlock(i); + if (x == ";") return make(SK::Empty, i, i + 1); + return toSemi(SK::Expr, i, limit); + } + if (tk.kind != TokKind::Ident) return toSemi(SK::Expr, i, limit); + + if (x == "if") { + auto s = parseCond(SK::If, i, limit); + if (s->kind == SK::If && !s->kids.empty() && s->end < limit && t(s->end) == "else" && + s->end + 1 < limit) { + s->elseTok = s->end; + auto e = parseStmt(s->end + 1, limit); + s->end = e->end; + adopt(*s, std::move(e)); + } + return s; + } + if (x == "while") return parseCond(SK::While, i, limit); + if (x == "switch") return parseCond(SK::Switch, i, limit); + if (x == "for") { + auto s = parseCond(SK::For, i, limit); + if (s->kind == SK::For) { + int semi = findSemi(s->condOpen + 1, s->condClose); + if (semi > s->condOpen + 1) { + auto init = make(SK::Expr, s->condOpen + 1, semi + 1); + if (tryDecl(*init, s->condOpen + 1, semi)) s->forInit = std::move(init); + } + } + return s; + } + if (x == "do") { + auto s = make(SK::Do, i, i + 1); + auto body = parseStmt(i + 1, limit); + int w = body->end; + adopt(*s, std::move(body)); + if (t(w) == "while" && punct(w + 1, "(")) { + int close = matchBracket(T, w + 1); + if (close > 0 && close < limit) { + s->condOpen = w + 1; + s->condClose = close; + s->end = punct(close + 1, ";") ? close + 2 : close + 1; + return s; + } + } + s->kind = SK::Other; + s->end = w; + s->kids.clear(); + return s; + } + if (x == "case") { + int depth = 0; + for (int j = i + 1; j < limit; ++j) { + if (punct(j, "(") || punct(j, "[")) depth++; + else if (punct(j, ")") || punct(j, "]")) depth--; + else if (punct(j, ":") && depth == 0) return make(SK::Case, i, j + 1); + } + return make(SK::Other, i, i + 1); + } + if (x == "default" && punct(i + 1, ":")) return make(SK::Default, i, i + 2); + if (x == "return" || x == "throw") return toSemi(SK::Return, i, limit); + if (x == "break") return toSemi(SK::Break, i, limit); + if (x == "continue") return toSemi(SK::Continue, i, limit); + if (x == "goto") return toSemi(SK::Goto, i, limit); + if (x == "__asm" || x == "_asm" || x == "asm") { + int e; + if (punct(i + 1, "{")) { + e = matchBracket(T, i + 1); + e = e < 0 ? limit : e + 1; + } else { + e = i + 1; + int line = tk.line; + while (e < limit && T[e].line == line && !punct(e, "}") && !punct(e, ";")) e++; + } + if (e < limit && punct(e, ";")) e++; + return make(SK::Asm, i, e); + } + if (x == "try" || x == "__try") { + auto s = make(SK::Try, i, i + 1); + int j = i + 1; + if (!punct(j, "{")) return toSemi(SK::Other, i, limit); + auto b = parseBlock(j); + j = b->end; + adopt(*s, std::move(b)); + while (j < limit && (t(j) == "catch" || t(j) == "__except" || t(j) == "__finally")) { + int k = j + 1; + if (punct(k, "(")) { + k = matchBracket(T, k); + if (k < 0) break; + k++; + } + if (!punct(k, "{")) break; + auto h = parseBlock(k); + j = h->end; + adopt(*s, std::move(h)); + } + s->end = j; + return s; + } + if (x == "typedef" || x == "using" || x == "template" || x == "namespace") + return toSemi(SK::Other, i, limit); + if ((x == "struct" || x == "class" || x == "union" || x == "enum") && + (punct(i + 1, "{") || punct(i + 2, "{") || punct(i + 2, ":"))) + return toSemi(SK::Other, i, limit); + if (!isKeyword(x) && punct(i + 1, ":")) return make(SK::Label, i, i + 2); + + int semi = findSemi(i, limit); + if (semi < 0) return make(SK::Other, i, findRecover(i, limit)); + auto s = make(SK::Expr, i, semi + 1); + tryDecl(*s, i, semi); + return s; + } +}; + +void setParents(Stmt& s) { + for (auto& k : s.kids) { + k->parent = &s; + setParents(*k); + } +} + +} // namespace + +std::unique_ptr parseFunc(const std::string& text, std::string& err) { + auto f = std::make_unique(); + f->text = text; + f->toks = lex(text); + auto& T = f->toks; + int n = (int)T.size(); + for (int i = 0; i < n; ++i) + if (T[i].text == "(" && T[i].kind == TokKind::Punct) { + f->paramOpen = i; + break; + } + if (f->paramOpen < 0) { + err = "no parameter list"; + return nullptr; + } + f->paramClose = matchBracket(T, f->paramOpen); + if (f->paramClose < 0) { + err = "unbalanced parameter list"; + return nullptr; + } + for (int i = f->paramClose + 1; i < n; ++i) { + if (T[i].text == "(") { + i = matchBracket(T, i); + if (i < 0) break; + continue; + } + if (T[i].text == "{") { + f->bodyOpen = i; + break; + } + } + if (f->bodyOpen < 0 || (f->bodyClose = matchBracket(T, f->bodyOpen)) < 0) { + err = "no function body"; + return nullptr; + } + + // parameter names: the last plain identifier of each parameter + int segStart = f->paramOpen + 1, depth = 0; + for (int i = f->paramOpen + 1; i <= f->paramClose; ++i) { + const std::string& x = T[i].text; + if (x == "(" || x == "[") depth++; + if ((x == ")" || x == "]") && i != f->paramClose) depth--; + if ((x == "," && depth == 0) || i == f->paramClose) { + std::string name; + int d2 = 0; + for (int j = segStart; j < i; ++j) { + if (T[j].text == "(" || T[j].text == "[") d2++; + if (T[j].text == ")" || T[j].text == "]") d2--; + if (T[j].text == "=" && d2 == 0) break; + if (T[j].kind == TokKind::Ident && !isKeyword(T[j].text)) name = T[j].text; + } + // a lone type ("Foo") has no name + int idents = 0; + for (int j = segStart; j < i; ++j) + if (T[j].kind == TokKind::Ident && T[j].text != "const" && T[j].text != "struct") + idents++; + if (!name.empty() && idents >= 2) { + f->params.push_back(name); + f->locals.insert(name); + } + segStart = i + 1; + } + } + + Parser p(*f); + f->body = p.parseBlock(f->bodyOpen); + setParents(*f->body); + return f; +} + +std::string dumpStmt(const Func& f, const Stmt& s, int depth) { + std::ostringstream o; + std::string head = f.tokText(s.begin, std::min(s.end, s.begin + 8)); + for (char& c : head) + if (c == '\n' || c == '\r' || c == '\t') c = ' '; + o << std::string(depth * 2, ' ') << skName(s.kind) << " L" << f.toks[s.begin].line << ": " + << head; + if (s.kind == SK::Decl) { + o << " ["; + for (auto& d : s.decls) o << " " << f.t(d.nameTok); + o << " ]"; + } + o << "\n"; + if (s.forInit) o << dumpStmt(f, *s.forInit, depth + 1); + for (auto& k : s.kids) o << dumpStmt(f, *k, depth + 1); + return o.str(); +} + +} // namespace perm diff --git a/src/parser.hpp b/src/parser.hpp new file mode 100644 index 0000000..49ffbce --- /dev/null +++ b/src/parser.hpp @@ -0,0 +1,89 @@ +// Statement-level parser for one function body. +// +// This is not a C++ front end. It only needs to find statements, blocks, +// local declarations and expressions inside a single function, and it must +// keep going on code it can't fully understand (MSVC extensions, macros, +// types from headers it never sees). Anything it can't classify becomes an +// "Other" statement, which the passes treat as a barrier. +#pragma once + +#include "lexer.hpp" + +#include +#include +#include +#include +#include + +namespace perm { + +enum class SK { + Block, Decl, Expr, If, For, While, Do, Switch, Case, Default, Label, + Return, Break, Continue, Goto, Asm, PP, Empty, Try, Other, +}; +const char* skName(SK k); + +struct Declarator { + enum Init { None, Assign, Paren, Brace }; + int nameTok = -1; + int begin = -1, end = -1; // the whole declarator, init included + int declEnd = -1; // end of the declarator without its init + int initBegin = -1, initEnd = -1; // init expression (inside parens/braces for Paren/Brace) + Init init = None; + bool isArray = false; + bool isRef = false; +}; + +struct Stmt { + SK kind = SK::Other; + int begin = 0, end = 0; // token range [begin, end) + std::vector> kids; + // If: kids[0] = then, kids[1] = else (optional). Loops/Switch: kids[0] = body. + // Try: the guarded block and the handler blocks. + int condOpen = -1, condClose = -1; // the '(' and ')' of if/while/for/switch/do-while + int elseTok = -1; + // Decl + int typeBegin = -1, typeEnd = -1; + std::vector decls; + bool isStatic = false; + bool isConst = false; + // For: the decl in the init clause, if any + std::unique_ptr forInit; + Stmt* parent = nullptr; +}; + +struct Func { + std::string text; // the function from its (qualified) name to the closing brace + std::vector toks; + int paramOpen = -1, paramClose = -1; + int bodyOpen = -1, bodyClose = -1; + std::unique_ptr body; + std::vector params; + std::set locals; // params and every local declared in the body + + std::string tokText(int b, int e) const { return joinTokens(toks, b, e); } + std::string textOf(const Stmt& s) const { return tokText(s.begin, s.end); } + const std::string& t(int i) const { return toks[i].text; } + bool isLocal(int i) const; + // Indentation of the line holding token i ("\n" + spaces), for inserted statements. + std::string indentOf(int i) const; +}; + +// Finds the definition of qualName ("Class::Method", "Method", "Class::~Class") +// in src. On success fills [start, end) with the byte range from the start of +// the qualified name to the closing brace (inclusive). +bool locateFunction(const std::string& src, const std::string& qualName, size_t& start, + size_t& end, std::string& err); + +// Parses the text produced by locateFunction. Returns null and sets err on failure. +std::unique_ptr parseFunc(const std::string& text, std::string& err); + +// Calls fn on every statement in pre-order (the body block included). +void forEachStmt(const Stmt& s, const std::function& fn); + +// Index of the token closing the bracket at i, or -1. +int matchBracket(const std::vector& toks, int i); + +std::string dumpStmt(const Func& f, const Stmt& s, int depth = 0); + +} // namespace perm diff --git a/src/passes.cpp b/src/passes.cpp new file mode 100644 index 0000000..c260b26 --- /dev/null +++ b/src/passes.cpp @@ -0,0 +1,875 @@ +#include "passes.hpp" + +#include "analysis.hpp" + +#include +#include + +namespace perm { + +// --------------------------------------------------------------------------- +// Rewriter + +Rewriter::Rewriter(const Func& f) : f_(f) { + size_t n = f.toks.size(); + lead_.resize(n); + text_.resize(n); + prefix_.resize(n); + touched_.assign(n, false); + for (size_t i = 0; i < n; ++i) { + lead_[i] = f.toks[i].lead; + text_[i] = f.toks[i].text; + } +} + +void Rewriter::replace(int b, int e, const std::string& text) { + for (int i = b; i < e; ++i) { + if (touched_[i]) overlap_ = true; + touched_[i] = true; + if (i != b) lead_[i].clear(); + text_[i].clear(); + } + if (b < e) text_[b] = text; + else overlap_ = true; +} + +void Rewriter::remove(int b, int e) { + replace(b, e, ""); + if (overlap_) return; + std::string& l = lead_[b]; + size_t nl = l.rfind('\n'); + l = nl == std::string::npos ? "" : l.substr(0, nl); +} + +void Rewriter::insertBefore(int tok, const std::string& stmtText) { + prefix_[tok] += stmtText + f_.indentOf(tok); +} + +std::string Rewriter::apply() const { + if (overlap_) return ""; + std::string out; + out.reserve(f_.text.size() + 256); + for (size_t i = 0; i < f_.toks.size(); ++i) { + out += lead_[i]; + out += prefix_[i]; + out += text_[i]; + } + return out; +} + +// --------------------------------------------------------------------------- +// helpers + +namespace { + +std::vector blocksOf(const Func& f) { + std::vector r; + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind == SK::Block) r.push_back(&s); + }); + return r; +} + +std::string norm(const std::string& s) { + std::string r; + for (char c : s) + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') r += c; + return r; +} + +std::string exprText(const Func& f, const Expr& x) { return f.tokText(x.b, x.e); } + +bool isPostfixLike(const Expr& x) { + return x.k == Expr::Primary || x.k == Expr::Paren || x.k == Expr::Call || + x.k == Expr::Index || x.k == Expr::Member || x.k == Expr::Postfix; +} + +int exprPrec(const Func& f, const Expr& x) { + switch (x.k) { + case Expr::Comma: return 1; + case Expr::Assign: return 2; + case Expr::Ternary: return 3; + case Expr::Binary: return binPrec(f.t(x.op)); + default: return 100; + } +} + +std::string paren(const std::string& s) { return "(" + s + ")"; } + +// Text of x for use as an operand of an operator with precedence prec. +std::string operandText(const Func& f, const Expr& x, int prec) { + std::string s = exprText(f, x); + return exprPrec(f, x) <= prec ? paren(s) : s; +} + +bool insideLoopNotContaining(const Func& f, int tok, const Stmt& s) { + bool r = false; + forEachStmt(*f.body, [&](const Stmt& l) { + if (l.kind != SK::For && l.kind != SK::While && l.kind != SK::Do) return; + if (tok >= l.begin && tok < l.end && !(s.begin >= l.begin && s.begin < l.end)) r = true; + }); + return r; +} + +std::string declTypeText(const Func& f, const Stmt& s) { return f.tokText(s.typeBegin, s.typeEnd); } + +// "T name" for declarator d of s, without its initializer. +std::string declNoInit(const Func& f, const Stmt& s, const Declarator& d) { + if (&d == &s.decls[0]) return f.tokText(s.typeBegin, d.declEnd); + std::string t = declTypeText(f, s); + std::string rest = f.tokText(d.begin, d.declEnd); + if (f.t(d.begin) == "*" || f.t(d.begin) == "&") return t + rest; + return t + " " + rest; +} + +// Whether moving an initializer with effects ie past code with effects between +// could change what it computes. Locals are tracked exactly; memory is only +// checked for calls, since a byte-for-byte match proves the rewrite anyway. +bool initBlocked(const Effects& ie, const Effects& between) { + for (auto& r : ie.reads) + if (between.writes.count(r)) return true; + if ((ie.call || ie.memRead) && between.call) return true; + if (ie.call && between.memWrite) return true; + return false; +} + +bool simpleSingleDecl(const Stmt& s) { + return s.kind == SK::Decl && !s.isStatic && s.decls.size() == 1 && s.parent && + s.parent->kind == SK::Block; +} + +int indexInParent(const Stmt& s) { + auto& k = s.parent->kids; + for (size_t i = 0; i < k.size(); ++i) + if (k[i].get() == &s) return (int)i; + return -1; +} + +// All orders of n items that keep every conflicting pair in its original order. +void topoOrders(int n, const std::vector>& conf, size_t cap, + std::vector>& out) { + std::vector cur; + std::vector used(n, false); + std::function rec = [&]() { + if (out.size() >= cap) return; + if ((int)cur.size() == n) { + out.push_back(cur); + return; + } + for (int i = 0; i < n; ++i) { + if (used[i]) continue; + bool ok = true; + for (int j = 0; j < i && ok; ++j) + if (conf[i][j] && !used[j]) ok = false; + if (!ok) continue; + used[i] = true; + cur.push_back(i); + rec(); + cur.pop_back(); + used[i] = false; + } + }; + rec(); +} + +std::vector randomTopo(int n, const std::vector>& conf, Rng& rng) { + std::vector cur; + std::vector used(n, false); + while ((int)cur.size() < n) { + std::vector avail; + for (int i = 0; i < n; ++i) { + if (used[i]) continue; + bool ok = true; + for (int j = 0; j < i && ok; ++j) + if (conf[i][j] && !used[j]) ok = false; + if (ok) avail.push_back(i); + } + int pick = avail[std::uniform_int_distribution(0, avail.size() - 1)(rng)]; + used[pick] = true; + cur.push_back(pick); + } + return cur; +} + +// --------------------------------------------------------------------------- +// reorder_saves + +struct Run { + std::vector items; + std::vector> conf; +}; + +bool isSave(const Func& f, const Stmt& s, const Effects& e) { + if (e.call || e.memWrite || e.barrier) return false; + if (s.kind == SK::Decl) return !s.isStatic; + if (s.kind == SK::Expr) return f.isLocal(s.begin) && f.t(s.begin + 1) == "="; + return false; +} + +std::vector saveRuns(const Func& f) { + std::vector runs; + for (const Stmt* b : blocksOf(f)) { + std::vector cur; + std::vector effs; + auto flush = [&]() { + if (cur.size() >= 2) { + Run r; + r.items = cur; + size_t n = cur.size(); + r.conf.assign(n, std::vector(n, false)); + for (size_t i = 0; i < n; ++i) + for (size_t j = 0; j < n; ++j) + if (i != j) r.conf[i][j] = conflicts(effs[i], effs[j]); + runs.push_back(std::move(r)); + } + cur.clear(); + effs.clear(); + }; + for (auto& k : b->kids) { + Effects e = effectsOfStmt(f, *k); + if (isSave(f, *k, e)) { + cur.push_back(k.get()); + effs.push_back(e); + } else { + flush(); + } + } + flush(); + } + return runs; +} + +std::string applyOrders(const Func& f, const std::vector& runs, + const std::vector>& orders) { + Rewriter rw(f); + for (size_t r = 0; r < runs.size(); ++r) { + auto& items = runs[r].items; + auto& ord = orders[r]; + for (size_t k = 0; k < items.size(); ++k) + if (ord[k] != (int)k) + rw.replace(items[k]->begin, items[k]->end, f.textOf(*items[ord[k]])); + } + return rw.apply(); +} + +void enumReorderSaves(const Func& f, const EmitFn& emit) { + std::vector runs = saveRuns(f); + if (runs.empty()) return; + std::vector>> perRun; + for (auto& r : runs) { + std::vector> o; + topoOrders((int)r.items.size(), r.conf, 40320, o); + perRun.push_back(std::move(o)); + } + // odometer over the cartesian product of every run's orders + auto shared = std::make_shared>(runs); + std::vector idx(runs.size(), 0); + while (true) { + std::vector> pick; + bool identity = true; + for (size_t r = 0; r < runs.size(); ++r) { + pick.push_back(perRun[r][idx[r]]); + for (size_t k = 0; k < pick.back().size(); ++k) + if (pick.back()[k] != (int)k) identity = false; + } + if (!identity && !emit([&f, shared, pick]() { return applyOrders(f, *shared, pick); })) + return; + size_t r = 0; + while (r < runs.size() && ++idx[r] == perRun[r].size()) idx[r++] = 0; + if (r == runs.size()) break; + } +} + +bool randomReorderSaves(const Func& f, Rng& rng, std::string& out) { + std::vector runs = saveRuns(f); + if (runs.empty()) return false; + for (int attempt = 0; attempt < 8; ++attempt) { + std::vector> orders; + size_t which = std::uniform_int_distribution(0, runs.size() - 1)(rng); + bool changed = false; + for (size_t r = 0; r < runs.size(); ++r) { + std::vector id(runs[r].items.size()); + std::iota(id.begin(), id.end(), 0); + if (r == which) { + auto o = randomTopo((int)id.size(), runs[r].conf, rng); + changed = o != id; + orders.push_back(o); + } else { + orders.push_back(id); + } + } + if (!changed) continue; + out = applyOrders(f, runs, orders); + return !out.empty(); + } + return false; +} + +// --------------------------------------------------------------------------- +// move_stmt + +void enumMoveStmt(const Func& f, const EmitFn& emit) { + for (const Stmt* b : blocksOf(f)) { + auto& kids = b->kids; + size_t n = kids.size(); + if (n < 2) continue; + std::vector effs; + for (auto& k : kids) effs.push_back(effectsOfStmt(f, *k)); + for (size_t i = 0; i < n; ++i) { + if (effs[i].barrier || kids[i]->kind == SK::Empty) continue; + for (int dir = -1; dir <= 1; dir += 2) { + for (long j = (long)i + dir; j >= 0 && j < (long)n; j += dir) { + if (conflictsRelaxed(effs[i], effs[j]) || kids[j]->kind == SK::Empty) break; + // move i to position j + std::vector order; + for (size_t k = 0; k < n; ++k) + if (k != i) order.push_back((int)k); + order.insert(order.begin() + j, (int)i); + size_t lo = std::min(i, j), hi = std::max(i, j); + if (!emit([&f, b, order, lo, hi]() { + Rewriter rw(f); + for (size_t k = lo; k <= hi; ++k) + rw.replace(b->kids[k]->begin, b->kids[k]->end, + f.textOf(*b->kids[order[k]])); + return rw.apply(); + })) + return; + } + } + } + } +} + +// --------------------------------------------------------------------------- +// inline_local + +void enumInlineLocal(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (simpleSingleDecl(s)) decls.push_back(&s); + }); + for (const Stmt* s : decls) { + const Declarator& d = s->decls[0]; + if (d.isArray) continue; + std::string name = f.t(d.nameTok); + std::vector uses = usesOf(f, name, s->end, f.bodyClose); + if (d.init != Declarator::Assign || f.t(d.initBegin) == "{") { + if (uses.empty() && d.init == Declarator::None) + emit([&f, s]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + return rw.apply(); + }); + continue; + } + Effects ie = effectsOf(f, d.initBegin, d.initEnd); + if (!ie.writes.empty() || ie.memWrite || ie.barrier) continue; + std::string init = f.tokText(d.initBegin, d.initEnd); + if (uses.empty()) { + // dead local: drop it, keeping a call for its side effects + bool keep = ie.call; + if (!emit([&f, s, init, keep]() { + Rewriter rw(f); + if (keep) rw.replace(s->begin, s->end, init + ";"); + else rw.remove(s->begin, s->end); + return rw.apply(); + })) + return; + continue; + } + bool anyWrite = false; + for (int u : uses) anyWrite |= isWriteUse(f, u); + if (anyWrite) continue; + bool pure = !ie.call; + if (uses.size() > 1 && !pure) continue; + Effects between = effectsOf(f, s->end, uses.back()); + bool trivially = pure && !ie.memRead && ie.reads.empty(); + if (!trivially && (initBlocked(ie, between) || (between.barrier && ie.call))) continue; + bool inLoop = false; + for (int u : uses) inLoop |= insideLoopNotContaining(f, u, *s); + if (inLoop && !trivially) continue; + + auto x = parseExpr(f, d.initBegin, d.initEnd); + bool needParen = !x || !isPostfixLike(*x); + std::string plain = needParen ? paren(init) : init; + std::string castT = declTypeText(f, *s) + f.tokText(d.begin, d.nameTok); + std::vector variants = {plain}; + if (uses.size() == 1 && !d.isRef && !s->isConst) variants.push_back("((" + castT + ")" + paren(init) + ")"); + for (auto& v : variants) { + if (!emit([&f, s, uses, v]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + for (int u : uses) rw.replace(u, u + 1, v); + return rw.apply(); + })) + return; + } + } +} + +// --------------------------------------------------------------------------- +// sink_decl + +void enumSinkDecl(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (simpleSingleDecl(s)) decls.push_back(&s); + }); + for (const Stmt* s : decls) { + const Stmt* P = s->parent; + int di = indexInParent(*s); + const Declarator& d = s->decls[0]; + std::string name = f.t(d.nameTok); + std::vector uses = usesOf(f, name, s->end, P->end); + if (uses.empty()) continue; + // innermost block holding every use + const Stmt* B = P; + forEachStmt(*P, [&](const Stmt& x) { + if (x.kind == SK::Block && x.begin < uses.front() && uses.back() < x.end && + x.end - x.begin < B->end - B->begin) + B = &x; + }); + auto itemAt = [&](const Stmt* blk) { + for (size_t k = 0; k < blk->kids.size(); ++k) + if (blk->kids[k]->end > uses.front()) return (int)k; + return -1; + }; + std::vector> targets; + int tb = itemAt(B); + if (tb >= 0 && (B != P || tb > di + 1)) targets.push_back({B, tb}); + int tp = itemAt(P); + if (B != P && tp > di + 1) targets.push_back({P, tp}); + Effects ie = d.init == Declarator::None ? Effects{} : effectsOf(f, d.initBegin, d.initEnd); + for (auto [blk, t] : targets) { + const Stmt& item = *blk->kids[t]; + int at = item.begin; + if (d.init != Declarator::None) { + Effects between = effectsOf(f, s->end, at); + if (initBlocked(ie, between)) continue; + if (insideLoopNotContaining(f, at, *s)) continue; + } else if (insideLoopNotContaining(f, at, *s)) { + // only when the loop body starts by assigning it + if (!(item.kind == SK::Expr && item.begin == uses.front() && f.t(at + 1) == "=")) + continue; + } + if (!emit([&f, s, at]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + rw.insertBefore(at, f.textOf(*s)); + return rw.apply(); + })) + return; + } + } +} + +// --------------------------------------------------------------------------- +// hoist_decl + +struct Hoist { + const Stmt* s; + std::string top; // declarations to place at the top + std::string leftover; // what stays in place ("" to remove) +}; + +bool makeHoist(const Func& f, const Stmt& s, Hoist& h) { + if (s.kind != SK::Decl || s.isStatic || !s.parent || s.parent->kind != SK::Block) return false; + h.s = &s; + for (auto& d : s.decls) { + if (d.init == Declarator::None) { + h.top += declNoInit(f, s, d) + ";"; + continue; + } + if (d.init != Declarator::Assign || s.isConst || d.isRef || d.isArray || + f.t(d.initBegin) == "{") + return false; + if (!h.top.empty()) h.top += " "; + h.top += declNoInit(f, s, d) + ";"; + if (!h.leftover.empty()) h.leftover += " "; + h.leftover += f.t(d.nameTok) + " = " + f.tokText(d.initBegin, d.initEnd) + ";"; + } + return true; +} + +// First item of blk that isn't a declaration, or -1. +int firstNonDecl(const Stmt& blk) { + for (size_t k = 0; k < blk.kids.size(); ++k) + if (blk.kids[k]->kind != SK::Decl && blk.kids[k]->kind != SK::Empty) return (int)k; + return -1; +} + +void applyHoist(Rewriter& rw, const Hoist& h, int at) { + if (h.leftover.empty()) rw.remove(h.s->begin, h.s->end); + else rw.replace(h.s->begin, h.s->end, h.leftover); + rw.insertBefore(at, h.top); +} + +void enumHoistDecl(const Func& f, const EmitFn& emit) { + std::vector hs; + forEachStmt(*f.body, [&](const Stmt& s) { + Hoist h; + if (makeHoist(f, s, h)) hs.push_back(h); + }); + int topIdx = firstNonDecl(*f.body); + if (topIdx < 0) return; + int topTok = f.body->kids[topIdx]->begin; + std::vector toTop; + for (auto& h : hs) { + const Stmt* P = h.s->parent; + bool inTopRun = P == f.body.get() && indexInParent(*h.s) < topIdx; + if (!inTopRun) { + toTop.push_back(&h); + if (!emit([&f, h, topTok]() { + Rewriter rw(f); + applyHoist(rw, h, topTok); + return rw.apply(); + })) + return; + } + if (P != f.body.get()) { + int bi = firstNonDecl(*P); + if (bi >= 0 && indexInParent(*h.s) > bi) { + int at = P->kids[bi]->begin; + if (!emit([&f, h, at]() { + Rewriter rw(f); + applyHoist(rw, h, at); + return rw.apply(); + })) + return; + } + } + } + if (toTop.size() >= 2) { + std::vector all; + for (auto* h : toTop) all.push_back(*h); + emit([&f, all, topTok]() { + Rewriter rw(f); + for (auto& h : all) applyHoist(rw, h, topTok); + return rw.apply(); + }); + } +} + +// --------------------------------------------------------------------------- +// merge_decl: "T x; ... x = e;" -> "T x = e;" + +void enumMergeDecl(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (simpleSingleDecl(s) && s.decls[0].init == Declarator::None && !s.decls[0].isArray) + decls.push_back(&s); + }); + for (const Stmt* s : decls) { + const Declarator& d = s->decls[0]; + std::string name = f.t(d.nameTok); + std::vector uses = usesOf(f, name, s->end, s->parent->end); + if (uses.empty() || f.t(uses[0] + 1) != "=") continue; + const Stmt* A = nullptr; + forEachStmt(*f.body, [&](const Stmt& x) { + if (x.kind == SK::Expr && x.begin == uses[0]) A = &x; + }); + if (!A || !A->parent || A->parent->kind != SK::Block) continue; + if (uses.back() >= A->parent->end) continue; + if (uses.size() > 1 && uses[1] < A->end) continue; // "x = x + 1" + std::string text = declNoInit(f, *s, d) + " = " + f.tokText(A->begin + 2, A->end - 1) + ";"; + if (!emit([&f, s, A, text]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + rw.replace(A->begin, A->end, text); + return rw.apply(); + })) + return; + } +} + +// --------------------------------------------------------------------------- +// expression passes + +template +void forEachSiteExpr(const Func& f, Fn fn) { + for (auto& site : exprSites(f)) { + auto x = parseExpr(f, site.b, site.e); + if (!x) continue; + std::shared_ptr root(std::move(x)); + bool stop = false; + forEachExpr(*root, [&](Expr& e) { + if (!stop && !fn(site, root, e)) stop = true; + }); + if (stop) return; + } +} + +void enumSwapOperands(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr root, Expr& x) { + if (x.k != Expr::Binary) return true; + const std::string& op = f.t(x.op); + if (op != "+" && op != "*" && op != "&" && op != "|" && op != "^" && op != "==" && op != "!=") + return true; + int p = binPrec(op); + const Expr& l = *x.kids[0]; + const Expr& r = *x.kids[1]; + bool assoc = op == "+" || op == "*" || op == "&" || op == "|" || op == "^"; + std::string lt = exprText(f, l); + if (exprPrec(f, l) < p || (exprPrec(f, l) == p && !(assoc && f.t(l.op) == op))) + lt = paren(lt); + std::string text = exprText(f, r) + " " + op + " " + lt; + int b = x.b, e = x.e; + return emit([&f, b, e, text, root]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }); +} + +void enumFlipCompare(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr root, Expr& x) { + if (x.k != Expr::Binary) return true; + const std::string& op = f.t(x.op); + std::string m = op == "<" ? ">" : op == ">" ? "<" : op == "<=" ? ">=" : op == ">=" ? "<=" : ""; + if (m.empty()) return true; + std::string text = exprText(f, *x.kids[1]) + " " + m + " " + operandText(f, *x.kids[0], 10); + int b = x.b, e = x.e; + return emit([&f, b, e, text, root]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }); +} + +std::string negate(const Func& f, int b, int e) { + auto x = parseExpr(f, b, e); + std::string all = f.tokText(b, e); + if (!x) return "!(" + all + ")"; + if (x->k == Expr::Binary) { + const std::string& op = f.t(x->op); + std::string n = op == "==" ? "!=" : op == "!=" ? "==" : op == "<" ? ">=" : op == ">=" ? "<" + : op == ">" ? "<=" : op == "<=" ? ">" : ""; + if (!n.empty()) return exprText(f, *x->kids[0]) + " " + n + " " + exprText(f, *x->kids[1]); + } + if (x->k == Expr::Unary && f.t(x->op) == "!") { + const Expr& o = *x->kids[0]; + if (o.k == Expr::Paren) return exprText(f, *o.kids[0]); + return exprText(f, o); + } + if (isPostfixLike(*x)) return "!" + all; + return "!(" + all + ")"; +} + +void enumInvertIf(const Func& f, const EmitFn& emit) { + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind != SK::If || s.kids.size() != 2) return; + const Stmt* th = s.kids[0].get(); + const Stmt* el = s.kids[1].get(); + std::string neg = negate(f, s.condOpen + 1, s.condClose); + std::string elText = f.textOf(*el); + if (el->kind == SK::If) { + std::string ind = f.indentOf(s.begin); + std::string inner; + for (char c : elText) inner += c == '\n' ? std::string("\n ") : std::string(1, c); + elText = "{" + ind + " " + inner + ind + "}"; + } + const Stmt* sp = &s; + emit([&f, sp, th, el, neg, elText]() { + Rewriter rw(f); + rw.replace(sp->condOpen + 1, sp->condClose, neg); + rw.replace(th->begin, th->end, elText); + rw.replace(el->begin, el->end, f.textOf(*th)); + return rw.apply(); + }); + }); +} + +// The one expression statement inside s (s itself, or a block holding just it). +const Stmt* singleStmt(const Stmt* s) { + while (s && s->kind == SK::Block) { + if (s->kids.size() != 1) return nullptr; + s = s->kids[0].get(); + } + return s; +} + +void enumTernary(const Func& f, const EmitFn& emit) { + auto emitReplace = [&](int b, int e, std::string text) { + return emit([&f, b, e, text]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }; + auto branch = [&](const Expr& x) { + return x.k == Expr::Assign || x.k == Expr::Comma ? paren(exprText(f, x)) : exprText(f, x); + }; + auto condText = [&](const Expr& c) { + return c.k == Expr::Paren ? exprText(f, *c.kids[0]) : exprText(f, c); + }; + // ternary -> if/else + for (auto& site : exprSites(f)) { + const Stmt& s = *site.stmt; + if (s.kind != SK::Expr && s.kind != SK::Return) continue; + auto x = parseExpr(f, site.b, site.e); + if (!x) continue; + std::string ind = f.indentOf(s.begin), in = ind + " "; + if (s.kind == SK::Expr && x->k == Expr::Assign && f.t(x->op) == "=" && + x->kids[1]->k == Expr::Ternary) { + const Expr& t = *x->kids[1]; + Effects le = effectsOf(f, x->kids[0]->b, x->kids[0]->e); + if (le.call || le.memWrite) continue; + std::string L = exprText(f, *x->kids[0]); + std::string text = "if (" + condText(*t.kids[0]) + ")" + ind + "{" + in + L + " = " + + exprText(f, *t.kids[1]) + ";" + ind + "}" + ind + "else" + ind + + "{" + in + L + " = " + exprText(f, *t.kids[2]) + ";" + ind + "}"; + if (!emitReplace(s.begin, s.end, text)) return; + } else if (s.kind == SK::Return && x->k == Expr::Ternary) { + std::string c = condText(*x->kids[0]); + std::string a = exprText(f, *x->kids[1]), b = exprText(f, *x->kids[2]); + std::string withElse = "if (" + c + ")" + ind + "{" + in + "return " + a + ";" + ind + + "}" + ind + "else" + ind + "{" + in + "return " + b + ";" + + ind + "}"; + std::string early = "if (" + c + ")" + ind + "{" + in + "return " + a + ";" + ind + + "}" + ind + "return " + b + ";"; + if (!emitReplace(s.begin, s.end, withElse)) return; + if (!emitReplace(s.begin, s.end, early)) return; + } + } + // if/else -> ternary + std::vector ifs; + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind == SK::If) ifs.push_back(&s); + }); + for (const Stmt* s : ifs) { + auto c = parseExpr(f, s->condOpen + 1, s->condClose); + if (!c) continue; + std::string ct = c->k == Expr::Assign || c->k == Expr::Ternary || c->k == Expr::Comma + ? paren(exprText(f, *c)) + : exprText(f, *c); + const Stmt* th = singleStmt(s->kids[0].get()); + const Stmt* el = s->kids.size() == 2 ? singleStmt(s->kids[1].get()) : nullptr; + if (!th) continue; + if (el && th->kind == SK::Expr && el->kind == SK::Expr) { + auto a = parseExpr(f, th->begin, th->end - 1); + auto b = parseExpr(f, el->begin, el->end - 1); + if (!a || !b || a->k != Expr::Assign || b->k != Expr::Assign) continue; + if (f.t(a->op) != "=" || f.t(b->op) != "=") continue; + std::string L = exprText(f, *a->kids[0]); + if (norm(L) != norm(exprText(f, *b->kids[0]))) continue; + std::string text = L + " = " + ct + " ? " + branch(*a->kids[1]) + " : " + + branch(*b->kids[1]) + ";"; + if (!emitReplace(s->begin, s->end, text)) return; + continue; + } + if (th->kind != SK::Return || f.t(th->begin) != "return") continue; + auto a = parseExpr(f, th->begin + 1, th->end - 1); + if (!a) continue; + const Stmt* other = el; + int end = s->end; + if (!other && s->parent && s->parent->kind == SK::Block) { + int i = indexInParent(*s); + if (i >= 0 && i + 1 < (int)s->parent->kids.size()) { + other = s->parent->kids[i + 1].get(); + end = other->end; + } + } + if (!other || other->kind != SK::Return || f.t(other->begin) != "return") continue; + auto b = parseExpr(f, other->begin + 1, other->end - 1); + if (!b) continue; + std::string text = "return " + ct + " ? " + branch(*a) + " : " + branch(*b) + ";"; + if (!emitReplace(s->begin, end, text)) return; + } +} + +void enumCompoundAssign(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr root, Expr& x) { + if (x.k != Expr::Assign) return true; + const std::string& op = f.t(x.op); + const Expr& L = *x.kids[0]; + const Expr& R = *x.kids[1]; + std::string lt = exprText(f, L); + Effects le = effectsOf(f, L.b, L.e); + if (le.call || le.memWrite) return true; + std::string text; + if (op == "=") { + if (R.k != Expr::Binary) return true; + const std::string& bop = f.t(R.op); + static const char* ok[] = {"+", "-", "*", "/", "%", "&", "|", "^", "<<", ">>"}; + if (std::find_if(std::begin(ok), std::end(ok), [&](const char* o) { return bop == o; }) == + std::end(ok)) + return true; + if (norm(exprText(f, *R.kids[0])) != norm(lt)) return true; + text = lt + " " + bop + "= " + exprText(f, *R.kids[1]); + } else { + std::string bop = op.substr(0, op.size() - 1); + text = lt + " = " + lt + " " + bop + " " + operandText(f, R, binPrec(bop)); + } + int b = x.b, e = x.e; + return emit([&f, b, e, text, root]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }); +} + +} // namespace + +// --------------------------------------------------------------------------- + +const std::vector& allPasses() { + static const std::vector passes = { + {"reorder_saves", + "Permute runs of consecutive local saves (declarations and assignments to locals " + "that don't call or store) in every order that keeps dependencies", + 30, enumReorderSaves, randomReorderSaves}, + {"move_stmt", + "Move a statement up or down past neighbours it doesn't depend on (locals and calls " + "are respected, memory aliasing is not)", + 20, + enumMoveStmt, nullptr}, + {"inline_local", + "Remove a local: substitute its initializer into its uses (plain and with a cast), " + "or drop it if unused", + 15, enumInlineLocal, nullptr}, + {"sink_decl", "Move a declaration into the innermost block holding all its uses, or to " + "just before its first use", + 15, enumSinkDecl, nullptr}, + {"hoist_decl", + "Move declarations to the top of their block or of the function (one, or all of " + "them), splitting the initializer into an assignment", + 15, enumHoistDecl, nullptr}, + {"merge_decl", "Join 'T x; ... x = e;' into 'T x = e;' at the assignment", 10, + enumMergeDecl, nullptr}, + {"swap_operands", "Swap the operands of a commutative operator (+ * & | ^ == !=)", 15, + enumSwapOperands, nullptr}, + {"flip_compare", "Mirror a comparison: 'a < b' -> 'b > a'", 10, enumFlipCompare, nullptr}, + {"invert_if", "Negate an if condition and swap the then/else branches", 10, enumInvertIf, + nullptr}, + {"ternary", "Turn 'x = c ? a : b' / 'return c ? a : b' into if/else and back", 8, + enumTernary, nullptr}, + {"compound_assign", "'x = x op y' <-> 'x op= y'", 8, enumCompoundAssign, nullptr}, + }; + return passes; +} + +const Pass* findPass(const std::string& name) { + for (auto& p : allPasses()) + if (p.name == name) return &p; + return nullptr; +} + +bool randomMutation(const Pass& p, const Func& f, Rng& rng, std::string& out) { + if (p.random) return p.random(f, rng, out); + // reservoir sample one mutation without materialising every text + Mutation chosen; + size_t seen = 0; + p.enumerate(f, [&](Mutation m) { + seen++; + if (std::uniform_int_distribution(0, seen - 1)(rng) == 0) chosen = std::move(m); + return seen < 20000; + }); + if (!chosen) return false; + out = chosen(); + return !out.empty(); +} + +} // namespace perm diff --git a/src/passes.hpp b/src/passes.hpp new file mode 100644 index 0000000..f52e3d9 --- /dev/null +++ b/src/passes.hpp @@ -0,0 +1,49 @@ +// Source mutation passes. Each pass looks at a parsed function and offers +// a set of rewrites ("mutations"); a mutation produces the new function text. +#pragma once + +#include "parser.hpp" + +#include +#include +#include +#include + +namespace perm { + +using Rng = std::mt19937_64; +using Mutation = std::function; +using EmitFn = std::function; // return false to stop enumerating + +struct Pass { + std::string name; + std::string description; + int weight = 10; // relative chance of being picked in random mode + std::function enumerate; + // Optional: a cheaper random pick than enumerating everything. + std::function random; +}; + +const std::vector& allPasses(); +const Pass* findPass(const std::string& name); + +// One random mutation of f by pass p. Returns false if p has nothing to offer. +bool randomMutation(const Pass& p, const Func& f, Rng& rng, std::string& out); + +// Token-range rewriting of a function. +class Rewriter { +public: + explicit Rewriter(const Func& f); + void replace(int b, int e, const std::string& text); + void remove(int b, int e); // also drops the line break before the range + void insertBefore(int tok, const std::string& stmtText); // on its own line + std::string apply() const; // empty if edits overlapped + +private: + const Func& f_; + std::vector lead_, text_, prefix_; + std::vector touched_; + bool overlap_ = false; +}; + +} // namespace perm diff --git a/src/runner.cpp b/src/runner.cpp new file mode 100644 index 0000000..177ad9d --- /dev/null +++ b/src/runner.cpp @@ -0,0 +1,99 @@ +#include "runner.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace perm { + +CmdResult runCommand(const std::string& cmd, int timeoutSec) { + CmdResult r; + int fds[2]; + if (pipe(fds) != 0) { + r.output = "pipe() failed"; + return r; + } + pid_t pid = fork(); + if (pid < 0) { + close(fds[0]); + close(fds[1]); + r.output = "fork() failed"; + return r; + } + if (pid == 0) { + setpgid(0, 0); + dup2(fds[1], 1); + dup2(fds[1], 2); + close(fds[0]); + close(fds[1]); + int devnull = open("/dev/null", O_RDONLY); + if (devnull >= 0) dup2(devnull, 0); + execl("/bin/sh", "sh", "-c", cmd.c_str(), (char*)nullptr); + _exit(127); + } + close(fds[1]); + // Wait for the child itself, not for EOF: daemons it spawns (wineserver) + // inherit the pipe and can keep it open long after the compile is done. + fcntl(fds[0], F_SETFL, fcntl(fds[0], F_GETFL) | O_NONBLOCK); + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeoutSec); + char buf[8192]; + int st = 0; + bool exited = false; + auto drain = [&]() { + ssize_t n; + while ((n = read(fds[0], buf, sizeof buf)) > 0) r.output.append(buf, (size_t)n); + }; + while (!exited) { + pollfd p{fds[0], POLLIN, 0}; + if (poll(&p, 1, 20) > 0 && (p.revents & POLLHUP)) usleep(5000); // closed: don't spin + drain(); + pid_t w = waitpid(pid, &st, WNOHANG); + if (w == pid) exited = true; + else if (timeoutSec > 0 && std::chrono::steady_clock::now() >= deadline) { + r.timedOut = true; + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + while (waitpid(pid, &st, 0) < 0 && errno == EINTR) {} + exited = true; + } + } + drain(); + close(fds[0]); + r.status = WIFEXITED(st) ? WEXITSTATUS(st) : -1; + if (r.timedOut) r.status = -1; + return r; +} + +std::string shellQuote(const std::string& s) { + std::string r = "'"; + for (char c : s) { + if (c == '\'') r += "'\\''"; + else r += c; + } + return r + "'"; +} + +std::string expandTemplate(const std::string& tmpl, const std::map& vars) { + std::string out; + for (size_t i = 0; i < tmpl.size();) { + if (tmpl[i] == '{') { + size_t e = tmpl.find('}', i); + if (e != std::string::npos) { + auto it = vars.find(tmpl.substr(i + 1, e - i - 1)); + if (it != vars.end()) { + out += shellQuote(it->second); + i = e + 1; + continue; + } + } + } + out += tmpl[i++]; + } + return out; +} + +} // namespace perm diff --git a/src/runner.hpp b/src/runner.hpp new file mode 100644 index 0000000..115067e --- /dev/null +++ b/src/runner.hpp @@ -0,0 +1,22 @@ +// Running shell commands with a timeout. +#pragma once + +#include +#include + +namespace perm { + +struct CmdResult { + int status = -1; // exit code, or -1 if it didn't exit normally + bool timedOut = false; + std::string output; // stdout and stderr together +}; + +CmdResult runCommand(const std::string& cmd, int timeoutSec); + +std::string shellQuote(const std::string& s); + +// Replaces {key} in tmpl with the shell-quoted value. +std::string expandTemplate(const std::string& tmpl, const std::map& vars); + +} // namespace perm diff --git a/src/scorer.cpp b/src/scorer.cpp new file mode 100644 index 0000000..2efc692 --- /dev/null +++ b/src/scorer.cpp @@ -0,0 +1,359 @@ +#include "scorer.hpp" + +#include "runner.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace perm { + +namespace { + +std::string trim(const std::string& s) { + size_t b = s.find_first_not_of(" \t\r"); + if (b == std::string::npos) return ""; + size_t e = s.find_last_not_of(" \t\r"); + return s.substr(b, e - b + 1); +} + +bool isHex(const std::string& s) { + return !s.empty() && std::all_of(s.begin(), s.end(), [](char c) { return std::isxdigit((unsigned char)c); }); +} + +struct RawInsn { + unsigned addr = 0; + std::vector bytes; + std::string mnem, args; + std::vector> relocs; // offset, symbol + std::vector relocTypes; +}; + +// Replaces the first whole-word occurrence of word in s. +bool replaceWord(std::string& s, const std::string& word, const std::string& with) { + size_t pos = 0; + while ((pos = s.find(word, pos)) != std::string::npos) { + bool lb = pos == 0 || !(std::isalnum((unsigned char)s[pos - 1]) || s[pos - 1] == '_'); + size_t e = pos + word.size(); + bool rb = e >= s.size() || !(std::isalnum((unsigned char)s[e]) || s[e] == '_'); + if (lb && rb) { + s.replace(pos, word.size(), with); + return true; + } + pos = e; + } + return false; +} + +std::string hexStr(unsigned v) { + char b[16]; + std::snprintf(b, sizeof b, "0x%x", v); + return b; +} + +} // namespace + +std::vector parseObjdump(const std::string& text, const std::string& symbol, + bool ignoreRelocNames) { + std::vector raw; + std::istringstream in(text); + std::string line; + bool inside = false; + std::string header = "<" + symbol + ">:"; + while (std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.find(">:") != std::string::npos && line.find('<') != std::string::npos && + !line.empty() && line[0] != ' ' && line[0] != '\t') { + inside = line.find(header) != std::string::npos; + continue; + } + if (!inside) continue; + std::string t = trim(line); + if (t.empty()) continue; + size_t colon = t.find(':'); + if (colon == std::string::npos || !isHex(t.substr(0, colon))) continue; + unsigned addr = (unsigned)std::stoul(t.substr(0, colon), nullptr, 16); + std::string rest = trim(t.substr(colon + 1)); + // relocation line: "IMAGE_REL_I386_DIR32\tsym" / "R_386_32\tsym" + if (rest.rfind("IMAGE_REL_", 0) == 0 || rest.rfind("R_386", 0) == 0 || + rest.rfind("R_X86", 0) == 0 || rest.rfind("dir32", 0) == 0 || + rest.rfind("DISP32", 0) == 0) { + size_t sp = rest.find_first_of(" \t"); + std::string type = rest.substr(0, sp); + std::string sym = sp == std::string::npos ? "" : trim(rest.substr(sp)); + if (!raw.empty()) { + raw.back().relocs.push_back({addr, sym}); + raw.back().relocTypes.push_back(type); + } + continue; + } + RawInsn r; + r.addr = addr; + // raw bytes are pairs of hex digits separated by single spaces, then a tab + size_t p = 0; + while (p + 2 <= rest.size() && isHex(rest.substr(p, 2)) && + (p + 2 == rest.size() || rest[p + 2] == ' ' || rest[p + 2] == '\t')) { + r.bytes.push_back((unsigned char)std::stoul(rest.substr(p, 2), nullptr, 16)); + p += 2; + if (p < rest.size() && rest[p] == '\t') break; + while (p < rest.size() && rest[p] == ' ') p++; + if (p < rest.size() && rest[p] == '\t') break; + } + std::string ins = trim(rest.substr(p)); + if (ins.empty()) { + // continuation of a long byte sequence (binutils) + if (!raw.empty()) raw.back().bytes.insert(raw.back().bytes.end(), r.bytes.begin(), r.bytes.end()); + continue; + } + size_t sp = ins.find_first_of(" \t"); + r.mnem = ins.substr(0, sp); + r.args = sp == std::string::npos ? "" : trim(ins.substr(sp)); + // prefixes such as "lock", "rep" are printed as their own mnemonic + raw.push_back(std::move(r)); + } + + std::map indexOf; + for (size_t i = 0; i < raw.size(); ++i) indexOf[raw[i].addr] = (int)i; + + std::vector out; + for (size_t i = 0; i < raw.size(); ++i) { + RawInsn& r = raw[i]; + std::string args = r.args; + size_t lt = args.find(" <"); + if (lt != std::string::npos) args = trim(args.substr(0, lt)); + bool branch = r.mnem[0] == 'j' || r.mnem.rfind("call", 0) == 0 || r.mnem.rfind("loop", 0) == 0; + bool handled = false; + for (size_t k = 0; k < r.relocs.size(); ++k) { + std::string sym = ignoreRelocNames ? "" : r.relocs[k].second; + bool rel = r.relocTypes[k].find("REL32") != std::string::npos || + r.relocTypes[k].find("PC32") != std::string::npos || + r.relocTypes[k] == "DISP32" || r.relocTypes[k].find("PLT") != std::string::npos; + if (rel && branch) { + args = sym; + handled = true; + continue; + } + unsigned off = r.relocs[k].first - r.addr; + unsigned addend = 0; + for (unsigned b = 0; b < 4 && off + b < r.bytes.size(); ++b) + addend |= (unsigned)r.bytes[off + b] << (8 * b); + std::string v = hexStr(addend); + if (!replaceWord(args, v, addend ? sym + "+" + v : sym)) args += " ; " + sym; + } + if (branch && !handled && r.relocs.empty() && args.rfind("0x", 0) == 0) { + unsigned target = (unsigned)std::stoul(args, nullptr, 16); + auto it = indexOf.find(target); + args = it != indexOf.end() ? "L" + std::to_string(it->second) : args; + } + if (ignoreRelocNames && r.relocs.empty()) { + // absolute addresses in code taken straight from an executable + std::string res; + size_t q = 0; + while (q < args.size()) { + if (args.compare(q, 2, "0x") == 0) { + size_t e = q + 2; + while (e < args.size() && std::isxdigit((unsigned char)args[e])) e++; + unsigned long v = std::stoul(args.substr(q + 2, e - q - 2), nullptr, 16); + res += v >= 0x400000 && v < 0x10000000 ? "" : args.substr(q, e - q); + q = e; + } else { + res += args[q++]; + } + } + args = res; + } + out.push_back({r.mnem, args}); + } + return out; +} + +bool disassemble(const ScoreConfig& cfg, const std::string& obj, const std::string& symbol, + std::vector& out, std::string& err) { + std::string cmd = shellQuote(cfg.objdump) + " -d -r --disassemble-symbols=" + + shellQuote(symbol) + " " + shellQuote(obj); + CmdResult r = runCommand(cmd, 60); + if (r.status != 0) { + err = "objdump failed: " + r.output; + return false; + } + out = parseObjdump(r.output, symbol, cfg.ignoreRelocNames); + if (out.empty()) { + err = "symbol '" + symbol + "' not found in " + obj; + return false; + } + return true; +} + +namespace { + +std::string squash(const std::string& s, bool numbers) { + std::string r; + for (size_t i = 0; i < s.size();) { + if (s[i] == '%') { + r += "%r"; + i++; + while (i < s.size() && std::isalnum((unsigned char)s[i])) i++; + } else if (numbers && std::isdigit((unsigned char)s[i])) { + r += "N"; + while (i < s.size() && (std::isxdigit((unsigned char)s[i]) || s[i] == 'x')) i++; + } else { + r += s[i++]; + } + } + return r; +} + +enum Op { Same, Sub, Del, Ins }; + +struct Alignment { + int score = 0; + std::vector>> steps; +}; + +int subCost(const ScoreConfig& cfg, const Insn& a, const Insn& b) { + if (a.mnem != b.mnem) return INT_MAX / 4; + if (a.args == b.args) return 0; + if (squash(a.args, false) == squash(b.args, false)) return cfg.penaltyRegalloc; + return cfg.penaltyArgs; +} + +Alignment align(const ScoreConfig& cfg, const std::vector& a, const std::vector& b) { + size_t n = a.size(), m = b.size(); + std::vector dp((n + 1) * (m + 1)); + auto at = [&](size_t i, size_t j) -> int& { return dp[i * (m + 1) + j]; }; + for (size_t i = 0; i <= n; ++i) at(i, 0) = (int)i * cfg.penaltyDelete; + for (size_t j = 0; j <= m; ++j) at(0, j) = (int)j * cfg.penaltyInsert; + for (size_t i = 1; i <= n; ++i) + for (size_t j = 1; j <= m; ++j) { + int best = std::min(at(i - 1, j) + cfg.penaltyDelete, at(i, j - 1) + cfg.penaltyInsert); + int s = subCost(cfg, a[i - 1], b[j - 1]); + if (s < INT_MAX / 4) best = std::min(best, at(i - 1, j - 1) + s); + at(i, j) = best; + } + Alignment al; + size_t i = n, j = m; + while (i > 0 || j > 0) { + if (i > 0 && j > 0) { + int s = subCost(cfg, a[i - 1], b[j - 1]); + if (s < INT_MAX / 4 && at(i, j) == at(i - 1, j - 1) + s) { + al.steps.push_back({s == 0 ? Same : Sub, {(int)i - 1, (int)j - 1}}); + i--, j--; + continue; + } + } + if (i > 0 && at(i, j) == at(i - 1, j) + cfg.penaltyDelete) { + al.steps.push_back({Del, {(int)i - 1, -1}}); + i--; + } else { + al.steps.push_back({Ins, {-1, (int)j - 1}}); + j--; + } + } + std::reverse(al.steps.begin(), al.steps.end()); + al.score = at(n, m); + return al; +} + +} // namespace + +int scoreInsns(const ScoreConfig& cfg, const std::vector& target, const std::vector& cand) { + Alignment al = align(cfg, target, cand); + // a deletion and an insertion of the same instruction is a reordering + std::multiset dels, ins; + for (auto& s : al.steps) { + if (s.first == Del) dels.insert(target[s.second.first].full()); + if (s.first == Ins) ins.insert(cand[s.second.second].full()); + } + int score = al.score; + for (auto& d : dels) { + auto it = ins.find(d); + if (it == ins.end()) continue; + ins.erase(it); + score -= cfg.penaltyDelete + cfg.penaltyInsert - cfg.penaltyReorder; + } + return score; +} + +std::string diffInsns(const ScoreConfig& cfg, const std::vector& target, + const std::vector& cand) { + Alignment al = align(cfg, target, cand); + std::ostringstream o; + char buf[512]; + std::snprintf(buf, sizeof buf, " %-44s %s\n", "TARGET", "CANDIDATE"); + o << buf; + for (auto& s : al.steps) { + std::string l = s.second.first >= 0 ? target[s.second.first].full() : ""; + std::string r = s.second.second >= 0 ? cand[s.second.second].full() : ""; + const char* mark = s.first == Same ? " " : s.first == Sub ? "r " : s.first == Del ? "- " : "+ "; + std::snprintf(buf, sizeof buf, "%s %-44s %s\n", mark, l.c_str(), r.c_str()); + o << buf; + } + return o.str(); +} + +std::vector listSymbols(const ScoreConfig& cfg, const std::string& obj, std::string& err) { + std::vector out; + CmdResult r = runCommand(shellQuote(cfg.objdump) + " -t " + shellQuote(obj), 60); + if (r.status != 0) { + err = r.output; + return out; + } + std::istringstream in(r.output); + std::string line; + while (std::getline(in, line)) { + // COFF: "[ 5](sec 1)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 ?foo@@YAHXZ" + // ELF: "00000000 g F .text 0000002a foo" + if (line.find("(sec") != std::string::npos || line.find(" F ") != std::string::npos || + line.find(" g ") != std::string::npos) { + size_t sp = line.find_last_of(" \t"); + if (sp != std::string::npos) out.push_back(line.substr(sp + 1)); + } + } + return out; +} + +std::string guessSymbol(const std::vector& syms, const std::string& qualName) { + std::vector comps; + size_t i = 0; + while (true) { + size_t j = qualName.find("::", i); + comps.push_back(qualName.substr(i, j == std::string::npos ? std::string::npos : j - i)); + if (j == std::string::npos) break; + i = j + 2; + } + std::string last = comps.back(); + std::vector prefixes; + // MSVC + { + // ctors/dtors are named by their class: "??0G@ns@@" for ns::G::G + std::string p; + if (!last.empty() && last[0] == '~') p = "??1"; + else if (comps.size() >= 2 && comps[comps.size() - 2] == last) p = "??0"; + else p = "?" + last + "@"; + for (size_t k = comps.size() - 1; k-- > 0;) p += comps[k] + "@"; + prefixes.push_back(p + "@"); + } + // Itanium + if (comps.size() == 1) { + prefixes.push_back("_Z" + std::to_string(last.size()) + last); + } else { + std::string p = "_ZN"; + for (auto& c : comps) { + if (!c.empty() && c[0] == '~') p += "D"; + else p += std::to_string(c.size()) + c; + } + prefixes.push_back(p); + } + for (auto& p : prefixes) + for (auto& s : syms) + if (s.rfind(p, 0) == 0) return s; + for (auto& s : syms) + if (s == last || s == "_" + last) return s; + return ""; +} + +} // namespace perm diff --git a/src/scorer.hpp b/src/scorer.hpp new file mode 100644 index 0000000..ea19a9c --- /dev/null +++ b/src/scorer.hpp @@ -0,0 +1,45 @@ +// Disassembles one function from a COFF/ELF object with llvm-objdump and +// scores it against a target (0 = identical after normalisation). +#pragma once + +#include +#include + +namespace perm { + +struct Insn { + std::string mnem, args; + std::string full() const { return args.empty() ? mnem : mnem + " " + args; } +}; + +struct ScoreConfig { + std::string objdump = "llvm-objdump"; + bool ignoreRelocNames = false; // compare relocated operands as "" + int penaltyRegalloc = 10; // same instruction, other registers + int penaltyArgs = 50; // same mnemonic, other operands + int penaltyReorder = 60; // an instruction that moved + int penaltyInsert = 100; + int penaltyDelete = 100; +}; + +bool disassemble(const ScoreConfig& cfg, const std::string& obj, const std::string& symbol, + std::vector& out, std::string& err); + +// Parses llvm-objdump -d -r output for one function. Exposed for tests. +std::vector parseObjdump(const std::string& text, const std::string& symbol, + bool ignoreRelocNames); + +int scoreInsns(const ScoreConfig& cfg, const std::vector& target, + const std::vector& cand); + +// Side-by-side alignment of target and candidate, for humans. +std::string diffInsns(const ScoreConfig& cfg, const std::vector& target, + const std::vector& cand); + +std::vector listSymbols(const ScoreConfig& cfg, const std::string& obj, + std::string& err); + +// Picks the symbol for a qualified C++ name (MSVC or Itanium mangling, or C). +std::string guessSymbol(const std::vector& syms, const std::string& qualName); + +} // namespace perm diff --git a/tests/e2e/ops_base.cpp b/tests/e2e/ops_base.cpp new file mode 100644 index 0000000..40f41a0 --- /dev/null +++ b/tests/e2e/ops_base.cpp @@ -0,0 +1,16 @@ +struct V { int x, y, z; }; +int g; + +int Sum(V* v, int k) +{ + int r = v->x + k * v->z; + if (r <= 10) + { + g = v->y; + } + else + { + g = r; + } + return r + v->y; +} diff --git a/tests/e2e/ops_target.cpp b/tests/e2e/ops_target.cpp new file mode 100644 index 0000000..de7a492 --- /dev/null +++ b/tests/e2e/ops_target.cpp @@ -0,0 +1,16 @@ +struct V { int x, y, z; }; +int g; + +int Sum(V* v, int k) +{ + int r = k * v->z + v->x; + if (r > 10) + { + g = r; + } + else + { + g = v->y; + } + return r + v->y; +} diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh new file mode 100755 index 0000000..c361110 --- /dev/null +++ b/tests/e2e/run.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# End-to-end: permute a wrongly ordered source until it compiles to the +# target. Uses clang's i686 MSVC target at -O0 as a stand-in for VC6 (at -O0 +# the statement order shows up directly in the code). +set -e +HERE=$(cd "$(dirname "$0")" && pwd) +PERMUTER=${PERMUTER:-$HERE/../../build/cpp_permuter} +CXX=${CXX_E2E:-clang++} +command -v "$CXX" >/dev/null || { echo "skip: $CXX not found"; exit 0; } +command -v llvm-objdump >/dev/null || { echo "skip: llvm-objdump not found"; exit 0; } +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +COMPILE="$CXX --target=i686-pc-windows-msvc -O0 -c {src} -o {obj}" + +check() { # name function mode passes [extra args] + cp "$HERE/$1_base.cpp" "$WORK/$1.cpp" + $CXX --target=i686-pc-windows-msvc -O0 -c "$HERE/$1_target.cpp" -o "$WORK/$1_target.obj" + if "$PERMUTER" -s "$WORK/$1.cpp" -f "$2" -c "$COMPILE" -t "$WORK/$1_target.obj" \ + -m "$3" -p "$4" -j 2 -n 400 --seed 1 -o "$WORK/out_$1_$3" $5 > "$WORK/log" 2>&1; then + echo "ok: $1 ($3)" + else + echo "FAIL: $1 ($3)" + cat "$WORK/log" + exit 1 + fi +} + +check saves Group::Promote exhaustive reorder_saves +check saves Group::Promote random reorder_saves,move_stmt +check ops Sum exhaustive swap_operands,invert_if "--depth 2" +check ops Sum random swap_operands,invert_if,flip_compare diff --git a/tests/e2e/saves_base.cpp b/tests/e2e/saves_base.cpp new file mode 100644 index 0000000..febedb6 --- /dev/null +++ b/tests/e2e/saves_base.cpp @@ -0,0 +1,18 @@ +// Base: decompiled with the saves in the wrong order. +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int pad[4]; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; void Promote(int idx); }; + +void Group::Promote(int idx) +{ + Weapon* leaderWeapon = leader->sel; + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon2 = leader->weapon; + Weapon* memberWeapon2 = list[idx]->weapon; + + leader->Copy(list[idx]); + leader->sel = leaderWeapon; + leader->weapon = leaderWeapon2; + list[idx]->sel = memberWeapon; + list[idx]->weapon = memberWeapon2; +} diff --git a/tests/e2e/saves_target.cpp b/tests/e2e/saves_target.cpp new file mode 100644 index 0000000..85e250a --- /dev/null +++ b/tests/e2e/saves_target.cpp @@ -0,0 +1,18 @@ +// Target: the "right" order of the local saves. +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int pad[4]; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; void Promote(int idx); }; + +void Group::Promote(int idx) +{ + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon2 = leader->weapon; + Weapon* leaderWeapon = leader->sel; + Weapon* memberWeapon2 = list[idx]->weapon; + + leader->Copy(list[idx]); + leader->sel = leaderWeapon; + leader->weapon = leaderWeapon2; + list[idx]->sel = memberWeapon; + list[idx]->weapon = memberWeapon2; +} diff --git a/tests/tests.cpp b/tests/tests.cpp new file mode 100644 index 0000000..4c123ff --- /dev/null +++ b/tests/tests.cpp @@ -0,0 +1,325 @@ +// Unit tests. No framework: each CHECK prints the failing line. + +#include "analysis.hpp" +#include "lexer.hpp" +#include "parser.hpp" +#include "passes.hpp" +#include "scorer.hpp" + +#include +#include + +using namespace perm; + +static int gFailures = 0; +#define CHECK(c) \ + do { \ + if (!(c)) { \ + std::cerr << __FILE__ << ":" << __LINE__ << ": CHECK(" #c ") failed\n"; \ + gFailures++; \ + } \ + } while (0) + +static std::unique_ptr parse(const std::string& src, const std::string& name) { + size_t b, e; + std::string err; + if (!locateFunction(src, name, b, e, err)) { + std::cerr << err << "\n"; + return nullptr; + } + return parseFunc(src.substr(b, e - b), err); +} + +static std::vector variants(const Func& f, const std::string& pass) { + std::vector out; + std::set seen; + findPass(pass)->enumerate(f, [&](Mutation m) { + std::string s = m(); + if (!s.empty() && seen.insert(s).second) out.push_back(s); + return true; + }); + return out; +} + +static bool contains(const std::string& s, const std::string& sub) { + return s.find(sub) != std::string::npos; +} + +static void testLexerRoundTrip() { + std::string src = "#include \"a.h\"\n// c\nint f(int a) { /* x */ return a >>= 2, L\"s\" '\\''; }\n" + "#define X(a) \\\n a\n__asm { mov eax, 1 ; don't }\n"; + auto toks = lex(src); + std::string back; + for (auto& t : toks) back += t.lead + t.text; + CHECK(back == src); + CHECK(toks[0].kind == TokKind::PP); + bool sawShift = false; + for (auto& t : toks) sawShift |= t.text == ">>="; + CHECK(sawShift); +} + +static void testLocate() { + std::string src = + "MATCH_FUNC(0x1)\nvoid A::f(int x);\nvoid g() { a.f(1); }\n" + "MATCH_FUNC(0x2)\nvoid A::f(int x)\n{\n x++;\n}\n" + "A::A() : m(1), n(2) { }\nA::~A() { }\n"; + size_t b, e; + std::string err; + CHECK(locateFunction(src, "A::f", b, e, err)); + CHECK(src.substr(b, 9) == "A::f(int "); + CHECK(src.substr(e - 1, 1) == "}"); + CHECK(locateFunction(src, "f", b, e, err)); // unqualified + CHECK(src.substr(b, 4) == "A::f"); + CHECK(locateFunction(src, "A::A", b, e, err)); + CHECK(locateFunction(src, "A::~A", b, e, err)); + CHECK(!locateFunction(src, "A::nope", b, e, err)); +} + +static void testParser() { + std::string src = R"( +void C::f(u8 idx, Foo* p) +{ + Ped* a = p->x; + s32 i, j = 2; + unsigned long k; + const Foo& r = *p; + a = 1; + p->x = 2; + idx * 2; + Foo::Bar(1); + for (int n = 0; n < 3; n++) { k += n; } + if (a) b(); else if (c) d(); else { e(); } + switch (idx) { case Enum::A: break; default: return; } +label: + __asm mov eax, 1 + __asm { int 3 } + do { i--; } while (i); + WIP_IMPLEMENTED; +} +)"; + auto f = parse(src, "C::f"); + CHECK(f != nullptr); + if (!f) return; + auto& k = f->body->kids; + std::vector kinds; + for (auto& s : k) kinds.push_back(s->kind); + std::vector want = {SK::Decl, SK::Decl, SK::Decl, SK::Decl, SK::Expr, SK::Expr, + SK::Expr, SK::Expr, SK::For, SK::If, SK::Switch, SK::Label, + SK::Asm, SK::Asm, SK::Do, SK::Expr}; + CHECK(kinds == want); + CHECK(f->params == (std::vector{"idx", "p"})); + CHECK(f->locals.count("a") && f->locals.count("i") && f->locals.count("j") && + f->locals.count("k") && f->locals.count("r") && f->locals.count("n")); + CHECK(k[1]->decls.size() == 2); + CHECK(k[3]->decls[0].isRef); + CHECK(k[9]->kids.size() == 2 && k[9]->kids[1]->kind == SK::If); +} + +static void testEffects() { + std::string src = "void f(P* p) { int a = p->x; int b = a + 1; p->y = b; g(a); int c = 3; }"; + auto f = parse(src, "f"); + auto& k = f->body->kids; + Effects e0 = effectsOfStmt(*f, *k[0]), e1 = effectsOfStmt(*f, *k[1]); + Effects e2 = effectsOfStmt(*f, *k[2]), e3 = effectsOfStmt(*f, *k[3]); + Effects e4 = effectsOfStmt(*f, *k[4]); + CHECK(e0.writes.count("a") && e0.memRead && !e0.memWrite && !e0.call); + CHECK(conflicts(e0, e1)); // b reads a + CHECK(e2.memWrite && e3.call); + CHECK(!conflicts(e0, e4)); + CHECK(conflicts(e2, e3)); + CHECK(!conflictsRelaxed(e0, e2)); +} + +static void testReorderSaves() { + std::string src = R"( +void G::f(u8 idx) +{ + Ped* t = Alloc(); + W* a = leader->w1; + W* b = list[idx]->w1; + W* c = leader->w2; + W* d = list[idx]->w2; + t->Copy(leader); +} +)"; + auto f = parse(src, "G::f"); + auto v = variants(*f, "reorder_saves"); + CHECK(v.size() == 23); // 4! - the original + // dependencies are kept + std::string src2 = "void f(P* p) { int a = p->x; int b = a + 1; int c = p->z; use(a, b, c); }"; + auto f2 = parse(src2, "f"); + auto v2 = variants(*f2, "reorder_saves"); + CHECK(v2.size() == 2); // abc, acb, cab keep a before b; two are new + for (auto& s : v2) CHECK(s.find("int a") < s.find("int b")); +} + +static void testInlineLocal() { + std::string src = "void f(P* p) { Q* q = p->q; q->x = 1; int u = 5; }"; + auto f = parse(src, "f"); + auto v = variants(*f, "inline_local"); + bool plain = false, cast = false, dead = false; + for (auto& s : v) { + plain |= contains(s, "p->q->x = 1;") && !contains(s, "Q* q"); + cast |= contains(s, "((Q*)(p->q))->x = 1;"); + dead |= !contains(s, "int u"); + } + CHECK(plain && cast && dead); + // a field read isn't moved past a call + std::string src2 = "void f(P* p) { int a = p->x; p->Update(); g = a; }"; + auto v2 = variants(*parse(src2, "f"), "inline_local"); + CHECK(v2.empty()); +} + +static void testSinkHoistMerge() { + std::string src = R"( +void f(int n) +{ + int a = 1; + int b; + g(); + if (n) + { + b = 2; + use(a, b); + } +} +)"; + auto f = parse(src, "f"); + auto sink = variants(*f, "sink_decl"); + bool intoBlock = false; + for (auto& s : sink) intoBlock |= contains(s, "{\n int b;\n b = 2;"); + CHECK(intoBlock); + auto merge = variants(*f, "merge_decl"); + CHECK(merge.size() == 1 && contains(merge[0], "int b = 2;") && !contains(merge[0], "int b;")); + + std::string src2 = R"( +void f(int n) +{ + int a = 1; + g(); + if (n) + { + int b = 2; + use(a, b); + } + int c = 3; + use(c); +} +)"; + auto hoist = variants(*parse(src2, "f"), "hoist_decl"); + bool all = false; + for (auto& s : hoist) + all |= contains(s, "int a = 1;\n int b;\n int c;\n g();") && + contains(s, " b = 2;") && contains(s, " c = 3;"); + CHECK(all); +} + +static void testExprPasses() { + std::string src = R"( +int f(int a, int b, int* p) +{ + int r = a + b * 2; + if (a < b) r = 1; else r = 2; + r = r + 3; + p[0] += a; + return a > 0 ? a : b; +} +)"; + auto f = parse(src, "f"); + auto sw = variants(*f, "swap_operands"); + bool s1 = false; + for (auto& s : sw) s1 |= contains(s, "int r = b * 2 + a;"); + CHECK(s1); + auto fl = variants(*f, "flip_compare"); + bool f1 = false; + for (auto& s : fl) f1 |= contains(s, "if (b > a)"); + CHECK(f1); + auto inv = variants(*f, "invert_if"); + CHECK(inv.size() == 1 && contains(inv[0], "if (a >= b) r = 2; else r = 1;")); + auto ca = variants(*f, "compound_assign"); + bool c1 = false, c2 = false; + for (auto& s : ca) { + c1 |= contains(s, "r += 3;"); + c2 |= contains(s, "p[0] = p[0] + a;"); + } + CHECK(c1 && c2); + auto te = variants(*f, "ternary"); + bool t1 = false, t2 = false; + for (auto& s : te) { + t1 |= contains(s, "r = a < b ? 1 : 2;"); + t2 |= contains(s, "if (a > 0)") && contains(s, "return a;") && contains(s, "return b;"); + } + CHECK(t1 && t2); +} + +static void testExprParser() { + std::string src = "void f(int a) { x = (Foo*)a->b + (a) - c[1] * sizeof(int) ? y : z; }"; + auto f = parse(src, "f"); + auto& s = *f->body->kids[0]; + auto x = parseExpr(*f, s.begin, s.end - 1); + CHECK(x && x->k == Expr::Assign && x->kids[1]->k == Expr::Ternary); +} + +static void testScorer() { + std::string dump = R"( +t.obj: file format coff-i386 + +Disassembly of section .text: + +00000000 : + 0: 57 pushl %edi + 2: 8b 4c 24 0c movl 0xc(%esp), %ecx + d: 7e 06 jle 0x15 + f: 89 3d 04 00 00 00 movl %edi, 0x4 + 00000011: IMAGE_REL_I386_DIR32 ?g@@3HA + 15: e8 00 00 00 00 calll 0x1a + 00000016: IMAGE_REL_I386_REL32 ?f@S@@QAEXXZ + 1a: c3 retl + +00000020 : + 20: c3 retl +)"; + auto in = parseObjdump(dump, "?foo@@YAHPAUS@@@Z", false); + CHECK(in.size() == 6); + if (in.size() == 6) { + CHECK(in[2].full() == "jle L4"); + CHECK(in[3].full() == "movl %edi, ?g@@3HA+0x4"); + CHECK(in[4].full() == "calll ?f@S@@QAEXXZ"); + } + ScoreConfig sc; + CHECK(scoreInsns(sc, in, in) == 0); + auto regs = in; + regs[1].args = "0xc(%esp), %edx"; + CHECK(scoreInsns(sc, in, regs) == sc.penaltyRegalloc); + auto moved = in; + std::swap(moved[0], moved[1]); + CHECK(scoreInsns(sc, in, moved) == sc.penaltyReorder); + auto fewer = in; + fewer.pop_back(); + CHECK(scoreInsns(sc, in, fewer) == sc.penaltyDelete); + + std::vector syms = {"?Promote@G@@QAEXE@Z", "??0G@@QAE@XZ", "_ZN1G7PromoteEh", "_cfunc"}; + CHECK(guessSymbol(syms, "G::Promote") == "?Promote@G@@QAEXE@Z"); + CHECK(guessSymbol(syms, "G::G") == "??0G@@QAE@XZ"); + CHECK(guessSymbol({"_ZN1G7PromoteEh"}, "G::Promote") == "_ZN1G7PromoteEh"); + CHECK(guessSymbol(syms, "cfunc") == "_cfunc"); +} + +int main() { + testLexerRoundTrip(); + testLocate(); + testParser(); + testEffects(); + testReorderSaves(); + testInlineLocal(); + testSinkHoistMerge(); + testExprPasses(); + testExprParser(); + testScorer(); + if (gFailures) { + std::cerr << gFailures << " check(s) failed\n"; + return 1; + } + std::cout << "all tests passed\n"; + return 0; +} From 568e3dd60343be06dc76a77d6bd8b7ff9e9fc5b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 17:39:33 +0000 Subject: [PATCH 2/8] Add pass combos and integration tests that prove matches Passes can now be combined: -p a+b tries each pass's variants and every in-order chain of them (exhaustive), or one mutation of each (random). -p can be given more than once. Integration tests (ctest 'integration', clang -O0 i686-msvc) check that the permuter turns a wrong base.cpp into code that matches the target, and that the matched source is exactly target.cpp. Negative controls run the wrong passes and must not match. Cases: var_reorder, combo (needs reorder_saves+invert_if; single passes fail), inline_local, ops. The 'vc6' suite does the same with real MSVC 6 under wine on gta2_re's PedGroup::PromoteMemberToLeader_4C9680: scrambled saves (and an inverted if) are permuted back to the original source. Skipped without GTA2_RE. inline_local no longer wraps a substituted initializer in parentheses when the use is a whole operand on its own. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 15 +- README.md | 50 ++++++- examples/gta2/README.md | 3 + src/main.cpp | 106 +++++--------- src/passes.cpp | 128 ++++++++++++++++- src/passes.hpp | 23 +++ tests/e2e/run.sh | 31 ---- tests/integration/combo/base.cpp | 27 ++++ tests/integration/combo/target.cpp | 27 ++++ tests/integration/inline_local/base.cpp | 10 ++ tests/integration/inline_local/target.cpp | 9 ++ .../ops_base.cpp => integration/ops/base.cpp} | 0 .../ops/target.cpp} | 0 tests/integration/run.sh | 115 +++++++++++++++ .../var_reorder/base.cpp} | 3 +- .../var_reorder/target.cpp} | 3 +- tests/integration/vc6.sh | 135 ++++++++++++++++++ tests/tests.cpp | 45 ++++++ 18 files changed, 608 insertions(+), 122 deletions(-) delete mode 100755 tests/e2e/run.sh create mode 100644 tests/integration/combo/base.cpp create mode 100644 tests/integration/combo/target.cpp create mode 100644 tests/integration/inline_local/base.cpp create mode 100644 tests/integration/inline_local/target.cpp rename tests/{e2e/ops_base.cpp => integration/ops/base.cpp} (100%) rename tests/{e2e/ops_target.cpp => integration/ops/target.cpp} (100%) create mode 100755 tests/integration/run.sh rename tests/{e2e/saves_target.cpp => integration/var_reorder/base.cpp} (91%) rename tests/{e2e/saves_base.cpp => integration/var_reorder/target.cpp} (90%) create mode 100644 tests/integration/vc6.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 08dec5e..c5b8733 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,15 @@ enable_testing() add_executable(permuter_tests tests/tests.cpp) target_link_libraries(permuter_tests PRIVATE permuter_core Threads::Threads) add_test(NAME unit COMMAND permuter_tests) -add_test(NAME end_to_end - COMMAND ${CMAKE_COMMAND} -E env PERMUTER=$ - sh ${CMAKE_SOURCE_DIR}/tests/e2e/run.sh) +foreach(suite integration vc6) + if(suite STREQUAL "integration") + set(script run.sh) + else() + set(script vc6.sh) + endif() + add_test(NAME ${suite} + COMMAND ${CMAKE_COMMAND} -E env PERMUTER=$ + sh ${CMAKE_SOURCE_DIR}/tests/integration/${script}) + # 77: the compiler isn't available here + set_tests_properties(${suite} PROPERTIES SKIP_RETURN_CODE 77) +endforeach() diff --git a/README.md b/README.md index e947a33..95280ac 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Needs CMake 3.16+ and a C++17 compiler. Scoring also needs `llvm-objdump` on `PA ```sh cmake -S . -B build -G Ninja # or leave out -G for make cmake --build build -ctest --test-dir build # unit tests, plus end-to-end tests with clang +ctest --test-dir build # unit and integration tests (see Tests below) ``` ## Usage @@ -45,9 +45,9 @@ cpp_permuter -s Source/PedGroup.cpp -f PedGroup::PromoteMemberToLeader_4C9680 \ | `--symbol`, `--target-symbol` | symbol names, if the guess from `--function` is wrong (MSVC `?Name@Class@@...`, Itanium, or C) | | `--score-cmd` | score with your own command instead (`{obj}`, `{src}`). It must print the score (0 = match) as the last number in its output | | `--ignore-reloc-names` | compare relocated operands without their symbol names, and treat absolute addresses as symbols. Use this when the target came from raw exe asm | -| `-m exhaustive` | try every candidate the selected passes produce. `--depth N` chains N mutations, `--max-candidates` caps the count | -| `-m random` | (default) apply 1..`--max-mutations` random mutations, starting from the base or from the best so far. Runs until a match, `-n` compiles, or Ctrl-C | -| `-p a,b` | passes to use (default: all) | +| `-m exhaustive` | try every candidate the selected passes and combos produce. `--depth N` chains N of them, `--max-candidates` caps the count | +| `-m random` | (default) apply 1..`--max-mutations` random passes or combos, starting from the base or from the best so far. Runs until a match, `-n` compiles, or Ctrl-C | +| `-p a,b` | passes to use (default: all). `-p` can be repeated. `a+b` combines passes, see below | | `-j N` | parallel compiles | | `--dry-run` | print the candidates as diffs without compiling. A quick check of what a pass would do | | `--show-ast` | print how the function was parsed | @@ -68,6 +68,23 @@ Candidate sources are written next to the original (`PedGroup.permuter0.cpp`, .. `#include`s still resolve. They are deleted on exit. `--candidate-dir` puts them somewhere else. +## Combining passes + +A function is often off in more than one way: the saves are in the wrong order *and* an `if` +is the wrong way round. A single pass can't fix both, so passes can be combined: + +```sh +-p reorder_saves,invert_if # each pass on its own: 23 + 4 candidates +-p reorder_saves+invert_if # a combo: 23 + 4 + 23*4 = 119 candidates +-p reorder_saves -p invert_if+swap_operands # -p can be repeated +``` + +In exhaustive mode, a combo `a+b+c` tries every variant of every pass in it, and every +in-order chain of them: a, b, c, a then b, a then c, b then c, and a then b then c. In random +mode a combo applies one mutation from each of its passes. `--depth N` chains whole passes or +combos too: `-p reorder_saves -p invert_if --depth 2` reaches the same candidates as the combo +above, plus each pass applied twice. + ## Passes `cpp_permuter --list-passes` prints these: @@ -118,6 +135,31 @@ Target and candidate are then aligned with an edit distance. Penalties: 10 for a difference, 50 for other operand differences, 60 for a moved instruction, 100 for an insertion or a deletion. The weights are the same idea as decomp-permuter's. +## Tests + +`ctest --test-dir build` runs three suites: + +- `unit` (`tests/tests.cpp`): the lexer, parser, effects, each pass, pass combos, and the + scorer. +- `integration` (`tests/integration/run.sh`): each case directory has a `base.cpp` that is + wrong in some way and the `target.cpp` that the target object is built from. The permuter + must match the target, and where it should, give back exactly `target.cpp`. Negative controls + run the wrong passes and must *not* match, which shows the match comes from the pass under + test. The cases are: + - `var_reorder`: local saves out of order; + - `combo`: saves out of order plus an inverted `if`. Single passes fail and the combo + matches; + - `inline_local`: a local the original didn't have; + - `ops`: swapped operands. + + The compiler is clang's i686 MSVC target at `-O0`, where statement order shows up directly in + the code. +- `vc6` (`tests/integration/vc6.sh`): the same proof with the real MSVC 6 under wine, on + gta2_re's `PedGroup::PromoteMemberToLeader_4C9680`. That function already matches, so VC6's + code for it is the original game's code. The test scrambles the four weapon saves (and + inverts an `if`) and checks that the permuter recovers the exact original source. It needs + `GTA2_RE` set and wine installed, and is skipped otherwise. + ## gta2_re `examples/gta2/` has a VC6-under-wine compile script and a walkthrough. See diff --git a/examples/gta2/README.md b/examples/gta2/README.md index b586d15..1c6ae98 100644 --- a/examples/gta2/README.md +++ b/examples/gta2/README.md @@ -52,6 +52,9 @@ Suggestions: - When you already suspect the kind of fix, use exhaustive mode with just that pass: `-m exhaustive -p reorder_saves` tries every order of every run of local saves. +- When it's off in two ways at once, combine the passes: `-p reorder_saves+invert_if` tries + each pass alone and every pair of their variants. `-p` can be repeated to run several + combos in one go. - `--dry-run` shows what a pass would produce without compiling anything. - Random mode (the default) mixes all passes and is good for leaving running. Add `-p reorder_saves,move_stmt` and similar to focus it. diff --git a/src/main.cpp b/src/main.cpp index 0f57068..583ce87 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -51,11 +51,13 @@ Finds a source permutation of one function that compiles to the target code. Search: -m, --mode MODE random (default) or exhaustive - -p, --passes LIST comma separated passes (default: all). See --list-passes - --depth N exhaustive: apply up to N mutations in a row (default 1) + -p, --passes LIST passes to use (default: all), see --list-passes. Commas + separate passes, '+' combines them: "a+b" tries a, b and + a then b. Can be given more than once + --depth N exhaustive: chain up to N passes/combos (default 1) --max-candidates N exhaustive: stop after N candidates (default 200000) -n, --iterations N random: stop after N compiles (default: run until match/Ctrl-C) - --max-mutations N random: up to N mutations per candidate (default 3) + --max-mutations N random: up to N passes/combos per candidate (default 3) --seed N random seed --keep-going don't stop at the first exact match -j, --jobs N parallel compiles (default 1) @@ -97,21 +99,6 @@ std::atomic gStop{false}; void onSignal(int) { gStop = true; } -std::vector split(const std::string& s, char sep) { - std::vector r; - std::string cur; - for (char c : s) { - if (c == sep) { - if (!cur.empty()) r.push_back(cur); - cur.clear(); - } else if (c != ' ') { - cur += c; - } - } - if (!cur.empty()) r.push_back(cur); - return r; -} - bool readFile(const std::string& path, std::string& out) { std::ifstream in(path, std::ios::binary); if (!in) return false; @@ -154,7 +141,7 @@ bool applyOption(Options& o, const std::string& key, const std::string& val, boo if (key == "objdump") return need() && (o.objdump = val, true); if (key == "ignore-reloc-names") return o.ignoreRelocNames = true; if (key == "mode" || key == "m") return need() && (o.mode = val, true); - if (key == "passes" || key == "p") return need() && (o.passes = split(val, ','), true); + if (key == "passes" || key == "p") return need() && (o.passes.push_back(val), true); if (key == "depth") return num(o.depth); if (key == "max-candidates") return num(o.maxCandidates); if (key == "iterations" || key == "n") return num(o.iterations); @@ -416,46 +403,26 @@ void printStatus(State& st) { << std::flush; } -std::vector pickPasses(const Options& o, std::string& err) { - std::vector r; - if (o.passes.empty()) { - for (auto& p : allPasses()) r.push_back(&p); - return r; - } - for (auto& n : o.passes) { - const Pass* p = findPass(n); - if (!p) { - err = "unknown pass '" + n + "' (see --list-passes)"; - return {}; - } - r.push_back(p); - } - return r; -} - // One random candidate derived from start; empty if nothing applied. -std::string randomCandidate(const Options& o, const std::vector& passes, +std::string randomCandidate(const Options& o, const std::vector& groups, const std::string& start, Rng& rng) { int total = 0; - for (auto* p : passes) total += p->weight; + for (auto& g : groups) total += g.weight(); int k = std::uniform_int_distribution(1, std::max(1, o.maxMutations))(rng); std::string text = start; for (int m = 0; m < k; ++m) { - std::string err; - auto f = parseFunc(text, err); - if (!f) break; for (int attempt = 0; attempt < 10; ++attempt) { int r = std::uniform_int_distribution(0, total - 1)(rng); - const Pass* p = passes.back(); - for (auto* q : passes) { - if (r < q->weight) { - p = q; + const PassGroup* g = &groups.back(); + for (auto& q : groups) { + if (r < q.weight()) { + g = &q; break; } - r -= q->weight; + r -= q.weight(); } std::string out; - if (randomMutation(*p, *f, rng, out) && out != text) { + if (randomGroupMutation(*g, text, rng, out)) { text = out; break; } @@ -465,7 +432,7 @@ std::string randomCandidate(const Options& o, const std::vector& pa } int runRandom(const Options& o, const Evaluator& ev, State& st, - const std::vector& passes) { + const std::vector& groups) { unsigned long long seed = o.seedSet ? o.seed : (unsigned long long)std::random_device{}(); std::vector workers; std::atomic idle{0}; @@ -482,7 +449,7 @@ int runRandom(const Options& o, const Evaluator& ev, State& st, std::uniform_int_distribution(0, 1)(rng) == 0; start = fromBest ? st.bestText : st.baseText; } - std::string cand = randomCandidate(o, passes, start, rng); + std::string cand = randomCandidate(o, groups, start, rng); bool fresh = false; if (!cand.empty()) { std::lock_guard lk(st.mu); @@ -511,7 +478,7 @@ int runRandom(const Options& o, const Evaluator& ev, State& st, } int runExhaustive(const Options& o, const Evaluator& ev, State& st, - const std::vector& passes) { + const std::vector& groups) { std::mutex qmu; std::condition_variable qcv; std::deque queue; @@ -546,14 +513,10 @@ int runExhaustive(const Options& o, const Evaluator& ev, State& st, for (int d = 1; d <= o.depth && !gStop; ++d) { std::vector next; for (auto& text : level) { - std::string err; - auto f = parseFunc(text, err); - if (!f) continue; - for (auto* p : passes) { - p->enumerate(*f, [&](Mutation m) { + for (auto& g : groups) { + if (gStop || produced >= o.maxCandidates) break; + enumerateGroup(g, text, [&](const std::string& s) { if (gStop || produced >= o.maxCandidates) return false; - std::string s = m(); - if (s.empty()) return true; { std::lock_guard lk(st.mu); if (!st.seen.insert(hashText(s)).second) return true; @@ -562,7 +525,7 @@ int runExhaustive(const Options& o, const Evaluator& ev, State& st, if (d < o.depth) next.push_back(s); std::unique_lock lk(qmu); qcv.wait(lk, [&] { return queue.size() < 256 || gStop; }); - queue.push_back(std::move(s)); + queue.push_back(s); qcv.notify_all(); return true; }); @@ -599,22 +562,19 @@ int runExhaustive(const Options& o, const Evaluator& ev, State& st, return 0; } -int dryRun(const Options& o, const std::string& base, const std::vector& passes) { - std::string err; - auto f = parseFunc(base, err); +int dryRun(const Options& o, const std::string& base, const std::vector& groups) { if (o.mode == "exhaustive") { long count = 0; std::unordered_set seen; - for (auto* p : passes) { + for (auto& g : groups) { long mine = 0; - p->enumerate(*f, [&](Mutation m) { - std::string s = m(); - if (s.empty() || !seen.insert(hashText(s)).second) return true; + enumerateGroup(g, base, [&](const std::string& s) { + if (!seen.insert(hashText(s)).second) return true; mine++; - if (++count <= 50) std::cout << "=== " << p->name << " #" << mine << "\n" << lineDiff(base, s); + if (++count <= 50) std::cout << "=== " << g.name() << " #" << mine << "\n" << lineDiff(base, s); return count < o.maxCandidates; }); - std::cerr << p->name << ": " << mine << " candidates\n"; + std::cerr << g.name() << ": " << mine << " candidates\n"; } std::cerr << count << " candidates in total\n"; return 0; @@ -622,7 +582,7 @@ int dryRun(const Options& o, const std::string& base, const std::vector groups; + if (!parsePassSpecs(o.passes, groups, err)) { std::cerr << "error: " << err << "\n"; return 2; } @@ -680,7 +640,7 @@ int main(int argc, char** argv) { std::cerr << "error: --mode must be random or exhaustive\n"; return 2; } - if (o.dryRun) return dryRun(o, funcText, passes); + if (o.dryRun) return dryRun(o, funcText, groups); if (o.compile.empty() || (o.targetObj.empty() && o.scoreCmd.empty())) { std::cerr << "error: --compile and --target-obj (or --score-cmd) are required\n"; @@ -753,8 +713,8 @@ int main(int argc, char** argv) { cleanup(); return 0; } - if (o.mode == "random") runRandom(o, ev, st, passes); - else runExhaustive(o, ev, st, passes); + if (o.mode == "random") runRandom(o, ev, st, groups); + else runExhaustive(o, ev, st, groups); cleanup(); std::cout << "\n"; diff --git a/src/passes.cpp b/src/passes.cpp index c260b26..28031ec 100644 --- a/src/passes.cpp +++ b/src/passes.cpp @@ -4,6 +4,8 @@ #include #include +#include +#include namespace perm { @@ -390,16 +392,30 @@ void enumInlineLocal(const Func& f, const EmitFn& emit) { if (inLoop && !trivially) continue; auto x = parseExpr(f, d.initBegin, d.initEnd); - bool needParen = !x || !isPostfixLike(*x); - std::string plain = needParen ? paren(init) : init; + bool postfixLike = x && isPostfixLike(*x); + bool commaFree = x && x->k != Expr::Comma; + // A use that is a whole operand on its own (an assignment's right side, + // an argument, a return value, an index) needs no parentheses. + auto standalone = [&](int u) { + const std::string& p = f.t(u - 1); + const std::string& n = f.t(u + 1); + bool before = isAssignOp(p) || p == "(" || p == "," || p == "return" || p == "["; + bool after = n == ";" || n == ")" || n == "," || n == "]"; + return before && after && commaFree; + }; std::string castT = declTypeText(f, *s) + f.tokText(d.begin, d.nameTok); - std::vector variants = {plain}; - if (uses.size() == 1 && !d.isRef && !s->isConst) variants.push_back("((" + castT + ")" + paren(init) + ")"); - for (auto& v : variants) { - if (!emit([&f, s, uses, v]() { + std::vector casts = {0}; + if (uses.size() == 1 && !d.isRef && !s->isConst) casts.push_back(1); + for (int cast : casts) { + std::vector texts; + for (int u : uses) { + if (cast) texts.push_back("((" + castT + ")" + paren(init) + ")"); + else texts.push_back(postfixLike || standalone(u) ? init : paren(init)); + } + if (!emit([&f, s, uses, texts]() { Rewriter rw(f); rw.remove(s->begin, s->end); - for (int u : uses) rw.replace(u, u + 1, v); + for (size_t k = 0; k < uses.size(); ++k) rw.replace(uses[k], uses[k] + 1, texts[k]); return rw.apply(); })) return; @@ -872,4 +888,102 @@ bool randomMutation(const Pass& p, const Func& f, Rng& rng, std::string& out) { return !out.empty(); } +// --------------------------------------------------------------------------- +// pass groups + +std::string PassGroup::name() const { + std::string r; + for (auto* p : passes) r += (r.empty() ? "" : "+") + p->name; + return r; +} + +int PassGroup::weight() const { + int w = 0; + for (auto* p : passes) w = std::max(w, p->weight); + return w; +} + +bool parsePassSpecs(const std::vector& specs, std::vector& out, + std::string& err) { + out.clear(); + auto addAll = [&]() { + for (auto& p : allPasses()) out.push_back({{&p}}); + }; + if (specs.empty()) { + addAll(); + return true; + } + for (auto& spec : specs) { + std::stringstream groups(spec); + std::string g; + while (std::getline(groups, g, ',')) { + g.erase(std::remove(g.begin(), g.end(), ' '), g.end()); + if (g.empty()) continue; + if (g == "all") { + addAll(); + continue; + } + PassGroup grp; + std::stringstream names(g); + std::string n; + while (std::getline(names, n, '+')) { + const Pass* p = findPass(n); + if (!p) { + err = "unknown pass '" + n + "' (see --list-passes)"; + return false; + } + grp.passes.push_back(p); + } + if (!grp.passes.empty()) out.push_back(grp); + } + } + if (out.empty()) { + err = "no passes given"; + return false; + } + return true; +} + +void enumerateGroup(const PassGroup& g, const std::string& text, + const std::function& emit) { + std::set seen = {text}; + bool stop = false; + std::function rec = [&](const std::string& cur, size_t from) { + for (size_t j = from; j < g.passes.size() && !stop; ++j) { + std::string err; + auto f = parseFunc(cur, err); + if (!f) return; + std::vector outs; + g.passes[j]->enumerate(*f, [&](Mutation m) { + std::string s = m(); + if (!s.empty() && seen.insert(s).second) outs.push_back(std::move(s)); + return true; + }); + for (auto& s : outs) { + if (stop) return; + if (!emit(s)) { + stop = true; + return; + } + rec(s, j + 1); + } + } + }; + rec(text, 0); +} + +bool randomGroupMutation(const PassGroup& g, const std::string& text, Rng& rng, std::string& out) { + std::string cur = text; + for (auto* p : g.passes) { + std::string err; + auto f = parseFunc(cur, err); + if (!f) break; + std::string next; + if (randomMutation(*p, *f, rng, next) && next != cur) cur = next; + } + if (cur == text) return false; + out = cur; + return true; +} + } // namespace perm diff --git a/src/passes.hpp b/src/passes.hpp index f52e3d9..0be549c 100644 --- a/src/passes.hpp +++ b/src/passes.hpp @@ -30,6 +30,29 @@ const Pass* findPass(const std::string& name); // One random mutation of f by pass p. Returns false if p has nothing to offer. bool randomMutation(const Pass& p, const Func& f, Rng& rng, std::string& out); +// A pass, or a combination of passes written "a+b+c" on the command line. +struct PassGroup { + std::vector passes; + std::string name() const; + int weight() const; +}; + +// Parses pass specs such as {"reorder_saves,invert_if", "swap_operands+flip_compare"}: +// commas separate groups, '+' combines passes into one group, "all" adds every +// pass as its own group. No specs means "all". +bool parsePassSpecs(const std::vector& specs, std::vector& out, + std::string& err); + +// Every distinct text made by applying one mutation of each pass in some +// non-empty, in-order subset of g's passes (for "a+b": a, b, and a then b). +// emit returns false to stop. +void enumerateGroup(const PassGroup& g, const std::string& text, + const std::function& emit); + +// One random mutation from each of g's passes, in order. Passes with nothing to +// offer are skipped. Returns false if nothing changed. +bool randomGroupMutation(const PassGroup& g, const std::string& text, Rng& rng, std::string& out); + // Token-range rewriting of a function. class Rewriter { public: diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh deleted file mode 100755 index c361110..0000000 --- a/tests/e2e/run.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# End-to-end: permute a wrongly ordered source until it compiles to the -# target. Uses clang's i686 MSVC target at -O0 as a stand-in for VC6 (at -O0 -# the statement order shows up directly in the code). -set -e -HERE=$(cd "$(dirname "$0")" && pwd) -PERMUTER=${PERMUTER:-$HERE/../../build/cpp_permuter} -CXX=${CXX_E2E:-clang++} -command -v "$CXX" >/dev/null || { echo "skip: $CXX not found"; exit 0; } -command -v llvm-objdump >/dev/null || { echo "skip: llvm-objdump not found"; exit 0; } -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT -COMPILE="$CXX --target=i686-pc-windows-msvc -O0 -c {src} -o {obj}" - -check() { # name function mode passes [extra args] - cp "$HERE/$1_base.cpp" "$WORK/$1.cpp" - $CXX --target=i686-pc-windows-msvc -O0 -c "$HERE/$1_target.cpp" -o "$WORK/$1_target.obj" - if "$PERMUTER" -s "$WORK/$1.cpp" -f "$2" -c "$COMPILE" -t "$WORK/$1_target.obj" \ - -m "$3" -p "$4" -j 2 -n 400 --seed 1 -o "$WORK/out_$1_$3" $5 > "$WORK/log" 2>&1; then - echo "ok: $1 ($3)" - else - echo "FAIL: $1 ($3)" - cat "$WORK/log" - exit 1 - fi -} - -check saves Group::Promote exhaustive reorder_saves -check saves Group::Promote random reorder_saves,move_stmt -check ops Sum exhaustive swap_operands,invert_if "--depth 2" -check ops Sum random swap_operands,invert_if,flip_compare diff --git a/tests/integration/combo/base.cpp b/tests/integration/combo/base.cpp new file mode 100644 index 0000000..010c48b --- /dev/null +++ b/tests/integration/combo/base.cpp @@ -0,0 +1,27 @@ +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int ammo; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; int count; void Swap(int idx); }; + +void Group::Swap(int idx) +{ + Weapon* memberWeapon2 = list[idx]->weapon; + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon = leader->sel; + Weapon* leaderWeapon2 = leader->weapon; + + leader->Copy(list[idx]); + leader->sel = memberWeapon; + leader->weapon = memberWeapon2; + list[idx]->sel = leaderWeapon; + list[idx]->weapon = leaderWeapon2; + + if (idx >= count - 1) + { + list[idx]->ammo = 0; + count--; + } + else + { + list[idx]->ammo = 99; + } +} diff --git a/tests/integration/combo/target.cpp b/tests/integration/combo/target.cpp new file mode 100644 index 0000000..d8bc8ab --- /dev/null +++ b/tests/integration/combo/target.cpp @@ -0,0 +1,27 @@ +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int ammo; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; int count; void Swap(int idx); }; + +void Group::Swap(int idx) +{ + Weapon* leaderWeapon = leader->sel; + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon2 = leader->weapon; + Weapon* memberWeapon2 = list[idx]->weapon; + + leader->Copy(list[idx]); + leader->sel = memberWeapon; + leader->weapon = memberWeapon2; + list[idx]->sel = leaderWeapon; + list[idx]->weapon = leaderWeapon2; + + if (idx < count - 1) + { + list[idx]->ammo = 99; + } + else + { + list[idx]->ammo = 0; + count--; + } +} diff --git a/tests/integration/inline_local/base.cpp b/tests/integration/inline_local/base.cpp new file mode 100644 index 0000000..dc97c7f --- /dev/null +++ b/tests/integration/inline_local/base.cpp @@ -0,0 +1,10 @@ +struct Car { int speed; int gear; }; +struct Driver { Car* car; int skill; }; + +int Boost(Driver* d, int amount) +{ + int boost = amount * d->skill; + d->car->speed += boost; + d->car->gear = 2; + return d->car->speed; +} diff --git a/tests/integration/inline_local/target.cpp b/tests/integration/inline_local/target.cpp new file mode 100644 index 0000000..e87e330 --- /dev/null +++ b/tests/integration/inline_local/target.cpp @@ -0,0 +1,9 @@ +struct Car { int speed; int gear; }; +struct Driver { Car* car; int skill; }; + +int Boost(Driver* d, int amount) +{ + d->car->speed += amount * d->skill; + d->car->gear = 2; + return d->car->speed; +} diff --git a/tests/e2e/ops_base.cpp b/tests/integration/ops/base.cpp similarity index 100% rename from tests/e2e/ops_base.cpp rename to tests/integration/ops/base.cpp diff --git a/tests/e2e/ops_target.cpp b/tests/integration/ops/target.cpp similarity index 100% rename from tests/e2e/ops_target.cpp rename to tests/integration/ops/target.cpp diff --git a/tests/integration/run.sh b/tests/integration/run.sh new file mode 100755 index 0000000..aca7b3a --- /dev/null +++ b/tests/integration/run.sh @@ -0,0 +1,115 @@ +#!/bin/sh +# Integration tests: each case directory holds base.cpp (the "decompiled" +# source, wrong somewhere) and target.cpp (the source the target object was +# built from). The two differ only inside the function. The permuter has to +# turn base.cpp into code that compiles to exactly the target. +# +# The compiler is clang's i686 MSVC target at -O0, standing in for VC6: at -O0 +# statement and declaration order show up directly in the code. See vc6.sh for +# the same check with the real VC6 on gta2_re. +# +# Each case also has negative controls: the wrong passes must NOT match, which +# shows the match comes from the pass under test, not from something else. +set -u +HERE=$(cd "$(dirname "$0")" && pwd) +PERMUTER=${PERMUTER:-$HERE/../../build/cpp_permuter} +CXX=${CXX_INTEGRATION:-clang++} +command -v "$CXX" >/dev/null || { echo "skip: $CXX not found"; exit 77; } +command -v llvm-objdump >/dev/null || { echo "skip: llvm-objdump not found"; exit 77; } +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +COMPILE="$CXX --target=i686-pc-windows-msvc -O0 -c {src} -o {obj}" +FAILED=0 +N=0 + +# run ; sets RC and OUT +run() { + case_=$1 fn=$2 name=$3 + shift 3 + N=$((N + 1)) + OUT="$WORK/out_$N" + mkdir -p "$WORK/src_$N" + cp "$HERE/$case_/base.cpp" "$WORK/src_$N/$case_.cpp" + $CXX --target=i686-pc-windows-msvc -O0 -c "$HERE/$case_/target.cpp" -o "$WORK/target_$N.obj" || exit 1 + "$PERMUTER" -s "$WORK/src_$N/$case_.cpp" -f "$fn" -c "$COMPILE" -t "$WORK/target_$N.obj" \ + -j 2 --seed 1 -o "$OUT" "$@" > "$WORK/log_$N" 2>&1 + RC=$? + LABEL="$case_: $name ($*)" +} + +fail() { + echo "FAIL: $LABEL: $1" + tr '\r' '\n' < "$WORK/log_$N" | grep -v '^iterations' | tail -30 + FAILED=$((FAILED + 1)) +} + +base_score() { tr '\r' '\n' < "$WORK/log_$N" | sed -n 's/^base score: //p'; } + +# The permuter found an exact match (exit 0, score 0, base was not a match). +expect_match() { + if [ "$RC" -ne 0 ]; then fail "expected a match, exit $RC"; return 1; fi + b=$(base_score) + if [ -z "$b" ] || [ "$b" -eq 0 ]; then fail "base already matched (score '$b')"; return 1; fi + if ! ls -d "$OUT"/output-0-* >/dev/null 2>&1; then fail "no output-0-* written"; return 1; fi + echo "ok: $LABEL: base $b -> 0" +} + +# Like expect_match, and the matched source is the target source (ignoring +# whitespace): the search recovered exactly what the target was built from. +expect_exact_source() { + expect_match || return + m=$(ls -d "$OUT"/output-0-* | head -1) + if ! diff -w "$m/source.cpp" "$HERE/$1/target.cpp" > "$WORK/srcdiff_$N"; then + fail "matched, but the source differs from target.cpp:" + cat "$WORK/srcdiff_$N" + return + fi + echo "ok: $LABEL: matched source == target.cpp" +} + +expect_no_match() { + if [ "$RC" -eq 0 ]; then fail "matched without the needed pass"; return; fi + if [ "$RC" -ne 3 ]; then fail "expected 'no match' (exit 3), got exit $RC"; return; fi + echo "ok: $LABEL: no match, as expected" +} + +# --- var_reorder: four local saves in the wrong order ----------------------- +run var_reorder Group::Promote exhaustive -m exhaustive -p reorder_saves +expect_exact_source var_reorder +run var_reorder Group::Promote random -m random -p reorder_saves -n 200 +expect_exact_source var_reorder +run var_reorder Group::Promote control -m exhaustive -p swap_operands,invert_if,flip_compare +expect_no_match + +# --- combo: saves in the wrong order AND an inverted if --------------------- +# Neither pass alone gets there; the combination does. +run combo Group::Swap single-passes -m exhaustive -p reorder_saves,invert_if +expect_no_match +run combo Group::Swap combo -m exhaustive -p reorder_saves+invert_if +expect_exact_source combo +run combo Group::Swap repeated-p-depth2 -m exhaustive -p reorder_saves -p invert_if --depth 2 +expect_exact_source combo +run combo Group::Swap random-combo -m random -p reorder_saves+invert_if -n 300 +expect_exact_source combo +run combo Group::Swap combo-plus-noise -m exhaustive -p swap_operands+reorder_saves+flip_compare+invert_if +expect_exact_source combo + +# --- inline_local: a local the original didn't have ------------------------- +# -j 1: the cast variant compiles to the same code; keep the plain one (tried first) +run inline_local Boost inline -m exhaustive -p inline_local -j 1 +expect_exact_source inline_local +run inline_local Boost control -m exhaustive -p reorder_saves,move_stmt +expect_no_match + +# --- ops: swapped operands and an inverted if -------------------------------- +run ops Sum combo -m exhaustive -p swap_operands+invert_if +expect_match +run ops Sum random -m random -p swap_operands,invert_if,flip_compare -n 400 +expect_match + +echo +if [ "$FAILED" -ne 0 ]; then + echo "$FAILED of $N integration checks failed" + exit 1 +fi +echo "all $N integration runs passed" diff --git a/tests/e2e/saves_target.cpp b/tests/integration/var_reorder/base.cpp similarity index 91% rename from tests/e2e/saves_target.cpp rename to tests/integration/var_reorder/base.cpp index 85e250a..2078092 100644 --- a/tests/e2e/saves_target.cpp +++ b/tests/integration/var_reorder/base.cpp @@ -1,13 +1,12 @@ -// Target: the "right" order of the local saves. struct Weapon; struct Ped { Weapon* sel; Weapon* weapon; int pad[4]; void Copy(Ped* other); }; struct Group { Ped* leader; Ped* list[8]; void Promote(int idx); }; void Group::Promote(int idx) { + Weapon* leaderWeapon = leader->sel; Weapon* memberWeapon = list[idx]->sel; Weapon* leaderWeapon2 = leader->weapon; - Weapon* leaderWeapon = leader->sel; Weapon* memberWeapon2 = list[idx]->weapon; leader->Copy(list[idx]); diff --git a/tests/e2e/saves_base.cpp b/tests/integration/var_reorder/target.cpp similarity index 90% rename from tests/e2e/saves_base.cpp rename to tests/integration/var_reorder/target.cpp index febedb6..463ff05 100644 --- a/tests/e2e/saves_base.cpp +++ b/tests/integration/var_reorder/target.cpp @@ -1,13 +1,12 @@ -// Base: decompiled with the saves in the wrong order. struct Weapon; struct Ped { Weapon* sel; Weapon* weapon; int pad[4]; void Copy(Ped* other); }; struct Group { Ped* leader; Ped* list[8]; void Promote(int idx); }; void Group::Promote(int idx) { - Weapon* leaderWeapon = leader->sel; Weapon* memberWeapon = list[idx]->sel; Weapon* leaderWeapon2 = leader->weapon; + Weapon* leaderWeapon = leader->sel; Weapon* memberWeapon2 = list[idx]->weapon; leader->Copy(list[idx]); diff --git a/tests/integration/vc6.sh b/tests/integration/vc6.sh new file mode 100644 index 0000000..eb3d385 --- /dev/null +++ b/tests/integration/vc6.sh @@ -0,0 +1,135 @@ +#!/bin/sh +# Integration test with the real compiler: MSVC 6 under wine, on gta2_re. +# +# PedGroup::PromoteMemberToLeader_4C9680 is a MATCH_FUNC, so VC6's code for +# the unmodified PedGroup.cpp is the original game's code. We compile that as +# the target, break the function the way a decompiler would (the four weapon +# saves in another order, an if inverted) and check that the permuter finds +# the way back to the exact original source. +# +# Needs GTA2_RE pointing at a gta2_re checkout with its compile tools +# submodule, and wine. Skips (exit 77) otherwise. +set -u +HERE=$(cd "$(dirname "$0")" && pwd) +ROOT=$HERE/../.. +PERMUTER=${PERMUTER:-$ROOT/build/cpp_permuter} +[ -n "${GTA2_RE:-}" ] || { echo "skip: GTA2_RE not set"; exit 77; } +[ -d "$GTA2_RE/3rdParty/gta2_re_compile_tools/VC98" ] || { echo "skip: no VC6 in $GTA2_RE"; exit 77; } +command -v wine >/dev/null || { echo "skip: wine not found"; exit 77; } +command -v llvm-objdump >/dev/null || { echo "skip: llvm-objdump not found"; exit 77; } +command -v python3 >/dev/null || { echo "skip: python3 not found"; exit 77; } +export WINEDEBUG=-all GTA2_RE +wineserver -p 2>/dev/null || true + +FN=PedGroup::PromoteMemberToLeader_4C9680 +ORIG=$GTA2_RE/Source/PedGroup.cpp +TEST_SRC=$GTA2_RE/Source/PedGroup_permuter_test.cpp +COMPILE="$ROOT/examples/gta2/compile.sh {src} {obj}" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"; rm -f "$TEST_SRC"' EXIT +FAILED=0 +N=0 + +"$ROOT/examples/gta2/compile.sh" "$ORIG" "$WORK/target.obj" > "$WORK/target.log" 2>&1 || + { echo "FAIL: PedGroup.cpp doesn't compile"; cat "$WORK/target.log"; exit 1; } + +# mutate : writes TEST_SRC from ORIG +mutate() { + python3 - "$ORIG" "$TEST_SRC" "$1" "$2" <<'EOF' +import sys +src, dst, order, invert = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1" +s = open(src).read() +saves = [ + " Weapon_30* leaderWeapon = field_2C_ped_leader->field_170_selected_weapon;", + " Weapon_30* memberWeapon = field_4_ped_list[idx]->field_170_selected_weapon;", + " Weapon_30* leaderWeapon2 = field_2C_ped_leader->field_174_pWeapon;", + " Weapon_30* memberWeapon2 = field_4_ped_list[idx]->field_174_pWeapon;", +] +old = "\n".join(saves) +assert old in s, "the saves in PedGroup.cpp changed; update vc6.sh" +s = s.replace(old, "\n".join(saves[int(c)] for c in order)) +if invert: + a = """ if (pTmp->field_238_ped_type == 5) + { + pTmp->PoolAllocate(); + pTmp->field_21C |= 0x400; + } + else + { + pTmp->PoolAllocate(); + }""" + b = """ if (pTmp->field_238_ped_type != 5) + { + pTmp->PoolAllocate(); + } + else + { + pTmp->PoolAllocate(); + pTmp->field_21C |= 0x400; + }""" + assert a in s, "the if in PedGroup.cpp changed; update vc6.sh" + s = s.replace(a, b) +open(dst, "w").write(s) +EOF +} + +# run