diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d0476ca --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# LF everywhere, also on Windows checkouts: the shell scripts need it, and the +# unit tests compare docs/passes.md byte for byte. +* text=auto eol=lf +*.cmd text eol=crlf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..54a7f4a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,59 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y ninja-build clang llvm + - name: Configure + run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release + - name: Build + run: cmake --build build + - name: Test + # unit and clang integration tests; gta2_parse and vc6 need a gta2_re + # checkout and wine, and report as skipped + run: ctest --test-dir build --output-on-failure + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: cpp_permuter-linux-x64 + path: | + build/cpp_permuter + examples/gta2/compile.sh + README.md + LICENSE + + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v5 + - name: Configure + run: cmake -S . -B build + - name: Build + run: cmake --build build --config Release + - name: Tools + shell: bash + run: | + # clang and llvm-objdump come with the runner's LLVM install + echo "C:\Program Files\LLVM\bin" >> "$GITHUB_PATH" + - name: Test + shell: bash + run: ctest --test-dir build -C Release --output-on-failure + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: cpp_permuter-windows-x64 + path: | + build/Release/cpp_permuter.exe + examples/gta2/compile.cmd + README.md + LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..44f58f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +build/ +permuter_out/ +*.permuter[0-9]*.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..8ff00c1 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,55 @@ +cmake_minimum_required(VERSION 3.16) +# static MSVC runtime (CMAKE_MSVC_RUNTIME_LIBRARY): the .exe runs without the +# Visual C++ redistributable +cmake_policy(SET CMP0091 NEW) +project(cpp_permuter CXX) +set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +find_package(Threads REQUIRED) + +add_library(permuter_core STATIC + src/lexer.cpp + src/parser.cpp + src/analysis.cpp + src/passes.cpp + src/passes_control.cpp + src/passes_more.cpp + src/scorer.cpp + src/runner.cpp + src/workspace.cpp + src/macros.cpp + src/examples.cpp +) +target_include_directories(permuter_core PUBLIC src) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(permuter_core PRIVATE -Wall -Wextra) +endif() + +add_executable(cpp_permuter src/main.cpp) +target_link_libraries(cpp_permuter PRIVATE permuter_core Threads::Threads) + +enable_testing() +add_executable(permuter_tests tests/tests.cpp) +target_link_libraries(permuter_tests PRIVATE permuter_core Threads::Threads) +target_compile_definitions(permuter_tests PRIVATE PERMUTER_SOURCE_DIR="${CMAKE_SOURCE_DIR}") +add_test(NAME unit COMMAND permuter_tests) +# integration: clang stand-in, runs anywhere clang and llvm-objdump are +# installed. gta2_parse and vc6 need GTA2_RE (and wine for vc6). Exit code 77 +# means "skipped": the tools or the checkout aren't available here. +foreach(suite integration gta2_parse vc6) + if(suite STREQUAL "integration") + set(script run.sh) + else() + set(script ${suite}.sh) + endif() + add_test(NAME ${suite} + COMMAND ${CMAKE_COMMAND} -E env PERMUTER=$ + sh ${CMAKE_SOURCE_DIR}/tests/integration/${script}) + set_tests_properties(${suite} PROPERTIES SKIP_RETURN_CODE 77) +endforeach() diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cee82e1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 CriminalRETeam and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2f98a55 --- /dev/null +++ b/README.md @@ -0,0 +1,381 @@ +# cpp_permuter + +[![CI](https://github.com/CriminalRETeam/cpp_permuter/actions/workflows/ci.yml/badge.svg)](https://github.com/CriminalRETeam/cpp_permuter/actions/workflows/ci.yml) + +Brute-forces source permutations of a single C++ function until its compiled code matches a +target object, or gets closer to it. It is built for matching decompilation projects such as +[gta2_re](https://github.com/CriminalRETeam/gta2_re) (MSVC 6). + +cpp_permuter is heavily inspired by Simon Lindholm's +[decomp-permuter](https://github.com/simonlindholm/decomp-permuter), which does the same for +C. It borrows decomp-permuter's design wholesale: random and exhaustive search, +weighted randomization passes, `PERM_*` macros, an objdump-based scorer, and outputs with a +score and a diff. It is a separate implementation, written in C++, for C++ code compiled by +MSVC. Like decomp-permuter, it is released under the [MIT license](LICENSE). + +How it works: + +1. Find the function in the `.cpp` and parse its body into statements, declarations and + expressions. +2. Make candidates by applying mutation passes (reorder local saves, inline a local, move + declarations in and out of blocks, swap operands, invert an `if`, ...). Optionally do the + same to the inline helpers the function calls. +3. Splice each candidate back into the file, compile it with your compile command, and + disassemble the function from the object with `llvm-objdump`. +4. Score it against the target: 0 means identical, lower is closer. Improvements are written + to the output directory. + +## Getting it + +Every CI run builds `cpp_permuter-linux-x64` and `cpp_permuter-windows-x64` (the Windows one +needs no Visual C++ redistributable). Download them from the run's artifacts on the +[Actions page](https://github.com/CriminalRETeam/cpp_permuter/actions/workflows/ci.yml), or build +from source: + +## Building + +Needs CMake 3.16+ and a C++17 compiler: GCC or clang on Linux, MSVC (or MinGW) on Windows. +Scoring also needs `llvm-objdump` on `PATH`. On Windows it comes with the LLVM installer. + +Windows runs natively. Commands go through `cmd.exe`, so use `examples/gta2/compile.cmd` +there. Where symlinks aren't allowed (no Developer Mode), the mirror uses hard links or +copies instead. CI builds and tests on Linux and Windows. + +```sh +cmake -S . -B build -G Ninja # or leave out -G for make +cmake --build build +ctest --test-dir build # unit and integration tests (see Tests below) +``` + +## Usage + +```sh +cpp_permuter -s Source/PedGroup.cpp -f PedGroup::PromoteMemberToLeader_4C9680 \ + -c 'examples/gta2/compile.sh {src} {obj}' \ + -t Scripts/asm/PedGroup.cpp.obj \ + -m exhaustive -p reorder_saves -j 4 +``` + +| option | meaning | +|---|---| +| `-s/--source` | the `.cpp` holding the function | +| `-f/--function` | `Class::Method`, `Method`, `Class::Class`, `Class::~Class` | +| `-c/--compile` | compile command. `{src}` is the candidate source, `{obj}` the object it must write, `{dir}` the candidate's directory, `{root}` the mirror's root (see below) | +| `-t/--target-obj` | object file holding the target code | +| `--symbol`, `--target-symbol` | symbol names, if the guess from `--function` is wrong (MSVC `?Name@Class@@...`, Itanium, or C) | +| `--score-cmd` | score with your own command instead (`{obj}`, `{src}`). It must print the score (0 = match) as the last number in its output | +| `--ignore-reloc-names` | compare relocated operands without their symbol names, and treat absolute addresses as symbols. Use this when the target came from raw exe asm | +| `-m exhaustive` | try every candidate the selected passes and combos produce. `--depth N` chains N of them, `--max-candidates` caps the count | +| `-m random` | (default) apply 1..`--max-mutations` random passes or combos, starting from the base or from the best so far. Runs until a match, `-n` compiles, or Ctrl-C | +| `-p a,b` | passes to use (default: all). `-p` can be repeated. `a+b` combines passes, see below | +| `-j N` | parallel compiles | +| `--dry-run` | print the candidates as diffs without compiling. A quick check of what a pass would do | +| `--show-ast` | print how the function was parsed | +| `--show-base-diff` | print the target/base asm alignment before starting | +| `--config FILE` | `key = value` lines, same keys as the long options. Handy for one config per function | +| `--keep-going` | keep searching after an exact match | +| `--keep-prob P` | random: start from the best candidate so far with probability P, else from the base (default 0.5) | +| `--weight NAME=N` | random: how often a pass or combo (as written in `-p`) is picked; `0` never | +| `--keep-ties` | also write candidates that tie the best score (different ways to the same code) | +| `--only-if-below N` | only write candidates scoring below N | +| `--base-only` | compile and score the unmodified function, print the score, exit | +| `--show-timings` | where the time went: permuting, writing, compiling, scoring | +| `-p none` | no passes: only the `PERM_*` macros (see below) | +| `-I DIR` | where to find headers for `--inline-callees`/`--also`, besides each file's own directory. The compiler's include paths stay in your compile command | +| `--inline-callees`, `--also [FILE:]NAME` | also permute inline helpers the function calls, see below | +| `--check-parse FILE...` | parse every function in the files and run every pass on each; see Tests | + +Each improvement goes to `permuter_out/output--/`: + +- `function.cpp`: the permuted function (and any helper that changed). +- `source.cpp`: the whole file. +- `
.hpp`: each header that changed, when helpers were permuted. +- `diff.txt`: what changed. +- `asm_diff.txt`: target and candidate asm aligned side by side. + +The exit status is 0 if a match was found and 3 if not. + +Candidates are compiled in a mirror of the source's directory under +`permuter_out/.work/w/`, one per worker. The mirror is built from symlinks to every entry, +plus real copies of the files the permuter edits. Your files are never written to. File names +stay the same, and every `#include "..."` resolves inside the mirror, so an edited header is +the one that every file includes. `--mirror-root DIR` mirrors a larger tree when the edited +files aren't all under the source's directory. + +## Inline helpers + +VC6 inlines small helpers (getters, `Fix16::Abs`, `Max`, ...) into their callers, so a +function can be off only because a helper is written differently. The permuter can work on +those too: + +```sh +--inline-callees # the inline functions this function calls +--also fix16.hpp:Max # a particular function, by file (relative to . or the source) +--also Fix16::Max # ... or looked up in the source and the headers it includes +--list-regions # print what would be permuted, then exit +``` + +`--inline-callees` looks for definitions of the called names in these places: + +- the headers the source reaches through `#include "..."` (searched in the including file's + directory, then in `-I` dirs); +- the source itself, for functions marked `inline`. + +Overloads whose parameter count can't fit any call are skipped. The first definition of +every name is taken before any second overload, up to `--max-callees` (default 8). + +Every pass then also works on those helpers. Half of the random mutations go to the target +function, and exhaustive mode tries every pass on every function. Only the target function +is scored. When a candidate matches, the output has the changed headers next to +`source.cpp`. + +## PERM macros + +When you already know the alternatives, write them into the function, as with decomp-permuter: + +```cpp +void PedGroup::f(u8 idx) +{ + PERM_LINESWAP( + Weapon_30* a = field_2C_ped_leader->field_170_selected_weapon; + Weapon_30* b = field_4_ped_list[idx]->field_170_selected_weapon; + ) + field_23C = PERM_GENERAL(99, 100); + if (PERM_GENERAL(x == 0, !x)) g(); +} +``` + +| macro | expands to | +|---|---| +| `PERM_GENERAL(a, b, ...)` | one of `a`, `b`, ... | +| `PERM_LINESWAP(lines)` | the non-blank lines in any order | +| `PERM_INT(lo, hi)` | an integer from `lo` to `hi` | +| `PERM_ONCE([key,] code)` | `code` at exactly one of the places that use `key` | +| `PERM_VAR(a, b)`, `PERM_VAR(a)` | set, and read, the meta-variable `a` | +| `PERM_RANDOMIZE(code)` | `code`, and turns the passes on | +| `PERM_FORCE_SAMELINE(code)` | `code` on one line | +| `PERM_IGNORE(code)` | `code`, with no macros expanded inside | +| `PERM_PRETEND(code)` | nothing | + +Arguments split at top-level commas, and `(,)` is a literal comma. Macros can be nested. + +Exhaustive mode tries every expansion, and random mode picks one at random. As in +decomp-permuter, a function with macros is only permuted through them, unless it uses +`PERM_RANDOMIZE` or you pass `-p`. In that case the passes run on top of each expansion, over +the whole function (not just inside the `PERM_RANDOMIZE` region). Macros are supported in +the target function only. + +## Combining passes + +A function is often off in more than one way: the saves are in the wrong order *and* an `if` +is the wrong way round. A single pass can't fix both, so passes can be combined: + +```sh +-p reorder_saves,invert_if # each pass on its own: 23 + 4 candidates +-p reorder_saves+invert_if # a combo: 23 + 4 + 23*4 = 119 candidates +-p reorder_saves -p invert_if+swap_operands # -p can be repeated +``` + +In exhaustive mode, a combo `a+b+c` tries every variant of every pass in it, and every +in-order chain of them: a, b, c, a then b, a then c, b then c, and a then b then c. In random +mode a combo applies one mutation from each of its passes. `--depth N` chains whole passes or +combos too: `-p reorder_saves -p invert_if --depth 2` reaches the same candidates as the combo +above, plus each pass applied twice. + +## Passes + +`cpp_permuter --list-passes` prints these. **[docs/passes.md](docs/passes.md) shows each one +on an example function**, generated from the passes themselves +(`cpp_permuter --pass-examples`). + +| pass | what it does | +|---|---| +| `reorder_saves` | Runs of consecutive "local saves" (declarations, or assignments to locals, that don't call or store) in every order that keeps dependencies. Four independent saves give 23 new orders (the `PedGroup::PromoteMemberToLeader_4C9680` quirk) | +| `move_stmt` | Move one statement up or down past neighbours it doesn't depend on. Field loads and stores are assumed not to alias; locals and calls are respected | +| `inline_local` | Remove a local by putting its initializer into its uses, plainly and with a cast to the local's type. A local that is never used is dropped | +| `sink_decl` | Move a declaration into the innermost block that holds all its uses, or to just before its first use | +| `hoist_decl` | Move a declaration to the top of its block or of the function, splitting `T x = e;` into `T x;` plus `x = e;`. Also offers one candidate that hoists every declaration at once (C89 style) | +| `merge_decl` | `T x; ... x = e;` becomes `T x = e;` at the assignment | +| `swap_operands` | `a + b` becomes `b + a`, for `+ * & \| ^ == !=` | +| `flip_compare` | `a < b` becomes `b > a` | +| `invert_if` | `if (c) A else B` becomes `if (!c) B else A`, negating comparisons directly | +| `ternary` | `x = c ? a : b` and `return c ? a : b` become `if`/`else` (and an early-return form), and back | +| `compound_assign` | `x = x op y` and `x op= y`, both ways | +| `bool_return` | `return a <= b;` and `if (a <= b) return true; return false;` (also with `else`, and with `1`/`0`), both ways | +| `ternary_arg` | `f(c ? a : b);` and `if (c) f(a); else f(b);`, both ways. Going back, two branches that differ in one sub-expression become one statement with a ternary | +| `switch_if` | `if (x == K) S else D` and `switch (x) { case K: S break; default: D }`, both ways (the one-case switch quirk) | +| `early_return` | `if (c) { ...; return; } else { B }` and `if (c) { ...; return; } B`, both ways (also `break`, `continue`, `goto`) | +| `branch_dup` | Move a statement that both branches start or end with out of the `if`, or copy the statement before or after an `if`/`else` into both branches | +| `cond_split` | `if (a && b) S` and `if (a) { if (b) S }`, both ways | +| `explicit_compare` | `if (x)` and `if (x != 0)`, `if (!x)` and `if (x == 0)`, in conditions | +| `negate_const` | `x - 4` and `x + -4`, `x -= 4` and `x += -4` (the `sub` vs `add -N` quirk) | +| `loop_form` | `for (i; c; s) B` and `i; while (c) { B s; }`, `while (c)` and `for (; c;)` | +| `reassociate` | `a + b + c` and `a + (b + c)`, for `+ * & \| ^` | +| `inequalities` | `x > 4` and `x >= 5`, `x < 4` and `x <= 3`, for integer constants (VC6 picks the `cmp` constant and jump from it) | +| `chain_assign` | `a = v; b = v;` and `a = b = v;` (or `b = v; a = b;`), which decides the register that holds `v` | +| `scope_block` | Wrap a declaration and the statements using it in a `{ }` block, or take a nested block apart (VC6 shares stack slots between blocks) | +| `local_type` | Change a local's integer type: `s32`, `u32`, `s16`, `u16`, `s8`, `u8` (or `int`/`unsigned`/`short`/`char`..., or `DWORD`/`WORD`/`BYTE`...) | +| `locals_to_array` | Consecutive locals of one type become one array, in order or back to front (stack slot order) | +| `reorder_cases` | Reorder a switch's case groups (case bodies are laid out in source order). A last group without `break` gets one when it moves | +| `split_case_labels` | `case 1: case 2: S` and `case 1: S case 2: S`, both ways (one jump table entry per case vs a byte index table) | +| `use_getter` | `p->field_1AC_cam.x` and `p->get_cam_x()`, both ways, using the inline getters found in the source and the headers it includes | +| `temp_for_expr` | Compute a cast, a local or `*p` into a new local first (`u8 tmp = (u8)p->a; g(tmp);`). Types come from the declarations; also "copy through a local to get a spill" | +| `remove_stmt` | Remove an expression statement (low weight) | + +decomp-permuter passes left out because they rely on IDO behaviour that VC6 `/O2` folds +away: `perm_xor_zero`, `perm_mult_zero`, `perm_add_mask`, `perm_dummy_comma_expr`, +`perm_factor_mult`/`shift`, `perm_float_literal`, `perm_sameline`, `perm_pad_var_decl`, +`perm_add_self_assignment` and `perm_duplicate_assignment`. + +The passes aim to keep behaviour the same, but they don't prove it. For example they assume +that two different fields don't alias. That is fine here, because a candidate that matches the +target byte for byte is correct by construction. A rewrite that changes behaviour just won't +match. + +## The parser + +This is not a full C++ front end, and that is deliberate. It works on the unpreprocessed +source and only rewrites the body of the target function. Everything else in the file stays +byte-identical, so candidates go through cl.exe with the original includes and macros. Inside +the body it recognises statements, local declarations and expressions. Anything it can't +classify (`__asm`, unknown macros, local types) becomes a fixed barrier that is never moved +across. If it misreads something, that candidate fails to compile and is skipped. + +Running clang on the whole file instead would mean the code has to compile under clang. MSVC 6 +code with its own headers, `__asm` and naked marker functions doesn't, and clang's error +recovery drops the very declarations we want to move. + +On gta2_re (`--check-parse Source/*.cpp Source/*.hpp`): + +- all 4,235 functions in 231 files parse; +- 2 of 53,552 expressions are not understood (a `decltype` cast and a function-pointer cast); +- 4 statements stay barriers (a `typedef`, two `using namespace`, a local `struct`); +- every candidate that every pass produces parses back. + +## Speed + +`--show-timings` on VC6 under wine, `PedGroup.cpp`, 4 workers: + +| stage | per candidate | +|---|---| +| permute (parse, mutate, splice) | 0.5 ms | +| write the mirror files | 1 ms | +| compile | 600 ms; 355 ms with precompiled headers | +| score (`llvm-objdump` plus diff) | 32 ms | + +The permuter keeps the original file and each function's byte range. A mutation only +re-lexes the function, and each candidate is spliced into in-memory copies, so practically +all the time is spent in the compiler. + +`examples/gta2/compile.sh` and `compile.cmd` use VC6 precompiled headers (`/YX`, one `.pch` +per worker), which halves the compile time. The code is identical to a normal compile for +every function in `PedGroup.cpp`. The mirror only rewrites files whose content changed, so +the `.pch` stays valid. While headers are being permuted, the permuter sets +`PERMUTER_NO_PCH=1`: VC6 checks the `.pch` by timestamp, and missed header edits made within +the same second. + +Stripping the other functions out of the file (decomp-permuter's `strip_other_fns.py`) +would only save another 10%, and it changed the code of 1 of PedGroup's 38 functions, so it +isn't offered. + +## Scoring + +The function is disassembled with `llvm-objdump -d -r --disassemble-symbols=`. Before +comparing: + +- relocated operands are replaced with their symbol; +- branch targets become instruction indices, so a size change upstream doesn't change every + jump; +- `<...>` annotations are stripped. + +Target and candidate are then aligned with an edit distance. Penalties: 10 for a register +difference, 50 for other operand differences, 60 for a moved instruction, 100 for an insertion +or a deletion. The weights are the same idea as decomp-permuter's. + +## Tests + +`ctest --test-dir build` runs four suites. CI runs `unit` and `integration` on Linux and +Windows. + +`unit` (`tests/tests.cpp`) covers: + +- the lexer and the parser, including listing every definition in a file; +- the effects analysis; +- every pass, in both directions where it has two; +- the `PERM_*` macro expander; +- that `docs/passes.md` is what the passes produce today; +- pass combos; +- the scorer; +- finding inline callees; +- the mirror, including a source tree made of symlinks. + +`integration` (`tests/integration/run.sh`) needs clang and llvm-objdump. It uses clang's +i686 MSVC target at `-O0`, where statement order and expression shape show up directly in the +code. Each case has a `base/` directory that is wrong in some way, and a `target/` directory +that the target object is built from: `main.cpp`, plus any headers. The permuter must match +the target, and where it should, give back exactly the target sources. Negative controls run +without the pass under test (usually: with every other pass) and must *not* match, which +shows the match comes from that pass. The cases are: + +- `var_reorder`: local saves out of order. +- `combo`: saves out of order plus an inverted `if`. Single passes fail and the combo matches. +- `inline_local`: a local the original didn't have. +- `ops`: swapped operands. +- One case for each of `bool_return`, `ternary_arg`, `switch_if`, `branch_dup`, `cond_split`, + `negate_const`, `reassociate`, `inequalities`, `chain_assign`, `local_type`, + `locals_to_array`, `reorder_cases`, `split_case_labels`, `use_getter`, `temp_for_expr` and + `remove_stmt`. +- `perm_macros`: `PERM_LINESWAP` and `PERM_GENERAL`. All 17 expansions are tried, and + without the macros (`-p none`) there's no match. +- `inline_callee`: the function is right, but the `__forceinline` helper in its header isn't. + It is found by `--inline-callees`, `--also NAME` and `--also FILE:NAME`. Without them there + is no match. + +`early_return`, `explicit_compare`, `loop_form` and `scope_block` compile identically at +clang `-O0`, so only the unit tests cover them. + +`gta2_parse` (`tests/integration/gta2_parse.sh`) runs `--check-parse` over every gta2_re +source and header. It fails if any of these happen: + +- a function doesn't parse; +- a candidate doesn't parse back, or is a no-op; +- more than 0.1% of expressions, or more than 10 statements, aren't understood. + +`vc6` (`tests/integration/vc6.sh`) uses real MSVC 6 under wine, on gta2_re functions that +already match, so VC6's code for them is the original game's code. Each case breaks a +function the way a decompiler might, and checks that the permuter recovers the exact original +source. A control without the needed pass must not match. It works in a symlinked copy of +`Source/` and never modifies the checkout. The cases are: + +- `PedGroup::PromoteMemberToLeader_4C9680`: the four weapon saves in three different orders; + and the saves out of order plus an inverted `if`, which only the combo fixes. +- `PedGroup::RemovePed_4C9970`: `ternary_arg`, the `if`/`else` around a call (from + `matching_quirks.md`). +- `sound_obj::IsTrainOrBoxcar_57F120`: `bool_return` (from `matching_quirks.md`). +- `Particle_4C::UpdateShortAnim_state_37_53B580`: `switch_if`. +- `Fix16::Max` with its comparison mirrored, and `Fix16::Abs` written as a ternary, in + `fix16.hpp`, as inlined into `PedGroup` functions. They are recovered through + `--inline-callees` and `--also`; permuting only the calling function doesn't match. +- `PedGroup::IsLeaderCloseToTargetCar_4CAD40`: `use_getter`, with `get_cam_y()` written + out as the field it reads. Two sources match here: the original, and one that reads the + car's x through `Car_BC::get_x_41E430()` instead. VC6 compiles both to the same code. +- `PedGroup::sub_4C9150`: `inequalities` (`< 0x28` written as `<= 0x27`). +- `PERM_LINESWAP` around the scrambled weapon saves. +- 60 random candidates from every pass on `RemovePed`: at most 10% may fail to compile. + None did. + +Some rewrites turned out to make no difference to VC6, so they have no VC6 case: + +- inverting `Fix16::Abs`'s `if`, and its early-return form; +- reordering the cases of the switches tried; +- `u8` instead of `s32` for a loop counter. + +Clang at `-O0` covers those passes instead. + +`gta2_parse` and `vc6` need `GTA2_RE` set to a gta2_re checkout, and `vc6` also needs wine. +Without them they are reported as skipped. + +## gta2_re + +`examples/gta2/` has a VC6-under-wine compile script and a walkthrough. See +[examples/gta2/README.md](examples/gta2/README.md). diff --git a/docs/passes.md b/docs/passes.md new file mode 100644 index 0000000..6fcf8ff --- /dev/null +++ b/docs/passes.md @@ -0,0 +1,852 @@ +# Passes + +Generated by `cpp_permuter --pass-examples` from the samples in `src/examples.cpp`; a unit test checks this file is up to date. Each example shows a small function and what the pass makes of it (some passes offer more candidates than shown). Most passes also work in the other direction. + +## `reorder_saves` + +Permute runs of consecutive local saves (declarations and assignments to locals that don't call or store) in every order that keeps dependencies. + +```cpp +void Group::f(int idx) +{ + Weapon* a = leader->weapon; + Weapon* b = list[idx]->weapon; + Weapon* c = leader->weapon2; + leader->Copy(list[idx]); +} +``` + +becomes + +```cpp +void Group::f(int idx) +{ + Weapon* a = leader->weapon; + Weapon* c = leader->weapon2; + Weapon* b = list[idx]->weapon; + leader->Copy(list[idx]); +} +``` + +becomes + +```cpp +void Group::f(int idx) +{ + Weapon* c = leader->weapon2; + Weapon* b = list[idx]->weapon; + Weapon* a = leader->weapon; + leader->Copy(list[idx]); +} +``` + +(5 candidates in all for this function.) + +## `move_stmt` + +Move a statement up or down past neighbours it doesn't depend on (locals and calls are respected, memory aliasing is not). + +```cpp +void f(Ped* p) +{ + p->a = 1; + p->b = 2; + g(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + p->b = 2; + p->a = 1; + g(); +} +``` + +## `inline_local` + +Remove a local: substitute its initializer into its uses (plain and with a cast), or drop it if unused. + +```cpp +void f(Ped* p) +{ + Car* car = p->car; + car->speed = 0; +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + p->car->speed = 0; +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + ((Car*)(p->car))->speed = 0; +} +``` + +## `sink_decl` + +Move a declaration into the innermost block holding all its uses, or to just before its first use. + +```cpp +void f(int n) +{ + int i; + g(); + if (n) + { + i = 2; + h(i); + } +} +``` + +becomes + +```cpp +void f(int n) +{ + g(); + if (n) + { + int i; + i = 2; + h(i); + } +} +``` + +(2 candidates in all for this function.) + +## `hoist_decl` + +Move declarations to the top of their block or of the function (one, or all of them), splitting the initializer into an assignment. + +```cpp +void f(int n) +{ + g(); + if (n) + { + int i = 2; + h(i); + } +} +``` + +becomes + +```cpp +void f(int n) +{ + int i; + g(); + if (n) + { + i = 2; + h(i); + } +} +``` + +## `merge_decl` + +Join 'T x; ... x = e;' into 'T x = e;' at the assignment. + +```cpp +void f(int n) +{ + int i; + g(); + i = n * 2; + h(i); +} +``` + +becomes + +```cpp +void f(int n) +{ + g(); + int i = n * 2; + h(i); +} +``` + +## `swap_operands` + +Swap the operands of a commutative operator (+ * & | ^ == !=). + +```cpp +int f(Ped* p) +{ + return p->a + p->b; +} +``` + +becomes + +```cpp +int f(Ped* p) +{ + return p->b + p->a; +} +``` + +## `flip_compare` + +Mirror a comparison: 'a < b' -> 'b > a'. + +```cpp +void f(Ped* p) +{ + if (p->a < p->b) + g(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->b > p->a) + g(); +} +``` + +## `invert_if` + +Negate an if condition and swap the then/else branches. + +```cpp +void f(Ped* p) +{ + if (p->a == 0) + g(); + else + h(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->a != 0) + h(); + else + g(); +} +``` + +## `ternary` + +Turn 'x = c ? a : b' / 'return c ? a : b' into if/else and back. + +```cpp +void f(Ped* p, int x) +{ + p->a = x > 0 ? 1 : 2; +} +``` + +becomes + +```cpp +void f(Ped* p, int x) +{ + if (x > 0) + { + p->a = 1; + } + else + { + p->a = 2; + } +} +``` + +## `compound_assign` + +'x = x op y' <-> 'x op= y'. + +```cpp +void f(Ped* p, int x) +{ + p->a = p->a + x; +} +``` + +becomes + +```cpp +void f(Ped* p, int x) +{ + p->a += x; +} +``` + +## `bool_return` + +'return a <= b;' <-> 'if (a <= b) return true; return false;' (with and without else, true/false and 1/0). + +```cpp +bool f(Ped* p) +{ + return p->a <= p->b; +} +``` + +becomes + +```cpp +bool f(Ped* p) +{ + if (p->a <= p->b) + { + return true; + } + return false; +} +``` + +becomes + +```cpp +bool f(Ped* p) +{ + if (p->a <= p->b) + { + return 1; + } + else + { + return 0; + } +} +``` + +(4 candidates in all for this function.) + +## `ternary_arg` + +'f(c ? a : b);' <-> 'if (c) f(a); else f(b);': a ternary inside a statement, and two branches that differ in one sub-expression. + +```cpp +void f(Ped* p) +{ + p->Set(p->state == 0 ? 1 : 0); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->state == 0) + { + p->Set(1); + } + else + { + p->Set(0); + } +} +``` + +## `switch_if` + +'if (x == K) S else D' <-> 'switch (x) { case K: S break; default: D }'. + +```cpp +void f(Ped* p, int notify) +{ + if (notify == 1) + { + g(p->a); + } +} +``` + +becomes + +```cpp +void f(Ped* p, int notify) +{ + switch (notify) + { + case 1: + g(p->a); + break; + } +} +``` + +## `early_return` + +'if (c) { ...return; } else { B }' <-> 'if (c) { ...return; } B' (also break, continue, goto). + +```cpp +int f(Ped* p) +{ + if (!p) + { + return 0; + } + else + { + g(p); + } + return 1; +} +``` + +becomes + +```cpp +int f(Ped* p) +{ + if (!p) + { + return 0; + } + g(p); + return 1; +} +``` + +## `branch_dup` + +Hoist a statement both branches start or end with out of the if, or copy the statement before/after an if/else into both branches. + +```cpp +void f(Ped* p) +{ + if (p->a) + { + g(1); + } + else + { + g(2); + } + p->c = 5; +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->a) + { + g(1); + p->c = 5; + } + else + { + g(2); + p->c = 5; + } +} +``` + +## `cond_split` + +'if (a && b) S' <-> 'if (a) { if (b) S }'. + +```cpp +void f(Ped* p) +{ + if (p->a && p->b) + g(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->a) + { + if (p->b) + g(); + } +} +``` + +## `explicit_compare` + +'if (x)' <-> 'if (x != 0)', 'if (!x)' <-> 'if (x == 0)'. + +```cpp +void f(Ped* p) +{ + if (!p->car) + g(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->car == 0) + g(); +} +``` + +## `negate_const` + +'x - 4' <-> 'x + -4', 'x -= 4' <-> 'x += -4'. + +```cpp +int f(int x) +{ + return x - 256; +} +``` + +becomes + +```cpp +int f(int x) +{ + return x + -256; +} +``` + +## `loop_form` + +for <-> while: 'for (i; c; s) B' <-> 'i; while (c) { B s; }', 'while (c)' <-> 'for (; c;)'. + +```cpp +void f(Ped* p) +{ + for (int i = 0; i < p->count; i++) + { + g(i); + } +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + int i = 0; + while (i < p->count) + { + g(i); + i++; + } +} +``` + +## `reassociate` + +'a + b + c' <-> 'a + (b + c)' (for + * & | ^). + +```cpp +int f(Ped* p) +{ + return p->a + p->b + p->c; +} +``` + +becomes + +```cpp +int f(Ped* p) +{ + return p->a + (p->b + p->c); +} +``` + +## `inequalities` + +'x > 4' <-> 'x >= 5', 'x < 4' <-> 'x <= 3' (integer constants). + +```cpp +void f(Ped* p) +{ + if (p->a > 4) + g(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + if (p->a >= 5) + g(); +} +``` + +## `chain_assign` + +'a = v; b = v;' <-> 'a = b = v;' (and 'b = v; a = b;'): which register holds the shared value. + +```cpp +void f(Ped* p) +{ + p->a = 0; + p->b = 0; +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + p->a = p->b = 0; +} +``` + +(2 candidates in all for this function.) + +## `scope_block` + +Wrap a declaration and the statements using it in their own { } block (VC6 shares stack slots between blocks), or take a nested block apart. + +```cpp +void f(Ped* p) +{ + int x = p->a; + g(x); + h(); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + { + int x = p->a; + g(x); + } + h(); +} +``` + +## `local_type` + +Change a local's integer type: s32 <-> u32 <-> s16 <-> u8 ... (or the builtin / Windows equivalents). + +```cpp +void f(Ped* p) +{ + s32 x = p->a; + g(x); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + s8 x = p->a; + g(x); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + u32 x = p->a; + g(x); +} +``` + +(5 candidates in all for this function.) + +## `locals_to_array` + +Turn consecutive locals of one type into an array, in order or back to front (stack slot order). + +```cpp +void f(Ped* p) +{ + s32 x = p->a; + s32 y = p->b; + g(x, y); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + s32 x_arr[2]; + x_arr[0] = p->a; + x_arr[1] = p->b; + g(x_arr[0], x_arr[1]); +} +``` + +(2 candidates in all for this function.) + +## `reorder_cases` + +Reorder a switch's case groups (case bodies are laid out in source order). + +```cpp +int f(int n) +{ + switch (n) + { + case 1: + return g(1); + case 2: + return g(2); + } + return 0; +} +``` + +becomes + +```cpp +int f(int n) +{ + switch (n) + { + case 2: + return g(2); + case 1: + return g(1); + } + return 0; +} +``` + +## `split_case_labels` + +'case 1: case 2: S' <-> 'case 1: S case 2: S' (a byte index table vs one entry per case). + +```cpp +int f(int n) +{ + switch (n) + { + case 1: + case 2: + return g(1); + } + return 0; +} +``` + +becomes + +```cpp +int f(int n) +{ + switch (n) + { + case 1: + return g(1); + case 2: + return g(1); + } + return 0; +} +``` + +## `use_getter` + +'p->field.x' <-> 'p->get_x()' using the inline getters in the headers the source includes. + +With `Fix16 get_cam_x() { return field_1AC_cam.x; }` in an included header: + +```cpp +Fix16 f(Ped* p) +{ + return p->field_1AC_cam.x; +} +``` + +becomes + +```cpp +Fix16 f(Ped* p) +{ + return p->get_cam_x(); +} +``` + +## `temp_for_expr` + +Compute a cast, a local or '*p' into a new local first (types from the declarations; also 'copy through a local to get a spill'). + +```cpp +void f(Ped* p) +{ + g((u8)p->a); +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + u8 tmp = (u8)p->a; + g(tmp); +} +``` + +(2 candidates in all for this function.) + +## `remove_stmt` + +Remove an expression statement. + +```cpp +void f(Ped* p) +{ + g(); + WIP_IMPLEMENTED; +} +``` + +becomes + +```cpp +void f(Ped* p) +{ + g(); +} +``` + +(2 candidates in all for this function.) + diff --git a/examples/gta2/PromoteMemberToLeader.conf b/examples/gta2/PromoteMemberToLeader.conf new file mode 100644 index 0000000..c0336ff --- /dev/null +++ b/examples/gta2/PromoteMemberToLeader.conf @@ -0,0 +1,10 @@ +# cpp_permuter --config PromoteMemberToLeader.conf +# Paths are relative to where cpp_permuter runs (here: the gta2_re checkout). +source = Source/PedGroup.cpp +function = PedGroup::PromoteMemberToLeader_4C9680 +compile = /path/to/cpp_permuter/examples/gta2/compile.sh {src} {obj} +target-obj = Scripts/asm/PedGroup.cpp.obj +mode = exhaustive +passes = reorder_saves +jobs = 4 +output-dir = permuter_out/PromoteMemberToLeader diff --git a/examples/gta2/README.md b/examples/gta2/README.md new file mode 100644 index 0000000..e9d8e66 --- /dev/null +++ b/examples/gta2/README.md @@ -0,0 +1,83 @@ +# Using cpp_permuter with gta2_re + +## Setup (Linux) + +Set up wine and the gta2_re VC6 toolchain as gta2_re's `CLAUDE.md` describes, so the +`3rdParty/gta2_re_compile_tools` submodule is checked out. Then: + +```sh +export GTA2_RE=/path/to/gta2_re +export WINEDEBUG=-all +wineserver -p # keep wineserver running: compiles drop from seconds to well under one +``` + +Check that a single TU builds: + +```sh +examples/gta2/compile.sh $GTA2_RE/Source/PedGroup.cpp /tmp/PedGroup.obj +``` + +`compile.sh` uses the `gta2_lib` flags (`/O2 /GX /ML ...`). Files that `cmake/vc6.cmake` gives +extra flags need `EXTRA_CFLAGS`, for example `EXTRA_CFLAGS=/Gz` for `Network_20324.cpp`. + +## A target object + +The permuter compares against an object file that holds the original function. It needs the +original `10.5.exe` in `Scripts/bin_comp/`, as for objdiff: + +```sh +cd $GTA2_RE +python3 Scripts/generate_target_asm_for_objs.py PedGroup.cpp # writes Scripts/asm/PedGroup.cpp.asm + make_objs.sh +(cd Scripts/asm && ./make_objs.sh) # -> Scripts/asm/PedGroup.cpp.obj +``` + +Calls in that asm are resolved to mangled names, so they compare with the symbols in our +object. If some operands are still raw addresses from the exe, add `--ignore-reloc-names`. + +## Running + +Remove `WIP_IMPLEMENTED` / `NOT_IMPLEMENTED` from the function first, since they add code. + +```sh +cpp_permuter -s $GTA2_RE/Source/PedGroup.cpp -f PedGroup::SomeFunction_4C9xxx \ + -c "$PWD/examples/gta2/compile.sh {src} {obj}" \ + -t $GTA2_RE/Scripts/asm/PedGroup.cpp.obj \ + --show-base-diff -j 4 +``` + +Or put the options in a file (see `PromoteMemberToLeader.conf`) and run +`cpp_permuter --config PromoteMemberToLeader.conf`. + +Suggestions: + +- When you already suspect the kind of fix, use exhaustive mode with just that pass: + `-m exhaustive -p reorder_saves` tries every order of every run of local saves. +- When it's off in two ways at once, combine the passes: `-p reorder_saves+invert_if` tries + each pass alone and every pair of their variants. `-p` can be repeated to run several + combos in one go. +- If the asm looks right apart from code that comes from an inline helper (a getter, a + `Fix16` operator, `Abs`/`Max`), add `--inline-callees`, or name the helper with `--also + fix16.hpp:Max`. The helper is permuted too. Check `--list-regions` to see which functions + were picked up. If a header change ends up in the match, it affects every TU that includes + that header, so run `compare_builds.py` before committing it. +- `--dry-run` shows what a pass would produce without compiling anything. +- Random mode (the default) mixes all passes and is good for leaving running. Add + `-p reorder_saves,move_stmt` and similar to focus it. +- Results go to `permuter_out/output--/`. Copy the function back from + `function.cpp`, then run `build.py` and `compare_builds.py` as usual. + +## Checking the setup without the original exe + +A `MATCH_FUNC` is its own target. Compile the unmodified TU, scramble the function, and the +permuter should find the way back: + +```sh +examples/gta2/compile.sh $GTA2_RE/Source/PedGroup.cpp /tmp/target.obj +# swap a few lines of PedGroup::PromoteMemberToLeader_4C9680 in a copy of the file, then +cpp_permuter -s $GTA2_RE/Source/PedGroup_scrambled.cpp -f PedGroup::PromoteMemberToLeader_4C9680 \ + -c "$PWD/examples/gta2/compile.sh {src} {obj}" -t /tmp/target.obj \ + -m exhaustive -p reorder_saves -j 4 +``` + +With the four weapon saves scrambled (base score 520), this tried the other 23 orders and found +the matching one in about 4 seconds. diff --git a/examples/gta2/compile.cmd b/examples/gta2/compile.cmd new file mode 100644 index 0000000..04ec645 --- /dev/null +++ b/examples/gta2/compile.cmd @@ -0,0 +1,46 @@ +@echo off +rem Native Windows version of compile.sh: compiles one gta2_re translation unit +rem with VC6, using the gta2_lib flags. +rem +rem compile.cmd +rem +rem GTA2_RE must point at a gta2_re checkout with the +rem 3rdParty\gta2_re_compile_tools submodule. EXTRA_CFLAGS adds per-file flags. +rem Precompiled headers are used unless NO_PCH or PERMUTER_NO_PCH is set (see +rem compile.sh for why). +rem +rem Use it as: cpp_permuter -c "examples\gta2\compile.cmd {src} {obj}" ... +setlocal +if "%GTA2_RE%"=="" ( + echo set GTA2_RE to your gta2_re checkout + exit /b 2 +) +set "TOOLS=%GTA2_RE%\3rdParty\gta2_re_compile_tools" +set "PATH=%TOOLS%\VC98\Bin;%TOOLS%\Common\MSDev98\Bin;%PATH%" +set "INCLUDE=%TOOLS%\VC98\ATL\Include;%TOOLS%\VC98\Include;%TOOLS%\VC98\MFC\Include" +set "LIB=%TOOLS%\VC98\Lib" +set "SRC=%~f1" +set "OBJ=%~f2" +set "PCH=%~dpn2.pch" +set "LOG=%~dpn2.log" +rem the source's own directory first, so a candidate in cpp_permuter's mirror +rem sees the mirror's headers ("." keeps the trailing backslash off the quote) +set "SRCDIR=%~dp1." +set FLAGS=/DWIN32 /D_WINDOWS /D_CRT_SECURE_NO_WARNINGS /D_CRT_NON_CONFORMING_SWPRINTFS /DIMGUI_DLL /W3 /EHsc /GX /ML /O2 /DNDEBUG %EXTRA_CFLAGS% + +if defined NO_PCH goto plain +if defined PERMUTER_NO_PCH goto plain +cl.exe /nologo /TP /c /I"%SRCDIR%" /I"%GTA2_RE%" /I"%TOOLS%" %FLAGS% /Zm500 /YX /Fp"%PCH%" /Fo"%OBJ%" "%SRC%" > "%LOG%" 2>&1 +if not errorlevel 1 ( + type "%LOG%" + exit /b 0 +) +rem only fall back when the precompiled header itself was the problem +findstr /R "C1060 C1076 C1083 C1852 C1853 C1859 C2859" "%LOG%" >nul +if errorlevel 1 ( + type "%LOG%" + exit /b 2 +) + +:plain +cl.exe /nologo /TP /c /I"%SRCDIR%" /I"%GTA2_RE%" /I"%TOOLS%" %FLAGS% /Zm1000 /Fo"%OBJ%" "%SRC%" diff --git a/examples/gta2/compile.sh b/examples/gta2/compile.sh new file mode 100755 index 0000000..23bb5b2 --- /dev/null +++ b/examples/gta2/compile.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Compiles one gta2_re translation unit with VC6 under wine, using the same +# flags as gta2_lib in gta2_re's CMakeLists.txt / build.py --single_cpp. +# +# compile.sh +# +# GTA2_RE must point at a gta2_re checkout with the 3rdParty/gta2_re_compile_tools +# submodule. EXTRA_CFLAGS adds per-file flags (cmake/vc6.cmake), e.g. "/Gz" for +# Network_20324.cpp. +# +# The source's own directory stands in for Source/ on the include path, so a +# candidate compiled in cpp_permuter's mirror of Source/ sees the mirror's +# (possibly edited) headers everywhere. +# +# Use it as: cpp_permuter -c 'examples/gta2/compile.sh {src} {obj}' ... +set -e +SRC=$1 +OBJ=$2 +: "${GTA2_RE:?set GTA2_RE to your gta2_re checkout}" +TOOLS="$GTA2_RE/3rdParty/gta2_re_compile_tools" + +winpath() { printf 'Z:%s' "$(realpath -m "$1")" | tr / '\\'; } + +export WINEDEBUG=-all +export WINEPATH="$(winpath "$TOOLS/VC98/Bin");$(winpath "$TOOLS/Common/MSDev98/Bin")" +export INCLUDE="$(winpath "$TOOLS/VC98/ATL/Include");$(winpath "$TOOLS/VC98/Include");$(winpath "$TOOLS/VC98/MFC/Include")" +export LIB="$(winpath "$TOOLS/VC98/Lib")" + +FLAGS="/DWIN32 /D_WINDOWS /D_CRT_SECURE_NO_WARNINGS /D_CRT_NON_CONFORMING_SWPRINTFS /DIMGUI_DLL \ + /W3 /EHsc /GX /ML /O2 /DNDEBUG $EXTRA_CFLAGS" +INCS="/I$(winpath "$(dirname "$SRC")") /I$(winpath "$GTA2_RE") /I$(winpath "$TOOLS")" + +# Precompiled headers (/YX, one .pch next to each object, so per permuter +# worker) halve the compile time. VC6 rebuilds the .pch when a header it holds +# changes, and the code is the same as without it (checked on every function of +# PedGroup.cpp). /Zm500 instead of /Zm1000: with /YX, /Zm1000 runs out of heap +# space under wine. NO_PCH=1 turns it off; if VC6 can't use the .pch the file is +# compiled again without it. +# +# cpp_permuter sets PERMUTER_NO_PCH=1 while it edits headers (--inline-callees, +# --also): VC6 decides whether the .pch is stale by timestamp, and a header +# edited within the same second as the .pch would be missed. +# +# cl.exe exits 0 on warnings and 2 on errors; its output goes to stdout. +if [ -z "${NO_PCH:-}" ] && [ -z "${PERMUTER_NO_PCH:-}" ]; then + LOG=$(wine cl.exe /nologo /TP /c $INCS $FLAGS /Zm500 /YX /Fp"$(winpath "${OBJ%.*}.pch")" \ + /Fo"$(winpath "$OBJ")" "$(winpath "$SRC")") && { printf '%s\n' "$LOG"; exit 0; } + if ! printf '%s' "$LOG" | grep -q -E "C1060|C1076|C1083|C1852|C1853|C1859|C2859"; then + printf '%s\n' "$LOG" + exit 2 + fi +fi +wine cl.exe /nologo /TP /c $INCS $FLAGS /Zm1000 /Fo"$(winpath "$OBJ")" "$(winpath "$SRC")" diff --git a/src/analysis.cpp b/src/analysis.cpp new file mode 100644 index 0000000..001e016 --- /dev/null +++ b/src/analysis.cpp @@ -0,0 +1,588 @@ +#include "analysis.hpp" + +namespace perm { + +int binPrec(const std::string& op) { + if (op == ",") return 1; + if (op == "||") return 4; + if (op == "&&") return 5; + if (op == "|") return 6; + if (op == "^") return 7; + if (op == "&") return 8; + if (op == "==" || op == "!=") return 9; + if (op == "<" || op == ">" || op == "<=" || op == ">=") return 10; + if (op == "<<" || op == ">>") return 11; + if (op == "+" || op == "-") return 12; + if (op == "*" || op == "/" || op == "%") return 13; + if (op == ".*" || op == "->*") return 14; + return 0; +} + +void forEachExpr(Expr& x, const std::function& fn) { + fn(x); + for (auto& k : x.kids) forEachExpr(*k, fn); +} + +namespace { + +// True if token i can end an operand, so a following '*', '&', '-' is binary. +bool operandEnd(const Func& f, int i) { + if (i < 0) return false; + const Token& t = f.toks[i]; + switch (t.kind) { + case TokKind::Number: + case TokKind::String: + case TokKind::Char: return true; + case TokKind::Ident: + return !isKeyword(t.text) || t.text == "this" || t.text == "true" || t.text == "false" || + t.text == "nullptr"; + case TokKind::Punct: return t.text == ")" || t.text == "]" || t.text == "++" || t.text == "--"; + default: return false; + } +} + +struct ExprParser { + const Func& f; + const std::vector& T; + int pos, end; + bool fail = false; + + ExprParser(const Func& fn, int b, int e) : f(fn), T(fn.toks), pos(b), end(e) {} + + bool at(const char* s) const { + return pos < end && T[pos].kind == TokKind::Punct && T[pos].text == s; + } + + std::unique_ptr node(Expr::K k, int b) { + auto x = std::make_unique(); + x->k = k; + x->b = b; + return x; + } + + bool isTypeTok(int i) const { + const Token& t = T[i]; + if (t.kind == TokKind::Ident) { + if (isBuiltinType(t.text)) return true; + if (t.text == "const" || t.text == "volatile" || t.text == "struct" || + t.text == "class" || t.text == "union" || t.text == "enum" || + t.text == "unsigned" || t.text == "signed") + return true; + return !isKeyword(t.text) && !f.locals.count(t.text); + } + if (t.kind == TokKind::Punct) + return t.text == "::" || t.text == "*" || t.text == "&" || t.text == "<" || + t.text == ">" || t.text == ","; + return false; + } + + // '(' at pos starts a C-style cast? + bool isCast() const { + int close = matchBracket(T, pos); + if (close < 0 || close >= end || close == pos + 1) return false; + int idents = 0; + bool ptr = false, builtin = false; + for (int i = pos + 1; i < close; ++i) { + if (!isTypeTok(i)) return false; + if (T[i].kind == TokKind::Ident) { + idents++; + if (isBuiltinType(T[i].text)) builtin = true; + } else if (T[i].text == "*" || T[i].text == "&") { + ptr = true; + } + } + if (idents == 0) return false; + if (close + 1 >= end) return false; + const Token& n = T[close + 1]; + if (n.kind == TokKind::Punct) { + const std::string& s = n.text; + bool ambiguous = s == "-" || s == "+" || s == "*" || s == "&"; + if (ambiguous && !ptr && !builtin && idents == 1) return false; + return s == "(" || s == "!" || s == "~" || s == "-" || s == "+" || s == "*" || + s == "&" || s == "++" || s == "--" || s == "::"; + } + if (n.kind == TokKind::Ident) return !isKeyword(n.text) || n.text == "this" || + n.text == "sizeof" || n.text == "true" || + n.text == "operator" || n.text == "new" || + n.text == "false" || isBuiltinType(n.text); + return n.kind != TokKind::PP && n.kind != TokKind::End; + } + + std::unique_ptr parse(int minPrec) { + auto lhs = unary(); + if (!lhs) return nullptr; + while (pos < end && !fail) { + const Token& t = T[pos]; + if (t.kind != TokKind::Punct) break; + const std::string& op = t.text; + int b = lhs->b; + if (isAssignOp(op)) { + if (minPrec > 2) break; + int o = pos++; + auto rhs = parse(2); + if (!rhs) return nullptr; + auto x = node(Expr::Assign, b); + x->op = o; + x->e = rhs->e; + x->kids.push_back(std::move(lhs)); + x->kids.push_back(std::move(rhs)); + lhs = std::move(x); + } else if (op == "?") { + if (minPrec > 3) break; + int q = pos++; + auto mid = parse(1); + if (!mid || !at(":")) return fail = true, nullptr; + int c = pos++; + auto rhs = parse(2); + if (!rhs) return nullptr; + auto x = node(Expr::Ternary, b); + x->op = q; + x->colon = c; + x->e = rhs->e; + x->kids.push_back(std::move(lhs)); + x->kids.push_back(std::move(mid)); + x->kids.push_back(std::move(rhs)); + lhs = std::move(x); + } else { + int p = binPrec(op); + if (p == 0 || p < minPrec) break; + int o = pos++; + auto rhs = parse(p + 1); + if (!rhs) return nullptr; + auto x = node(op == "," ? Expr::Comma : Expr::Binary, b); + x->op = o; + x->e = rhs->e; + x->kids.push_back(std::move(lhs)); + x->kids.push_back(std::move(rhs)); + lhs = std::move(x); + } + } + return fail ? nullptr : std::move(lhs); + } + + std::unique_ptr unary() { + if (pos >= end) return fail = true, nullptr; + const Token& t = T[pos]; + int b = pos; + if (t.kind == TokKind::Punct) { + const std::string& s = t.text; + if (s == "+" || s == "-" || s == "!" || s == "~" || s == "*" || s == "&" || + s == "++" || s == "--") { + pos++; + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Unary, b); + x->op = b; + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + if (s == "(" && isCast()) { + int close = matchBracket(T, pos); + pos = close + 1; + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Cast, b); + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + } else if (t.kind == TokKind::Ident) { + if (t.text == "sizeof") { + pos++; + if (at("(")) { + // sizeof(type) or sizeof(expr): treat the group as opaque + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + pos = close + 1; + auto x = node(Expr::Primary, b); + x->e = pos; + return postfix(std::move(x)); + } + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Unary, b); + x->op = b; + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + if (t.text == "delete") { + // delete p, delete[] p + pos++; + if (at("[") && pos + 1 < end && T[pos + 1].text == "]") pos += 2; + auto o = unary(); + if (!o) return nullptr; + auto x = node(Expr::Unary, b); + x->op = b; + x->e = o->e; + x->kids.push_back(std::move(o)); + return x; + } + if (t.text == "new") { + // new T, new T(args), new T[n], new (place) T(...) + pos++; + if (at("(")) { + int c = matchBracket(T, pos); + if (c < 0 || c >= end) return fail = true, nullptr; + pos = c + 1; + } + if (at("::")) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + while (pos < end && (isBuiltinType(T[pos].text) || T[pos].text == "const")) pos++; + if (pos < end && T[pos].kind == TokKind::Ident && !isBuiltinType(T[pos].text)) { + pos++; + while (at("::") && pos + 1 < end && T[pos + 1].kind == TokKind::Ident) pos += 2; + } + while (at("*")) pos++; + auto x = node(Expr::Call, b); + if (at("(") || at("[")) { + bool idx = at("["); + int c = matchBracket(T, pos); + if (c < 0 || c >= end) return fail = true, nullptr; + if (c > pos + 1) { + ExprParser a(f, pos + 1, c); + auto arg = a.parse(idx ? 1 : 1); + if (!arg || a.pos != c) return fail = true, nullptr; + x->kids.push_back(std::move(arg)); + } + pos = c + 1; + } + x->e = pos; + return x; + } + if (t.text == "throw") return fail = true, nullptr; + } + return postfix(primary()); + } + + std::unique_ptr primary() { + if (pos >= end) return fail = true, nullptr; + const Token& t = T[pos]; + int b = pos; + if (t.kind == TokKind::Number || t.kind == TokKind::Char) { + pos++; + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::String) { + while (pos < end && T[pos].kind == TokKind::String) pos++; + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::Ident || (t.kind == TokKind::Punct && t.text == "::")) { + const std::string& s = t.text; + if (s == "static_cast" || s == "reinterpret_cast" || s == "const_cast" || + s == "dynamic_cast") { + pos++; + if (!at("<")) return fail = true, nullptr; + int depth = 0; + while (pos < end) { + if (T[pos].text == "<") depth++; + if (T[pos].text == ">" && --depth == 0) break; + pos++; + } + pos++; + if (!at("(")) return fail = true, nullptr; + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + ExprParser inner(f, pos + 1, close); + auto in = inner.parse(1); + if (!in || inner.pos != close) return fail = true, nullptr; + pos = close + 1; + auto x = node(Expr::Cast, b); + x->e = pos; + x->kids.push_back(std::move(in)); + return x; + } + if (s == "operator" && pos + 1 < end) { + // operator new(...), operator delete(...), operator==(...) + pos += 2; + if (at("[") && pos + 1 < end && T[pos + 1].text == "]") pos += 2; + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::Ident && isKeyword(s) && s != "this" && s != "true" && + s != "false" && s != "nullptr" && !isBuiltinType(s)) + return fail = true, nullptr; + if (t.kind == TokKind::Punct) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + pos++; + while (at("::") && pos + 1 < end) { + pos++; + if (at("~")) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + pos++; + } + auto x = node(Expr::Primary, b); + x->e = pos; + return x; + } + if (t.kind == TokKind::Punct && t.text == "(") { + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + ExprParser inner(f, pos + 1, close); + auto in = inner.parse(1); + if (!in || inner.pos != close) return fail = true, nullptr; + pos = close + 1; + auto x = node(Expr::Paren, b); + x->e = pos; + x->kids.push_back(std::move(in)); + return x; + } + return fail = true, nullptr; + } + + std::unique_ptr postfix(std::unique_ptr x) { + if (!x) return nullptr; + while (pos < end && !fail) { + int b = x->b; + if (at("(")) { + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + auto c = node(Expr::Call, b); + c->kids.push_back(std::move(x)); + int p = pos + 1; + while (p < close) { + ExprParser a(f, p, close); + auto arg = a.parse(2); + if (!arg) { + // not an expression (a type passed to a macro such as + // va_arg): keep it as an opaque operand + int q = p, depth = 0; + for (; q < close; ++q) { + const std::string& x = T[q].text; + if (x == "(" || x == "[" || x == "{") depth++; + else if (x == ")" || x == "]" || x == "}") depth--; + else if (x == "," && depth == 0) break; + } + if (q == p) return fail = true, nullptr; + arg = node(Expr::Primary, p); + arg->e = q; + a.pos = q; + } + c->kids.push_back(std::move(arg)); + p = a.pos; + if (p < close) { + if (T[p].text != ",") return fail = true, nullptr; + p++; + } + } + pos = close + 1; + c->e = pos; + x = std::move(c); + } else if (at("[")) { + int close = matchBracket(T, pos); + if (close < 0 || close >= end) return fail = true, nullptr; + ExprParser a(f, pos + 1, close); + auto idx = a.parse(1); + if (!idx || a.pos != close) return fail = true, nullptr; + auto c = node(Expr::Index, b); + c->kids.push_back(std::move(x)); + c->kids.push_back(std::move(idx)); + pos = close + 1; + c->e = pos; + x = std::move(c); + } else if (at(".") || at("->")) { + pos++; + if (at("~")) pos++; + if (pos >= end || T[pos].kind != TokKind::Ident) return fail = true, nullptr; + pos++; + while (at("::") && pos + 1 < end && T[pos + 1].kind == TokKind::Ident) pos += 2; + auto c = node(Expr::Member, b); + c->kids.push_back(std::move(x)); + c->e = pos; + x = std::move(c); + } else if (at("++") || at("--")) { + auto c = node(Expr::Postfix, b); + c->op = pos++; + c->kids.push_back(std::move(x)); + c->e = pos; + x = std::move(c); + } else { + break; + } + } + return fail ? nullptr : std::move(x); + } +}; + +} // namespace + +std::unique_ptr parseExpr(const Func& f, int b, int e) { + if (b >= e) return nullptr; + ExprParser p(f, b, e); + auto x = p.parse(1); + if (!x || p.pos != e) return nullptr; + return x; +} + +std::vector exprSites(const Func& f) { + std::vector out; + auto addDecl = [&](const Stmt& s) { + for (auto& d : s.decls) + if (d.init == Declarator::Assign && f.t(d.initBegin) != "{") + out.push_back({d.initBegin, d.initEnd, &s}); + }; + forEachStmt(*f.body, [&](const Stmt& s) { + switch (s.kind) { + case SK::Expr: + if (s.end - 1 > s.begin && f.t(s.end - 1) == ";") out.push_back({s.begin, s.end - 1, &s}); + break; + case SK::Return: + if (f.t(s.begin) == "return" && f.t(s.end - 1) == ";" && s.end - 1 > s.begin + 1) + out.push_back({s.begin + 1, s.end - 1, &s}); + break; + case SK::Decl: addDecl(s); break; + case SK::If: + case SK::While: + case SK::Switch: + case SK::Do: + if (s.condOpen >= 0) out.push_back({s.condOpen + 1, s.condClose, &s}); + break; + case SK::For: { + int depth = 0, segStart = s.condOpen + 1, seg = 0; + for (int i = s.condOpen + 1; i <= s.condClose; ++i) { + const std::string& x = f.t(i); + if (i < s.condClose && (x == "(" || x == "[" || x == "{")) depth++; + else if (i < s.condClose && (x == ")" || x == "]" || x == "}")) depth--; + if ((x == ";" && depth == 0) || i == s.condClose) { + if (seg == 0 && s.forInit) addDecl(*s.forInit); + else if (i > segStart) out.push_back({segStart, i, &s}); + segStart = i + 1; + seg++; + } + } + break; + } + default: break; + } + }); + return out; +} + +bool isWriteUse(const Func& f, int i) { + int n = (int)f.toks.size(); + const std::string& next = i + 1 < n ? f.t(i + 1) : ""; + const std::string& prev = i > 0 ? f.t(i - 1) : ""; + bool prevUnary = i < 2 || !operandEnd(f, i - 2); + if (isAssignOp(next) && !(prev == "*" && prevUnary)) return true; + if (next == "++" || next == "--" || prev == "++" || prev == "--") return true; + if (prev == "&" && prevUnary) return true; // address taken + return false; +} + +Effects effectsOf(const Func& f, int b, int e) { + Effects r; + for (int i = b; i < e; ++i) { + const Token& t = f.toks[i]; + if (t.kind == TokKind::PP) { + r.barrier = true; + continue; + } + if (t.kind == TokKind::Ident) { + const std::string& x = t.text; + if (x == "return" || x == "break" || x == "continue" || x == "goto" || x == "throw" || + x == "__asm" || x == "_asm" || x == "asm" || x == "case" || x == "default" || + x == "__leave") { + r.barrier = true; + continue; + } + if (x == "new" || x == "delete") { + r.call = true; + continue; + } + if (f.isLocal(i)) { + if (isWriteUse(f, i)) { + r.writes.insert(x); + if (f.t(i + 1) != "=") r.reads.insert(x); + } else { + r.reads.insert(x); + } + continue; + } + if (isKeyword(x)) continue; + if (i + 1 < (int)f.toks.size() && f.t(i + 1) == "(") { + r.call = true; + continue; + } + r.memRead = true; // member, global or something we can't see + continue; + } + if (t.kind != TokKind::Punct) continue; + const std::string& x = t.text; + if (x == "->" || x == "[") r.memRead = true; + else if (x == "*" && !operandEnd(f, i - 1)) r.memRead = true; + else if (isAssignOp(x)) { + bool plainLocal = f.isLocal(i - 1) && + !(i >= 2 && (f.t(i - 2) == "*" || f.t(i - 2) == "&") && + !operandEnd(f, i - 3)); + if (!plainLocal) r.memWrite = true; + } else if (x == "++" || x == "--") { + bool local = f.isLocal(i + 1) || (f.isLocal(i - 1)); + if (!local) r.memWrite = true; + } + } + return r; +} + +Effects effectsOfStmt(const Func& f, const Stmt& s) { + Effects r = effectsOf(f, s.begin, s.end); + switch (s.kind) { + case SK::Return: + case SK::Break: + case SK::Continue: + case SK::Goto: + case SK::Label: + case SK::Case: + case SK::Default: + case SK::Asm: + case SK::PP: + case SK::Other: + case SK::Try: r.barrier = true; break; + case SK::Decl: + if (s.isStatic) r.barrier = true; + for (auto& d : s.decls) { + r.writes.insert(f.t(d.nameTok)); + r.reads.erase(f.t(d.nameTok)); + } + break; + default: break; + } + return r; +} + +bool conflicts(const Effects& a, const Effects& b) { + if (a.barrier || b.barrier) return true; + for (auto& w : a.writes) + if (b.reads.count(w) || b.writes.count(w)) return true; + for (auto& w : b.writes) + if (a.reads.count(w)) return true; + bool aw = a.call || a.memWrite, bw = b.call || b.memWrite; + if (aw && (bw || b.memRead)) return true; + if (bw && a.memRead) return true; + return false; +} + +bool conflictsRelaxed(const Effects& a, const Effects& b) { + if (a.barrier || b.barrier) return true; + for (auto& w : a.writes) + if (b.reads.count(w) || b.writes.count(w)) return true; + for (auto& w : b.writes) + if (a.reads.count(w)) return true; + if (a.call && (b.call || b.memWrite || b.memRead)) return true; + if (b.call && (a.memWrite || a.memRead)) return true; + return false; +} + +std::vector usesOf(const Func& f, const std::string& name, int b, int e) { + std::vector r; + for (int i = b; i < e; ++i) + if (f.toks[i].kind == TokKind::Ident && f.toks[i].text == name && f.isLocal(i)) r.push_back(i); + return r; +} + +} // namespace perm diff --git a/src/analysis.hpp b/src/analysis.hpp new file mode 100644 index 0000000..d7838e8 --- /dev/null +++ b/src/analysis.hpp @@ -0,0 +1,49 @@ +// Expression parsing and read/write effects over token ranges. +#pragma once + +#include "parser.hpp" + +#include +#include +#include + +namespace perm { + +struct Expr { + enum K { Primary, Paren, Call, Index, Member, Postfix, Unary, Cast, Binary, Assign, Ternary, Comma } k; + int b = 0, e = 0; // token range [b, e) + int op = -1; // operator token (Binary/Assign/Unary/Postfix/Ternary '?') + int colon = -1; // Ternary ':' + std::vector> kids; +}; + +// Parses tokens [b, e) as one expression. Returns null if the whole range +// doesn't parse. +std::unique_ptr parseExpr(const Func& f, int b, int e); +int binPrec(const std::string& op); // 0 = not a binary operator +void forEachExpr(Expr& x, const std::function& fn); + +// Token ranges in the function that hold a complete expression. +struct ExprSite { + int b, e; + const Stmt* stmt; +}; +std::vector exprSites(const Func& f); + +struct Effects { + std::set reads, writes; + bool memRead = false, memWrite = false, call = false; + bool barrier = false; // control flow, labels, asm, preprocessor +}; +Effects effectsOf(const Func& f, int b, int e); +Effects effectsOfStmt(const Func& f, const Stmt& s); +bool conflicts(const Effects& a, const Effects& b); +// Like conflicts(), but two statements that only read or write memory (no +// calls) may swap: field stores and loads are assumed not to alias. +bool conflictsRelaxed(const Effects& a, const Effects& b); + +// Number of token uses of a local name in [b, e). +std::vector usesOf(const Func& f, const std::string& name, int b, int e); +bool isWriteUse(const Func& f, int tok); + +} // namespace perm diff --git a/src/examples.cpp b/src/examples.cpp new file mode 100644 index 0000000..badd206 --- /dev/null +++ b/src/examples.cpp @@ -0,0 +1,270 @@ +#include "examples.hpp" + +#include "passes.hpp" + +#include +#include + +namespace perm { + +namespace { + +struct Sample { + const char* pass; + const char* code; // a function named f + std::vector show; // which candidates to show (indexes into the pass's output) +}; + +// One small function per pass, written so the shown candidates are the typical +// rewrites. docs/passes.md is generated from these (--pass-examples), and a +// unit test checks it is up to date. +const std::vector& samples() { + static const std::vector s = { + {"reorder_saves", R"(void Group::f(int idx) +{ + Weapon* a = leader->weapon; + Weapon* b = list[idx]->weapon; + Weapon* c = leader->weapon2; + leader->Copy(list[idx]); +})", {0, 4}}, + {"move_stmt", R"(void f(Ped* p) +{ + p->a = 1; + p->b = 2; + g(); +})", {0}}, + {"inline_local", R"(void f(Ped* p) +{ + Car* car = p->car; + car->speed = 0; +})", {0, 1}}, + {"sink_decl", R"(void f(int n) +{ + int i; + g(); + if (n) + { + i = 2; + h(i); + } +})", {0}}, + {"hoist_decl", R"(void f(int n) +{ + g(); + if (n) + { + int i = 2; + h(i); + } +})", {0}}, + {"merge_decl", R"(void f(int n) +{ + int i; + g(); + i = n * 2; + h(i); +})", {0}}, + {"swap_operands", R"(int f(Ped* p) +{ + return p->a + p->b; +})", {0}}, + {"flip_compare", R"(void f(Ped* p) +{ + if (p->a < p->b) + g(); +})", {0}}, + {"invert_if", R"(void f(Ped* p) +{ + if (p->a == 0) + g(); + else + h(); +})", {0}}, + {"ternary", R"(void f(Ped* p, int x) +{ + p->a = x > 0 ? 1 : 2; +})", {0}}, + {"compound_assign", R"(void f(Ped* p, int x) +{ + p->a = p->a + x; +})", {0}}, + {"bool_return", R"(bool f(Ped* p) +{ + return p->a <= p->b; +})", {0, 3}}, + {"ternary_arg", R"(void f(Ped* p) +{ + p->Set(p->state == 0 ? 1 : 0); +})", {0}}, + {"switch_if", R"(void f(Ped* p, int notify) +{ + if (notify == 1) + { + g(p->a); + } +})", {0}}, + {"early_return", R"(int f(Ped* p) +{ + if (!p) + { + return 0; + } + else + { + g(p); + } + return 1; +})", {0}}, + {"branch_dup", R"(void f(Ped* p) +{ + if (p->a) + { + g(1); + } + else + { + g(2); + } + p->c = 5; +})", {0}}, + {"cond_split", R"(void f(Ped* p) +{ + if (p->a && p->b) + g(); +})", {0}}, + {"explicit_compare", R"(void f(Ped* p) +{ + if (!p->car) + g(); +})", {0}}, + {"negate_const", R"(int f(int x) +{ + return x - 256; +})", {0}}, + {"loop_form", R"(void f(Ped* p) +{ + for (int i = 0; i < p->count; i++) + { + g(i); + } +})", {0}}, + {"reassociate", R"(int f(Ped* p) +{ + return p->a + p->b + p->c; +})", {0}}, + {"inequalities", R"(void f(Ped* p) +{ + if (p->a > 4) + g(); +})", {0}}, + {"chain_assign", R"(void f(Ped* p) +{ + p->a = 0; + p->b = 0; +})", {0}}, + {"scope_block", R"(void f(Ped* p) +{ + int x = p->a; + g(x); + h(); +})", {0}}, + {"local_type", R"(void f(Ped* p) +{ + s32 x = p->a; + g(x); +})", {0, 4}}, + {"locals_to_array", R"(void f(Ped* p) +{ + s32 x = p->a; + s32 y = p->b; + g(x, y); +})", {0}}, + {"reorder_cases", R"(int f(int n) +{ + switch (n) + { + case 1: + return g(1); + case 2: + return g(2); + } + return 0; +})", {0}}, + {"split_case_labels", R"(int f(int n) +{ + switch (n) + { + case 1: + case 2: + return g(1); + } + return 0; +})", {0}}, + {"use_getter", R"(Fix16 f(Ped* p) +{ + return p->field_1AC_cam.x; +})", {0}}, + {"temp_for_expr", R"(void f(Ped* p) +{ + g((u8)p->a); +})", {0}}, + {"remove_stmt", R"(void f(Ped* p) +{ + g(); + WIP_IMPLEMENTED; +})", {1}}, + }; + return s; +} + +// The function's lines, without the signature and braces, for showing. +std::string body(const std::string& f) { + std::istringstream in(f); + std::string line, out; + while (std::getline(in, line)) out += line + "\n"; + return out; +} + +} // namespace + +std::string passExamplesMarkdown() { + std::ostringstream o; + o << "# Passes\n\n" + "Generated by `cpp_permuter --pass-examples` from the samples in `src/examples.cpp`; " + "a unit test checks this file is up to date. Each example shows a small function " + "and what the pass makes of it (some passes offer more candidates than shown). Most " + "passes also work in the other direction.\n\n"; + // use_getter needs a getter to know about + setGetters({{"get_cam_x", "field_1AC_cam.x"}}); + for (auto& s : samples()) { + const Pass* p = findPass(s.pass); + if (!p) continue; + o << "## `" << s.pass << "`\n\n" << p->description << ".\n\n"; + if (std::string(s.pass) == "use_getter") + o << "With `Fix16 get_cam_x() { return field_1AC_cam.x; }` in an included header:\n\n"; + std::string err; + auto f = parseFunc(s.code, err); + std::vector outs; + std::set seen; + if (f) + p->enumerate(*f, [&](Mutation m) { + std::string t = m(); + if (!t.empty() && seen.insert(t).second) outs.push_back(t); + return outs.size() < 100; + }); + o << "```cpp\n" << body(s.code) << "```\n\n"; + for (int i : s.show) { + if (i >= (int)outs.size()) { + o << "(no candidate " << i << ")\n\n"; + continue; + } + o << "becomes\n\n```cpp\n" << body(outs[i]) << "```\n\n"; + } + if (outs.size() > s.show.size()) + o << "(" << outs.size() << " candidates in all for this function.)\n\n"; + } + setGetters({}); + return o.str(); +} + +} // namespace perm diff --git a/src/examples.hpp b/src/examples.hpp new file mode 100644 index 0000000..b521cef --- /dev/null +++ b/src/examples.hpp @@ -0,0 +1,11 @@ +// docs/passes.md: an example of every pass, generated from the passes +// themselves so the documentation shows exactly what they do. +#pragma once + +#include + +namespace perm { + +std::string passExamplesMarkdown(); + +} // namespace perm diff --git a/src/lexer.cpp b/src/lexer.cpp new file mode 100644 index 0000000..362bc72 --- /dev/null +++ b/src/lexer.cpp @@ -0,0 +1,201 @@ +#include "lexer.hpp" + +#include +#include +#include + +namespace perm { + +namespace { + +const char* kPuncts[] = { + ">>=", "<<=", "->*", "...", "::", "->", "++", "--", "<<", ">>", "<=", ">=", "==", "!=", + "&&", "||", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", ".*", "##", +}; + +bool identStart(char c) { return std::isalpha((unsigned char)c) || c == '_' || c == '$'; } +bool identChar(char c) { return std::isalnum((unsigned char)c) || c == '_' || c == '$'; } + +} // namespace + +std::vector lex(const std::string& s) { + std::vector out; + size_t i = 0, n = s.size(); + int line = 1; + bool lineStart = true; // only whitespace seen on this line so far + std::string lead; + + auto advance = [&](size_t to) { + for (size_t k = i; k < to && k < n; ++k) + if (s[k] == '\n') line++; + i = to; + }; + + while (i < n) { + char c = s[i]; + // whitespace + if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v') { + if (c == '\n') lineStart = true; + lead += c; + advance(i + 1); + continue; + } + // line continuation outside of PP lines + if (c == '\\' && i + 1 < n && (s[i + 1] == '\n' || s[i + 1] == '\r')) { + lead += c; + advance(i + 1); + continue; + } + // comments + if (c == '/' && i + 1 < n && s[i + 1] == '/') { + size_t e = s.find('\n', i); + if (e == std::string::npos) e = n; + lead.append(s, i, e - i); + advance(e); + continue; + } + if (c == '/' && i + 1 < n && s[i + 1] == '*') { + size_t e = s.find("*/", i + 2); + e = (e == std::string::npos) ? n : e + 2; + lead.append(s, i, e - i); + advance(e); + continue; + } + + Token t; + t.line = line; + t.offset = i; + t.lead = std::move(lead); + lead.clear(); + size_t start = i, end = i; + + if (c == '#' && lineStart) { + // preprocessor line, with backslash continuations + end = i; + while (end < n) { + if (s[end] == '\n') { + size_t b = end; + if (b > start && s[b - 1] == '\r') b--; + if (b > start && s[b - 1] == '\\') { + end++; + continue; + } + break; + } + // a block comment can span lines inside a directive + if (s[end] == '/' && end + 1 < n && s[end + 1] == '*') { + size_t e = s.find("*/", end + 2); + end = (e == std::string::npos) ? n : e + 2; + continue; + } + end++; + } + // keep the trailing \r out of the token text + while (end > start && (s[end - 1] == '\r')) end--; + t.kind = TokKind::PP; + } else if (identStart(c)) { + end = i; + while (end < n && identChar(s[end])) end++; + std::string id = s.substr(start, end - start); + if (end < n && (s[end] == '"' || s[end] == '\'') && + (id == "L" || id == "u" || id == "U" || id == "u8")) { + char q = s[end]; + end++; + while (end < n && s[end] != q) { + if (s[end] == '\\') end++; + end++; + } + if (end < n) end++; + t.kind = q == '"' ? TokKind::String : TokKind::Char; + } else { + t.kind = TokKind::Ident; + } + } else if (std::isdigit((unsigned char)c) || + (c == '.' && i + 1 < n && std::isdigit((unsigned char)s[i + 1]))) { + end = i; + bool hex = c == '0' && i + 1 < n && (s[i + 1] == 'x' || s[i + 1] == 'X'); + while (end < n) { + char d = s[end]; + if (identChar(d) || d == '.') { + end++; + continue; + } + if ((d == '+' || d == '-') && !hex && end > start && + (s[end - 1] == 'e' || s[end - 1] == 'E')) { + end++; + continue; + } + break; + } + t.kind = TokKind::Number; + } else if (c == '"' || c == '\'') { + end = i + 1; + while (end < n && s[end] != c && s[end] != '\n') { + if (s[end] == '\\') end++; + end++; + } + if (end < n && s[end] == c) end++; + t.kind = c == '"' ? TokKind::String : TokKind::Char; + } else { + t.kind = TokKind::Punct; + end = i + 1; + for (const char* p : kPuncts) { + size_t len = std::strlen(p); + if (s.compare(i, len, p) == 0) { + end = i + len; + break; + } + } + } + t.text = s.substr(start, end - start); + advance(end); + lineStart = false; + out.push_back(std::move(t)); + } + Token e; + e.kind = TokKind::End; + e.lead = std::move(lead); + e.line = line; + e.offset = n; + out.push_back(std::move(e)); + return out; +} + +std::string joinTokens(const std::vector& toks, int b, int e) { + std::string r; + for (int i = b; i < e; ++i) { + if (i != b) r += toks[i].lead; + r += toks[i].text; + } + return r; +} + +bool isKeyword(const std::string& s) { + static const std::unordered_set kw = { + "if", "else", "for", "while", "do", "switch", "case", "default", "return", "break", + "continue", "goto", "sizeof", "new", "delete", "this", "true", "false", "static", + "const", "volatile", "register", "extern", "mutable", "inline", "typedef", "struct", + "class", "union", "enum", "template", "typename", "using", "namespace", "try", + "catch", "throw", "operator", "static_cast", "reinterpret_cast", "const_cast", + "dynamic_cast", "__asm", "_asm", "asm", "__try", "__except", "__finally", "__leave", + "__forceinline", "__inline", "__declspec", "__stdcall", "__cdecl", "__fastcall", + "__thiscall", "virtual", "public", "private", "protected", "friend", "explicit", + "auto", "typeid", "nullptr", + }; + return kw.count(s) || isBuiltinType(s); +} + +bool isBuiltinType(const std::string& s) { + static const std::unordered_set t = { + "void", "char", "short", "int", "long", "float", "double", "signed", "unsigned", + "bool", "wchar_t", "__int8", "__int16", "__int32", "__int64", + }; + return t.count(s) > 0; +} + +bool isAssignOp(const std::string& s) { + return s == "=" || s == "+=" || s == "-=" || s == "*=" || s == "/=" || s == "%=" || + s == "&=" || s == "|=" || s == "^=" || s == "<<=" || s == ">>="; +} + +} // namespace perm diff --git a/src/lexer.hpp b/src/lexer.hpp new file mode 100644 index 0000000..247b3e1 --- /dev/null +++ b/src/lexer.hpp @@ -0,0 +1,30 @@ +// Tolerant C++ tokenizer. Every token keeps the whitespace and comments that +// precede it ("lead"), so joining lead + text over all tokens reproduces the +// input exactly. Preprocessor lines become single PP tokens. +#pragma once + +#include +#include + +namespace perm { + +enum class TokKind { Ident, Number, String, Char, Punct, PP, End }; + +struct Token { + TokKind kind = TokKind::End; + std::string text; + std::string lead; // whitespace/comments before the token + int line = 0; // 1-based line of the token text + size_t offset = 0; // byte offset of text in the source +}; + +std::vector lex(const std::string& src); + +// Joins tokens [b, e): lead of b is omitted, leads of the rest are kept. +std::string joinTokens(const std::vector& toks, int b, int e); + +bool isKeyword(const std::string& s); +bool isBuiltinType(const std::string& s); +bool isAssignOp(const std::string& s); + +} // namespace perm diff --git a/src/macros.cpp b/src/macros.cpp new file mode 100644 index 0000000..2f83432 --- /dev/null +++ b/src/macros.cpp @@ -0,0 +1,291 @@ +#include "macros.hpp" + +#include +#include +#include +#include + +namespace perm { + +// A template: text pieces and macro calls. +struct MacroExpander::Node { + struct Piece { + std::string text; // when name is empty + std::string name; // PERM_... + std::vector> args; + std::vector rawArgs; // the arguments as written + }; + std::vector pieces; +}; + +namespace { + +using Node = MacroExpander::Node; + +// Skips a string/char literal or comment starting at i; returns the index after it. +size_t skipLiteral(const std::string& s, size_t i) { + char c = s[i]; + if (c == '"' || c == '\'') { + size_t j = i + 1; + while (j < s.size() && s[j] != c && s[j] != '\n') j += s[j] == '\\' ? 2 : 1; + return std::min(s.size(), j + 1); + } + if (c == '/' && i + 1 < s.size() && s[i + 1] == '/') { + size_t e = s.find('\n', i); + return e == std::string::npos ? s.size() : e; + } + if (c == '/' && i + 1 < s.size() && s[i + 1] == '*') { + size_t e = s.find("*/", i + 2); + return e == std::string::npos ? s.size() : e + 2; + } + return i; +} + +std::string trim(const std::string& s) { + size_t b = s.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) return ""; + size_t e = s.find_last_not_of(" \t\r\n"); + return s.substr(b, e - b + 1); +} + +std::shared_ptr parse(const std::string& s, bool& has, bool& randomize, std::string& err); + +// Splits the text between the parentheses of a macro call at top-level commas. +std::vector splitArgs(const std::string& s) { + std::vector out; + std::string cur; + int depth = 0; + for (size_t i = 0; i < s.size();) { + size_t j = skipLiteral(s, i); + if (j != i) { + cur += s.substr(i, j - i); + i = j; + continue; + } + if (s.compare(i, 3, "(,)") == 0) { + cur += "(,)"; // unescaped after parsing the argument + i += 3; + continue; + } + char c = s[i]; + if (c == '(' || c == '[' || c == '{') depth++; + if (c == ')' || c == ']' || c == '}') depth--; + if (c == ',' && depth == 0) { + out.push_back(cur); + cur.clear(); + } else { + cur += c; + } + i++; + } + out.push_back(cur); + return out; +} + +void unescape(Node& n) { + for (auto& p : n.pieces) { + if (p.name.empty()) { + size_t pos; + while ((pos = p.text.find("(,)")) != std::string::npos) p.text.replace(pos, 3, ","); + } + for (auto& a : p.args) unescape(*a); + } +} + +std::shared_ptr parse(const std::string& s, bool& has, bool& randomize, std::string& err) { + auto n = std::make_shared(); + std::string text; + size_t i = 0; + while (i < s.size()) { + size_t j = skipLiteral(s, i); + if (j != i) { + text += s.substr(i, j - i); + i = j; + continue; + } + bool wordStart = i == 0 || !(std::isalnum((unsigned char)s[i - 1]) || s[i - 1] == '_'); + if (wordStart && s.compare(i, 5, "PERM_") == 0) { + size_t e = i + 5; + while (e < s.size() && (std::isalnum((unsigned char)s[e]) || s[e] == '_')) e++; + std::string name = s.substr(i, e - i); + size_t open = e; + while (open < s.size() && (s[open] == ' ' || s[open] == '\t')) open++; + if (open < s.size() && s[open] == '(') { + // find the matching ')' + int depth = 0; + size_t k = open; + for (; k < s.size();) { + size_t l = skipLiteral(s, k); + if (l != k) { + k = l; + continue; + } + if (s.compare(k, 3, "(,)") == 0) { + k += 3; + continue; + } + if (s[k] == '(') depth++; + if (s[k] == ')' && --depth == 0) break; + k++; + } + if (k >= s.size()) { + err = name + ": no closing parenthesis"; + return n; + } + if (!text.empty()) n->pieces.push_back({text, "", {}, {}}); + text.clear(); + Node::Piece p; + p.name = name; + p.rawArgs = splitArgs(s.substr(open + 1, k - open - 1)); + // value-like arguments are trimmed; code blocks keep their layout + if (name == "PERM_GENERAL" || name == "PERM_INT" || name == "PERM_VAR" || name == "PERM_ONCE") + for (auto& a : p.rawArgs) a = trim(a); + for (auto& a : p.rawArgs) { + if (name == "PERM_IGNORE") { + auto t = std::make_shared(); + t->pieces.push_back({a, "", {}, {}}); + p.args.push_back(t); + } else { + p.args.push_back(parse(a, has, randomize, err)); + } + } + has = true; + if (name == "PERM_RANDOMIZE") randomize = true; + static const std::set known = { + "PERM_GENERAL", "PERM_LINESWAP", "PERM_LINESWAP_TEXT", "PERM_INT", + "PERM_ONCE", "PERM_VAR", "PERM_RANDOMIZE", "PERM_FORCE_SAMELINE", + "PERM_IGNORE", "PERM_PRETEND"}; + if (!known.count(name)) err = "unknown macro " + name; + n->pieces.push_back(std::move(p)); + i = k + 1; + continue; + } + } + text += s[i++]; + } + if (!text.empty()) n->pieces.push_back({text, "", {}, {}}); + return n; +} + +void collectOnce(const Node& n, std::map& counts) { + for (auto& p : n.pieces) { + if (p.name == "PERM_ONCE") counts[trim(p.rawArgs[0])]++; + for (auto& a : p.args) collectOnce(*a, counts); + } +} + +struct Eval { + const std::function& choose; + std::map vars; + std::map onceTotal, onceSeen, onceChosen; + + int pick(int n) { + if (n <= 1) return 0; + int c = choose(n); + return std::max(0, std::min(n - 1, c)); + } + + std::string run(const Node& n) { + std::string out; + for (auto& p : n.pieces) out += p.name.empty() ? p.text : call(p); + return out; + } + + std::string call(const Node::Piece& p) { + const std::string& m = p.name; + auto arg = [&](size_t i) { return i < p.args.size() ? run(*p.args[i]) : std::string(); }; + if (m == "PERM_GENERAL") return arg(pick((int)p.args.size())); + if (m == "PERM_RANDOMIZE" || m == "PERM_IGNORE") return arg(0); + if (m == "PERM_PRETEND") return ""; + if (m == "PERM_INT") { + long lo = std::stol(trim(arg(0))), hi = std::stol(trim(arg(1))); + if (hi < lo) return std::to_string(lo); + return std::to_string(lo + pick((int)(hi - lo + 1))); + } + if (m == "PERM_FORCE_SAMELINE") { + std::string s = arg(0), r; + for (char c : s) r += c == '\n' ? ' ' : c; + return r; + } + if (m == "PERM_LINESWAP" || m == "PERM_LINESWAP_TEXT") { + std::string s = arg(0); + std::vector lines; + size_t b = 0; + while (b <= s.size()) { + size_t e = s.find('\n', b); + std::string l = s.substr(b, e == std::string::npos ? std::string::npos : e - b); + if (!trim(l).empty()) lines.push_back(l); + if (e == std::string::npos) break; + b = e + 1; + } + std::string out = "\n"; + while (!lines.empty()) { + int k = pick((int)lines.size()); + out += lines[k] + "\n"; + lines.erase(lines.begin() + k); + } + return out; + } + if (m == "PERM_VAR") { + std::string name = trim(arg(0)); + if (p.args.size() >= 2) { + vars[name] = arg(1); + return ""; + } + return vars[name]; + } + if (m == "PERM_ONCE") { + std::string key = trim(p.rawArgs[0]); + int idx = onceSeen[key]++; + if (!onceChosen.count(key)) onceChosen[key] = pick(onceTotal[key]); + if (idx != onceChosen[key]) return ""; + return p.args.size() >= 2 ? arg(1) : arg(0); + } + return ""; + } +}; + +} // namespace + +MacroExpander::MacroExpander(const std::string& text) { + root_ = parse(text, hasMacros_, randomize_, error_); + unescape(*root_); +} + +std::string MacroExpander::expand(const std::function& choose) const { + Eval ev{choose, {}, {}, {}, {}}; + collectOnce(*root_, ev.onceTotal); + return ev.run(*root_); +} + +std::string MacroExpander::first() const { + return expand([](int) { return 0; }); +} + +void MacroExpander::enumerate(size_t cap, const std::function& emit) const { + // walk the tree of choices: replay a prefix, take option 0 after it, then + // advance the last choice that still has options left + std::vector prefix; + std::set seen; + size_t produced = 0; + while (true) { + std::vector taken, counts; + std::string out = expand([&](int n) { + size_t i = taken.size(); + int c = i < prefix.size() ? prefix[i] : 0; + taken.push_back(c); + counts.push_back(n); + return c; + }); + if (seen.insert(out).second) { + if (!emit(out) || ++produced >= cap) return; + } + int j = (int)taken.size() - 1; + while (j >= 0 && taken[j] + 1 >= counts[j]) j--; + if (j < 0) return; + prefix.assign(taken.begin(), taken.begin() + j); + prefix.push_back(taken[j] + 1); + } +} + +} // namespace perm diff --git a/src/macros.hpp b/src/macros.hpp new file mode 100644 index 0000000..0a5009a --- /dev/null +++ b/src/macros.hpp @@ -0,0 +1,51 @@ +// decomp-permuter's PERM_* macros: hand-written alternatives inside the +// function, expanded before anything is compiled. +// +// PERM_GENERAL(a, b, ...) one of a, b, ... +// PERM_LINESWAP(lines) the non-blank lines in any order +// PERM_INT(lo, hi) an integer from lo to hi +// PERM_ONCE([key,] code) code at exactly one of the places using key +// PERM_VAR(a, b) / (a) set / read the meta-variable a +// PERM_RANDOMIZE(code) code; also lets the passes run (see below) +// PERM_FORCE_SAMELINE(code) code joined onto one line +// PERM_IGNORE(code) code, unexpanded +// PERM_PRETEND(code) nothing (it only helps decomp-permuter's C parser) +// +// Arguments are split at top-level commas; "(,)" is a literal comma. Macros +// nest. Like decomp-permuter, a function using any PERM macro is only permuted +// through them unless it also has PERM_RANDOMIZE, which turns the passes on +// (for the whole function, not just that region). +#pragma once + +#include +#include +#include +#include + +namespace perm { + +class MacroExpander { +public: + explicit MacroExpander(const std::string& text); + + bool hasMacros() const { return hasMacros_; } + bool randomize() const { return randomize_; } // PERM_RANDOMIZE used + const std::string& error() const { return error_; } + + // One expansion; choose(n) picks one of n options at each choice point. + std::string expand(const std::function& choose) const; + // The expansion taking the first option everywhere. + std::string first() const; + // Every expansion (each choice combination once), until emit returns false + // or cap expansions were produced. + void enumerate(size_t cap, const std::function& emit) const; + + struct Node; + +private: + std::shared_ptr root_; + bool hasMacros_ = false, randomize_ = false; + std::string error_; +}; + +} // namespace perm diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..dc79660 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,1187 @@ +// cpp_permuter: brute-forces source permutations of one C++ function until +// its compiled code matches (or gets closer to) a target object. + +#include "analysis.hpp" +#include "examples.hpp" +#include "parser.hpp" +#include "passes.hpp" +#include "runner.hpp" +#include "scorer.hpp" +#include "macros.hpp" +#include "workspace.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace perm; + +namespace { + +const char* kUsage = R"(usage: cpp_permuter [options] + +Finds a source permutation of one function that compiles to the target code. + + cpp_permuter -s FILE -f NAME -c CMD -t OBJ [options] + cpp_permuter --check-parse FILE... [--check-limit N] + +Required: + -s, --source FILE .cpp file holding the function + -f, --function NAME function to permute, e.g. PedGroup::PromoteMemberToLeader_4C9680 + -c, --compile CMD compile command. {src} is the candidate source, {obj} the + object to write, {dir} the candidate's directory and {root} + the mirror of --mirror-root + -t, --target-obj FILE object file with the target code (not needed with --score-cmd) + +Scoring: + --symbol SYM symbol of the function in the compiled object + (default: guessed from --function) + --target-symbol SYM symbol in the target object (default: same as --symbol) + --score-cmd CMD score with this command instead ({obj}, {src}); it must + print the score (lower is better, 0 = match) as its last number + --objdump PATH llvm-objdump to use (default: llvm-objdump) + --ignore-reloc-names compare relocated operands without their symbol names + (for targets built from raw executable asm) + +Search: + -m, --mode MODE random (default) or exhaustive + -p, --passes LIST passes to use (default: all), see --list-passes. Commas + separate passes, '+' combines them: "a+b" tries a, b and + a then b. Can be given more than once + --depth N exhaustive: chain up to N passes/combos (default 1) + --max-candidates N exhaustive: stop after N candidates (default 200000) + -n, --iterations N random: stop after N compiles (default: run until match/Ctrl-C) + --max-mutations N random: up to N passes/combos per candidate (default 3) + --keep-prob P random: start from the best candidate so far with + probability P, else from the base (default 0.5) + --seed N random seed + --keep-going don't stop at the first exact match + --weight NAME=N random: how often a pass or combo (as written in -p) is + picked, relative to the others; 0 never. Repeatable + -j, --jobs N parallel compiles (default 1) + --timeout SEC compile timeout (default 120) + +Other functions (for helpers VC6 inlines into the target): + --also [FILE:]NAME also permute this function. FILE is relative to the current + directory or the source's; without it NAME is looked up in + the source and the headers it includes. Repeatable + --inline-callees also permute the inline functions the target calls: their + definitions in the source (if marked inline) and in the + headers it includes + --max-callees N at most N of those (default 8) + -I, --include-dir DIR where to look for #include "..." headers besides the + including file's directory (default: the source's) + --mirror-root DIR candidates are compiled in a mirror of this directory + (symlinks plus the changed files), so changed headers are + the ones every #include sees (default: the source's dir) + --list-regions print the functions that would be permuted and exit + +Output: + -o, --output-dir DIR where improvements go (default: permuter_out) + --keep-ties also write candidates that tie the best score + --only-if-below N only write candidates scoring below N + --base-only compile and score the unmodified function, then exit + --config FILE read "key = value" lines as if they were --key value + --list-passes list the passes and exit + --pass-examples print an example of every pass (docs/passes.md) and exit + --show-ast print the parsed statement tree and exit + --dry-run print the candidates' diffs instead of compiling + --show-base-diff print the asm diff of the unmodified function first + --show-timings print where the time went: permuting, writing, compiling, scoring + -v, --verbose print compile errors + +Checking the parser: + --check-parse FILE... parse every function in the files and run every pass on + it; report functions that don't parse and candidates + that don't parse back. Exit 1 on any + --check-limit N candidates checked per pass and function (default 25) + --list-definitions FILE... print each function definition's byte range and name +)"; + +struct Options { + std::string source, function, compile, targetObj, symbol, targetSymbol, scoreCmd; + std::string objdump = "llvm-objdump"; + bool ignoreRelocNames = false; + std::string mode = "random"; + std::vector passes; + int depth = 1; + long maxCandidates = 200000; + long iterations = 0; + int maxMutations = 3; + double keepProb = 0.5; + std::vector weights; // NAME=N + bool keepTies = false, baseOnly = false; + long onlyBelow = -1; + unsigned long long seed = 0; + bool seedSet = false; + bool keepGoing = false; + int jobs = 1; + int timeout = 120; + std::string outputDir = "permuter_out"; + std::vector also, includeDirs, checkParse; + bool inlineCallees = false, listRegions = false, checkMode = false, showTimings = false, listDefs = false, + passExamples = false; + int maxCallees = 8; + long checkLimit = 25; + std::string mirrorRoot; + bool listPasses = false, showAst = false, dryRun = false, showBaseDiff = false, verbose = false; +}; + +std::atomic gStop{false}; + +void onSignal(int) { gStop = true; } + +bool readFile(const std::string& path, std::string& out) { + std::ifstream in(path, std::ios::binary); + if (!in) return false; + std::ostringstream ss; + ss << in.rdbuf(); + out = ss.str(); + return true; +} + +bool writeFile(const std::string& path, const std::string& data) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) return false; + out << data; + return (bool)out; +} + +bool applyOption(Options& o, const std::string& key, const std::string& val, bool hasVal, + std::string& err) { + auto need = [&]() { + if (!hasVal) err = "--" + key + " needs a value"; + return hasVal; + }; + auto num = [&](auto& dst) { + if (!need()) return false; + try { + dst = (std::remove_reference_t)std::stoll(val); + } catch (...) { + err = "--" + key + ": not a number: " + val; + return false; + } + return true; + }; + if (key == "source" || key == "s") return need() && (o.source = val, true); + if (key == "function" || key == "f") return need() && (o.function = val, true); + if (key == "compile" || key == "c") return need() && (o.compile = val, true); + if (key == "target-obj" || key == "t") return need() && (o.targetObj = val, true); + if (key == "symbol") return need() && (o.symbol = val, true); + if (key == "target-symbol") return need() && (o.targetSymbol = val, true); + if (key == "score-cmd") return need() && (o.scoreCmd = val, true); + if (key == "objdump") return need() && (o.objdump = val, true); + if (key == "ignore-reloc-names") return o.ignoreRelocNames = true; + if (key == "mode" || key == "m") return need() && (o.mode = val, true); + if (key == "passes" || key == "p") return need() && (o.passes.push_back(val), true); + if (key == "depth") return num(o.depth); + if (key == "max-candidates") return num(o.maxCandidates); + if (key == "iterations" || key == "n") return num(o.iterations); + if (key == "max-mutations") return num(o.maxMutations); + if (key == "weight") return need() && (o.weights.push_back(val), true); + if (key == "keep-ties") return o.keepTies = true; + if (key == "base-only") return o.baseOnly = true; + if (key == "only-if-below") return num(o.onlyBelow); + if (key == "keep-prob") { + if (!need()) return false; + try { + o.keepProb = std::stod(val); + } catch (...) { + err = "--keep-prob: not a number: " + val; + return false; + } + return true; + } + if (key == "seed") return num(o.seed) && (o.seedSet = true); + if (key == "keep-going") return o.keepGoing = true; + if (key == "jobs" || key == "j") return num(o.jobs); + if (key == "timeout") return num(o.timeout); + if (key == "output-dir" || key == "o") return need() && (o.outputDir = val, true); + if (key == "also") return need() && (o.also.push_back(val), true); + if (key == "inline-callees") return o.inlineCallees = true; + if (key == "max-callees") return num(o.maxCallees); + if (key == "include-dir" || key == "I") return need() && (o.includeDirs.push_back(val), true); + if (key == "mirror-root") return need() && (o.mirrorRoot = val, true); + if (key == "list-regions") return o.listRegions = true; + if (key == "show-timings") return o.showTimings = true; + if (key == "list-definitions") return o.listDefs = true; + if (key == "pass-examples") return o.passExamples = true; + if (key == "check-parse") return o.checkMode = true; + if (key == "check-limit") return num(o.checkLimit); + if (key == "list-passes") return o.listPasses = true; + if (key == "show-ast") return o.showAst = true; + if (key == "dry-run") return o.dryRun = true; + if (key == "show-base-diff") return o.showBaseDiff = true; + if (key == "verbose" || key == "v") return o.verbose = true; + err = "unknown option --" + key; + return false; +} + +bool isFlag(const std::string& k) { + static const std::set flags = { + "ignore-reloc-names", "keep-going", "list-passes", "show-ast", "dry-run", + "show-base-diff", "verbose", "v", "inline-callees", "list-regions", "check-parse", "show-timings", "list-definitions", "keep-ties", "base-only", "pass-examples", + }; + return flags.count(k) > 0; +} + +bool loadConfig(Options& o, const std::string& path, std::string& err) { + std::string text; + if (!readFile(path, text)) { + err = "can't read config " + path; + return false; + } + std::istringstream in(text); + std::string line; + while (std::getline(in, line)) { + size_t b = line.find_first_not_of(" \t"); + if (b == std::string::npos || line[b] == '#') continue; + line = line.substr(b); + while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) line.pop_back(); + size_t eq = line.find('='); + std::string key = line.substr(0, eq), val; + while (!key.empty() && key.back() == ' ') key.pop_back(); + if (eq != std::string::npos) { + val = line.substr(eq + 1); + size_t vb = val.find_first_not_of(" \t"); + val = vb == std::string::npos ? "" : val.substr(vb); + if (val.size() >= 2 && val.front() == '"' && val.back() == '"') val = val.substr(1, val.size() - 2); + } + if (!applyOption(o, key, val, eq != std::string::npos, err)) return false; + } + return true; +} + +bool parseArgs(int argc, char** argv, Options& o, std::string& err) { + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "-h" || a == "--help") { + std::cout << kUsage; + std::exit(0); + } + std::string key, val; + bool hasVal = false; + if (a.rfind("--", 0) == 0) { + key = a.substr(2); + size_t eq = key.find('='); + if (eq != std::string::npos) { + val = key.substr(eq + 1); + key = key.substr(0, eq); + hasVal = true; + } + } else if (a.size() == 2 && a[0] == '-') { + key = a.substr(1); + } else { + o.checkParse.push_back(a); // files for --check-parse + continue; + } + if (!hasVal && !isFlag(key) && i + 1 < argc) { + val = argv[++i]; + hasVal = true; + } + if (key == "config") { + if (!hasVal || !loadConfig(o, val, err)) { + if (err.empty()) err = "--config needs a value"; + return false; + } + continue; + } + if (!applyOption(o, key, val, hasVal, err)) return false; + } + return true; +} + +// Line diff (LCS) with a little context, for the output files. +std::string lineDiff(const std::string& a, const std::string& b) { + auto lines = [](const std::string& s) { + std::vector r; + std::istringstream in(s); + std::string l; + while (std::getline(in, l)) r.push_back(l); + return r; + }; + auto A = lines(a), B = lines(b); + size_t n = A.size(), m = B.size(); + std::vector> L(n + 1, std::vector(m + 1, 0)); + for (size_t i = n; i-- > 0;) + for (size_t j = m; j-- > 0;) + L[i][j] = A[i] == B[j] ? L[i + 1][j + 1] + 1 : std::max(L[i + 1][j], L[i][j + 1]); + std::vector> ops; + size_t i = 0, j = 0; + while (i < n || j < m) { + if (i < n && j < m && A[i] == B[j]) ops.push_back({' ', A[i++]}), j++; + else if (j < m && (i == n || L[i][j + 1] >= L[i + 1][j])) ops.push_back({'+', B[j++]}); + else ops.push_back({'-', A[i++]}); + } + std::string out; + const int ctx = 2; + std::vector show(ops.size(), false); + for (size_t k = 0; k < ops.size(); ++k) + if (ops[k].first != ' ') + for (long d = -ctx; d <= ctx; ++d) + if ((long)k + d >= 0 && (long)k + d < (long)ops.size()) show[k + d] = true; + bool gap = false; + for (size_t k = 0; k < ops.size(); ++k) { + if (!show[k]) { + if (!gap && !out.empty()) out += "...\n"; + gap = true; + continue; + } + gap = false; + out += ops[k].first; + out += ops[k].second + "\n"; + } + return out; +} + +double lastNumber(const std::string& s, bool& ok) { + ok = false; + double v = 0; + for (size_t i = 0; i < s.size(); ++i) { + if (std::isdigit((unsigned char)s[i]) || (s[i] == '-' && i + 1 < s.size() && std::isdigit((unsigned char)s[i + 1]))) { + size_t used = 0; + try { + v = std::stod(s.substr(i), &used); + ok = true; + i += used - 1; + } catch (...) { + } + } + } + return v; +} + +// One text per region; [0] is the target function. +using Cand = std::vector; + +// Where the time goes (--show-timings), in microseconds summed over workers. +struct Timings { + std::atomic permute{0}, write{0}, compile{0}, score{0}; + std::atomic generated{0}; +} gTimes; + +struct Stopwatch { + std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now(); + long long us() const { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + } +}; + +size_t hashCand(const Cand& c) { + std::string all; + for (auto& t : c) all += t + '\x01'; + return std::hash()(all); +} + +struct Workspace { + std::vector regions; + Cand base; + std::shared_ptr macro; // PERM_* macros in the target function + bool passesOn = true; // false: only the macros are expanded + std::map files; // original contents of the files with regions + std::string root; + + // New contents of every file that holds a region. + std::map render(const Cand& c) const { + std::map out; + for (auto& [file, text] : files) { + std::vector> parts; + for (size_t i = 0; i < regions.size(); ++i) + if (regions[i].file == file) parts.push_back({®ions[i], &c[i]}); + out[file] = spliceRegions(text, parts); + } + return out; + } +}; + +struct Evaluator { + const Options& o; + const Workspace& ws; + ScoreConfig sc; + std::string symbol, targetSymbol; + std::vector target; + std::vector mirrors; // one per worker + // What each worker's mirror holds now: files are only rewritten when their + // content changes, so unchanged headers keep their timestamps and the + // compiler's precompiled header stays valid. + mutable std::vector> written; + + std::string srcPath(int w) const { return mirrors[w].map(ws.regions[0].file); } + std::string objPath(int w) const { + return (fs::path(o.outputDir) / ".work" / ("w" + std::to_string(w) + ".obj")).string(); + } + + bool compile(int w, const Cand& c, std::string& log) const { + Stopwatch sw; + for (auto& [file, text] : ws.render(c)) { + auto it = written[w].find(file); + if (it != written[w].end() && it->second == text) continue; + std::string path = mirrors[w].map(file); + if (!writeFile(path, text)) { + log = "can't write " + path; + written[w].erase(file); + return false; + } + written[w][file] = text; + } + std::string src = srcPath(w), obj = objPath(w); + std::error_code ec; + fs::remove(obj, ec); + gTimes.write += sw.us(); + Stopwatch cw; + std::string cmd = expandTemplate(o.compile, {{"src", src}, + {"obj", obj}, + {"dir", fs::path(src).parent_path().string()}, + {"root", mirrors[w].dir()}}); + // Precompiled headers can't be trusted while headers are being edited: + // VC6 checks them by timestamp, and an edit within the same second as + // the .pch is missed. Compile scripts honour PERMUTER_NO_PCH. + std::map env; + if (ws.files.size() > 1) env["PERMUTER_NO_PCH"] = "1"; + CmdResult r = runCommand(cmd, o.timeout, env); + gTimes.compile += cw.us(); + log = r.output; + if (r.timedOut) log += "\n(timed out)"; + return r.status == 0 && fs::exists(obj); + } + + // Score of a compiled candidate; -1 if it couldn't be scored. + long score(int w, std::vector* insns, std::string& log) const { + Stopwatch sw; + struct Add { + Stopwatch& sw; + ~Add() { gTimes.score += sw.us(); } + } add{sw}; + std::string obj = objPath(w); + if (!o.scoreCmd.empty()) { + CmdResult r = runCommand(expandTemplate(o.scoreCmd, {{"obj", obj}, {"src", srcPath(w)}}), o.timeout); + bool ok; + double v = lastNumber(r.output, ok); + if (r.status != 0 || !ok) { + log = r.output; + return -1; + } + return (long)(v + 0.5); + } + std::vector mine; + if (!disassemble(sc, obj, symbol, mine, log)) return -1; + if (insns) *insns = mine; + return scoreInsns(sc, target, mine); + } + + long eval(int w, const Cand& c, std::string& log, std::vector* insns = nullptr) const { + if (!compile(w, c, log)) return -1; + return score(w, insns, log); + } +}; + +struct State { + std::mutex mu; + Cand base, best; + long baseScore = 0, bestScore = 0; + long compiles = 0, failures = 0, improvements = 0; + std::unordered_set seen; + int outputs = 0, ties = 0; +}; + +// Diff of every region that changed, with a header per region when there are several. +std::string candDiff(const Workspace& ws, const Cand& c) { + std::string out; + for (size_t i = 0; i < c.size(); ++i) { + if (c[i] == ws.base[i]) continue; + if (ws.regions.size() > 1) out += "@@ " + ws.regions[i].name + " (" + fs::path(ws.regions[i].file).filename().string() + ")\n"; + out += lineDiff(ws.base[i], c[i]); + } + return out; +} + +void writeOutput(const Options& o, const Evaluator& ev, State& st, const Cand& c, long score, + const std::vector& insns) { + const Workspace& ws = ev.ws; + std::string dir = (fs::path(o.outputDir) / ("output-" + std::to_string(score) + "-" + + std::to_string(++st.outputs))) + .string(); + fs::create_directories(dir); + std::string funcs; + for (size_t i = 0; i < c.size(); ++i) { + if (i > 0 && c[i] == ws.base[i]) continue; + if (ws.regions.size() > 1) funcs += "// " + ws.regions[i].name + " in " + ws.regions[i].file + "\n"; + funcs += c[i] + "\n\n"; + } + writeFile(dir + "/function.cpp", funcs); + auto rendered = ws.render(c); + writeFile(dir + "/source.cpp", rendered[ws.regions[0].file]); + for (auto& [file, text] : rendered) // changed headers, under their own names + if (file != ws.regions[0].file && text != ws.files.at(file)) + writeFile(dir + "/" + fs::path(file).filename().string(), text); + std::string diff = candDiff(ws, c); + writeFile(dir + "/diff.txt", diff); + if (!insns.empty()) writeFile(dir + "/asm_diff.txt", diffInsns(ev.sc, ev.target, insns)); + writeFile(dir + "/score.txt", std::to_string(score) + "\n"); + std::cout << "\n[" << score << "] " << (score < st.bestScore ? "new best (was " + std::to_string(st.bestScore) + ")" : std::string("tie")) + << ", written to " << dir + << "\n" + << diff << std::flush; +} + +// Records a scored candidate. Returns true if it was a match and we should stop. +bool report(const Options& o, const Evaluator& ev, State& st, const Cand& c, long score, + const std::vector& insns, const std::string& log) { + std::lock_guard lk(st.mu); + st.compiles++; + if (score < 0) { + st.failures++; + if (o.verbose) std::cout << "\ncandidate failed:\n" << log << "\n"; + return false; + } + bool wanted = o.onlyBelow < 0 || score < o.onlyBelow; + if (score < st.bestScore) { + if (wanted) writeOutput(o, ev, st, c, score, insns); + st.bestScore = score; + st.best = c; + st.improvements++; + } else if (o.keepTies && score == st.bestScore && score < st.baseScore && wanted && st.ties < 200) { + st.ties++; + writeOutput(o, ev, st, c, score, insns); + } + return score == 0 && !o.keepGoing; +} + +void printStatus(State& st) { + std::lock_guard lk(st.mu); + std::cout << "\riterations: " << st.compiles << ", failed: " << st.failures + << ", base: " << st.baseScore << ", best: " << st.bestScore << " " + << std::flush; +} + +// One random candidate derived from start; empty if nothing applied. With +// several regions, half the mutations go to the target function. +Cand randomCandidate(const Options& o, const std::vector& groups, const Cand& start, + Rng& rng) { + int total = 0; + for (auto& g : groups) total += g.weight(); + if (total <= 0) return {}; + int k = std::uniform_int_distribution(1, std::max(1, o.maxMutations))(rng); + Cand c = start; + for (int m = 0; m < k; ++m) { + size_t r = 0; + if (c.size() > 1 && std::uniform_int_distribution(0, 1)(rng) == 1) + r = std::uniform_int_distribution(1, c.size() - 1)(rng); + for (int attempt = 0; attempt < 10; ++attempt) { + int x = std::uniform_int_distribution(0, total - 1)(rng); + const PassGroup* g = &groups.back(); + for (auto& q : groups) { + if (x < q.weight()) { + g = &q; + break; + } + x -= q.weight(); + } + std::string out; + if (randomGroupMutation(*g, c[r], rng, out)) { + c[r] = out; + break; + } + } + } + return c == start ? Cand{} : c; +} + +void enumerateCand(const std::vector& groups, const Cand& c, + const std::function& emit); + +bool hasMacros(const Workspace& ws) { return ws.macro && ws.macro->hasMacros(); } + +// One random candidate: from the best so far (with --keep-prob) or from the +// base with a fresh random expansion of the PERM macros, plus random passes. +// Empty if nothing changed. +Cand nextRandom(const Options& o, const Workspace& ws, const std::vector& groups, + const Cand& best, Rng& rng) { + std::uniform_real_distribution coin(0, 1); + Cand start = ws.base; + if (ws.passesOn && best != ws.base && coin(rng) < o.keepProb) { + start = best; + } else if (hasMacros(ws)) { + start[0] = ws.macro->expand([&](int n) { return std::uniform_int_distribution(0, n - 1)(rng); }); + // a third of the time, the expansion on its own + if (!ws.passesOn || (start != ws.base && coin(rng) < 1.0 / 3)) return start == ws.base ? Cand{} : start; + } + Cand c = randomCandidate(o, groups, start, rng); + if (c.empty() && start != ws.base) return start; + return c; +} + +// Every candidate of exhaustive mode, in order: each expansion of the PERM +// macros, then (when the passes are on) up to --depth passes or combos applied +// on top. emit returns 0 to stop, 1 for a new candidate, 2 for a repeat. +void exhaustiveCands(const Options& o, const Workspace& ws, const std::vector& groups, + const std::function& emit) { + std::vector level; + bool stop = false; + if (hasMacros(ws)) { + ws.macro->enumerate((size_t)o.maxCandidates, [&](const std::string& text) { + Cand c = ws.base; + c[0] = text; + level.push_back(c); + if (c != ws.base && emit(c, "PERM macros") == 0) stop = true; + return !stop; + }); + } else { + level.push_back(ws.base); + } + if (!ws.passesOn) return; + for (int d = 1; d <= o.depth && !stop; ++d) { + std::vector next; + for (auto& c : level) { + if (stop) break; + enumerateCand(groups, c, [&](const Cand& n, const std::string& g) { + int r = emit(n, g); + if (r == 0) stop = true; + if (r == 1 && d < o.depth) next.push_back(n); + return !stop; + }); + } + level = std::move(next); + } +} + +int runRandom(const Options& o, const Evaluator& ev, State& st, const std::vector& groups) { + unsigned long long seed = o.seedSet ? o.seed : (unsigned long long)std::random_device{}(); + std::vector workers; + std::atomic idle{0}; + for (int w = 0; w < o.jobs; ++w) { + workers.emplace_back([&, w]() { + Rng rng(seed + (unsigned long long)w * 7919); + int misses = 0; + while (!gStop) { + Cand best; + { + std::lock_guard lk(st.mu); + if (o.iterations && st.compiles >= o.iterations) break; + best = st.best; + } + Stopwatch gen; + Cand cand = nextRandom(o, ev.ws, groups, best, rng); + gTimes.permute += gen.us(); + gTimes.generated++; + bool fresh = false; + if (!cand.empty()) { + std::lock_guard lk(st.mu); + fresh = st.seen.insert(hashCand(cand)).second; + } + if (!fresh) { + if (++misses > 2000) break; // nothing new left to try + continue; + } + misses = 0; + std::string log; + std::vector insns; + long s = ev.eval(w, cand, log, &insns); + if (report(o, ev, st, cand, s, insns, log)) gStop = true; + } + idle++; + }); + } + while (idle < o.jobs) { + for (int i = 0; i < 20 && idle < o.jobs; ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + printStatus(st); + } + for (auto& t : workers) t.join(); + return 0; +} + +// Every candidate one pass or combo away from c, in any region. +void enumerateCand(const std::vector& groups, const Cand& c, + const std::function& emit) { + bool stop = false; + for (size_t r = 0; r < c.size() && !stop; ++r) + for (auto& g : groups) { + if (stop) break; + enumerateGroup(g, c[r], [&](const std::string& s) { + Cand n = c; + n[r] = s; + if (!emit(n, g.name())) stop = true; + return !stop; + }); + } +} + +int runExhaustive(const Options& o, const Evaluator& ev, State& st, const std::vector& groups) { + std::mutex qmu; + std::condition_variable qcv; + std::deque queue; + bool done = false; + long produced = 0; + + auto worker = [&](int w) { + while (true) { + Cand cand; + { + std::unique_lock lk(qmu); + qcv.wait(lk, [&] { return !queue.empty() || done || gStop; }); + if (gStop || (queue.empty() && done)) return; + cand = std::move(queue.front()); + queue.pop_front(); + } + qcv.notify_all(); + std::string log; + std::vector insns; + long s = ev.eval(w, cand, log, &insns); + if (report(o, ev, st, cand, s, insns, log)) { + gStop = true; + qcv.notify_all(); + } + } + }; + std::vector workers; + for (int w = 0; w < o.jobs; ++w) workers.emplace_back(worker, w); + + std::thread producer([&]() { + Stopwatch gen; + long long waited = 0; + exhaustiveCands(o, ev.ws, groups, [&](const Cand& n, const std::string&) { + gTimes.generated++; + if (gStop || produced >= o.maxCandidates) return 0; + { + std::lock_guard lk(st.mu); + if (!st.seen.insert(hashCand(n)).second) return 2; + } + produced++; + Stopwatch wait; + std::unique_lock lk(qmu); + qcv.wait(lk, [&] { return queue.size() < 256 || gStop; }); + queue.push_back(n); + qcv.notify_all(); + waited += wait.us(); + return 1; + }); + gTimes.permute += gen.us() - waited; + std::lock_guard lk(qmu); + done = true; + qcv.notify_all(); + }); + + while (true) { + { + std::lock_guard lk(qmu); + if ((done && queue.empty()) || gStop) break; + } + for (int i = 0; i < 20; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::lock_guard lk(qmu); + if ((done && queue.empty()) || gStop) break; + } + printStatus(st); + } + producer.join(); + { + std::lock_guard lk(qmu); + done = true; + } + qcv.notify_all(); + for (auto& t : workers) t.join(); + printStatus(st); + std::cout << "\n" << produced << " candidates generated\n"; + return 0; +} + +int dryRun(const Options& o, const Workspace& ws, const std::vector& groups) { + if (o.mode == "exhaustive") { + long count = 0; + std::unordered_set seen = {hashCand(ws.base)}; + std::map perGroup; + exhaustiveCands(o, ws, groups, [&](const Cand& c, const std::string& g) { + if (!seen.insert(hashCand(c)).second) return 2; + perGroup[g]++; + if (++count <= 50) std::cout << "=== " << g << " #" << perGroup[g] << "\n" << candDiff(ws, c); + return count < o.maxCandidates ? 1 : 0; + }); + for (auto& [g, n] : perGroup) std::cerr << g << ": " << n << " candidates\n"; + std::cerr << count << " candidates in total\n"; + return 0; + } + Rng rng(o.seedSet ? o.seed : 1); + long n = o.iterations ? o.iterations : 10; + for (long i = 0; i < n; ++i) { + Cand c = nextRandom(o, ws, groups, ws.base, rng); + std::cout << "=== random #" << i + 1 << "\n" << (c.empty() ? "(no change)\n" : candDiff(ws, c)); + } + return 0; +} + +// --check-parse: every function in the files must parse, and every pass's +// candidates must parse back. +int checkParse(const Options& o) { + long funcs = 0, parseFails = 0, others = 0, sites = 0, siteFails = 0, cands = 0, badCands = 0; + std::map perPass; + for (auto& path : o.checkParse) { + std::string src; + if (!readFile(path, src)) { + std::cerr << "can't read " << path << "\n"; + parseFails++; + continue; + } + for (auto& d : listDefinitions(src)) { + funcs++; + std::string text = src.substr(d.start, d.end - d.start); + std::string err; + auto f = parseFunc(text, err); + if (!f) { + parseFails++; + std::cout << "PARSE FAIL " << path << ": " << d.name << ": " << err << "\n"; + continue; + } + forEachStmt(*f->body, [&](const Stmt& s) { + if (s.kind == SK::Other) { + others++; + if (o.verbose) std::cout << "other " << path << ": " << d.name << ": " << f->textOf(s).substr(0, 60) << "\n"; + } + }); + for (auto& site : exprSites(*f)) { + sites++; + if (!parseExpr(*f, site.b, site.e)) { + siteFails++; + if (o.verbose) std::cout << "expr " << path << ": " << d.name << ": " << f->tokText(site.b, site.e).substr(0, 60) << "\n"; + } + } + for (auto& p : allPasses()) { + long n = 0; + p.enumerate(*f, [&](Mutation m) { + std::string out = m(); + cands++; + perPass[p.name]++; + std::string e2; + if (out.empty() || out == text || !parseFunc(out, e2)) { + badCands++; + std::cout << "BAD CANDIDATE " << path << ": " << d.name << " (" << p.name << ")" + << (out.empty() ? ": empty" : out == text ? ": unchanged" : ": " + e2) << "\n"; + } + return ++n < o.checkLimit; + }); + } + } + } + std::cout << o.checkParse.size() << " files, " << funcs << " functions, " << parseFails + << " failed to parse\n" + << others << " statements left unclassified (barriers), " << siteFails << " of " + << sites << " expressions not understood\n" + << cands << " candidates checked, " << badCands << " bad\n"; + for (auto& [name, n] : perPass) std::cout << " " << name << ": " << n << "\n"; + return parseFails == 0 && badCands == 0 ? 0 : 1; +} + +// Resolves "--also [FILE:]NAME" to a region. +bool findAlso(const std::string& spec, const std::string& srcPath, const std::vector& dirs, + Region& r, std::string& err) { + std::string file, name = spec; + size_t colon = spec.find(':'); + while (colon != std::string::npos && colon + 1 < spec.size() && spec[colon + 1] == ':') + colon = spec.find(':', colon + 2); // skip "::" + if (colon != std::string::npos) { + file = spec.substr(0, colon); + name = spec.substr(colon + 1); + } + std::vector files; + if (!file.empty()) { + // relative to the current directory, else to the source's + fs::path p = fs::absolute(file); + if (!fs::exists(p)) p = fs::path(srcPath).parent_path() / file; + files.push_back(fs::absolute(p).lexically_normal().string()); + } + else { + files.push_back(srcPath); + for (auto& h : includedFiles(srcPath, dirs)) files.push_back(h); + } + for (auto& path : files) { + std::string text; + if (!readFile(path, text)) continue; + auto defs = findDefinitions(text, name); + if (defs.empty()) continue; + r = {path, defs[0].first, defs[0].second, name}; + return true; + } + err = "--also " + spec + ": definition not found"; + return false; +} + +} // namespace + +int main(int argc, char** argv) { + Options o; + std::string err; + if (!parseArgs(argc, argv, o, err)) { + std::cerr << "error: " << err << "\n\n" << kUsage; + return 2; + } + if (o.listPasses) { + for (auto& p : allPasses()) std::cout << " " << p.name << "\n " << p.description << "\n"; + return 0; + } + if (o.passExamples) { + std::cout << passExamplesMarkdown(); + return 0; + } + if (o.listDefs) { + for (auto& path : o.checkParse) { + std::string text; + if (!readFile(path, text)) { + std::cerr << "error: can't read " << path << "\n"; + return 1; + } + for (auto& d : listDefinitions(text)) std::cout << d.start << " " << d.end << " " << d.name << "\n"; + } + return 0; + } + if (o.checkMode) { + if (o.checkParse.empty()) { + std::cerr << "error: --check-parse needs files\n"; + return 2; + } + return checkParse(o); + } + if (!o.checkParse.empty()) { + std::cerr << "error: unexpected argument: " << o.checkParse[0] << "\n"; + return 2; + } + if (o.source.empty() || o.function.empty()) { + std::cerr << "error: --source and --function are required\n\n" << kUsage; + return 2; + } + std::string srcPath = fs::absolute(o.source).lexically_normal().string(); + std::string src; + if (!readFile(srcPath, src)) { + std::cerr << "error: can't read " << o.source << "\n"; + return 1; + } + size_t start, end; + if (!locateFunction(src, o.function, start, end, err)) { + std::cerr << "error: " << err << "\n"; + return 1; + } + if (!err.empty()) std::cerr << err << "\n"; + std::string funcText = src.substr(start, end - start); + auto func = parseFunc(funcText, err); + if (!func) { + std::cerr << "error: can't parse " << o.function << ": " << err << "\n"; + return 1; + } + if (o.showAst) { + std::cout << dumpStmt(*func, *func->body); + std::cout << "params:"; + for (auto& p : func->params) std::cout << " " << p; + std::cout << "\nlocals:"; + for (auto& l : func->locals) std::cout << " " << l; + std::cout << "\n"; + return 0; + } + std::vector groups; + if (!parsePassSpecs(o.passes, groups, err)) { + std::cerr << "error: " << err << "\n"; + return 2; + } + for (auto& w : o.weights) { + size_t eq = w.find('='); + bool found = false; + if (eq != std::string::npos) + for (auto& g : groups) + if (g.name() == w.substr(0, eq)) { + try { + g.weightOverride = std::stoi(w.substr(eq + 1)); + found = true; + } catch (...) { + } + } + if (!found) { + std::cerr << "error: --weight " << w << ": expected NAME=N with NAME one of the -p passes/combos\n"; + return 2; + } + } + if (o.mode != "random" && o.mode != "exhaustive") { + std::cerr << "error: --mode must be random or exhaustive\n"; + return 2; + } + + // the functions to permute + Workspace ws; + std::vector dirs = o.includeDirs; + if (dirs.empty()) dirs.push_back(fs::path(srcPath).parent_path().string()); + ws.root = o.mirrorRoot.empty() ? fs::path(srcPath).parent_path().string() + : fs::absolute(o.mirrorRoot).lexically_normal().string(); + ws.regions.push_back({srcPath, start, end, o.function}); + auto addRegion = [&](const Region& r) { + for (auto& x : ws.regions) + if (x.file == r.file && x.start < r.end && r.start < x.end) return; // overlaps + ws.regions.push_back(r); + }; + for (auto& spec : o.also) { + Region r; + if (!findAlso(spec, srcPath, dirs, r, err)) { + std::cerr << "error: " << err << "\n"; + return 1; + } + addRegion(r); + } + if (o.inlineCallees) + for (auto& r : inlineCallees(srcPath, start, end, *func, dirs, (size_t)o.maxCallees)) addRegion(r); + for (auto& r : ws.regions) { + if (!ws.files.count(r.file) && !readFile(r.file, ws.files[r.file])) { + std::cerr << "error: can't read " << r.file << "\n"; + return 1; + } + std::string text = ws.files[r.file].substr(r.start, r.end - r.start); + if (ws.base.empty()) { + ws.macro = std::make_shared(text); + if (!ws.macro->error().empty()) { + std::cerr << "error: " << r.name << ": " << ws.macro->error() << "\n"; + return 1; + } + if (ws.macro->hasMacros()) { + text = ws.macro->first(); + // like decomp-permuter: macros alone, unless PERM_RANDOMIZE or -p + ws.passesOn = ws.macro->randomize() || !o.passes.empty(); + } + } else if (MacroExpander(text).hasMacros()) { + std::cerr << "error: " << r.name << ": PERM macros are only supported in the target function\n"; + return 1; + } + if (groups.empty()) ws.passesOn = false; + if (!parseFunc(text, err)) { + std::cerr << "error: can't parse " << r.name << ": " << err << "\n"; + return 1; + } + ws.base.push_back(text); + } + // inline getters for use_getter: in the source and the headers it includes + { + std::vector gs; + std::vector files = {srcPath}; + for (auto& h : includedFiles(srcPath, dirs)) files.push_back(h); + for (auto& file : files) { + std::string text; + if (readFile(file, text)) + for (auto& g : findGetters(text)) gs.push_back(g); + } + setGetters(gs); + } + if (o.listRegions || ws.regions.size() > 1) { + std::cout << "permuting:\n"; + for (auto& r : ws.regions) std::cout << " " << r.name << " (" << r.file << ")\n"; + if (o.listRegions) return 0; + } + if (o.dryRun) return dryRun(o, ws, groups); + + if (o.compile.empty() || (o.targetObj.empty() && o.scoreCmd.empty())) { + std::cerr << "error: --compile and --target-obj (or --score-cmd) are required\n"; + return 2; + } + if (o.jobs < 1) o.jobs = 1; + + Evaluator ev{o, ws}; + ev.sc.objdump = o.objdump; + ev.sc.ignoreRelocNames = o.ignoreRelocNames; + std::vector realFiles; + for (auto& [file, text] : ws.files) realFiles.push_back(file); + for (int w = 0; w < o.jobs; ++w) { + Mirror m; + std::string dir = (fs::path(o.outputDir) / ".work" / ("w" + std::to_string(w))).string(); + if (!m.create(ws.root, dir, realFiles, err)) { + std::cerr << "error: " << err << "\n"; + return 1; + } + ev.mirrors.push_back(m); + ev.written.push_back(ws.files); // the mirror starts with copies of the originals + } + + std::signal(SIGINT, onSignal); + std::signal(SIGTERM, onSignal); + auto cleanup = [&]() { + std::error_code ec; + for (auto& m : ev.mirrors) fs::remove_all(m.dir(), ec); + }; + + // the unmodified functions + std::string log; + if (!ev.compile(0, ws.base, log)) { + std::cerr << "error: the unmodified source doesn't compile:\n" << log << "\n"; + cleanup(); + return 1; + } + if (o.scoreCmd.empty()) { + if (o.symbol.empty()) { + auto syms = listSymbols(ev.sc, ev.objPath(0), err); + ev.symbol = guessSymbol(syms, o.function); + if (ev.symbol.empty()) { + std::cerr << "error: can't find the symbol for " << o.function + << " in the compiled object; pass --symbol\n"; + cleanup(); + return 1; + } + } else { + ev.symbol = o.symbol; + } + ev.targetSymbol = o.targetSymbol.empty() ? ev.symbol : o.targetSymbol; + if (!disassemble(ev.sc, o.targetObj, ev.targetSymbol, ev.target, err)) { + std::cerr << "error: target: " << err << "\n"; + cleanup(); + return 1; + } + std::cout << "symbol: " << ev.symbol << " (" << ev.target.size() << " target instructions)\n"; + } + std::vector baseInsns; + long base = ev.score(0, &baseInsns, log); + if (base < 0) { + std::cerr << "error: can't score the unmodified source:\n" << log << "\n"; + cleanup(); + return 1; + } + if (o.showBaseDiff && !baseInsns.empty()) std::cout << diffInsns(ev.sc, ev.target, baseInsns); + std::cout << "base score: " << base << "\n"; + if (o.baseOnly) { + cleanup(); + return base == 0 ? 0 : 3; + } + + State st; + st.base = st.best = ws.base; + st.baseScore = st.bestScore = base; + st.seen.insert(hashCand(ws.base)); + if (base == 0) { + std::cout << "already matching\n"; + cleanup(); + return 0; + } + if (o.mode == "random") runRandom(o, ev, st, groups); + else runExhaustive(o, ev, st, groups); + cleanup(); + + std::cout << "\n"; + if (st.bestScore == 0) std::cout << "MATCH found"; + else if (st.bestScore < base) std::cout << "best score " << st.bestScore << " (base " << base << ")"; + else std::cout << "no improvement over base score " << base; + std::cout << " after " << st.compiles << " compiles (" << st.failures << " failed)\n"; + if (o.showTimings) { + long n = std::max(1, st.compiles); + auto ms = [](long long us) { return std::to_string(us / 1000) + " ms"; }; + std::cout << "timings (summed over " << o.jobs << " workers):\n" + << " permute: " << ms(gTimes.permute) << " for " << gTimes.generated + << " candidates generated\n" + << " write: " << ms(gTimes.write) << " (" << ms(gTimes.write / n) << " per compile)\n" + << " compile: " << ms(gTimes.compile) << " (" << ms(gTimes.compile / n) << " per compile)\n" + << " score: " << ms(gTimes.score) << " (" << ms(gTimes.score / n) << " per compile)\n"; + } + return st.bestScore == 0 ? 0 : 3; +} diff --git a/src/parser.cpp b/src/parser.cpp new file mode 100644 index 0000000..42b694f --- /dev/null +++ b/src/parser.cpp @@ -0,0 +1,680 @@ +#include "parser.hpp" + +#include + +namespace perm { + +const char* skName(SK k) { + switch (k) { + case SK::Block: return "Block"; + case SK::Decl: return "Decl"; + case SK::Expr: return "Expr"; + case SK::If: return "If"; + case SK::For: return "For"; + case SK::While: return "While"; + case SK::Do: return "Do"; + case SK::Switch: return "Switch"; + case SK::Case: return "Case"; + case SK::Default: return "Default"; + case SK::Label: return "Label"; + case SK::Return: return "Return"; + case SK::Break: return "Break"; + case SK::Continue: return "Continue"; + case SK::Goto: return "Goto"; + case SK::Asm: return "Asm"; + case SK::PP: return "PP"; + case SK::Empty: return "Empty"; + case SK::Try: return "Try"; + case SK::Other: return "Other"; + } + return "?"; +} + +int matchBracket(const std::vector& toks, int i) { + const std::string& o = toks[i].text; + std::string c = o == "(" ? ")" : o == "[" ? "]" : o == "{" ? "}" : ""; + if (c.empty()) return -1; + int depth = 0; + for (int j = i; j < (int)toks.size(); ++j) { + if (toks[j].kind != TokKind::Punct) continue; + const std::string& t = toks[j].text; + if (t == "(" || t == "[" || t == "{") depth++; + else if (t == ")" || t == "]" || t == "}") { + depth--; + if (depth == 0) return t == c ? j : -1; + } + } + return -1; +} + +bool Func::isLocal(int i) const { + if (i < 0 || i >= (int)toks.size() || toks[i].kind != TokKind::Ident) return false; + if (!locals.count(toks[i].text)) return false; + if (i > 0) { + const std::string& p = toks[i - 1].text; + if (p == "." || p == "->" || p == "::") return false; + } + if (i + 1 < (int)toks.size() && toks[i + 1].text == "::") return false; + return true; +} + +std::string Func::indentOf(int i) const { + for (int k = i; k >= 0; --k) { + const std::string& lead = toks[k].lead; + size_t nl = lead.rfind('\n'); + if (nl == std::string::npos) continue; + std::string ind = "\n"; + for (size_t j = nl + 1; j < lead.size() && (lead[j] == ' ' || lead[j] == '\t'); ++j) + ind += lead[j]; + return ind; + } + return "\n"; +} + +void forEachStmt(const Stmt& s, const std::function& fn) { + fn(s); + for (auto& k : s.kids) forEachStmt(*k, fn); +} + +// --------------------------------------------------------------------------- +// Locating the function + +namespace { + +std::vector nameTokens(const std::string& qual) { + std::vector out; + size_t i = 0; + while (i <= qual.size()) { + size_t j = qual.find("::", i); + std::string comp = qual.substr(i, j == std::string::npos ? std::string::npos : j - i); + if (!out.empty()) out.push_back("::"); + if (!comp.empty() && comp[0] == '~') { + out.push_back("~"); + comp = comp.substr(1); + } + out.push_back(comp); + if (j == std::string::npos) break; + i = j + 2; + } + return out; +} + +} // namespace + +namespace { + +// For a name whose parameter list opens at toks[open]: the '{' of its body if +// this is a definition, else -1. Between ')' and '{' only qualifiers such as +// const/throw(...) and a constructor's initializer list may appear, so macro +// invocations like "MATCH_FUNC(0x1) void f() {" aren't mistaken for one. +int definitionBody(const std::vector& toks, int open) { + int n = (int)toks.size(); + int close = matchBracket(toks, open); + if (close < 0) return -1; + int j = close + 1; + bool initList = false; + while (j < n) { + const Token& tk = toks[j]; + const std::string& t = tk.text; + if (tk.kind == TokKind::Punct) { + if (t == "{") { + // in an initializer list "m{...}" is a member init, not the body + if (initList && j > 0 && toks[j - 1].kind == TokKind::Ident && + toks[j - 1].text != "const") { + j = matchBracket(toks, j); + if (j < 0) return -1; + j++; + continue; + } + return j; + } + if (t == "(" || t == "[") { + if (!initList && !(j > 0 && (toks[j - 1].text == "throw" || + toks[j - 1].text == "noexcept" || + toks[j - 1].text == "__declspec"))) + return -1; + j = matchBracket(toks, j); + if (j < 0) return -1; + j++; + continue; + } + if (t == ":") { + initList = true; + j++; + continue; + } + if (initList && (t == "," || t == "::" || t == "<" || t == ">")) { + j++; + continue; + } + return -1; + } + if (tk.kind != TokKind::Ident) return -1; + if (!initList && t != "const" && t != "volatile" && t != "throw" && t != "noexcept" && + t != "override" && t != "final" && t != "__declspec") + return -1; + j++; + } + return -1; +} + +bool keywordCall(const std::string& s) { + return s == "if" || s == "while" || s == "for" || s == "switch" || s == "return" || + s == "sizeof" || s == "catch" || s == "__except" || s == "defined" || + s == "static_cast" || s == "reinterpret_cast" || s == "const_cast" || + s == "dynamic_cast" || s == "throw" || s == "__declspec" || s == "operator"; +} + +} // namespace + +std::vector> findDefinitions(const std::string& src, + const std::string& qualName) { + std::vector> out; + std::vector toks = lex(src); + std::vector pat = nameTokens(qualName); + int n = (int)toks.size(); + for (int k = 0; k + (int)pat.size() < n; ++k) { + bool ok = true; + for (size_t p = 0; p < pat.size() && ok; ++p) ok = toks[k + p].text == pat[p]; + if (!ok) continue; + int open = k + (int)pat.size(); + if (toks[open].text != "(") continue; + // extend backwards over qualifiers the caller left out + int s = k; + while (s >= 2 && toks[s - 1].text == "::" && toks[s - 2].kind == TokKind::Ident) s -= 2; + if (s >= 1 && (toks[s - 1].text == "." || toks[s - 1].text == "->")) continue; + int body = definitionBody(toks, open); + if (body < 0) continue; + int bodyEnd = matchBracket(toks, body); + if (bodyEnd < 0) continue; + out.push_back({toks[s].offset, toks[bodyEnd].offset + 1}); + } + return out; +} + +std::vector listDefinitions(const std::string& src) { + std::vector out; + std::vector toks = lex(src); + int n = (int)toks.size(); + for (int k = 0; k + 1 < n; ++k) { + if (toks[k].kind != TokKind::Ident || toks[k + 1].text != "(" || keywordCall(toks[k].text)) + continue; + int s = k; + if (s >= 1 && toks[s - 1].text == "~") s--; + while (s >= 2 && toks[s - 1].text == "::" && toks[s - 2].kind == TokKind::Ident) s -= 2; + if (s >= 1 && (toks[s - 1].text == "." || toks[s - 1].text == "->")) continue; + int body = definitionBody(toks, k + 1); + if (body < 0) continue; + int bodyEnd = matchBracket(toks, body); + if (bodyEnd < 0) continue; + std::string name; + for (int i = s; i <= k; ++i) name += toks[i].text; + out.push_back({name, toks[s].offset, toks[bodyEnd].offset + 1}); + k = bodyEnd; // nothing inside a body is a definition we want + } + return out; +} + +bool locateFunction(const std::string& src, const std::string& qualName, size_t& start, + size_t& end, std::string& err) { + auto defs = findDefinitions(src, qualName); + if (defs.empty()) { + err = "definition of '" + qualName + "' not found"; + return false; + } + start = defs[0].first; + end = defs[0].second; + if (defs.size() > 1) + err = "warning: " + std::to_string(defs.size()) + " definitions of '" + qualName + + "' found, using the first"; + return true; +} + +// --------------------------------------------------------------------------- +// Statement parser + +namespace { + +struct Parser { + Func& f; + const std::vector& T; + + explicit Parser(Func& fn) : f(fn), T(fn.toks) {} + + const std::string& t(int i) const { + static const std::string empty; + return i >= 0 && i < (int)T.size() ? T[i].text : empty; + } + bool punct(int i, const char* p) const { + return i >= 0 && i < (int)T.size() && T[i].kind == TokKind::Punct && T[i].text == p; + } + + // Index of the ';' ending the statement starting at i, or -1 if a '}' at + // depth 0 (or limit) comes first. + int findSemi(int i, int limit) const { + int depth = 0; + for (int j = i; j < limit; ++j) { + if (T[j].kind != TokKind::Punct) continue; + const std::string& s = T[j].text; + if (s == "(" || s == "[" || s == "{") depth++; + else if (s == ")" || s == "]" || s == "}") { + if (depth == 0) return -1; + depth--; + } else if (s == ";" && depth == 0) return j; + } + return -1; + } + + // End (exclusive) of an unterminated statement: the next '}' at depth 0. + int findRecover(int i, int limit) const { + int depth = 0; + for (int j = i; j < limit; ++j) { + if (T[j].kind != TokKind::Punct) continue; + const std::string& s = T[j].text; + if (s == "(" || s == "[" || s == "{") depth++; + else if (s == ")" || s == "]" || s == "}") { + if (depth == 0) return j > i ? j : i + 1; + depth--; + } + } + return limit > i ? limit : i + 1; + } + + std::unique_ptr make(SK k, int b, int e) { + auto s = std::make_unique(); + s->kind = k; + s->begin = b; + s->end = e; + return s; + } + + std::unique_ptr toSemi(SK k, int i, int limit) { + int semi = findSemi(i, limit); + if (semi < 0) return make(SK::Other, i, findRecover(i, limit)); + return make(k, i, semi + 1); + } + + void adopt(Stmt& p, std::unique_ptr c) { + c->parent = &p; + p.kids.push_back(std::move(c)); + } + + std::unique_ptr parseBlock(int open) { + int close = matchBracket(T, open); + if (close < 0) close = (int)T.size() - 1; + auto s = make(SK::Block, open, close + 1); + int i = open + 1; + while (i < close) { + auto c = parseStmt(i, close); + if (c->end <= i) c->end = i + 1; + i = c->end; + adopt(*s, std::move(c)); + } + return s; + } + + int matchAngle(int i, int limit) const { + int depth = 0; + for (int j = i; j < limit; ++j) { + const std::string& s = T[j].text; + if (s == "<") depth++; + else if (s == ">") { + if (--depth == 0) return j; + } else if (s == ">>") { + depth -= 2; + if (depth <= 0) return depth == 0 ? j : -1; + } else if (s == ";" || s == "{" || s == "}" || s == "&&" || s == "||") + return -1; + } + return -1; + } + + // Tries to read tokens [b, e) (e is the ';' or the end of a for-init) as a + // declaration. + bool tryDecl(Stmt& s, int b, int e) { + int i = b; + bool isStatic = false, isConst = false; + auto isStorage = [&](const std::string& x) { + return x == "static" || x == "const" || x == "volatile" || x == "register" || + x == "extern" || x == "mutable" || x == "typename"; + }; + while (i < e && isStorage(t(i))) { + if (t(i) == "static") isStatic = true; + if (t(i) == "const") isConst = true; + i++; + } + if (t(i) == "struct" || t(i) == "class" || t(i) == "union" || t(i) == "enum") i++; + if (punct(i, "::")) i++; + if (i >= e) return false; + if (isBuiltinType(t(i))) { + while (i < e && isBuiltinType(t(i))) i++; + } else if (T[i].kind == TokKind::Ident && (t(i) == "auto" || !isKeyword(t(i))) && + !f.locals.count(t(i))) { + i++; + while (i < e) { + if (punct(i, "<")) { + int j = matchAngle(i, e); + if (j < 0) return false; + i = j + 1; + continue; + } + if (punct(i, "::") && i + 1 < e && T[i + 1].kind == TokKind::Ident) { + i += 2; + continue; + } + break; + } + } else { + return false; + } + while (i < e && (t(i) == "const" || t(i) == "volatile")) { + if (t(i) == "const") isConst = true; + i++; + } + int typeEnd = i; + std::vector decls; + while (true) { + Declarator d; + d.begin = i; + while (i < e && (punct(i, "*") || punct(i, "&") || t(i) == "const" || + t(i) == "volatile" || t(i) == "__stdcall" || t(i) == "__cdecl" || + t(i) == "__fastcall" || t(i) == "__thiscall")) { + if (punct(i, "&")) d.isRef = true; + i++; + } + if (punct(i, "(") && (punct(i + 1, "*") || punct(i + 1, "&"))) { + // function pointer: (*name)(params) + int close = matchBracket(T, i); + if (close < 0 || close >= e) return false; + int j = i + 1; + while (j < close && (punct(j, "*") || punct(j, "&") || t(j) == "const")) j++; + if (j >= close || T[j].kind != TokKind::Ident) return false; + d.nameTok = j; + i = close + 1; + if (!punct(i, "(")) return false; + int pc = matchBracket(T, i); + if (pc < 0 || pc >= e) return false; + i = pc + 1; + } else if (i < e && T[i].kind == TokKind::Ident && !isKeyword(t(i))) { + d.nameTok = i++; + } else { + return false; + } + while (punct(i, "[")) { + int close = matchBracket(T, i); + if (close < 0 || close >= e) return false; + d.isArray = true; + i = close + 1; + } + d.declEnd = i; + if (punct(i, "=")) { + i++; + d.initBegin = i; + int depth = 0; + while (i < e) { + if (T[i].kind == TokKind::Punct) { + const std::string& x = T[i].text; + if (x == "(" || x == "[" || x == "{") depth++; + else if (x == ")" || x == "]" || x == "}") depth--; + else if (x == "," && depth == 0) break; + } + i++; + } + d.initEnd = i; + d.init = Declarator::Assign; + if (d.initBegin == d.initEnd) return false; + } else if (punct(i, "(") || punct(i, "{")) { + int close = matchBracket(T, i); + if (close < 0 || close >= e) return false; + d.init = punct(i, "(") ? Declarator::Paren : Declarator::Brace; + d.initBegin = i + 1; + d.initEnd = close; + i = close + 1; + } + d.end = i; + decls.push_back(d); + if (i == e) break; + if (!punct(i, ",")) return false; + i++; + } + s.kind = SK::Decl; + s.typeBegin = b; + s.typeEnd = typeEnd; + s.decls = std::move(decls); + s.isStatic = isStatic; + s.isConst = isConst; + for (auto& d : s.decls) f.locals.insert(t(d.nameTok)); + return true; + } + + std::unique_ptr parseCond(SK k, int i, int limit) { + // keyword ( cond ) stmt + if (!punct(i + 1, "(")) return toSemi(SK::Other, i, limit); + int close = matchBracket(T, i + 1); + if (close < 0 || close >= limit) return make(SK::Other, i, findRecover(i, limit)); + auto s = make(k, i, close + 1); + s->condOpen = i + 1; + s->condClose = close; + if (close + 1 >= limit) return s; + auto body = parseStmt(close + 1, limit); + s->end = body->end; + adopt(*s, std::move(body)); + return s; + } + + std::unique_ptr parseStmt(int i, int limit) { + const Token& tk = T[i]; + const std::string& x = tk.text; + if (tk.kind == TokKind::PP) return make(SK::PP, i, i + 1); + if (tk.kind == TokKind::Punct) { + if (x == "{") return parseBlock(i); + if (x == ";") return make(SK::Empty, i, i + 1); + return toSemi(SK::Expr, i, limit); + } + if (tk.kind != TokKind::Ident) return toSemi(SK::Expr, i, limit); + + if (x == "if") { + auto s = parseCond(SK::If, i, limit); + if (s->kind == SK::If && !s->kids.empty() && s->end < limit && t(s->end) == "else" && + s->end + 1 < limit) { + s->elseTok = s->end; + auto e = parseStmt(s->end + 1, limit); + s->end = e->end; + adopt(*s, std::move(e)); + } + return s; + } + if (x == "while") return parseCond(SK::While, i, limit); + if (x == "switch") return parseCond(SK::Switch, i, limit); + if (x == "for") { + auto s = parseCond(SK::For, i, limit); + if (s->kind == SK::For) { + int semi = findSemi(s->condOpen + 1, s->condClose); + if (semi > s->condOpen + 1) { + auto init = make(SK::Expr, s->condOpen + 1, semi + 1); + if (tryDecl(*init, s->condOpen + 1, semi)) s->forInit = std::move(init); + } + } + return s; + } + if (x == "do") { + auto s = make(SK::Do, i, i + 1); + auto body = parseStmt(i + 1, limit); + int w = body->end; + adopt(*s, std::move(body)); + if (t(w) == "while" && punct(w + 1, "(")) { + int close = matchBracket(T, w + 1); + if (close > 0 && close < limit) { + s->condOpen = w + 1; + s->condClose = close; + s->end = punct(close + 1, ";") ? close + 2 : close + 1; + return s; + } + } + s->kind = SK::Other; + s->end = w; + s->kids.clear(); + return s; + } + if (x == "case") { + int depth = 0; + for (int j = i + 1; j < limit; ++j) { + if (punct(j, "(") || punct(j, "[")) depth++; + else if (punct(j, ")") || punct(j, "]")) depth--; + else if (punct(j, ":") && depth == 0) return make(SK::Case, i, j + 1); + } + return make(SK::Other, i, i + 1); + } + if (x == "default" && punct(i + 1, ":")) return make(SK::Default, i, i + 2); + if (x == "return" || x == "throw") return toSemi(SK::Return, i, limit); + if (x == "break") return toSemi(SK::Break, i, limit); + if (x == "continue") return toSemi(SK::Continue, i, limit); + if (x == "goto") return toSemi(SK::Goto, i, limit); + if (x == "__asm" || x == "_asm" || x == "asm") { + int e; + if (punct(i + 1, "{")) { + e = matchBracket(T, i + 1); + e = e < 0 ? limit : e + 1; + } else { + e = i + 1; + int line = tk.line; + while (e < limit && T[e].line == line && !punct(e, "}") && !punct(e, ";")) e++; + } + if (e < limit && punct(e, ";")) e++; + return make(SK::Asm, i, e); + } + if (x == "try" || x == "__try") { + auto s = make(SK::Try, i, i + 1); + int j = i + 1; + if (!punct(j, "{")) return toSemi(SK::Other, i, limit); + auto b = parseBlock(j); + j = b->end; + adopt(*s, std::move(b)); + while (j < limit && (t(j) == "catch" || t(j) == "__except" || t(j) == "__finally")) { + int k = j + 1; + if (punct(k, "(")) { + k = matchBracket(T, k); + if (k < 0) break; + k++; + } + if (!punct(k, "{")) break; + auto h = parseBlock(k); + j = h->end; + adopt(*s, std::move(h)); + } + s->end = j; + return s; + } + if (x == "typedef" || x == "using" || x == "template" || x == "namespace") + return toSemi(SK::Other, i, limit); + if ((x == "struct" || x == "class" || x == "union" || x == "enum") && + (punct(i + 1, "{") || punct(i + 2, "{") || punct(i + 2, ":"))) + return toSemi(SK::Other, i, limit); + if (!isKeyword(x) && punct(i + 1, ":")) return make(SK::Label, i, i + 2); + + int semi = findSemi(i, limit); + if (semi < 0) return make(SK::Other, i, findRecover(i, limit)); + auto s = make(SK::Expr, i, semi + 1); + tryDecl(*s, i, semi); + return s; + } +}; + +void setParents(Stmt& s) { + for (auto& k : s.kids) { + k->parent = &s; + setParents(*k); + } +} + +} // namespace + +std::unique_ptr parseFunc(const std::string& text, std::string& err) { + auto f = std::make_unique(); + f->text = text; + f->toks = lex(text); + auto& T = f->toks; + int n = (int)T.size(); + for (int i = 0; i < n; ++i) + if (T[i].text == "(" && T[i].kind == TokKind::Punct) { + f->paramOpen = i; + break; + } + if (f->paramOpen < 0) { + err = "no parameter list"; + return nullptr; + } + f->paramClose = matchBracket(T, f->paramOpen); + if (f->paramClose < 0) { + err = "unbalanced parameter list"; + return nullptr; + } + for (int i = f->paramClose + 1; i < n; ++i) { + if (T[i].text == "(") { + i = matchBracket(T, i); + if (i < 0) break; + continue; + } + if (T[i].text == "{") { + f->bodyOpen = i; + break; + } + } + if (f->bodyOpen < 0 || (f->bodyClose = matchBracket(T, f->bodyOpen)) < 0) { + err = "no function body"; + return nullptr; + } + + // parameter names: the last plain identifier of each parameter + int segStart = f->paramOpen + 1, depth = 0; + for (int i = f->paramOpen + 1; i <= f->paramClose; ++i) { + const std::string& x = T[i].text; + if (x == "(" || x == "[") depth++; + if ((x == ")" || x == "]") && i != f->paramClose) depth--; + if ((x == "," && depth == 0) || i == f->paramClose) { + std::string name; + int d2 = 0; + for (int j = segStart; j < i; ++j) { + if (T[j].text == "(" || T[j].text == "[") d2++; + if (T[j].text == ")" || T[j].text == "]") d2--; + if (T[j].text == "=" && d2 == 0) break; + if (T[j].kind == TokKind::Ident && !isKeyword(T[j].text)) name = T[j].text; + } + // a lone type ("Foo") has no name + int idents = 0; + for (int j = segStart; j < i; ++j) + if (T[j].kind == TokKind::Ident && T[j].text != "const" && T[j].text != "struct") + idents++; + if (!name.empty() && idents >= 2) { + f->params.push_back(name); + f->locals.insert(name); + } + segStart = i + 1; + } + } + + Parser p(*f); + f->body = p.parseBlock(f->bodyOpen); + setParents(*f->body); + return f; +} + +std::string dumpStmt(const Func& f, const Stmt& s, int depth) { + std::ostringstream o; + std::string head = f.tokText(s.begin, std::min(s.end, s.begin + 8)); + for (char& c : head) + if (c == '\n' || c == '\r' || c == '\t') c = ' '; + o << std::string(depth * 2, ' ') << skName(s.kind) << " L" << f.toks[s.begin].line << ": " + << head; + if (s.kind == SK::Decl) { + o << " ["; + for (auto& d : s.decls) o << " " << f.t(d.nameTok); + o << " ]"; + } + o << "\n"; + if (s.forInit) o << dumpStmt(f, *s.forInit, depth + 1); + for (auto& k : s.kids) o << dumpStmt(f, *k, depth + 1); + return o.str(); +} + +} // namespace perm diff --git a/src/parser.hpp b/src/parser.hpp new file mode 100644 index 0000000..b89f1fc --- /dev/null +++ b/src/parser.hpp @@ -0,0 +1,100 @@ +// Statement-level parser for one function body. +// +// This is not a C++ front end. It only needs to find statements, blocks, +// local declarations and expressions inside a single function, and it must +// keep going on code it can't fully understand (MSVC extensions, macros, +// types from headers it never sees). Anything it can't classify becomes an +// "Other" statement, which the passes treat as a barrier. +#pragma once + +#include "lexer.hpp" + +#include +#include +#include +#include +#include + +namespace perm { + +enum class SK { + Block, Decl, Expr, If, For, While, Do, Switch, Case, Default, Label, + Return, Break, Continue, Goto, Asm, PP, Empty, Try, Other, +}; +const char* skName(SK k); + +struct Declarator { + enum Init { None, Assign, Paren, Brace }; + int nameTok = -1; + int begin = -1, end = -1; // the whole declarator, init included + int declEnd = -1; // end of the declarator without its init + int initBegin = -1, initEnd = -1; // init expression (inside parens/braces for Paren/Brace) + Init init = None; + bool isArray = false; + bool isRef = false; +}; + +struct Stmt { + SK kind = SK::Other; + int begin = 0, end = 0; // token range [begin, end) + std::vector> kids; + // If: kids[0] = then, kids[1] = else (optional). Loops/Switch: kids[0] = body. + // Try: the guarded block and the handler blocks. + int condOpen = -1, condClose = -1; // the '(' and ')' of if/while/for/switch/do-while + int elseTok = -1; + // Decl + int typeBegin = -1, typeEnd = -1; + std::vector decls; + bool isStatic = false; + bool isConst = false; + // For: the decl in the init clause, if any + std::unique_ptr forInit; + Stmt* parent = nullptr; +}; + +struct Func { + std::string text; // the function from its (qualified) name to the closing brace + std::vector toks; + int paramOpen = -1, paramClose = -1; + int bodyOpen = -1, bodyClose = -1; + std::unique_ptr body; + std::vector params; + std::set locals; // params and every local declared in the body + + std::string tokText(int b, int e) const { return joinTokens(toks, b, e); } + std::string textOf(const Stmt& s) const { return tokText(s.begin, s.end); } + const std::string& t(int i) const { return toks[i].text; } + bool isLocal(int i) const; + // Indentation of the line holding token i ("\n" + spaces), for inserted statements. + std::string indentOf(int i) const; +}; + +// Finds the definition of qualName ("Class::Method", "Method", "Class::~Class") +// in src. On success fills [start, end) with the byte range from the start of +// the qualified name to the closing brace (inclusive). +bool locateFunction(const std::string& src, const std::string& qualName, size_t& start, + size_t& end, std::string& err); + +// Byte ranges of every definition (with a body) of qualName in src. +std::vector> findDefinitions(const std::string& src, + const std::string& qualName); + +struct Definition { + std::string name; // as written, e.g. "Ped::GetX" or "GetX" inside a class + size_t start, end; +}; +// Every function definition in src (not nested in another function's body). +std::vector listDefinitions(const std::string& src); + +// Parses the text produced by locateFunction. Returns null and sets err on failure. +std::unique_ptr parseFunc(const std::string& text, std::string& err); + +// Calls fn on every statement in pre-order (the body block included). +void forEachStmt(const Stmt& s, const std::function& fn); + +// Index of the token closing the bracket at i, or -1. +int matchBracket(const std::vector& toks, int i); + +std::string dumpStmt(const Func& f, const Stmt& s, int depth = 0); + +} // namespace perm diff --git a/src/pass_util.hpp b/src/pass_util.hpp new file mode 100644 index 0000000..dda4b2c --- /dev/null +++ b/src/pass_util.hpp @@ -0,0 +1,71 @@ +// Helpers shared by the pass implementations (passes.cpp, passes_control.cpp). +// Internal: not part of the permuter's interface. +#pragma once + +#include "analysis.hpp" +#include "passes.hpp" + +#include +#include +#include + +namespace perm::util { + +std::vector blocksOf(const Func& f); +std::string norm(const std::string& s); // without whitespace, for comparing code +std::string exprText(const Func& f, const Expr& x); +bool isPostfixLike(const Expr& x); +int exprPrec(const Func& f, const Expr& x); +std::string paren(const std::string& s); +std::string operandText(const Func& f, const Expr& x, int prec); +int indexInParent(const Stmt& s); +const Stmt* singleStmt(const Stmt* s); +std::string negate(const Func& f, int b, int e); + +// passes_control.cpp: rewriting and layout helpers +bool emitReplace(const Func& f, const EmitFn& emit, int b, int e, const std::string& text); +std::string lineInd(const Func& f, int tok); // indentation (spaces) of tok's line +void replaceAll(std::string& s, const std::string& from, const std::string& to); +std::vector branchStmts(const Stmt* s); // a block's statements, or s +// consecutive statements re-indented so their first line sits at ind +std::string stmtsText(const Func& f, const std::vector& v, const std::string& ind); +std::string stmtText(const Func& f, const Stmt* s, const std::string& ind); +std::string bodyText(const Func& f, const Stmt* s, const std::string& ind); +std::string blockText(const Func& f, const std::vector& v, const std::string& ind, + const std::vector& extraBefore = {}, + const std::vector& extraAfter = {}); +bool isJump(const Stmt* s); +bool hasToken(const Func& f, const Stmt& s, const char* tok); +std::vector stmtsOfKind(const Func& f, SK k); +bool inBlock(const Stmt& s); + +template +void forEachSiteExpr(const Func& f, Fn fn) { + for (auto& site : exprSites(f)) { + auto x = parseExpr(f, site.b, site.e); + if (!x) continue; + std::shared_ptr root(std::move(x)); + bool stop = false; + forEachExpr(*root, [&](Expr& e) { + if (!stop && !fn(site, root, e)) stop = true; + }); + if (stop) return; + } +} + +// passes_control.cpp +void enumBoolReturn(const Func& f, const EmitFn& emit); +void enumTernaryArg(const Func& f, const EmitFn& emit); +void enumSwitchIf(const Func& f, const EmitFn& emit); +void enumEarlyReturn(const Func& f, const EmitFn& emit); +void enumBranchDup(const Func& f, const EmitFn& emit); +void enumCondSplit(const Func& f, const EmitFn& emit); +void enumExplicitCompare(const Func& f, const EmitFn& emit); +void enumNegateConst(const Func& f, const EmitFn& emit); +void enumLoopForm(const Func& f, const EmitFn& emit); +void enumReassociate(const Func& f, const EmitFn& emit); + +// passes_more.cpp +void registerMorePasses(std::vector& passes); + +} // namespace perm::util diff --git a/src/passes.cpp b/src/passes.cpp new file mode 100644 index 0000000..9776b15 --- /dev/null +++ b/src/passes.cpp @@ -0,0 +1,1016 @@ +#include "passes.hpp" + +#include "analysis.hpp" +#include "pass_util.hpp" + +#include +#include +#include +#include + +namespace perm { + +// --------------------------------------------------------------------------- +// Rewriter + +Rewriter::Rewriter(const Func& f) : f_(f) { + size_t n = f.toks.size(); + lead_.resize(n); + text_.resize(n); + prefix_.resize(n); + touched_.assign(n, false); + for (size_t i = 0; i < n; ++i) { + lead_[i] = f.toks[i].lead; + text_[i] = f.toks[i].text; + } +} + +void Rewriter::replace(int b, int e, const std::string& text) { + for (int i = b; i < e; ++i) { + if (touched_[i]) overlap_ = true; + touched_[i] = true; + if (i != b) lead_[i].clear(); + text_[i].clear(); + } + if (b < e) text_[b] = text; + else overlap_ = true; +} + +void Rewriter::remove(int b, int e) { + replace(b, e, ""); + if (overlap_) return; + std::string& l = lead_[b]; + size_t nl = l.rfind('\n'); + l = nl == std::string::npos ? "" : l.substr(0, nl); +} + +void Rewriter::insertBefore(int tok, const std::string& stmtText) { + prefix_[tok] += stmtText + f_.indentOf(tok); +} + +std::string Rewriter::apply() const { + if (overlap_) return ""; + std::string out; + out.reserve(f_.text.size() + 256); + for (size_t i = 0; i < f_.toks.size(); ++i) { + out += lead_[i]; + out += prefix_[i]; + out += text_[i]; + } + return out; +} + +// --------------------------------------------------------------------------- +// helpers (shared with the other pass files through pass_util.hpp) + +namespace util { + +std::vector blocksOf(const Func& f) { + std::vector r; + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind == SK::Block) r.push_back(&s); + }); + return r; +} + +std::string norm(const std::string& s) { + std::string r; + for (char c : s) + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') r += c; + return r; +} + +std::string exprText(const Func& f, const Expr& x) { return f.tokText(x.b, x.e); } + +bool isPostfixLike(const Expr& x) { + return x.k == Expr::Primary || x.k == Expr::Paren || x.k == Expr::Call || + x.k == Expr::Index || x.k == Expr::Member || x.k == Expr::Postfix; +} + +int exprPrec(const Func& f, const Expr& x) { + switch (x.k) { + case Expr::Comma: return 1; + case Expr::Assign: return 2; + case Expr::Ternary: return 3; + case Expr::Binary: return binPrec(f.t(x.op)); + default: return 100; + } +} + +std::string paren(const std::string& s) { return "(" + s + ")"; } + +// Text of x for use as an operand of an operator with precedence prec. +std::string operandText(const Func& f, const Expr& x, int prec) { + std::string s = exprText(f, x); + return exprPrec(f, x) <= prec ? paren(s) : s; +} + +bool insideLoopNotContaining(const Func& f, int tok, const Stmt& s) { + bool r = false; + forEachStmt(*f.body, [&](const Stmt& l) { + if (l.kind != SK::For && l.kind != SK::While && l.kind != SK::Do) return; + if (tok >= l.begin && tok < l.end && !(s.begin >= l.begin && s.begin < l.end)) r = true; + }); + return r; +} + +std::string declTypeText(const Func& f, const Stmt& s) { return f.tokText(s.typeBegin, s.typeEnd); } + +// "T name" for declarator d of s, without its initializer. +std::string declNoInit(const Func& f, const Stmt& s, const Declarator& d) { + if (&d == &s.decls[0]) return f.tokText(s.typeBegin, d.declEnd); + std::string t = declTypeText(f, s); + std::string rest = f.tokText(d.begin, d.declEnd); + if (f.t(d.begin) == "*" || f.t(d.begin) == "&") return t + rest; + return t + " " + rest; +} + +// Whether moving an initializer with effects ie past code with effects between +// could change what it computes. Locals are tracked exactly; memory is only +// checked for calls, since a byte-for-byte match proves the rewrite anyway. +bool initBlocked(const Effects& ie, const Effects& between) { + for (auto& r : ie.reads) + if (between.writes.count(r)) return true; + if ((ie.call || ie.memRead) && between.call) return true; + if (ie.call && between.memWrite) return true; + return false; +} + +bool simpleSingleDecl(const Stmt& s) { + return s.kind == SK::Decl && !s.isStatic && s.decls.size() == 1 && s.parent && + s.parent->kind == SK::Block; +} + +int indexInParent(const Stmt& s) { + auto& k = s.parent->kids; + for (size_t i = 0; i < k.size(); ++i) + if (k[i].get() == &s) return (int)i; + return -1; +} + +// All orders of n items that keep every conflicting pair in its original order. +void topoOrders(int n, const std::vector>& conf, size_t cap, + std::vector>& out) { + std::vector cur; + std::vector used(n, false); + std::function rec = [&]() { + if (out.size() >= cap) return; + if ((int)cur.size() == n) { + out.push_back(cur); + return; + } + for (int i = 0; i < n; ++i) { + if (used[i]) continue; + bool ok = true; + for (int j = 0; j < i && ok; ++j) + if (conf[i][j] && !used[j]) ok = false; + if (!ok) continue; + used[i] = true; + cur.push_back(i); + rec(); + cur.pop_back(); + used[i] = false; + } + }; + rec(); +} + +std::vector randomTopo(int n, const std::vector>& conf, Rng& rng) { + std::vector cur; + std::vector used(n, false); + while ((int)cur.size() < n) { + std::vector avail; + for (int i = 0; i < n; ++i) { + if (used[i]) continue; + bool ok = true; + for (int j = 0; j < i && ok; ++j) + if (conf[i][j] && !used[j]) ok = false; + if (ok) avail.push_back(i); + } + int pick = avail[std::uniform_int_distribution(0, avail.size() - 1)(rng)]; + used[pick] = true; + cur.push_back(pick); + } + return cur; +} + +// --------------------------------------------------------------------------- +// reorder_saves + +struct Run { + std::vector items; + std::vector> conf; +}; + +bool isSave(const Func& f, const Stmt& s, const Effects& e) { + if (e.call || e.memWrite || e.barrier) return false; + if (s.kind == SK::Decl) return !s.isStatic; + if (s.kind == SK::Expr) return f.isLocal(s.begin) && f.t(s.begin + 1) == "="; + return false; +} + +std::vector saveRuns(const Func& f) { + std::vector runs; + for (const Stmt* b : blocksOf(f)) { + std::vector cur; + std::vector effs; + auto flush = [&]() { + if (cur.size() >= 2) { + Run r; + r.items = cur; + size_t n = cur.size(); + r.conf.assign(n, std::vector(n, false)); + for (size_t i = 0; i < n; ++i) + for (size_t j = 0; j < n; ++j) + if (i != j) r.conf[i][j] = conflicts(effs[i], effs[j]); + runs.push_back(std::move(r)); + } + cur.clear(); + effs.clear(); + }; + for (auto& k : b->kids) { + Effects e = effectsOfStmt(f, *k); + if (isSave(f, *k, e)) { + cur.push_back(k.get()); + effs.push_back(e); + } else { + flush(); + } + } + flush(); + } + return runs; +} + +std::string applyOrders(const Func& f, const std::vector& runs, + const std::vector>& orders) { + Rewriter rw(f); + for (size_t r = 0; r < runs.size(); ++r) { + auto& items = runs[r].items; + auto& ord = orders[r]; + for (size_t k = 0; k < items.size(); ++k) + if (ord[k] != (int)k) + rw.replace(items[k]->begin, items[k]->end, f.textOf(*items[ord[k]])); + } + return rw.apply(); +} + +void enumReorderSaves(const Func& f, const EmitFn& emit) { + std::vector runs = saveRuns(f); + if (runs.empty()) return; + std::vector>> perRun; + for (auto& r : runs) { + std::vector> o; + topoOrders((int)r.items.size(), r.conf, 40320, o); + perRun.push_back(std::move(o)); + } + // odometer over the cartesian product of every run's orders + auto shared = std::make_shared>(runs); + std::vector idx(runs.size(), 0); + while (true) { + std::vector> pick; + bool identity = true; + for (size_t r = 0; r < runs.size(); ++r) { + pick.push_back(perRun[r][idx[r]]); + for (size_t k = 0; k < pick.back().size(); ++k) + if (pick.back()[k] != (int)k) identity = false; + } + if (!identity && !emit([&f, shared, pick]() { return applyOrders(f, *shared, pick); })) + return; + size_t r = 0; + while (r < runs.size() && ++idx[r] == perRun[r].size()) idx[r++] = 0; + if (r == runs.size()) break; + } +} + +bool randomReorderSaves(const Func& f, Rng& rng, std::string& out) { + std::vector runs = saveRuns(f); + if (runs.empty()) return false; + for (int attempt = 0; attempt < 8; ++attempt) { + std::vector> orders; + size_t which = std::uniform_int_distribution(0, runs.size() - 1)(rng); + bool changed = false; + for (size_t r = 0; r < runs.size(); ++r) { + std::vector id(runs[r].items.size()); + std::iota(id.begin(), id.end(), 0); + if (r == which) { + auto o = randomTopo((int)id.size(), runs[r].conf, rng); + changed = o != id; + orders.push_back(o); + } else { + orders.push_back(id); + } + } + if (!changed) continue; + out = applyOrders(f, runs, orders); + return !out.empty(); + } + return false; +} + +// --------------------------------------------------------------------------- +// move_stmt + +void enumMoveStmt(const Func& f, const EmitFn& emit) { + for (const Stmt* b : blocksOf(f)) { + auto& kids = b->kids; + size_t n = kids.size(); + if (n < 2) continue; + std::vector effs; + for (auto& k : kids) effs.push_back(effectsOfStmt(f, *k)); + for (size_t i = 0; i < n; ++i) { + if (effs[i].barrier || kids[i]->kind == SK::Empty) continue; + for (int dir = -1; dir <= 1; dir += 2) { + for (long j = (long)i + dir; j >= 0 && j < (long)n; j += dir) { + if (conflictsRelaxed(effs[i], effs[j]) || kids[j]->kind == SK::Empty) break; + // move i to position j + std::vector order; + for (size_t k = 0; k < n; ++k) + if (k != i) order.push_back((int)k); + order.insert(order.begin() + j, (int)i); + size_t lo = std::min(i, j), hi = std::max(i, j); + if (!emit([&f, b, order, lo, hi]() { + Rewriter rw(f); + for (size_t k = lo; k <= hi; ++k) + rw.replace(b->kids[k]->begin, b->kids[k]->end, + f.textOf(*b->kids[order[k]])); + return rw.apply(); + })) + return; + } + } + } + } +} + +// --------------------------------------------------------------------------- +// inline_local + +void enumInlineLocal(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (simpleSingleDecl(s)) decls.push_back(&s); + }); + for (const Stmt* s : decls) { + const Declarator& d = s->decls[0]; + if (d.isArray) continue; + std::string name = f.t(d.nameTok); + std::vector uses = usesOf(f, name, s->end, f.bodyClose); + if (d.init != Declarator::Assign || f.t(d.initBegin) == "{") { + if (uses.empty() && d.init == Declarator::None) + emit([&f, s]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + return rw.apply(); + }); + continue; + } + Effects ie = effectsOf(f, d.initBegin, d.initEnd); + if (!ie.writes.empty() || ie.memWrite || ie.barrier) continue; + std::string init = f.tokText(d.initBegin, d.initEnd); + if (uses.empty()) { + // dead local: drop it, keeping a call for its side effects + bool keep = ie.call; + if (!emit([&f, s, init, keep]() { + Rewriter rw(f); + if (keep) rw.replace(s->begin, s->end, init + ";"); + else rw.remove(s->begin, s->end); + return rw.apply(); + })) + return; + continue; + } + bool anyWrite = false; + for (int u : uses) anyWrite |= isWriteUse(f, u); + if (anyWrite) continue; + bool pure = !ie.call; + if (uses.size() > 1 && !pure) continue; + Effects between = effectsOf(f, s->end, uses.back()); + bool trivially = pure && !ie.memRead && ie.reads.empty(); + if (!trivially && (initBlocked(ie, between) || (between.barrier && ie.call))) continue; + bool inLoop = false; + for (int u : uses) inLoop |= insideLoopNotContaining(f, u, *s); + if (inLoop && !trivially) continue; + + auto x = parseExpr(f, d.initBegin, d.initEnd); + bool postfixLike = x && isPostfixLike(*x); + bool commaFree = x && x->k != Expr::Comma; + // A use that is a whole operand on its own (an assignment's right side, + // an argument, a return value, an index) needs no parentheses. + auto standalone = [&](int u) { + const std::string& p = f.t(u - 1); + const std::string& n = f.t(u + 1); + bool before = isAssignOp(p) || p == "(" || p == "," || p == "return" || p == "["; + bool after = n == ";" || n == ")" || n == "," || n == "]"; + return before && after && commaFree; + }; + std::string castT = declTypeText(f, *s) + f.tokText(d.begin, d.nameTok); + std::vector casts = {0}; + if (uses.size() == 1 && !d.isRef && !s->isConst) casts.push_back(1); + for (int cast : casts) { + std::vector texts; + for (int u : uses) { + if (cast) texts.push_back("((" + castT + ")" + paren(init) + ")"); + else texts.push_back(postfixLike || standalone(u) ? init : paren(init)); + } + if (!emit([&f, s, uses, texts]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + for (size_t k = 0; k < uses.size(); ++k) rw.replace(uses[k], uses[k] + 1, texts[k]); + return rw.apply(); + })) + return; + } + } +} + +// --------------------------------------------------------------------------- +// sink_decl + +void enumSinkDecl(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (simpleSingleDecl(s)) decls.push_back(&s); + }); + for (const Stmt* s : decls) { + const Stmt* P = s->parent; + int di = indexInParent(*s); + const Declarator& d = s->decls[0]; + std::string name = f.t(d.nameTok); + std::vector uses = usesOf(f, name, s->end, P->end); + if (uses.empty()) continue; + // innermost block holding every use + const Stmt* B = P; + forEachStmt(*P, [&](const Stmt& x) { + if (x.kind == SK::Block && x.begin < uses.front() && uses.back() < x.end && + x.end - x.begin < B->end - B->begin) + B = &x; + }); + auto itemAt = [&](const Stmt* blk) { + for (size_t k = 0; k < blk->kids.size(); ++k) + if (blk->kids[k]->end > uses.front()) return (int)k; + return -1; + }; + std::vector> targets; + int tb = itemAt(B); + if (tb >= 0 && (B != P || tb > di + 1)) targets.push_back({B, tb}); + int tp = itemAt(P); + if (B != P && tp > di + 1) targets.push_back({P, tp}); + Effects ie = d.init == Declarator::None ? Effects{} : effectsOf(f, d.initBegin, d.initEnd); + for (auto [blk, t] : targets) { + const Stmt& item = *blk->kids[t]; + int at = item.begin; + if (d.init != Declarator::None) { + Effects between = effectsOf(f, s->end, at); + if (initBlocked(ie, between)) continue; + if (insideLoopNotContaining(f, at, *s)) continue; + } else if (insideLoopNotContaining(f, at, *s)) { + // only when the loop body starts by assigning it + if (!(item.kind == SK::Expr && item.begin == uses.front() && f.t(at + 1) == "=")) + continue; + } + if (!emit([&f, s, at]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + rw.insertBefore(at, f.textOf(*s)); + return rw.apply(); + })) + return; + } + } +} + +// --------------------------------------------------------------------------- +// hoist_decl + +struct Hoist { + const Stmt* s; + std::string top; // declarations to place at the top + std::string leftover; // what stays in place ("" to remove) +}; + +bool makeHoist(const Func& f, const Stmt& s, Hoist& h) { + if (s.kind != SK::Decl || s.isStatic || !s.parent || s.parent->kind != SK::Block) return false; + h.s = &s; + for (auto& d : s.decls) { + if (d.init == Declarator::None) { + h.top += declNoInit(f, s, d) + ";"; + continue; + } + if (d.init != Declarator::Assign || s.isConst || d.isRef || d.isArray || + f.t(d.initBegin) == "{") + return false; + if (!h.top.empty()) h.top += " "; + h.top += declNoInit(f, s, d) + ";"; + if (!h.leftover.empty()) h.leftover += " "; + h.leftover += f.t(d.nameTok) + " = " + f.tokText(d.initBegin, d.initEnd) + ";"; + } + return true; +} + +// First item of blk that isn't a declaration, or -1. +int firstNonDecl(const Stmt& blk) { + for (size_t k = 0; k < blk.kids.size(); ++k) + if (blk.kids[k]->kind != SK::Decl && blk.kids[k]->kind != SK::Empty) return (int)k; + return -1; +} + +void applyHoist(Rewriter& rw, const Hoist& h, int at) { + if (h.leftover.empty()) rw.remove(h.s->begin, h.s->end); + else rw.replace(h.s->begin, h.s->end, h.leftover); + rw.insertBefore(at, h.top); +} + +void enumHoistDecl(const Func& f, const EmitFn& emit) { + std::vector hs; + forEachStmt(*f.body, [&](const Stmt& s) { + Hoist h; + if (makeHoist(f, s, h)) hs.push_back(h); + }); + int topIdx = firstNonDecl(*f.body); + if (topIdx < 0) return; + int topTok = f.body->kids[topIdx]->begin; + std::vector toTop; + for (auto& h : hs) { + const Stmt* P = h.s->parent; + bool inTopRun = P == f.body.get() && indexInParent(*h.s) < topIdx; + if (!inTopRun) { + toTop.push_back(&h); + if (!emit([&f, h, topTok]() { + Rewriter rw(f); + applyHoist(rw, h, topTok); + return rw.apply(); + })) + return; + } + if (P != f.body.get()) { + int bi = firstNonDecl(*P); + if (bi >= 0 && indexInParent(*h.s) > bi) { + int at = P->kids[bi]->begin; + if (!emit([&f, h, at]() { + Rewriter rw(f); + applyHoist(rw, h, at); + return rw.apply(); + })) + return; + } + } + } + if (toTop.size() >= 2) { + std::vector all; + for (auto* h : toTop) all.push_back(*h); + emit([&f, all, topTok]() { + Rewriter rw(f); + for (auto& h : all) applyHoist(rw, h, topTok); + return rw.apply(); + }); + } +} + +// --------------------------------------------------------------------------- +// merge_decl: "T x; ... x = e;" -> "T x = e;" + +void enumMergeDecl(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (simpleSingleDecl(s) && s.decls[0].init == Declarator::None && !s.decls[0].isArray) + decls.push_back(&s); + }); + for (const Stmt* s : decls) { + const Declarator& d = s->decls[0]; + std::string name = f.t(d.nameTok); + std::vector uses = usesOf(f, name, s->end, s->parent->end); + if (uses.empty() || f.t(uses[0] + 1) != "=") continue; + const Stmt* A = nullptr; + forEachStmt(*f.body, [&](const Stmt& x) { + if (x.kind == SK::Expr && x.begin == uses[0]) A = &x; + }); + if (!A || !A->parent || A->parent->kind != SK::Block) continue; + if (uses.back() >= A->parent->end) continue; + if (uses.size() > 1 && uses[1] < A->end) continue; // "x = x + 1" + std::string text = declNoInit(f, *s, d) + " = " + f.tokText(A->begin + 2, A->end - 1) + ";"; + if (!emit([&f, s, A, text]() { + Rewriter rw(f); + rw.remove(s->begin, s->end); + rw.replace(A->begin, A->end, text); + return rw.apply(); + })) + return; + } +} + +// --------------------------------------------------------------------------- +// expression passes + +void enumSwapOperands(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr root, Expr& x) { + if (x.k != Expr::Binary) return true; + const std::string& op = f.t(x.op); + if (op != "+" && op != "*" && op != "&" && op != "|" && op != "^" && op != "==" && op != "!=") + return true; + int p = binPrec(op); + const Expr& l = *x.kids[0]; + const Expr& r = *x.kids[1]; + if (norm(exprText(f, l)) == norm(exprText(f, r))) return true; // "x * x" + bool assoc = op == "+" || op == "*" || op == "&" || op == "|" || op == "^"; + std::string lt = exprText(f, l); + if (exprPrec(f, l) < p || (exprPrec(f, l) == p && !(assoc && f.t(l.op) == op))) + lt = paren(lt); + std::string text = exprText(f, r) + " " + op + " " + lt; + int b = x.b, e = x.e; + return emit([&f, b, e, text, root]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }); +} + +void enumFlipCompare(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr root, Expr& x) { + if (x.k != Expr::Binary) return true; + const std::string& op = f.t(x.op); + std::string m = op == "<" ? ">" : op == ">" ? "<" : op == "<=" ? ">=" : op == ">=" ? "<=" : ""; + if (m.empty()) return true; + std::string text = exprText(f, *x.kids[1]) + " " + m + " " + operandText(f, *x.kids[0], 10); + int b = x.b, e = x.e; + return emit([&f, b, e, text, root]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }); +} + +std::string negate(const Func& f, int b, int e) { + auto x = parseExpr(f, b, e); + std::string all = f.tokText(b, e); + if (!x) return "!(" + all + ")"; + if (x->k == Expr::Binary) { + const std::string& op = f.t(x->op); + std::string n = op == "==" ? "!=" : op == "!=" ? "==" : op == "<" ? ">=" : op == ">=" ? "<" + : op == ">" ? "<=" : op == "<=" ? ">" : ""; + if (!n.empty()) return exprText(f, *x->kids[0]) + " " + n + " " + exprText(f, *x->kids[1]); + } + if (x->k == Expr::Unary && f.t(x->op) == "!") { + const Expr& o = *x->kids[0]; + if (o.k == Expr::Paren) return exprText(f, *o.kids[0]); + return exprText(f, o); + } + if (isPostfixLike(*x)) return "!" + all; + return "!(" + all + ")"; +} + +void enumInvertIf(const Func& f, const EmitFn& emit) { + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind != SK::If || s.kids.size() != 2) return; + const Stmt* th = s.kids[0].get(); + const Stmt* el = s.kids[1].get(); + std::string neg = negate(f, s.condOpen + 1, s.condClose); + std::string elText = f.textOf(*el); + if (el->kind == SK::If) { + std::string ind = f.indentOf(s.begin); + std::string inner; + for (char c : elText) inner += c == '\n' ? std::string("\n ") : std::string(1, c); + elText = "{" + ind + " " + inner + ind + "}"; + } + const Stmt* sp = &s; + emit([&f, sp, th, el, neg, elText]() { + Rewriter rw(f); + rw.replace(sp->condOpen + 1, sp->condClose, neg); + rw.replace(th->begin, th->end, elText); + rw.replace(el->begin, el->end, f.textOf(*th)); + return rw.apply(); + }); + }); +} + +// The one expression statement inside s (s itself, or a block holding just it). +const Stmt* singleStmt(const Stmt* s) { + while (s && s->kind == SK::Block) { + if (s->kids.size() != 1) return nullptr; + s = s->kids[0].get(); + } + return s; +} + +void enumTernary(const Func& f, const EmitFn& emit) { + auto emitReplace = [&](int b, int e, std::string text) { + return emit([&f, b, e, text]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }; + auto branch = [&](const Expr& x) { + return x.k == Expr::Assign || x.k == Expr::Comma ? paren(exprText(f, x)) : exprText(f, x); + }; + auto condText = [&](const Expr& c) { + return c.k == Expr::Paren ? exprText(f, *c.kids[0]) : exprText(f, c); + }; + // ternary -> if/else + for (auto& site : exprSites(f)) { + const Stmt& s = *site.stmt; + if (s.kind != SK::Expr && s.kind != SK::Return) continue; + auto x = parseExpr(f, site.b, site.e); + if (!x) continue; + std::string ind = f.indentOf(s.begin), in = ind + " "; + if (s.kind == SK::Expr && x->k == Expr::Assign && f.t(x->op) == "=" && + x->kids[1]->k == Expr::Ternary) { + const Expr& t = *x->kids[1]; + Effects le = effectsOf(f, x->kids[0]->b, x->kids[0]->e); + if (le.call || le.memWrite) continue; + std::string L = exprText(f, *x->kids[0]); + std::string text = "if (" + condText(*t.kids[0]) + ")" + ind + "{" + in + L + " = " + + exprText(f, *t.kids[1]) + ";" + ind + "}" + ind + "else" + ind + + "{" + in + L + " = " + exprText(f, *t.kids[2]) + ";" + ind + "}"; + if (!emitReplace(s.begin, s.end, text)) return; + } else if (s.kind == SK::Return && x->k == Expr::Ternary) { + std::string c = condText(*x->kids[0]); + std::string a = exprText(f, *x->kids[1]), b = exprText(f, *x->kids[2]); + std::string withElse = "if (" + c + ")" + ind + "{" + in + "return " + a + ";" + ind + + "}" + ind + "else" + ind + "{" + in + "return " + b + ";" + + ind + "}"; + std::string early = "if (" + c + ")" + ind + "{" + in + "return " + a + ";" + ind + + "}" + ind + "return " + b + ";"; + if (!emitReplace(s.begin, s.end, withElse)) return; + if (!emitReplace(s.begin, s.end, early)) return; + } + } + // if/else -> ternary + std::vector ifs; + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind == SK::If) ifs.push_back(&s); + }); + for (const Stmt* s : ifs) { + auto c = parseExpr(f, s->condOpen + 1, s->condClose); + if (!c) continue; + std::string ct = c->k == Expr::Assign || c->k == Expr::Ternary || c->k == Expr::Comma + ? paren(exprText(f, *c)) + : exprText(f, *c); + const Stmt* th = singleStmt(s->kids[0].get()); + const Stmt* el = s->kids.size() == 2 ? singleStmt(s->kids[1].get()) : nullptr; + if (!th) continue; + if (el && th->kind == SK::Expr && el->kind == SK::Expr) { + auto a = parseExpr(f, th->begin, th->end - 1); + auto b = parseExpr(f, el->begin, el->end - 1); + if (!a || !b || a->k != Expr::Assign || b->k != Expr::Assign) continue; + if (f.t(a->op) != "=" || f.t(b->op) != "=") continue; + std::string L = exprText(f, *a->kids[0]); + if (norm(L) != norm(exprText(f, *b->kids[0]))) continue; + std::string text = L + " = " + ct + " ? " + branch(*a->kids[1]) + " : " + + branch(*b->kids[1]) + ";"; + if (!emitReplace(s->begin, s->end, text)) return; + continue; + } + if (th->kind != SK::Return || f.t(th->begin) != "return") continue; + auto a = parseExpr(f, th->begin + 1, th->end - 1); + if (!a) continue; + const Stmt* other = el; + int end = s->end; + if (!other && s->parent && s->parent->kind == SK::Block) { + int i = indexInParent(*s); + if (i >= 0 && i + 1 < (int)s->parent->kids.size()) { + other = s->parent->kids[i + 1].get(); + end = other->end; + } + } + if (!other || other->kind != SK::Return || f.t(other->begin) != "return") continue; + auto b = parseExpr(f, other->begin + 1, other->end - 1); + if (!b) continue; + std::string text = "return " + ct + " ? " + branch(*a) + " : " + branch(*b) + ";"; + if (!emitReplace(s->begin, end, text)) return; + } +} + +void enumCompoundAssign(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr root, Expr& x) { + if (x.k != Expr::Assign) return true; + const std::string& op = f.t(x.op); + const Expr& L = *x.kids[0]; + const Expr& R = *x.kids[1]; + std::string lt = exprText(f, L); + Effects le = effectsOf(f, L.b, L.e); + if (le.call || le.memWrite) return true; + std::string text; + if (op == "=") { + if (R.k != Expr::Binary) return true; + const std::string& bop = f.t(R.op); + static const char* ok[] = {"+", "-", "*", "/", "%", "&", "|", "^", "<<", ">>"}; + if (std::find_if(std::begin(ok), std::end(ok), [&](const char* o) { return bop == o; }) == + std::end(ok)) + return true; + if (norm(exprText(f, *R.kids[0])) != norm(lt)) return true; + text = lt + " " + bop + "= " + exprText(f, *R.kids[1]); + } else { + std::string bop = op.substr(0, op.size() - 1); + text = lt + " = " + lt + " " + bop + " " + operandText(f, R, binPrec(bop)); + } + int b = x.b, e = x.e; + return emit([&f, b, e, text, root]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); + }); +} + +} // namespace util + +// --------------------------------------------------------------------------- + +const std::vector& allPasses() { + static const std::vector passes = [] { + using namespace util; + std::vector passes = { + {"reorder_saves", + "Permute runs of consecutive local saves (declarations and assignments to locals " + "that don't call or store) in every order that keeps dependencies", + 30, enumReorderSaves, randomReorderSaves}, + {"move_stmt", + "Move a statement up or down past neighbours it doesn't depend on (locals and calls " + "are respected, memory aliasing is not)", + 20, + enumMoveStmt, nullptr}, + {"inline_local", + "Remove a local: substitute its initializer into its uses (plain and with a cast), " + "or drop it if unused", + 15, enumInlineLocal, nullptr}, + {"sink_decl", "Move a declaration into the innermost block holding all its uses, or to " + "just before its first use", + 15, enumSinkDecl, nullptr}, + {"hoist_decl", + "Move declarations to the top of their block or of the function (one, or all of " + "them), splitting the initializer into an assignment", + 15, enumHoistDecl, nullptr}, + {"merge_decl", "Join 'T x; ... x = e;' into 'T x = e;' at the assignment", 10, + enumMergeDecl, nullptr}, + {"swap_operands", "Swap the operands of a commutative operator (+ * & | ^ == !=)", 15, + enumSwapOperands, nullptr}, + {"flip_compare", "Mirror a comparison: 'a < b' -> 'b > a'", 10, enumFlipCompare, nullptr}, + {"invert_if", "Negate an if condition and swap the then/else branches", 10, enumInvertIf, + nullptr}, + {"ternary", "Turn 'x = c ? a : b' / 'return c ? a : b' into if/else and back", 8, + enumTernary, nullptr}, + {"compound_assign", "'x = x op y' <-> 'x op= y'", 8, enumCompoundAssign, nullptr}, + {"bool_return", + "'return a <= b;' <-> 'if (a <= b) return true; return false;' (with and without " + "else, true/false and 1/0)", + 8, enumBoolReturn, nullptr}, + {"ternary_arg", + "'f(c ? a : b);' <-> 'if (c) f(a); else f(b);': a ternary inside a statement, and two " + "branches that differ in one sub-expression", + 8, enumTernaryArg, nullptr}, + {"switch_if", "'if (x == K) S else D' <-> 'switch (x) { case K: S break; default: D }'", 6, + enumSwitchIf, nullptr}, + {"early_return", + "'if (c) { ...return; } else { B }' <-> 'if (c) { ...return; } B' (also break, " + "continue, goto)", + 10, enumEarlyReturn, nullptr}, + {"branch_dup", + "Hoist a statement both branches start or end with out of the if, or copy the " + "statement before/after an if/else into both branches", + 10, enumBranchDup, nullptr}, + {"cond_split", "'if (a && b) S' <-> 'if (a) { if (b) S }'", 8, enumCondSplit, nullptr}, + {"explicit_compare", "'if (x)' <-> 'if (x != 0)', 'if (!x)' <-> 'if (x == 0)'", 8, + enumExplicitCompare, nullptr}, + {"negate_const", "'x - 4' <-> 'x + -4', 'x -= 4' <-> 'x += -4'", 6, enumNegateConst, nullptr}, + {"loop_form", + "for <-> while: 'for (i; c; s) B' <-> 'i; while (c) { B s; }', 'while (c)' <-> " + "'for (; c;)'", + 6, enumLoopForm, nullptr}, + {"reassociate", "'a + b + c' <-> 'a + (b + c)' (for + * & | ^)", 6, enumReassociate, + nullptr}, + }; + registerMorePasses(passes); + return passes; + }(); + return passes; +} + +const Pass* findPass(const std::string& name) { + for (auto& p : allPasses()) + if (p.name == name) return &p; + return nullptr; +} + +bool randomMutation(const Pass& p, const Func& f, Rng& rng, std::string& out) { + if (p.random) return p.random(f, rng, out); + // reservoir sample one mutation without materialising every text + Mutation chosen; + size_t seen = 0; + p.enumerate(f, [&](Mutation m) { + seen++; + if (std::uniform_int_distribution(0, seen - 1)(rng) == 0) chosen = std::move(m); + return seen < 20000; + }); + if (!chosen) return false; + out = chosen(); + return !out.empty(); +} + +// --------------------------------------------------------------------------- +// pass groups + +std::string PassGroup::name() const { + std::string r; + for (auto* p : passes) r += (r.empty() ? "" : "+") + p->name; + return r; +} + +int PassGroup::weight() const { + if (weightOverride >= 0) return weightOverride; + int w = 0; + for (auto* p : passes) w = std::max(w, p->weight); + return w; +} + +bool parsePassSpecs(const std::vector& specs, std::vector& out, + std::string& err) { + out.clear(); + bool none = false; + auto addAll = [&]() { + for (auto& p : allPasses()) out.push_back({{&p}}); + }; + if (specs.empty()) { + addAll(); + return true; + } + for (auto& spec : specs) { + std::stringstream groups(spec); + std::string g; + while (std::getline(groups, g, ',')) { + g.erase(std::remove(g.begin(), g.end(), ' '), g.end()); + if (g.empty()) continue; + if (g == "all") { + addAll(); + continue; + } + if (g == "none") { + none = true; + continue; + } + PassGroup grp; + std::stringstream names(g); + std::string n; + while (std::getline(names, n, '+')) { + const Pass* p = findPass(n); + if (!p) { + err = "unknown pass '" + n + "' (see --list-passes)"; + return false; + } + grp.passes.push_back(p); + } + if (!grp.passes.empty()) out.push_back(grp); + } + } + if (out.empty() && !none) { + err = "no passes given"; + return false; + } + return true; +} + +void enumerateGroup(const PassGroup& g, const std::string& text, + const std::function& emit) { + std::set seen = {text}; + bool stop = false; + std::function rec = [&](const std::string& cur, size_t from) { + for (size_t j = from; j < g.passes.size() && !stop; ++j) { + std::string err; + auto f = parseFunc(cur, err); + if (!f) return; + std::vector outs; + g.passes[j]->enumerate(*f, [&](Mutation m) { + std::string s = m(); + if (!s.empty() && seen.insert(s).second) outs.push_back(std::move(s)); + return true; + }); + for (auto& s : outs) { + if (stop) return; + if (!emit(s)) { + stop = true; + return; + } + rec(s, j + 1); + } + } + }; + rec(text, 0); +} + +bool randomGroupMutation(const PassGroup& g, const std::string& text, Rng& rng, std::string& out) { + std::string cur = text; + for (auto* p : g.passes) { + std::string err; + auto f = parseFunc(cur, err); + if (!f) break; + std::string next; + if (randomMutation(*p, *f, rng, next) && next != cur) cur = next; + } + if (cur == text) return false; + out = cur; + return true; +} + +} // namespace perm diff --git a/src/passes.hpp b/src/passes.hpp new file mode 100644 index 0000000..cc69452 --- /dev/null +++ b/src/passes.hpp @@ -0,0 +1,82 @@ +// Source mutation passes. Each pass looks at a parsed function and offers +// a set of rewrites ("mutations"); a mutation produces the new function text. +#pragma once + +#include "parser.hpp" + +#include +#include +#include +#include + +namespace perm { + +using Rng = std::mt19937_64; +using Mutation = std::function; +using EmitFn = std::function; // return false to stop enumerating + +struct Pass { + std::string name; + std::string description; + int weight = 10; // relative chance of being picked in random mode + std::function enumerate; + // Optional: a cheaper random pick than enumerating everything. + std::function random; +}; + +const std::vector& allPasses(); +const Pass* findPass(const std::string& name); + +// One random mutation of f by pass p. Returns false if p has nothing to offer. +bool randomMutation(const Pass& p, const Func& f, Rng& rng, std::string& out); + +// An inline getter found in a header: "get_cam_x" returning "field_1AC_cam.x" +// (the member path with whitespace removed). use_getter swaps between the two. +struct Getter { + std::string name, expr; +}; +std::vector findGetters(const std::string& text); +void setGetters(std::vector g); // before the passes run +const std::vector& getters(); + +// A pass, or a combination of passes written "a+b+c" on the command line. +struct PassGroup { + std::vector passes; + int weightOverride = -1; // --weight + std::string name() const; + int weight() const; +}; + +// Parses pass specs such as {"reorder_saves,invert_if", "swap_operands+flip_compare"}: +// commas separate groups, '+' combines passes into one group, "all" adds every +// pass as its own group, "none" adds nothing (no passes). No specs means "all". +bool parsePassSpecs(const std::vector& specs, std::vector& out, + std::string& err); + +// Every distinct text made by applying one mutation of each pass in some +// non-empty, in-order subset of g's passes (for "a+b": a, b, and a then b). +// emit returns false to stop. +void enumerateGroup(const PassGroup& g, const std::string& text, + const std::function& emit); + +// One random mutation from each of g's passes, in order. Passes with nothing to +// offer are skipped. Returns false if nothing changed. +bool randomGroupMutation(const PassGroup& g, const std::string& text, Rng& rng, std::string& out); + +// Token-range rewriting of a function. +class Rewriter { +public: + explicit Rewriter(const Func& f); + void replace(int b, int e, const std::string& text); + void remove(int b, int e); // also drops the line break before the range + void insertBefore(int tok, const std::string& stmtText); // on its own line + std::string apply() const; // empty if edits overlapped + +private: + const Func& f_; + std::vector lead_, text_, prefix_; + std::vector touched_; + bool overlap_ = false; +}; + +} // namespace perm diff --git a/src/passes_control.cpp b/src/passes_control.cpp new file mode 100644 index 0000000..348b667 --- /dev/null +++ b/src/passes_control.cpp @@ -0,0 +1,596 @@ +// Control-flow and expression-shape passes. Most of them come straight from +// the VC6 codegen notes in gta2_re's docs/matching_quirks.md. + +#include "pass_util.hpp" + +#include +#include + +namespace perm::util { + +bool emitReplace(const Func& f, const EmitFn& emit, int b, int e, const std::string& text) { + return emit([&f, b, e, text]() { + Rewriter rw(f); + rw.replace(b, e, text); + return rw.apply(); + }); +} + +// Indentation (spaces only) of the line holding token i. +std::string lineInd(const Func& f, int i) { return f.indentOf(i).substr(1); } + +void replaceAll(std::string& s, const std::string& from, const std::string& to) { + if (from.empty()) return; + for (size_t p = 0; (p = s.find(from, p)) != std::string::npos; p += to.size()) + s.replace(p, from.size(), to); +} + +// Statements of a branch: the block's statements, or the statement itself. +std::vector branchStmts(const Stmt* s) { + std::vector r; + if (s->kind == SK::Block) + for (auto& k : s->kids) r.push_back(k.get()); + else + r.push_back(s); + return r; +} + +// Text of consecutive statements, re-indented so their first line sits at ind. +std::string stmtsText(const Func& f, const std::vector& v, const std::string& ind) { + if (v.empty()) return ""; + std::string text = f.tokText(v.front()->begin, v.back()->end); + replaceAll(text, f.indentOf(v.front()->begin), "\n" + ind); + return text; +} + +std::string stmtText(const Func& f, const Stmt* s, const std::string& ind) { + return stmtsText(f, {s}, ind); +} + +// A controlled statement (if/loop body) for a header at indentation ind, +// starting with its line break: a block sits at ind, a single statement one +// level deeper. +std::string bodyText(const Func& f, const Stmt* s, const std::string& ind) { + std::string in = s->kind == SK::Block ? ind : ind + " "; + return "\n" + in + stmtText(f, s, in); +} + +// Smallest expression node in x covering tokens [b, e), or null. +const Expr* coveringNode(const Expr& x, int b, int e) { + if (x.b > b || x.e < e) return nullptr; + for (auto& k : x.kids) + if (const Expr* r = coveringNode(*k, b, e)) return r; + return &x; +} + +const Expr* nodeAt(const Expr& x, int b, int e) { + if (x.b == b && x.e == e) return &x; + for (auto& k : x.kids) + if (const Expr* r = nodeAt(*k, b, e)) return r; + return nullptr; +} + +// "{ stmts }" laid out at indentation ind (the brace's own line). +std::string blockText(const Func& f, const std::vector& v, const std::string& ind, + const std::vector& extraBefore, + const std::vector& extraAfter) { + std::string in = ind + " "; + std::string r = "{"; + for (auto& x : extraBefore) r += "\n" + in + x; + if (!v.empty()) r += "\n" + in + stmtsText(f, v, in); + for (auto& x : extraAfter) r += "\n" + in + x; + return r + "\n" + ind + "}"; +} + +std::string ifElseText(const std::string& cond, const std::string& thenBlock, + const std::string& elseBlock, const std::string& ind) { + std::string r = "if (" + cond + ")\n" + ind + thenBlock; + if (!elseBlock.empty()) r += "\n" + ind + "else\n" + ind + elseBlock; + return r; +} + +bool isJump(const Stmt* s) { + return s && (s->kind == SK::Return || s->kind == SK::Break || s->kind == SK::Continue || + s->kind == SK::Goto); +} + +bool hasToken(const Func& f, const Stmt& s, const char* tok) { + for (int i = s.begin; i < s.end; ++i) + if (f.t(i) == tok) return true; + return false; +} + +bool isTrue(const std::string& s) { return s == "true" || s == "1" || s == "TRUE"; } +bool isFalse(const std::string& s) { return s == "false" || s == "0" || s == "FALSE"; } +bool isZero(const std::string& s) { return s == "0" || s == "NULL" || s == "nullptr"; } + +// Value of "return v;" as text, or "" when s isn't one. +std::string returnValue(const Func& f, const Stmt* s) { + if (!s || s->kind != SK::Return || f.t(s->begin) != "return" || s->end - 1 <= s->begin + 1) + return ""; + return f.tokText(s->begin + 1, s->end - 1); +} + +std::vector stmtsOfKind(const Func& f, SK k) { + std::vector r; + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind == k) r.push_back(&s); + }); + return r; +} + +bool inBlock(const Stmt& s) { return s.parent && s.parent->kind == SK::Block; } + +// --------------------------------------------------------------------------- +// bool_return: "return a <= b;" <-> "if (a <= b) return true; return false;" + +void enumBoolReturn(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::Return)) { + std::string v = returnValue(f, s); + if (v.empty()) continue; + auto x = parseExpr(f, s->begin + 1, s->end - 1); + if (!x) continue; + bool boolish = (x->k == Expr::Binary && binPrec(f.t(x->op)) >= 4 && + binPrec(f.t(x->op)) <= 10 && f.t(x->op) != "|" && f.t(x->op) != "^" && + f.t(x->op) != "&") || + (x->k == Expr::Unary && f.t(x->op) == "!"); + if (!boolish) continue; + std::string ind = lineInd(f, s->begin); + std::string c = x->k == Expr::Paren ? exprText(f, *x->kids[0]) : v; + for (auto [t, fl] : {std::pair{"true", "false"}, {"1", "0"}}) { + std::string ret = std::string("return ") + t + ";"; + std::string retF = std::string("return ") + fl + ";"; + std::string early = ifElseText(c, "{\n" + ind + " " + ret + "\n" + ind + "}", "", ind) + + "\n" + ind + retF; + std::string withElse = ifElseText(c, "{\n" + ind + " " + ret + "\n" + ind + "}", + "{\n" + ind + " " + retF + "\n" + ind + "}", ind); + if (!emitReplace(f, emit, s->begin, s->end, early)) return; + if (!emitReplace(f, emit, s->begin, s->end, withElse)) return; + } + } + for (const Stmt* s : stmtsOfKind(f, SK::If)) { + std::string a = returnValue(f, singleStmt(s->kids[0].get())); + if (a.empty()) continue; + std::string b; + int end = s->end; + if (s->kids.size() == 2) { + b = returnValue(f, singleStmt(s->kids[1].get())); + } else if (inBlock(*s)) { + int i = indexInParent(*s); + if (i + 1 < (int)s->parent->kids.size()) { + const Stmt* n = s->parent->kids[i + 1].get(); + b = returnValue(f, n); + end = n->end; + } + } + std::string cond; + if (isTrue(a) && isFalse(b)) cond = f.tokText(s->condOpen + 1, s->condClose); + else if (isFalse(a) && isTrue(b)) cond = negate(f, s->condOpen + 1, s->condClose); + else continue; + if (!emitReplace(f, emit, s->begin, end, "return " + cond + ";")) return; + } +} + +// --------------------------------------------------------------------------- +// ternary_arg: "f(c ? a : b);" <-> "if (c) f(a); else f(b);" + +void enumTernaryArg(const Func& f, const EmitFn& emit) { + for (auto& site : exprSites(f)) { + const Stmt& s = *site.stmt; + if (s.kind != SK::Expr && s.kind != SK::Return) continue; + if (!inBlock(s) && s.parent && s.parent->kind != SK::If) continue; + auto root = parseExpr(f, site.b, site.e); + if (!root) continue; + std::vector ternaries; + forEachExpr(*root, [&](Expr& x) { + if (x.k == Expr::Ternary && &x != root.get()) ternaries.push_back(&x); + }); + std::string ind = lineInd(f, s.begin); + for (const Expr* t : ternaries) { + auto with = [&](const Expr& branch) { + std::string v = exprText(f, branch); + if (branch.k == Expr::Comma) v = paren(v); + return f.tokText(s.begin, t->b) + f.toks[t->b].lead + v + f.toks[t->e].lead + + f.tokText(t->e, s.end); + }; + const Expr& c = *t->kids[0]; + std::string ct = c.k == Expr::Paren ? exprText(f, *c.kids[0]) : exprText(f, c); + std::string in = ind + " "; + std::string text = ifElseText(ct, "{\n" + in + with(*t->kids[1]) + "\n" + ind + "}", + "{\n" + in + with(*t->kids[2]) + "\n" + ind + "}", ind); + if (!emitReplace(f, emit, s.begin, s.end, text)) return; + } + } + // the other way: two branches that differ in one sub-expression + for (const Stmt* s : stmtsOfKind(f, SK::If)) { + if (s->kids.size() != 2) continue; + const Stmt* a = singleStmt(s->kids[0].get()); + const Stmt* b = singleStmt(s->kids[1].get()); + if (!a || !b || a->kind != b->kind || (a->kind != SK::Expr && a->kind != SK::Return)) continue; + int na = a->end - a->begin, nb = b->end - b->begin; + int p = 0; + while (p < na && p < nb && f.t(a->begin + p) == f.t(b->begin + p)) p++; + int q = 0; + while (q < na - p && q < nb - p && f.t(a->end - 1 - q) == f.t(b->end - 1 - q)) q++; + if (p == na && p == nb) continue; // identical + int ab = a->begin + p, ae = a->end - q, bb = b->begin + p, be = b->end - q; + if (ab >= ae || bb >= be) continue; + // widen the difference to whole sub-expressions of both statements + int skip = a->kind == SK::Return ? 1 : 0; + auto ta = parseExpr(f, a->begin + skip, a->end - 1); + auto tb = parseExpr(f, b->begin + skip, b->end - 1); + if (!ta || !tb) continue; + const Expr* ea = coveringNode(*ta, ab, ae); + if (!ea) continue; + const Expr* eb = nodeAt(*tb, b->begin + (ea->b - a->begin), b->end - (a->end - ea->e)); + if (!eb || ea->k == Expr::Comma || eb->k == Expr::Comma) continue; + ab = ea->b, ae = ea->e, bb = eb->b, be = eb->e; + auto c = parseExpr(f, s->condOpen + 1, s->condClose); + std::string ct = f.tokText(s->condOpen + 1, s->condClose); + if (!c || exprPrec(f, *c) <= 3) ct = paren(ct); + std::string tern = ct + " ? " + f.tokText(ab, ae) + " : " + f.tokText(bb, be); + const std::string& before = f.t(ab - 1); + const std::string& after = f.t(ae); + bool standalone = (before == "(" || before == "," || before == "return" || isAssignOp(before)) && + (after == ")" || after == "," || after == ";"); + if (!standalone) tern = paren(tern); + std::string text = f.tokText(a->begin, ab) + f.toks[ab].lead + tern + f.toks[ae].lead + + f.tokText(ae, a->end); + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } +} + +// --------------------------------------------------------------------------- +// switch_if: "if (x == K) S" <-> "switch (x) { case K: S break; }" + +void enumSwitchIf(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::If)) { + auto c = parseExpr(f, s->condOpen + 1, s->condClose); + if (!c || c->k != Expr::Binary || f.t(c->op) != "==") continue; + const Expr* v = c->kids[0].get(); + const Expr* k = c->kids[1].get(); + // how much an operand looks like a case label: 2 a literal, 1 an + // enumerator-like name (qualified or ALL_CAPS), 0 anything else + auto constness = [&](const Expr& x) { + if (x.k == Expr::Unary && f.t(x.op) == "-" && x.kids[0]->k == Expr::Primary && + f.toks[x.kids[0]->b].kind == TokKind::Number) + return 2; + if (x.k != Expr::Primary) return 0; + const Token& t = f.toks[x.b]; + if (t.kind == TokKind::Number || t.kind == TokKind::Char) return 2; + if (t.kind != TokKind::Ident || f.isLocal(x.b)) return 0; + if (x.e - x.b > 1) return 1; // Enum::Value + bool caps = true; + for (char ch : t.text) caps &= !std::islower((unsigned char)ch); + return caps ? 1 : 0; + }; + int cv = constness(*v), ck = constness(*k); + if (cv == ck) continue; + if (cv > ck) std::swap(v, k); + bool breaks = hasToken(f, *s->kids[0], "break") || + (s->kids.size() == 2 && hasToken(f, *s->kids[1], "break")); + if (breaks) continue; + std::string ind = lineInd(f, s->begin), in = ind + " ", in2 = in + " "; + auto caseBody = [&](const Stmt* br) { + std::string r; + auto v2 = branchStmts(br); + if (!v2.empty()) r += "\n" + in2 + stmtsText(f, v2, in2); + if (v2.empty() || !isJump(v2.back())) r += "\n" + in2 + "break;"; + return r; + }; + std::string text = "switch (" + exprText(f, *v) + ")\n" + ind + "{\n" + in + "case " + + exprText(f, *k) + ":" + caseBody(s->kids[0].get()); + if (s->kids.size() == 2) text += "\n" + in + "default:" + caseBody(s->kids[1].get()); + text += "\n" + ind + "}"; + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } + for (const Stmt* s : stmtsOfKind(f, SK::Switch)) { + if (s->kids.empty() || s->kids[0]->kind != SK::Block) continue; + auto& kids = s->kids[0]->kids; + // segments: label index, then statements up to the next label + std::vector>> segs; + bool ok = true; + for (auto& k : kids) { + if (k->kind == SK::Case || k->kind == SK::Default) segs.push_back({k.get(), {}}); + else if (segs.empty()) ok = false; + else segs.back().second.push_back(k.get()); + } + if (!ok || segs.empty() || segs.size() > 2 || segs[0].first->kind != SK::Case) continue; + if (segs.size() == 2 && segs[1].first->kind != SK::Default) continue; + std::vector> bodies; + for (size_t i = 0; i < segs.size() && ok; ++i) { + auto body = segs[i].second; + bool ends = !body.empty() && (body.back()->kind == SK::Break || isJump(body.back())); + if (!body.empty() && body.back()->kind == SK::Break) body.pop_back(); + for (auto* b : body) + if (hasToken(f, *b, "break")) ok = false; + if (i + 1 < segs.size() && !ends) ok = false; // falls through + bodies.push_back(body); + } + if (!ok) continue; + const Stmt* label = segs[0].first; + std::string k = f.tokText(label->begin + 1, label->end - 1); + std::string v = f.tokText(s->condOpen + 1, s->condClose); + auto vx = parseExpr(f, s->condOpen + 1, s->condClose); + if (!vx || exprPrec(f, *vx) <= 9) v = paren(v); + std::string ind = lineInd(f, s->begin); + std::string text = ifElseText(v + " == " + k, blockText(f, bodies[0], ind), + bodies.size() == 2 ? blockText(f, bodies[1], ind) : "", ind); + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } +} + +// --------------------------------------------------------------------------- +// early_return: "if (c) { A; return; } else { B; }" <-> "if (c) { A; return; } B;" + +void enumEarlyReturn(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::If)) { + if (!inBlock(*s)) continue; + std::string ind = lineInd(f, s->begin); + auto thenStmts = branchStmts(s->kids[0].get()); + bool thenJumps = !thenStmts.empty() && isJump(thenStmts.back()); + if (s->kids.size() == 2 && thenJumps) { + // drop the else + const Stmt* el = s->kids[1].get(); + std::string body = stmtsText(f, branchStmts(el), ind); + if (el->kind == SK::Block && el->kids.empty()) body = ""; + const Stmt* sp = s; + if (!emit([&f, sp, body]() { + Rewriter rw(f); + if (body.empty()) rw.remove(sp->elseTok, sp->end); + else rw.replace(sp->elseTok, sp->end, body); + return rw.apply(); + })) + return; + } else if (s->kids.size() == 1 && thenJumps) { + // wrap what follows in an else + auto& sib = s->parent->kids; + int i = indexInParent(*s); + std::vector rest; + bool ok = true; + for (size_t k = i + 1; k < sib.size(); ++k) { + SK kk = sib[k]->kind; + if (kk == SK::Case || kk == SK::Default || kk == SK::Label) ok = false; + rest.push_back(sib[k].get()); + } + if (!ok || rest.empty()) continue; + std::string text = f.textOf(*s) + "\n" + ind + "else\n" + ind + blockText(f, rest, ind); + if (!emitReplace(f, emit, s->begin, rest.back()->end, text)) return; + } + } +} + +// --------------------------------------------------------------------------- +// branch_dup: move a statement shared by both branches out of the if, or a +// neighbouring statement into both branches. + +void enumBranchDup(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::If)) { + if (s->kids.size() != 2 || s->kids[1]->kind == SK::If || !inBlock(*s)) continue; + std::string ind = lineInd(f, s->begin); + std::string cond = f.tokText(s->condOpen + 1, s->condClose); + Effects ce = effectsOf(f, s->condOpen + 1, s->condClose); + auto th = branchStmts(s->kids[0].get()); + auto el = branchStmts(s->kids[1].get()); + auto movable = [&](const Stmt* x) { + return x->kind == SK::Expr || x->kind == SK::Empty; + }; + auto build = [&](std::vector t, std::vector e, + std::vector pre, std::vector post) { + return ifElseText(cond, blockText(f, t, ind, pre, post), blockText(f, e, ind, pre, post), ind); + }; + // shared tail -> after the if + if (!th.empty() && !el.empty() && movable(th.back()) && + norm(f.textOf(*th.back())) == norm(f.textOf(*el.back()))) { + std::string tail = f.textOf(*th.back()); + std::string text = build({th.begin(), th.end() - 1}, {el.begin(), el.end() - 1}, {}, {}) + + "\n" + ind + tail; + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } + // shared head -> before the if + if (!th.empty() && !el.empty() && movable(th.front()) && + norm(f.textOf(*th.front())) == norm(f.textOf(*el.front())) && + !conflictsRelaxed(effectsOfStmt(f, *th.front()), ce)) { + std::string head = f.textOf(*th.front()); + std::string text = head + "\n" + ind + + build({th.begin() + 1, th.end()}, {el.begin() + 1, el.end()}, {}, {}); + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } + auto& sib = s->parent->kids; + int i = indexInParent(*s); + // the next statement -> end of both branches + bool thJumps = !th.empty() && isJump(th.back()), elJumps = !el.empty() && isJump(el.back()); + if (i + 1 < (int)sib.size() && movable(sib[i + 1].get()) && !thJumps && !elJumps) { + const Stmt* n = sib[i + 1].get(); + std::string text = build(th, el, {}, {f.textOf(*n)}); + if (!emitReplace(f, emit, s->begin, n->end, text)) return; + } + // the previous statement -> start of both branches + if (i > 0 && movable(sib[i - 1].get()) && + !conflictsRelaxed(effectsOfStmt(f, *sib[i - 1]), ce)) { + const Stmt* p = sib[i - 1].get(); + std::string text = build(th, el, {f.textOf(*p)}, {}); + if (!emitReplace(f, emit, p->begin, s->end, text)) return; + } + } +} + +// --------------------------------------------------------------------------- +// cond_split: "if (a && b) S" <-> "if (a) { if (b) S }" + +void enumCondSplit(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::If)) { + if (s->kids.size() != 1) continue; + std::string ind = lineInd(f, s->begin), in = ind + " "; + auto c = parseExpr(f, s->condOpen + 1, s->condClose); + if (c && c->k == Expr::Binary && f.t(c->op) == "&&") { + const Stmt* th = s->kids[0].get(); + std::string inner = "if (" + exprText(f, *c->kids[1]) + ")" + bodyText(f, th, in); + std::string text = "if (" + exprText(f, *c->kids[0]) + ")\n" + ind + "{\n" + in + inner + + "\n" + ind + "}"; + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } + const Stmt* inner = singleStmt(s->kids[0].get()); + if (inner && inner->kind == SK::If && inner->kids.size() == 1) { + auto a = parseExpr(f, s->condOpen + 1, s->condClose); + auto b = parseExpr(f, inner->condOpen + 1, inner->condClose); + if (!a || !b) continue; + std::string text = "if (" + operandText(f, *a, 4) + " && " + operandText(f, *b, 5) + + ")" + bodyText(f, inner->kids[0].get(), ind); + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } + } +} + +// --------------------------------------------------------------------------- +// explicit_compare: "if (x)" <-> "if (x != 0)", "if (!x)" <-> "if (x == 0)" + +void enumExplicitCompare(const Func& f, const EmitFn& emit) { + std::vector> conds; + forEachStmt(*f.body, [&](const Stmt& s) { + if ((s.kind == SK::If || s.kind == SK::While || s.kind == SK::Do) && s.condOpen >= 0) + conds.push_back({s.condOpen + 1, s.condClose}); + }); + for (auto [b, e] : conds) { + auto root = parseExpr(f, b, e); + if (!root) continue; + bool stop = false; + // ctx: precedence of the operator the node is an operand of (0 = whole condition) + std::function walk = [&](const Expr& x, int ctx) { + if (stop) return; + auto replaceWith = [&](const std::string& t, int prec) { + std::string text = prec <= ctx ? paren(t) : t; + if (!emitReplace(f, emit, x.b, x.e, text)) stop = true; + }; + if (x.k == Expr::Binary && (f.t(x.op) == "&&" || f.t(x.op) == "||")) { + int p = binPrec(f.t(x.op)); + walk(*x.kids[0], p); + walk(*x.kids[1], p); + return; + } + if (x.k == Expr::Paren) { + walk(*x.kids[0], 0); + return; + } + if (x.k == Expr::Unary && f.t(x.op) == "!") { + replaceWith(operandText(f, *x.kids[0], 9) + " == 0", 9); + return; + } + if (x.k == Expr::Binary && (f.t(x.op) == "!=" || f.t(x.op) == "==") && + isZero(norm(exprText(f, *x.kids[1])))) { + const Expr& o = *x.kids[0]; + if (f.t(x.op) == "!=") replaceWith(operandText(f, o, ctx == 100 ? 99 : ctx), 100); + else replaceWith("!" + (isPostfixLike(o) ? exprText(f, o) : paren(exprText(f, o))), 99); + return; + } + if (isPostfixLike(x) || x.k == Expr::Cast) + replaceWith(operandText(f, x, 9) + " != 0", 9); + }; + walk(*root, 0); + if (stop) return; + } +} + +// --------------------------------------------------------------------------- +// negate_const: "x - 4" <-> "x + -4", "x -= 4" <-> "x += -4" + +void enumNegateConst(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr, Expr& x) { + if (x.k != Expr::Binary && x.k != Expr::Assign) return true; + const std::string& op = f.t(x.op); + const Expr& r = *x.kids[1]; + std::string lt = exprText(f, *x.kids[0]); + bool num = r.k == Expr::Primary && f.toks[r.b].kind == TokKind::Number; + bool negNum = r.k == Expr::Unary && f.t(r.op) == "-" && r.kids[0]->k == Expr::Primary && + f.toks[r.kids[0]->b].kind == TokKind::Number; + std::string text; + if (op == "-" && num) text = lt + " + -" + exprText(f, r); + else if (op == "+" && negNum) text = lt + " - " + exprText(f, *r.kids[0]); + else if (op == "-=" && num) text = lt + " += -" + exprText(f, r); + else if (op == "+=" && negNum) text = lt + " -= " + exprText(f, *r.kids[0]); + else return true; + return emitReplace(f, emit, x.b, x.e, text); + }); +} + +// --------------------------------------------------------------------------- +// loop_form: for <-> while + +void enumLoopForm(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::For)) { + if (!inBlock(*s) || s->kids.empty()) continue; + std::vector parts; + int depth = 0, segStart = s->condOpen + 1; + for (int i = s->condOpen + 1; i <= s->condClose; ++i) { + const std::string& x = f.t(i); + if (i < s->condClose && (x == "(" || x == "[")) depth++; + if (i < s->condClose && (x == ")" || x == "]")) depth--; + if ((x == ";" && depth == 0) || i == s->condClose) { + parts.push_back(i > segStart ? f.tokText(segStart, i) : ""); + segStart = i + 1; + } + } + if (parts.size() != 3) continue; + const Stmt* body = s->kids[0].get(); + if (!parts[2].empty() && hasToken(f, *body, "continue")) continue; + std::string ind = lineInd(f, s->begin); + std::string text; + if (!parts[0].empty()) text += parts[0] + ";\n" + ind; + std::vector post; + if (!parts[2].empty()) post.push_back(parts[2] + ";"); + text += "while (" + (parts[1].empty() ? std::string("1") : parts[1]) + ")\n" + ind + + blockText(f, branchStmts(body), ind, {}, post); + if (!emitReplace(f, emit, s->begin, s->end, text)) return; + } + for (const Stmt* s : stmtsOfKind(f, SK::While)) { + if (!inBlock(*s) || s->kids.empty()) continue; + std::string c = f.tokText(s->condOpen + 1, s->condClose); + std::string ind = lineInd(f, s->begin); + const Stmt* body = s->kids[0].get(); + std::string head = norm(c) == "1" || norm(c) == "true" ? "for (;;)" : "for (; " + c + ";)"; + if (!emitReplace(f, emit, s->begin, s->end, head + bodyText(f, body, ind))) return; + // "init; while (c) { ...; step; }" -> "for (init; c; step) { ... }" + int i = indexInParent(*s); + if (i <= 0 || body->kind != SK::Block || body->kids.empty()) continue; + const Stmt* init = s->parent->kids[i - 1].get(); + const Stmt* step = body->kids.back().get(); + if ((init->kind != SK::Expr && init->kind != SK::Decl) || step->kind != SK::Expr) continue; + if (hasToken(f, *body, "continue")) continue; + Effects ie = effectsOfStmt(f, *init), se = effectsOfStmt(f, *step); + bool shared = false; + for (auto& w : ie.writes) shared |= se.writes.count(w) > 0; + if (!shared) continue; + std::vector rest; + for (size_t k = 0; k + 1 < body->kids.size(); ++k) rest.push_back(body->kids[k].get()); + std::string text = "for (" + f.tokText(init->begin, init->end - 1) + "; " + c + "; " + + f.tokText(step->begin, step->end - 1) + ")\n" + ind + blockText(f, rest, ind); + if (!emitReplace(f, emit, init->begin, s->end, text)) return; + } +} + +// --------------------------------------------------------------------------- +// reassociate: "a + b + c" <-> "a + (b + c)" + +void enumReassociate(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr, Expr& x) { + if (x.k != Expr::Binary) return true; + const std::string& op = f.t(x.op); + if (op != "+" && op != "*" && op != "&" && op != "|" && op != "^") return true; + const Expr& l = *x.kids[0]; + const Expr& r = *x.kids[1]; + if (l.k == Expr::Binary && f.t(l.op) == op) { + std::string text = exprText(f, *l.kids[0]) + " " + op + " (" + exprText(f, *l.kids[1]) + + " " + op + " " + exprText(f, r) + ")"; + if (!emitReplace(f, emit, x.b, x.e, text)) return false; + } + if (r.k == Expr::Paren && r.kids[0]->k == Expr::Binary && f.t(r.kids[0]->op) == op) { + const Expr& in = *r.kids[0]; + std::string text = exprText(f, l) + " " + op + " " + exprText(f, *in.kids[0]) + " " + op + + " " + operandText(f, *in.kids[1], binPrec(op)); + if (!emitReplace(f, emit, x.b, x.e, text)) return false; + } + return true; + }); +} + +} // namespace perm::util diff --git a/src/passes_more.cpp b/src/passes_more.cpp new file mode 100644 index 0000000..e74058b --- /dev/null +++ b/src/passes_more.cpp @@ -0,0 +1,633 @@ +// More passes, from gta2_re's docs/matching_quirks.md and decomp-permuter's +// randomizer: comparisons, chained assignments, scopes, local types and +// arrays, switch case layout, inline getters, temporaries, removals. + +#include "pass_util.hpp" + +#include +#include +#include +#include + +namespace perm { + +namespace { +std::vector gGetters; +} // namespace + +void setGetters(std::vector g) { gGetters = std::move(g); } +const std::vector& getters() { return gGetters; } + +std::vector findGetters(const std::string& text) { + std::vector out; + for (auto& d : listDefinitions(text)) { + std::string err; + auto f = parseFunc(text.substr(d.start, d.end - d.start), err); + if (!f) continue; + // no parameters + int po = f->paramOpen, pc = f->paramClose; + if (!(pc == po + 1 || (pc == po + 2 && f->t(po + 1) == "void"))) continue; + auto& k = f->body->kids; + if (k.size() != 1 || k[0]->kind != SK::Return || f->t(k[0]->begin) != "return") continue; + int b = k[0]->begin + 1, e = k[0]->end - 1; + if (f->t(b) == "this" && f->t(b + 1) == "->") b += 2; + if (b >= e || f->toks[b].kind != TokKind::Ident) continue; + // a plain member path: a.b->c[1].d + bool path = true; + for (int i = b; i < e && path; ++i) { + const Token& t = f->toks[i]; + path = t.kind == TokKind::Ident || t.kind == TokKind::Number || t.text == "." || + t.text == "->" || t.text == "[" || t.text == "]"; + } + if (!path || isKeyword(f->t(b))) continue; + std::string name = d.name.substr(d.name.rfind(':') == std::string::npos ? 0 : d.name.rfind(':') + 1); + out.push_back({name, util::norm(f->tokText(b, e))}); + } + return out; +} + +namespace util { + +namespace { + +// --- inequalities ----------------------------------------------------------- + +bool intLiteral(const Func& f, const Expr& x, long long& v, std::string& prefix, std::string& suffix) { + if (x.k != Expr::Primary || x.e != x.b + 1 || f.toks[x.b].kind != TokKind::Number) return false; + std::string s = f.t(x.b); + if (s.find('.') != std::string::npos) return false; + bool hex = s.size() > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X'); + if (!hex && (s.find('e') != std::string::npos || s.find('E') != std::string::npos || + s.find('f') != std::string::npos || s.find('F') != std::string::npos)) + return false; + size_t end = s.size(); + while (end > 0 && std::string("uUlL").find(s[end - 1]) != std::string::npos) end--; + suffix = s.substr(end); + prefix = hex ? s.substr(0, 2) : ""; + try { + v = std::stoll(s.substr(0, end), nullptr, hex ? 16 : (s.size() > 1 && s[0] == '0' ? 8 : 10)); + } catch (...) { + return false; + } + return true; +} + +std::string formatLike(long long v, const std::string& prefix, const std::string& suffix) { + if (prefix.empty()) return std::to_string(v) + suffix; + char buf[32]; + std::snprintf(buf, sizeof buf, prefix[1] == 'X' ? "%llX" : "%llx", (unsigned long long)v); + return prefix + buf + suffix; +} + +void enumInequalities(const Func& f, const EmitFn& emit) { + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr, Expr& x) { + if (x.k != Expr::Binary) return true; + const std::string& op = f.t(x.op); + if (op != "<" && op != ">" && op != "<=" && op != ">=") return true; + long long v; + std::string pre, suf; + bool right = intLiteral(f, *x.kids[1], v, pre, suf); + if (!right && !intLiteral(f, *x.kids[0], v, pre, suf)) return true; + // with the constant on the right: a > N == a >= N+1, a >= N == a > N-1, ... + // on the left the same rules apply to the mirrored operator + std::string mop = right ? op : (op == "<" ? ">" : op == ">" ? "<" : op == "<=" ? ">=" : "<="); + long long nv = (mop == ">" || mop == "<=") ? v + 1 : v - 1; + std::string nop = mop == ">" ? ">=" : mop == ">=" ? ">" : mop == "<" ? "<=" : "<"; + if (nv < 0 && v >= 0) return true; // "x >= 0" -> "x > -1" breaks unsigned x + if (!right) nop = nop == "<" ? ">" : nop == ">" ? "<" : nop == "<=" ? ">=" : "<="; + std::string lit = formatLike(nv, pre, suf); + std::string text = right ? exprText(f, *x.kids[0]) + " " + nop + " " + lit + : lit + " " + nop + " " + exprText(f, *x.kids[1]); + return emitReplace(f, emit, x.b, x.e, text); + }); +} + +// --- chain_assign ------------------------------------------------------------- + +struct SimpleAssign { + std::string lhs, rhs; +}; + +bool simpleAssign(const Func& f, const Stmt& s, SimpleAssign& a) { + if (s.kind != SK::Expr) return false; + auto x = parseExpr(f, s.begin, s.end - 1); + if (!x || x->k != Expr::Assign || f.t(x->op) != "=") return false; + const Expr& r = *x->kids[1]; + if (r.k == Expr::Assign) return false; + Effects e = effectsOf(f, r.b, r.e); + if (e.call || e.memWrite || !e.writes.empty()) return false; + a = {exprText(f, *x->kids[0]), exprText(f, r)}; + return true; +} + +void enumChainAssign(const Func& f, const EmitFn& emit) { + for (const Stmt* b : blocksOf(f)) { + auto& k = b->kids; + for (size_t i = 0; i + 1 < k.size(); ++i) { + SimpleAssign a0; + if (!simpleAssign(f, *k[i], a0)) continue; + std::vector run = {a0}; + size_t j = i + 1; + SimpleAssign aj; + while (j < k.size() && simpleAssign(f, *k[j], aj) && norm(aj.rhs) == norm(a0.rhs)) { + run.push_back(aj); + j++; + } + if (run.size() < 2) continue; + // two at a time, both orders, and the whole run + for (size_t p = 0; p + 1 < run.size(); ++p) { + const SimpleAssign &x = run[p], &y = run[p + 1]; + if (!emitReplace(f, emit, k[i + p]->begin, k[i + p + 1]->end, x.lhs + " = " + y.lhs + " = " + x.rhs + ";")) + return; + if (!emitReplace(f, emit, k[i + p]->begin, k[i + p + 1]->end, y.lhs + " = " + x.lhs + " = " + x.rhs + ";")) + return; + } + if (run.size() > 2) { + std::string text; + for (auto& r : run) text += r.lhs + " = "; + if (!emitReplace(f, emit, k[i]->begin, k[j - 1]->end, text + a0.rhs + ";")) return; + } + i = j - 1; + } + } + // a = b = v; -> b = v; a = v; / b = v; a = b; + for (const Stmt* s : stmtsOfKind(f, SK::Expr)) { + if (!inBlock(*s)) continue; + auto x = parseExpr(f, s->begin, s->end - 1); + if (!x || x->k != Expr::Assign || f.t(x->op) != "=" || x->kids[1]->k != Expr::Assign) continue; + const Expr& in = *x->kids[1]; + if (f.t(in.op) != "=") continue; + Effects e = effectsOf(f, in.kids[1]->b, in.kids[1]->e); + std::string a = exprText(f, *x->kids[0]), bb = exprText(f, *in.kids[0]), v = exprText(f, *in.kids[1]); + std::string ind = "\n" + lineInd(f, s->begin); + if (!emitReplace(f, emit, s->begin, s->end, bb + " = " + v + ";" + ind + a + " = " + bb + ";")) return; + if (!e.call && !e.memWrite && e.writes.empty()) + if (!emitReplace(f, emit, s->begin, s->end, bb + " = " + v + ";" + ind + a + " = " + v + ";")) return; + } +} + +// --- scope_block -------------------------------------------------------------- + +std::set declaredIn(const Func& f, const Stmt& s) { + std::set r; + forEachStmt(s, [&](const Stmt& x) { + if (x.kind == SK::Decl) + for (auto& d : x.decls) r.insert(f.t(d.nameTok)); + }); + return r; +} + +void enumScopeBlock(const Func& f, const EmitFn& emit) { + for (const Stmt* b : blocksOf(f)) { + auto& k = b->kids; + // wrap a declaration and the statements using it in their own block + for (size_t i = 0; i < k.size(); ++i) { + if (k[i]->kind != SK::Decl || k[i]->isStatic) continue; + std::set names = declaredIn(f, *k[i]); + size_t j = i; + bool ok = true; + for (bool grew = true; grew && ok;) { + grew = false; + for (size_t m = j + 1; m < k.size(); ++m) { + bool uses = false; + for (auto& n : names) uses |= !usesOf(f, n, k[m]->begin, k[m]->end).empty(); + if (uses) { + for (size_t q = j + 1; q <= m; ++q) + for (auto& n : declaredIn(f, *k[q])) grew |= names.insert(n).second; + j = m; + } + } + } + for (size_t q = i; q <= j; ++q) + if (k[q]->kind == SK::Case || k[q]->kind == SK::Default || k[q]->kind == SK::Label) ok = false; + if (!ok || (i == 0 && j + 1 == k.size())) continue; + std::vector range; + for (size_t q = i; q <= j; ++q) range.push_back(k[q].get()); + std::string ind = lineInd(f, k[i]->begin); + if (!emitReplace(f, emit, k[i]->begin, k[j]->end, blockText(f, range, ind))) return; + } + // or take a plain nested block apart + for (size_t i = 0; i < k.size(); ++i) { + if (k[i]->kind != SK::Block || k[i]->kids.empty()) continue; + std::set inner = declaredIn(f, *k[i]); + bool clash = false; + for (size_t q = 0; q < k.size(); ++q) + if (q != i && k[q]->kind == SK::Decl) + for (auto& d : k[q]->decls) clash |= inner.count(f.t(d.nameTok)) > 0; + if (clash) continue; + std::vector inside; + for (auto& s : k[i]->kids) inside.push_back(s.get()); + if (!emitReplace(f, emit, k[i]->begin, k[i]->end, stmtsText(f, inside, lineInd(f, k[i]->begin)))) return; + } + } +} + +// --- local_type --------------------------------------------------------------- + +const std::vector>& typeFamilies() { + static const std::vector> fams = { + {"s8", "u8", "s16", "u16", "s32", "u32"}, + {"char", "unsigned char", "signed char", "short", "unsigned short", "int", "unsigned int", "long", + "unsigned long", "unsigned"}, + {"BYTE", "WORD", "DWORD", "INT", "UINT", "SHORT", "USHORT", "LONG", "ULONG"}, + }; + return fams; +} + +void enumLocalType(const Func& f, const EmitFn& emit) { + std::vector decls; + forEachStmt(*f.body, [&](const Stmt& s) { + if (s.kind == SK::Decl) decls.push_back(&s); + if (s.forInit) decls.push_back(s.forInit.get()); + }); + for (const Stmt* s : decls) { + if (s->isStatic) continue; + int b = s->typeBegin, e = s->typeEnd; + auto cv = [&](int i) { + const std::string& t = f.t(i); + return t == "const" || t == "volatile" || t == "register"; + }; + while (b < e && cv(b)) b++; + while (e > b && cv(e - 1)) e--; + if (b >= e) continue; + std::string core; + for (int i = b; i < e; ++i) core += (i > b ? " " : "") + f.t(i); + for (auto& fam : typeFamilies()) { + if (std::find(fam.begin(), fam.end(), core) == fam.end()) continue; + for (auto& alt : fam) { + if (alt == core || alt == "unsigned") continue; + if (!emitReplace(f, emit, b, e, alt)) return; + } + } + } +} + +// --- locals_to_array ---------------------------------------------------------- + +void enumLocalsToArray(const Func& f, const EmitFn& emit) { + for (const Stmt* b : blocksOf(f)) { + auto& k = b->kids; + auto eligible = [&](const Stmt& s) { + if (s.kind != SK::Decl || s.isStatic || s.isConst || s.decls.size() != 1) return false; + const Declarator& d = s.decls[0]; + return !d.isArray && !d.isRef && d.init != Declarator::Paren && d.init != Declarator::Brace && + !(d.init == Declarator::Assign && f.t(d.initBegin) == "{"); + }; + auto typeKey = [&](const Stmt& s) { + return norm(f.tokText(s.typeBegin, s.typeEnd) + f.tokText(s.decls[0].begin, s.decls[0].nameTok)); + }; + for (size_t i = 0; i < k.size(); ++i) { + if (!eligible(*k[i])) continue; + size_t j = i + 1; + while (j < k.size() && eligible(*k[j]) && typeKey(*k[j]) == typeKey(*k[i])) j++; + if (j - i < 2) continue; + std::vector run; + for (size_t q = i; q < j; ++q) run.push_back(k[q].get()); + // the whole run and each adjacent pair, in order and back to front + std::vector> spans = {{0, run.size()}}; + if (run.size() > 2) + for (size_t q = 0; q + 1 < run.size(); ++q) spans.push_back({q, q + 2}); + for (auto [lo, hi] : spans) { + std::vector part(run.begin() + lo, run.begin() + hi); + std::set names; + for (auto* s : part) names.insert(f.t(s->decls[0].nameTok)); + bool selfRef = false; + for (auto* s : part) + if (s->decls[0].init == Declarator::Assign) + for (auto& n : names) selfRef |= !usesOf(f, n, s->decls[0].initBegin, s->decls[0].initEnd).empty(); + if (selfRef) continue; + std::string arr = f.t(part[0]->decls[0].nameTok) + "_arr"; + while (f.locals.count(arr)) arr += "_"; + for (int reversed = 0; reversed < 2; ++reversed) { + std::vector idx; + for (size_t q = 0; q < part.size(); ++q) + idx.push_back(arr + "[" + std::to_string(reversed ? part.size() - 1 - q : q) + "]"); + const Stmt* first = part.front(); + std::string ind = "\n" + lineInd(f, first->begin); + std::string text = f.tokText(first->typeBegin, first->typeEnd) + " " + + f.tokText(first->decls[0].begin, first->decls[0].nameTok) + arr + "[" + + std::to_string(part.size()) + "];"; + for (size_t q = 0; q < part.size(); ++q) + if (part[q]->decls[0].init == Declarator::Assign) + text += ind + idx[q] + " = " + + f.tokText(part[q]->decls[0].initBegin, part[q]->decls[0].initEnd) + ";"; + std::vector> subst; + for (size_t q = 0; q < part.size(); ++q) + for (int u : usesOf(f, f.t(part[q]->decls[0].nameTok), part.back()->end, f.bodyClose)) + subst.push_back({u, idx[q]}); + int pb = part.front()->begin, pe = part.back()->end; + if (!emit([&f, pb, pe, text, subst]() { + Rewriter rw(f); + rw.replace(pb, pe, text); + for (auto& [u, t] : subst) rw.replace(u, u + 1, t); + return rw.apply(); + })) + return; + } + } + i = j - 1; + } + } +} + +// --- reorder_cases / split_case_labels ------------------------------------------ + +struct CaseGroup { + std::vector labels, body; +}; + +// The case groups of a switch body, or false if it has statements before the +// first label. +bool caseGroups(const Stmt& sw, std::vector& out) { + if (sw.kids.empty() || sw.kids[0]->kind != SK::Block) return false; + out.clear(); + for (auto& k : sw.kids[0]->kids) { + bool label = k->kind == SK::Case || k->kind == SK::Default; + if (label && (out.empty() || !out.back().body.empty())) out.push_back({}); + if (out.empty()) return false; + (label ? out.back().labels : out.back().body).push_back(k.get()); + } + return !out.empty(); +} + +std::string groupText(const Func& f, const CaseGroup& g, bool addBreak) { + const Stmt* last = g.body.empty() ? g.labels.back() : g.body.back(); + std::string t = f.tokText(g.labels.front()->begin, last->end); + if (addBreak) t += "\n" + lineInd(f, last->begin) + (g.body.empty() ? " " : "") + "break;"; + return t; +} + +void enumReorderCases(const Func& f, const EmitFn& emit) { + for (const Stmt* sw : stmtsOfKind(f, SK::Switch)) { + std::vector gs; + if (!caseGroups(*sw, gs) || gs.size() < 2) continue; + bool ok = true; + for (size_t i = 0; i + 1 < gs.size(); ++i) ok &= !gs[i].body.empty() && isJump(gs[i].body.back()); + if (!ok) continue; // fall-through: order matters + bool lastOpen = gs.back().body.empty() || !isJump(gs.back().body.back()); + size_t n = gs.size(); + auto emitOrder = [&](const std::vector& ord) { + std::vector, std::string>> edits; + for (size_t slot = 0; slot < n; ++slot) { + if (ord[slot] == slot) continue; + const CaseGroup& g = gs[ord[slot]]; + const CaseGroup& at = gs[slot]; + const Stmt* atLast = at.body.empty() ? at.labels.back() : at.body.back(); + edits.push_back({{at.labels.front()->begin, atLast->end}, + groupText(f, g, lastOpen && ord[slot] == n - 1)}); + } + return emit([&f, edits]() { + Rewriter rw(f); + for (auto& [r, t] : edits) rw.replace(r.first, r.second, t); + return rw.apply(); + }); + }; + std::vector ord(n); + for (size_t i = 0; i < n; ++i) ord[i] = i; + if (n <= 5) { + while (std::next_permutation(ord.begin(), ord.end())) + if (!emitOrder(ord)) return; + } else { + for (size_t a = 0; a < n; ++a) + for (size_t c = 0; c < n; ++c) { + if (a == c) continue; + std::vector o2; + for (size_t i = 0; i < n; ++i) + if (i != a) o2.push_back(i); + o2.insert(o2.begin() + c, a); + if (!emitOrder(o2)) return; + } + } + } +} + +void enumSplitCaseLabels(const Func& f, const EmitFn& emit) { + for (const Stmt* sw : stmtsOfKind(f, SK::Switch)) { + std::vector gs; + if (!caseGroups(*sw, gs)) continue; + for (size_t i = 0; i < gs.size(); ++i) { + const CaseGroup& g = gs[i]; + bool closed = !g.body.empty() && isJump(g.body.back()); + // case 1: case 2: body -> case 1: body case 2: body + if (g.labels.size() >= 2 && (closed || i + 1 == gs.size()) && !g.body.empty()) { + std::string li = lineInd(f, g.labels.front()->begin), bi = lineInd(f, g.body.front()->begin); + std::string body = stmtsText(f, g.body, bi); + std::string text; + for (size_t q = 0; q < g.labels.size(); ++q) { + if (q) text += "\n" + li; + text += f.textOf(*g.labels[q]) + "\n" + bi + body; + if (!closed && q + 1 < g.labels.size()) text += "\n" + bi + "break;"; + } + if (!emitReplace(f, emit, g.labels.front()->begin, g.body.back()->end, text)) return; + } + // two groups with the same body -> one group with both labels + if (i + 1 < gs.size() && closed && !gs[i + 1].body.empty()) { + const CaseGroup& h = gs[i + 1]; + if (norm(f.tokText(g.body.front()->begin, g.body.back()->end)) != + norm(f.tokText(h.body.front()->begin, h.body.back()->end))) + continue; + std::string li = lineInd(f, g.labels.front()->begin); + std::string text = f.tokText(g.labels.front()->begin, g.labels.back()->end); + for (auto* l : h.labels) text += "\n" + li + f.textOf(*l); + text += "\n" + lineInd(f, g.body.front()->begin) + + stmtsText(f, g.body, lineInd(f, g.body.front()->begin)); + if (!emitReplace(f, emit, g.labels.front()->begin, h.body.back()->end, text)) return; + } + } + } +} + +// --- use_getter ----------------------------------------------------------------- + +void enumUseGetter(const Func& f, const EmitFn& emit) { + const auto& gs = getters(); + if (gs.empty()) return; + std::map> byExpr, byName; + for (auto& g : gs) { + byExpr[g.expr].push_back(&g); + byName[g.name].push_back(&g); + } + std::set> done; + forEachSiteExpr(f, [&](const ExprSite&, std::shared_ptr, Expr& x) { + // obj->path -> obj->getter() (and a bare member path in a method) + if (x.k == Expr::Member || (x.k == Expr::Primary && !f.isLocal(x.b))) { + std::string whole = norm(exprText(f, x)); + // try every split "obj" + "->"/"." + "path" + for (int i = x.b; i < x.e; ++i) { + bool sep = f.t(i) == "->" || f.t(i) == "."; + int from = sep ? i + 1 : (i == x.b && x.k == Expr::Primary ? i : -1); + if (from < 0) continue; + auto it = byExpr.find(norm(f.tokText(from, x.e))); + if (it == byExpr.end()) continue; + std::string obj = sep ? f.tokText(x.b, i) + f.t(i) : ""; + for (auto* g : it->second) { + std::string text = obj + g->name + "()"; + if (!done.insert({x.b, text}).second) continue; + if (!emitReplace(f, emit, x.b, x.e, text)) return false; + } + } + (void)whole; + } + // obj->getter() -> obj->path + if (x.k == Expr::Call && x.kids.size() == 1) { + const Expr& callee = *x.kids[0]; + int nameTok = callee.e - 1; + auto it = byName.find(f.t(nameTok)); + if (it == byName.end()) return true; + std::string obj = callee.k == Expr::Member ? f.tokText(callee.b, nameTok) : ""; + for (auto* g : it->second) { + std::string path; + // the stored path has no spaces; it is valid code as is + path = g->expr; + if (!emitReplace(f, emit, x.b, x.e, obj + path)) return false; + } + } + return true; + }); +} + +// --- temp_for_expr ------------------------------------------------------------- + +// Declared types of the locals and parameters: "s32", "Ped*", ... +std::map localTypes(const Func& f) { + std::map out; + forEachStmt(*f.body, [&](const Stmt& s) { + std::vector ds; + if (s.kind == SK::Decl) ds.push_back(&s); + if (s.forInit) ds.push_back(s.forInit.get()); + for (auto* d : ds) { + if (d->isStatic) continue; + int b = d->typeBegin; + while (b < d->typeEnd && (f.t(b) == "register" || f.t(b) == "const" || f.t(b) == "volatile")) b++; + for (auto& dc : d->decls) + if (!dc.isArray && !dc.isRef) + out[f.t(dc.nameTok)] = f.tokText(b, d->typeEnd) + f.tokText(dc.begin, dc.nameTok); + } + }); + int seg = f.paramOpen + 1, depth = 0; + for (int i = f.paramOpen + 1; i <= f.paramClose; ++i) { + const std::string& t = f.t(i); + if (i < f.paramClose && (t == "(" || t == "[")) depth++; + if (i < f.paramClose && (t == ")" || t == "]")) depth--; + if ((t == "," && depth == 0) || i == f.paramClose) { + int name = -1; + for (int j = seg; j < i && f.t(j) != "="; ++j) + if (f.toks[j].kind == TokKind::Ident && !isKeyword(f.t(j))) name = j; + bool simple = name > seg; + for (int j = seg; j < name && simple; ++j) simple = f.t(j) != "&" && f.t(j) != "("; + if (simple && f.locals.count(f.t(name))) { + int b = seg; + while (b < name && (f.t(b) == "const" || f.t(b) == "volatile")) b++; + out[f.t(name)] = f.tokText(b, name); + } + seg = i + 1; + } + } + for (auto& [n, t] : out) { + std::string c = t; + while (!c.empty() && c.back() == ' ') c.pop_back(); + t = c; + } + return out; +} + +void enumTempForExpr(const Func& f, const EmitFn& emit) { + auto types = localTypes(f); + int n = 0; + std::string tmp = "tmp"; + while (f.locals.count(tmp)) tmp = "tmp" + std::to_string(++n); + for (auto& site : exprSites(f)) { + const Stmt& s = *site.stmt; + if (!inBlock(s) || s.kind == SK::While || s.kind == SK::For || s.kind == SK::Do) continue; + auto root = parseExpr(f, site.b, site.e); + if (!root) continue; + std::vector> picks; + forEachExpr(*root, [&](Expr& x) { + if (x.k == Expr::Cast && f.t(x.b) == "(") { + int close = matchBracket(f.toks, x.b); + if (close > 0 && close < x.e) picks.push_back({&x, f.tokText(x.b + 1, close)}); + } else if (x.k == Expr::Primary && f.isLocal(x.b) && x.e == x.b + 1 && !isWriteUse(f, x.b) && + types.count(f.t(x.b))) { + picks.push_back({&x, types[f.t(x.b)]}); + } else if (x.k == Expr::Unary && f.t(x.op) == "*" && x.kids[0]->k == Expr::Primary && + f.isLocal(x.kids[0]->b) && types.count(f.t(x.kids[0]->b))) { + std::string t = types[f.t(x.kids[0]->b)]; + if (!t.empty() && t.back() == '*') { + t.pop_back(); + while (!t.empty() && t.back() == ' ') t.pop_back(); + // "*p" as an assignment target isn't a value to copy + if (!isAssignOp(f.t(x.e))) picks.push_back({&x, t}); + } + } + }); + for (auto& [x, type] : picks) { + if (x->b == site.b && x->e == site.e && s.kind == SK::Expr) continue; + std::string decl = type + " " + tmp + " = " + exprText(f, *x) + ";"; + int b = x->b, e = x->e, at = s.begin; + std::string name = tmp; + if (!emit([&f, b, e, at, decl, name]() { + Rewriter rw(f); + rw.insertBefore(at, decl); + rw.replace(b, e, name); + return rw.apply(); + })) + return; + } + } +} + +// --- remove_stmt ---------------------------------------------------------------- + +void enumRemoveStmt(const Func& f, const EmitFn& emit) { + for (const Stmt* s : stmtsOfKind(f, SK::Expr)) { + if (!inBlock(*s)) continue; + const Stmt* sp = s; + if (!emit([&f, sp]() { + Rewriter rw(f); + rw.remove(sp->begin, sp->end); + return rw.apply(); + })) + return; + } +} + +} // namespace + +void registerMorePasses(std::vector& passes) { + passes.push_back({"inequalities", "'x > 4' <-> 'x >= 5', 'x < 4' <-> 'x <= 3' (integer constants)", 6, + enumInequalities, nullptr}); + passes.push_back({"chain_assign", + "'a = v; b = v;' <-> 'a = b = v;' (and 'b = v; a = b;'): which register holds " + "the shared value", + 8, enumChainAssign, nullptr}); + passes.push_back({"scope_block", + "Wrap a declaration and the statements using it in their own { } block (VC6 " + "shares stack slots between blocks), or take a nested block apart", + 8, enumScopeBlock, nullptr}); + passes.push_back({"local_type", "Change a local's integer type: s32 <-> u32 <-> s16 <-> u8 ... (or the " + "builtin / Windows equivalents)", + 8, enumLocalType, nullptr}); + passes.push_back({"locals_to_array", + "Turn consecutive locals of one type into an array, in order or back to front " + "(stack slot order)", + 4, enumLocalsToArray, nullptr}); + passes.push_back({"reorder_cases", "Reorder a switch's case groups (case bodies are laid out in source order)", + 8, enumReorderCases, nullptr}); + passes.push_back({"split_case_labels", + "'case 1: case 2: S' <-> 'case 1: S case 2: S' (a byte index table vs one entry " + "per case)", + 5, enumSplitCaseLabels, nullptr}); + passes.push_back({"use_getter", + "'p->field.x' <-> 'p->get_x()' using the inline getters in the headers the " + "source includes", + 8, enumUseGetter, nullptr}); + passes.push_back({"temp_for_expr", + "Compute a cast, a local or '*p' into a new local first (types from the " + "declarations; also 'copy through a local to get a spill')", + 8, enumTempForExpr, nullptr}); + passes.push_back({"remove_stmt", "Remove an expression statement", 2, enumRemoveStmt, nullptr}); +} + +} // namespace util + +} // namespace perm diff --git a/src/runner.cpp b/src/runner.cpp new file mode 100644 index 0000000..a14c3b2 --- /dev/null +++ b/src/runner.cpp @@ -0,0 +1,222 @@ +#include "runner.hpp" + +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#else +#include +#include +#include +#include +#include +#include +#endif + +namespace perm { + +#ifdef _WIN32 + +// cmd.exe /S /C "cmd", stdout and stderr into one pipe. The process runs in a +// job object so a timeout kills everything it started. +CmdResult runCommand(const std::string& cmd, int timeoutSec, + const std::map& env) { + CmdResult r; + SECURITY_ATTRIBUTES sa{sizeof sa, nullptr, TRUE}; + HANDLE readEnd = nullptr, writeEnd = nullptr; + if (!CreatePipe(&readEnd, &writeEnd, &sa, 0)) { + r.output = "CreatePipe failed"; + return r; + } + SetHandleInformation(readEnd, HANDLE_FLAG_INHERIT, 0); + HANDLE nul = CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, nullptr); + + // environment block: ours plus the extra variables + std::string block; + if (!env.empty()) { + LPCH cur = GetEnvironmentStringsA(); + for (LPCH p = cur; *p; p += strlen(p) + 1) { + std::string kv = p; + size_t eq = kv.find('=', 1); + if (eq != std::string::npos && env.count(kv.substr(0, eq))) continue; + block += kv; + block += '\0'; + } + FreeEnvironmentStringsA(cur); + for (auto& [k, v] : env) { + block += k + "=" + v; + block += '\0'; + } + block += '\0'; + } + + STARTUPINFOA si{}; + si.cb = sizeof si; + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdOutput = writeEnd; + si.hStdError = writeEnd; + si.hStdInput = nul; + PROCESS_INFORMATION pi{}; + std::string line = "cmd.exe /S /C \"" + cmd + "\""; + std::vector buf(line.begin(), line.end()); + buf.push_back('\0'); + HANDLE job = CreateJobObjectA(nullptr, nullptr); + JOBOBJECT_EXTENDED_LIMIT_INFORMATION lim{}; + lim.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (job) SetInformationJobObject(job, JobObjectExtendedLimitInformation, &lim, sizeof lim); + BOOL ok = CreateProcessA(nullptr, buf.data(), nullptr, nullptr, TRUE, CREATE_SUSPENDED | CREATE_NO_WINDOW, + block.empty() ? nullptr : (LPVOID)block.data(), nullptr, &si, &pi); + CloseHandle(writeEnd); + if (nul != INVALID_HANDLE_VALUE) CloseHandle(nul); + if (!ok) { + CloseHandle(readEnd); + if (job) CloseHandle(job); + r.output = "CreateProcess failed"; + return r; + } + if (job) AssignProcessToJobObject(job, pi.hProcess); + ResumeThread(pi.hThread); + + // wait for the process itself, not for the pipe to close: anything it + // left running would keep the pipe open + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeoutSec); + auto drain = [&]() { + DWORD avail = 0; + while (PeekNamedPipe(readEnd, nullptr, 0, nullptr, &avail, nullptr) && avail > 0) { + char chunk[8192]; + DWORD got = 0; + if (!ReadFile(readEnd, chunk, (DWORD)std::min(avail, sizeof chunk), &got, nullptr) || !got) break; + r.output.append(chunk, got); + } + }; + while (true) { + drain(); + if (WaitForSingleObject(pi.hProcess, 20) == WAIT_OBJECT_0) break; + if (timeoutSec > 0 && std::chrono::steady_clock::now() >= deadline) { + r.timedOut = true; + if (job) TerminateJobObject(job, 1); + TerminateProcess(pi.hProcess, 1); + WaitForSingleObject(pi.hProcess, 5000); + break; + } + } + drain(); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + r.status = r.timedOut ? -1 : (int)code; + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + CloseHandle(readEnd); + if (job) CloseHandle(job); + return r; +} + +std::string shellQuote(const std::string& s) { + // for cmd.exe and the usual argv parsing: "..." with \" for quotes + std::string r = "\""; + for (char c : s) { + if (c == '"') r += "\\\""; + else r += c; + } + return r + "\""; +} + +#else + +CmdResult runCommand(const std::string& cmd, int timeoutSec, + const std::map& env) { + CmdResult r; + int fds[2]; + if (pipe(fds) != 0) { + r.output = "pipe() failed"; + return r; + } + pid_t pid = fork(); + if (pid < 0) { + close(fds[0]); + close(fds[1]); + r.output = "fork() failed"; + return r; + } + if (pid == 0) { + setpgid(0, 0); + dup2(fds[1], 1); + dup2(fds[1], 2); + close(fds[0]); + close(fds[1]); + int devnull = open("/dev/null", O_RDONLY); + if (devnull >= 0) dup2(devnull, 0); + for (auto& [k, v] : env) setenv(k.c_str(), v.c_str(), 1); + execl("/bin/sh", "sh", "-c", cmd.c_str(), (char*)nullptr); + _exit(127); + } + close(fds[1]); + // Wait for the child itself, not for EOF: daemons it spawns (wineserver) + // inherit the pipe and can keep it open long after the compile is done. + fcntl(fds[0], F_SETFL, fcntl(fds[0], F_GETFL) | O_NONBLOCK); + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeoutSec); + char buf[8192]; + int st = 0; + bool exited = false; + auto drain = [&]() { + ssize_t n; + while ((n = read(fds[0], buf, sizeof buf)) > 0) r.output.append(buf, (size_t)n); + }; + while (!exited) { + pollfd p{fds[0], POLLIN, 0}; + if (poll(&p, 1, 20) > 0 && (p.revents & POLLHUP)) usleep(5000); // closed: don't spin + drain(); + pid_t w = waitpid(pid, &st, WNOHANG); + if (w == pid) exited = true; + else if (timeoutSec > 0 && std::chrono::steady_clock::now() >= deadline) { + r.timedOut = true; + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + while (waitpid(pid, &st, 0) < 0 && errno == EINTR) {} + exited = true; + } + } + drain(); + close(fds[0]); + r.status = WIFEXITED(st) ? WEXITSTATUS(st) : -1; + if (r.timedOut) r.status = -1; + return r; +} + +std::string shellQuote(const std::string& s) { + std::string r = "'"; + for (char c : s) { + if (c == '\'') r += "'\\''"; + else r += c; + } + return r + "'"; +} + +#endif + +std::string expandTemplate(const std::string& tmpl, const std::map& vars) { + std::string out; + for (size_t i = 0; i < tmpl.size();) { + if (tmpl[i] == '{') { + size_t e = tmpl.find('}', i); + if (e != std::string::npos) { + auto it = vars.find(tmpl.substr(i + 1, e - i - 1)); + if (it != vars.end()) { + out += shellQuote(it->second); + i = e + 1; + continue; + } + } + } + out += tmpl[i++]; + } + return out; +} + +} // namespace perm diff --git a/src/runner.hpp b/src/runner.hpp new file mode 100644 index 0000000..a7c05d5 --- /dev/null +++ b/src/runner.hpp @@ -0,0 +1,24 @@ +// Running shell commands with a timeout. +#pragma once + +#include +#include + +namespace perm { + +struct CmdResult { + int status = -1; // exit code, or -1 if it didn't exit normally + bool timedOut = false; + std::string output; // stdout and stderr together +}; + +// Runs cmd through the shell, with extra environment variables set. +CmdResult runCommand(const std::string& cmd, int timeoutSec, + const std::map& env = {}); + +std::string shellQuote(const std::string& s); + +// Replaces {key} in tmpl with the shell-quoted value. +std::string expandTemplate(const std::string& tmpl, const std::map& vars); + +} // namespace perm diff --git a/src/scorer.cpp b/src/scorer.cpp new file mode 100644 index 0000000..2efc692 --- /dev/null +++ b/src/scorer.cpp @@ -0,0 +1,359 @@ +#include "scorer.hpp" + +#include "runner.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace perm { + +namespace { + +std::string trim(const std::string& s) { + size_t b = s.find_first_not_of(" \t\r"); + if (b == std::string::npos) return ""; + size_t e = s.find_last_not_of(" \t\r"); + return s.substr(b, e - b + 1); +} + +bool isHex(const std::string& s) { + return !s.empty() && std::all_of(s.begin(), s.end(), [](char c) { return std::isxdigit((unsigned char)c); }); +} + +struct RawInsn { + unsigned addr = 0; + std::vector bytes; + std::string mnem, args; + std::vector> relocs; // offset, symbol + std::vector relocTypes; +}; + +// Replaces the first whole-word occurrence of word in s. +bool replaceWord(std::string& s, const std::string& word, const std::string& with) { + size_t pos = 0; + while ((pos = s.find(word, pos)) != std::string::npos) { + bool lb = pos == 0 || !(std::isalnum((unsigned char)s[pos - 1]) || s[pos - 1] == '_'); + size_t e = pos + word.size(); + bool rb = e >= s.size() || !(std::isalnum((unsigned char)s[e]) || s[e] == '_'); + if (lb && rb) { + s.replace(pos, word.size(), with); + return true; + } + pos = e; + } + return false; +} + +std::string hexStr(unsigned v) { + char b[16]; + std::snprintf(b, sizeof b, "0x%x", v); + return b; +} + +} // namespace + +std::vector parseObjdump(const std::string& text, const std::string& symbol, + bool ignoreRelocNames) { + std::vector raw; + std::istringstream in(text); + std::string line; + bool inside = false; + std::string header = "<" + symbol + ">:"; + while (std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.find(">:") != std::string::npos && line.find('<') != std::string::npos && + !line.empty() && line[0] != ' ' && line[0] != '\t') { + inside = line.find(header) != std::string::npos; + continue; + } + if (!inside) continue; + std::string t = trim(line); + if (t.empty()) continue; + size_t colon = t.find(':'); + if (colon == std::string::npos || !isHex(t.substr(0, colon))) continue; + unsigned addr = (unsigned)std::stoul(t.substr(0, colon), nullptr, 16); + std::string rest = trim(t.substr(colon + 1)); + // relocation line: "IMAGE_REL_I386_DIR32\tsym" / "R_386_32\tsym" + if (rest.rfind("IMAGE_REL_", 0) == 0 || rest.rfind("R_386", 0) == 0 || + rest.rfind("R_X86", 0) == 0 || rest.rfind("dir32", 0) == 0 || + rest.rfind("DISP32", 0) == 0) { + size_t sp = rest.find_first_of(" \t"); + std::string type = rest.substr(0, sp); + std::string sym = sp == std::string::npos ? "" : trim(rest.substr(sp)); + if (!raw.empty()) { + raw.back().relocs.push_back({addr, sym}); + raw.back().relocTypes.push_back(type); + } + continue; + } + RawInsn r; + r.addr = addr; + // raw bytes are pairs of hex digits separated by single spaces, then a tab + size_t p = 0; + while (p + 2 <= rest.size() && isHex(rest.substr(p, 2)) && + (p + 2 == rest.size() || rest[p + 2] == ' ' || rest[p + 2] == '\t')) { + r.bytes.push_back((unsigned char)std::stoul(rest.substr(p, 2), nullptr, 16)); + p += 2; + if (p < rest.size() && rest[p] == '\t') break; + while (p < rest.size() && rest[p] == ' ') p++; + if (p < rest.size() && rest[p] == '\t') break; + } + std::string ins = trim(rest.substr(p)); + if (ins.empty()) { + // continuation of a long byte sequence (binutils) + if (!raw.empty()) raw.back().bytes.insert(raw.back().bytes.end(), r.bytes.begin(), r.bytes.end()); + continue; + } + size_t sp = ins.find_first_of(" \t"); + r.mnem = ins.substr(0, sp); + r.args = sp == std::string::npos ? "" : trim(ins.substr(sp)); + // prefixes such as "lock", "rep" are printed as their own mnemonic + raw.push_back(std::move(r)); + } + + std::map indexOf; + for (size_t i = 0; i < raw.size(); ++i) indexOf[raw[i].addr] = (int)i; + + std::vector out; + for (size_t i = 0; i < raw.size(); ++i) { + RawInsn& r = raw[i]; + std::string args = r.args; + size_t lt = args.find(" <"); + if (lt != std::string::npos) args = trim(args.substr(0, lt)); + bool branch = r.mnem[0] == 'j' || r.mnem.rfind("call", 0) == 0 || r.mnem.rfind("loop", 0) == 0; + bool handled = false; + for (size_t k = 0; k < r.relocs.size(); ++k) { + std::string sym = ignoreRelocNames ? "" : r.relocs[k].second; + bool rel = r.relocTypes[k].find("REL32") != std::string::npos || + r.relocTypes[k].find("PC32") != std::string::npos || + r.relocTypes[k] == "DISP32" || r.relocTypes[k].find("PLT") != std::string::npos; + if (rel && branch) { + args = sym; + handled = true; + continue; + } + unsigned off = r.relocs[k].first - r.addr; + unsigned addend = 0; + for (unsigned b = 0; b < 4 && off + b < r.bytes.size(); ++b) + addend |= (unsigned)r.bytes[off + b] << (8 * b); + std::string v = hexStr(addend); + if (!replaceWord(args, v, addend ? sym + "+" + v : sym)) args += " ; " + sym; + } + if (branch && !handled && r.relocs.empty() && args.rfind("0x", 0) == 0) { + unsigned target = (unsigned)std::stoul(args, nullptr, 16); + auto it = indexOf.find(target); + args = it != indexOf.end() ? "L" + std::to_string(it->second) : args; + } + if (ignoreRelocNames && r.relocs.empty()) { + // absolute addresses in code taken straight from an executable + std::string res; + size_t q = 0; + while (q < args.size()) { + if (args.compare(q, 2, "0x") == 0) { + size_t e = q + 2; + while (e < args.size() && std::isxdigit((unsigned char)args[e])) e++; + unsigned long v = std::stoul(args.substr(q + 2, e - q - 2), nullptr, 16); + res += v >= 0x400000 && v < 0x10000000 ? "" : args.substr(q, e - q); + q = e; + } else { + res += args[q++]; + } + } + args = res; + } + out.push_back({r.mnem, args}); + } + return out; +} + +bool disassemble(const ScoreConfig& cfg, const std::string& obj, const std::string& symbol, + std::vector& out, std::string& err) { + std::string cmd = shellQuote(cfg.objdump) + " -d -r --disassemble-symbols=" + + shellQuote(symbol) + " " + shellQuote(obj); + CmdResult r = runCommand(cmd, 60); + if (r.status != 0) { + err = "objdump failed: " + r.output; + return false; + } + out = parseObjdump(r.output, symbol, cfg.ignoreRelocNames); + if (out.empty()) { + err = "symbol '" + symbol + "' not found in " + obj; + return false; + } + return true; +} + +namespace { + +std::string squash(const std::string& s, bool numbers) { + std::string r; + for (size_t i = 0; i < s.size();) { + if (s[i] == '%') { + r += "%r"; + i++; + while (i < s.size() && std::isalnum((unsigned char)s[i])) i++; + } else if (numbers && std::isdigit((unsigned char)s[i])) { + r += "N"; + while (i < s.size() && (std::isxdigit((unsigned char)s[i]) || s[i] == 'x')) i++; + } else { + r += s[i++]; + } + } + return r; +} + +enum Op { Same, Sub, Del, Ins }; + +struct Alignment { + int score = 0; + std::vector>> steps; +}; + +int subCost(const ScoreConfig& cfg, const Insn& a, const Insn& b) { + if (a.mnem != b.mnem) return INT_MAX / 4; + if (a.args == b.args) return 0; + if (squash(a.args, false) == squash(b.args, false)) return cfg.penaltyRegalloc; + return cfg.penaltyArgs; +} + +Alignment align(const ScoreConfig& cfg, const std::vector& a, const std::vector& b) { + size_t n = a.size(), m = b.size(); + std::vector dp((n + 1) * (m + 1)); + auto at = [&](size_t i, size_t j) -> int& { return dp[i * (m + 1) + j]; }; + for (size_t i = 0; i <= n; ++i) at(i, 0) = (int)i * cfg.penaltyDelete; + for (size_t j = 0; j <= m; ++j) at(0, j) = (int)j * cfg.penaltyInsert; + for (size_t i = 1; i <= n; ++i) + for (size_t j = 1; j <= m; ++j) { + int best = std::min(at(i - 1, j) + cfg.penaltyDelete, at(i, j - 1) + cfg.penaltyInsert); + int s = subCost(cfg, a[i - 1], b[j - 1]); + if (s < INT_MAX / 4) best = std::min(best, at(i - 1, j - 1) + s); + at(i, j) = best; + } + Alignment al; + size_t i = n, j = m; + while (i > 0 || j > 0) { + if (i > 0 && j > 0) { + int s = subCost(cfg, a[i - 1], b[j - 1]); + if (s < INT_MAX / 4 && at(i, j) == at(i - 1, j - 1) + s) { + al.steps.push_back({s == 0 ? Same : Sub, {(int)i - 1, (int)j - 1}}); + i--, j--; + continue; + } + } + if (i > 0 && at(i, j) == at(i - 1, j) + cfg.penaltyDelete) { + al.steps.push_back({Del, {(int)i - 1, -1}}); + i--; + } else { + al.steps.push_back({Ins, {-1, (int)j - 1}}); + j--; + } + } + std::reverse(al.steps.begin(), al.steps.end()); + al.score = at(n, m); + return al; +} + +} // namespace + +int scoreInsns(const ScoreConfig& cfg, const std::vector& target, const std::vector& cand) { + Alignment al = align(cfg, target, cand); + // a deletion and an insertion of the same instruction is a reordering + std::multiset dels, ins; + for (auto& s : al.steps) { + if (s.first == Del) dels.insert(target[s.second.first].full()); + if (s.first == Ins) ins.insert(cand[s.second.second].full()); + } + int score = al.score; + for (auto& d : dels) { + auto it = ins.find(d); + if (it == ins.end()) continue; + ins.erase(it); + score -= cfg.penaltyDelete + cfg.penaltyInsert - cfg.penaltyReorder; + } + return score; +} + +std::string diffInsns(const ScoreConfig& cfg, const std::vector& target, + const std::vector& cand) { + Alignment al = align(cfg, target, cand); + std::ostringstream o; + char buf[512]; + std::snprintf(buf, sizeof buf, " %-44s %s\n", "TARGET", "CANDIDATE"); + o << buf; + for (auto& s : al.steps) { + std::string l = s.second.first >= 0 ? target[s.second.first].full() : ""; + std::string r = s.second.second >= 0 ? cand[s.second.second].full() : ""; + const char* mark = s.first == Same ? " " : s.first == Sub ? "r " : s.first == Del ? "- " : "+ "; + std::snprintf(buf, sizeof buf, "%s %-44s %s\n", mark, l.c_str(), r.c_str()); + o << buf; + } + return o.str(); +} + +std::vector listSymbols(const ScoreConfig& cfg, const std::string& obj, std::string& err) { + std::vector out; + CmdResult r = runCommand(shellQuote(cfg.objdump) + " -t " + shellQuote(obj), 60); + if (r.status != 0) { + err = r.output; + return out; + } + std::istringstream in(r.output); + std::string line; + while (std::getline(in, line)) { + // COFF: "[ 5](sec 1)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 ?foo@@YAHXZ" + // ELF: "00000000 g F .text 0000002a foo" + if (line.find("(sec") != std::string::npos || line.find(" F ") != std::string::npos || + line.find(" g ") != std::string::npos) { + size_t sp = line.find_last_of(" \t"); + if (sp != std::string::npos) out.push_back(line.substr(sp + 1)); + } + } + return out; +} + +std::string guessSymbol(const std::vector& syms, const std::string& qualName) { + std::vector comps; + size_t i = 0; + while (true) { + size_t j = qualName.find("::", i); + comps.push_back(qualName.substr(i, j == std::string::npos ? std::string::npos : j - i)); + if (j == std::string::npos) break; + i = j + 2; + } + std::string last = comps.back(); + std::vector prefixes; + // MSVC + { + // ctors/dtors are named by their class: "??0G@ns@@" for ns::G::G + std::string p; + if (!last.empty() && last[0] == '~') p = "??1"; + else if (comps.size() >= 2 && comps[comps.size() - 2] == last) p = "??0"; + else p = "?" + last + "@"; + for (size_t k = comps.size() - 1; k-- > 0;) p += comps[k] + "@"; + prefixes.push_back(p + "@"); + } + // Itanium + if (comps.size() == 1) { + prefixes.push_back("_Z" + std::to_string(last.size()) + last); + } else { + std::string p = "_ZN"; + for (auto& c : comps) { + if (!c.empty() && c[0] == '~') p += "D"; + else p += std::to_string(c.size()) + c; + } + prefixes.push_back(p); + } + for (auto& p : prefixes) + for (auto& s : syms) + if (s.rfind(p, 0) == 0) return s; + for (auto& s : syms) + if (s == last || s == "_" + last) return s; + return ""; +} + +} // namespace perm diff --git a/src/scorer.hpp b/src/scorer.hpp new file mode 100644 index 0000000..ea19a9c --- /dev/null +++ b/src/scorer.hpp @@ -0,0 +1,45 @@ +// Disassembles one function from a COFF/ELF object with llvm-objdump and +// scores it against a target (0 = identical after normalisation). +#pragma once + +#include +#include + +namespace perm { + +struct Insn { + std::string mnem, args; + std::string full() const { return args.empty() ? mnem : mnem + " " + args; } +}; + +struct ScoreConfig { + std::string objdump = "llvm-objdump"; + bool ignoreRelocNames = false; // compare relocated operands as "" + int penaltyRegalloc = 10; // same instruction, other registers + int penaltyArgs = 50; // same mnemonic, other operands + int penaltyReorder = 60; // an instruction that moved + int penaltyInsert = 100; + int penaltyDelete = 100; +}; + +bool disassemble(const ScoreConfig& cfg, const std::string& obj, const std::string& symbol, + std::vector& out, std::string& err); + +// Parses llvm-objdump -d -r output for one function. Exposed for tests. +std::vector parseObjdump(const std::string& text, const std::string& symbol, + bool ignoreRelocNames); + +int scoreInsns(const ScoreConfig& cfg, const std::vector& target, + const std::vector& cand); + +// Side-by-side alignment of target and candidate, for humans. +std::string diffInsns(const ScoreConfig& cfg, const std::vector& target, + const std::vector& cand); + +std::vector listSymbols(const ScoreConfig& cfg, const std::string& obj, + std::string& err); + +// Picks the symbol for a qualified C++ name (MSVC or Itanium mangling, or C). +std::string guessSymbol(const std::vector& syms, const std::string& qualName); + +} // namespace perm diff --git a/src/workspace.cpp b/src/workspace.cpp new file mode 100644 index 0000000..2c0a741 --- /dev/null +++ b/src/workspace.cpp @@ -0,0 +1,289 @@ +#include "workspace.hpp" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace perm { + +namespace { + +bool readAll(const std::string& path, std::string& out) { + std::ifstream in(path, std::ios::binary); + if (!in) return false; + std::ostringstream ss; + ss << in.rdbuf(); + out = ss.str(); + return true; +} + +// Absolute and normalised, but symlinks are kept: a source tree made of +// symlinks (like the mirror itself) must not be resolved back to its origin. +std::string canon(const fs::path& p) { return fs::absolute(p).lexically_normal().string(); } + +// Whether the definition starting at byte offset start in src is marked inline. +bool markedInline(const std::vector& toks, size_t start) { + int i = 0; + while (i < (int)toks.size() && toks[i].offset < start) i++; + for (int k = i - 1; k >= 0; --k) { + const Token& t = toks[k]; + if (t.kind == TokKind::PP) break; + if (t.kind == TokKind::Punct && (t.text == ";" || t.text == "}" || t.text == "{")) break; + if (t.text == "inline" || t.text == "__inline" || t.text == "__forceinline") return true; + } + return false; +} + +// Argument counts of every call to name in f's body. +std::set callArgCounts(const Func& f, const std::string& name) { + std::set out; + for (int i = f.bodyOpen + 1; i < f.bodyClose; ++i) { + if (f.t(i) != name || f.t(i + 1) != "(") continue; + int close = matchBracket(f.toks, i + 1); + if (close < 0) continue; + int n = close > i + 2 ? 1 : 0, depth = 0; + for (int j = i + 2; j < close; ++j) { + const std::string& x = f.t(j); + if (x == "(" || x == "[" || x == "{") depth++; + else if (x == ")" || x == "]" || x == "}") depth--; + else if (x == "," && depth == 0) n++; + } + out.insert(n); + } + return out; +} + +// Whether a definition (text starting at its name) takes n arguments, +// counting defaulted and variadic parameters. +bool takesArgs(const std::string& def, int n) { + std::vector t = lex(def); + int open = -1; + for (int i = 0; i < (int)t.size(); ++i) + if (t[i].text == "(") { + open = i; + break; + } + if (open < 0) return true; + int close = matchBracket(t, open); + if (close < 0) return true; + if (close == open + 1 || (close == open + 2 && t[open + 1].text == "void")) return n == 0; + int params = 1, defaults = 0, depth = 0; + bool variadic = false; + for (int j = open + 1; j < close; ++j) { + const std::string& x = t[j].text; + if (x == "(" || x == "[" || x == "{" || x == "<") depth++; + else if (x == ")" || x == "]" || x == "}" || x == ">") depth--; + else if (depth == 0 && x == ",") params++; + else if (depth == 0 && x == "=") defaults++; + else if (x == "...") variadic = true; + } + if (variadic) return n >= params - 1; + return n >= params - defaults && n <= params; +} + +} // namespace + +std::set calledNames(const Func& f) { + std::set out; + for (int i = f.bodyOpen + 1; i < f.bodyClose; ++i) + if (f.toks[i].kind == TokKind::Ident && !isKeyword(f.t(i)) && f.t(i + 1) == "(") + out.insert(f.t(i)); + return out; +} + +std::vector includedFiles(const std::string& file, const std::vector& dirs) { + std::vector out; + std::set seen = {canon(file)}; + std::vector todo = {canon(file)}; + while (!todo.empty()) { + std::string cur = todo.back(); + todo.pop_back(); + std::string text; + if (!readAll(cur, text)) continue; + for (auto& t : lex(text)) { + if (t.kind != TokKind::PP) continue; + size_t inc = t.text.find("include"); + size_t q1 = t.text.find('"'); + if (inc == std::string::npos || q1 == std::string::npos || q1 < inc) continue; + if (t.text.find_first_not_of(" \t", 1) != t.text.find("include")) continue; + size_t q2 = t.text.find('"', q1 + 1); + if (q2 == std::string::npos) continue; + std::string name = t.text.substr(q1 + 1, q2 - q1 - 1); + std::vector tries = {fs::path(cur).parent_path() / name}; + for (auto& d : dirs) tries.push_back(fs::path(d) / name); + for (auto& p : tries) { + if (!fs::is_regular_file(p)) continue; + std::string c = canon(p); + if (seen.insert(c).second) { + out.push_back(c); + todo.push_back(c); + } + break; + } + } + } + return out; +} + +std::vector inlineCallees(const std::string& srcPath, size_t mainStart, size_t mainEnd, + const Func& main, const std::vector& includeDirs, + size_t maxRegions) { + // calls in the order they first appear + std::vector names; + std::set want = calledNames(main); + for (int i = main.bodyOpen + 1; i < main.bodyClose; ++i) + if (want.count(main.t(i)) && main.t(i + 1) == "(" && + std::find(names.begin(), names.end(), main.t(i)) == names.end()) + names.push_back(main.t(i)); + + std::string src = canon(srcPath); + std::vector files = {src}; + for (auto& h : includedFiles(src, includeDirs)) files.push_back(h); + + // every reachable definition, per called name + std::vector> perName; + std::map texts; + std::map> toks; + for (auto& name : names) { + perName.emplace_back(); + std::set counts = callArgCounts(main, name); + for (auto& file : files) { + if (!texts.count(file)) { + readAll(file, texts[file]); + toks[file] = lex(texts[file]); + } + const std::string& text = texts[file]; + if (text.find(name) == std::string::npos) continue; + for (auto [b, e] : findDefinitions(text, name)) { + if (file == src && b == mainStart && e == mainEnd) continue; + if (file == src && !markedInline(toks[file], b)) continue; + // skip overloads no call could reach + bool fits = false; + for (int n : counts) fits |= takesArgs(text.substr(b, e - b), n); + if (!fits) continue; + // the qualified name as written + std::string q = text.substr(b, text.find('(', b) - b); + q.erase(std::remove_if(q.begin(), q.end(), ::isspace), q.end()); + perName.back().push_back({file, b, e, q}); + } + } + } + // the first definition of every name before any second overload, so a + // heavily overloaded name can't use up maxRegions on its own + std::vector out; + for (size_t k = 0;; ++k) { + bool any = false; + for (auto& defs : perName) { + if (k >= defs.size()) continue; + any = true; + const Region& r = defs[k]; + bool dup = false; + for (auto& o : out) dup |= o.file == r.file && o.start < r.end && r.start < o.end; + if (dup) continue; + out.push_back(r); + if (out.size() >= maxRegions) return out; + } + if (!any) break; + } + return out; +} + +std::string spliceRegions(const std::string& original, + std::vector> parts) { + std::sort(parts.begin(), parts.end(), + [](auto& a, auto& b) { return a.first->start < b.first->start; }); + std::string out; + size_t pos = 0; + for (auto& [r, text] : parts) { + out.append(original, pos, r->start - pos); + out += *text; + pos = r->end; + } + out.append(original, pos, std::string::npos); + return out; +} + +bool Mirror::create(const std::string& root, const std::string& dir, + const std::vector& realFiles, std::string& err) { + root_ = canon(root); + dir_ = fs::absolute(dir).string(); + std::error_code ec; + fs::remove_all(dir_, ec); + fs::create_directories(dir_, ec); + if (ec) { + err = "can't create " + dir_ + ": " + ec.message(); + return false; + } + // Each entry becomes a symlink. Where symlinks aren't allowed (Windows + // without Developer Mode) a directory is mirrored entry by entry and a + // file becomes a hard link, or a copy across volumes. The mirror never + // writes to these; files it changes are replaced by copies below. + std::function linkEntries = [&](const fs::path& from, + const fs::path& to) { + for (auto& e : fs::directory_iterator(from, ec)) { + fs::path dst = to / e.path().filename(); + std::error_code lec; + bool dir = fs::is_directory(e.path(), lec); + if (dir) fs::create_directory_symlink(e.path(), dst, lec); + else fs::create_symlink(e.path(), dst, lec); + if (!lec) continue; + if (dir) { + fs::create_directory(dst, lec); + if (lec || !linkEntries(e.path(), dst)) { + if (err.empty()) err = "can't mirror " + e.path().string() + ": " + lec.message(); + return false; + } + continue; + } + lec.clear(); + fs::create_hard_link(e.path(), dst, lec); + if (lec) { + lec.clear(); + fs::copy_file(e.path(), dst, lec); + } + if (lec) { + err = "can't mirror " + e.path().string() + ": " + lec.message(); + return false; + } + } + return true; + }; + if (!linkEntries(root_, dir_)) return false; + for (auto& f : realFiles) { + fs::path rel = fs::path(canon(f)).lexically_relative(root_); + if (rel.empty() || *rel.begin() == "..") { + err = f + " is outside " + root_ + " (use --mirror-root)"; + return false; + } + // replace symlinked directories on the way with real ones + fs::path cur = dir_, orig = root_; + for (auto it = rel.begin(); it != rel.end(); ++it) { + cur /= *it; + orig /= *it; + if (std::next(it) == rel.end()) break; + if (fs::is_symlink(cur)) { + fs::remove(cur, ec); + fs::create_directory(cur, ec); + if (!linkEntries(orig, cur)) return false; + } + } + fs::remove(cur, ec); + fs::copy_file(orig, cur, ec); + if (ec) { + err = "can't copy " + orig.string() + ": " + ec.message(); + return false; + } + } + return true; +} + +std::string Mirror::map(const std::string& original) const { + return (fs::path(dir_) / fs::path(canon(original)).lexically_relative(root_)).string(); +} + +} // namespace perm diff --git a/src/workspace.hpp b/src/workspace.hpp new file mode 100644 index 0000000..e45d341 --- /dev/null +++ b/src/workspace.hpp @@ -0,0 +1,55 @@ +// Everything the permuter edits: the target function plus, optionally, other +// functions (typically inline helpers in headers) whose code VC6 inlines into +// it. Candidates are compiled in a per-worker mirror of the source tree so +// edited headers are seen by every #include. +#pragma once + +#include "parser.hpp" + +#include +#include +#include + +namespace perm { + +// A function being permuted. +struct Region { + std::string file; // absolute path of the file holding it + size_t start = 0; // byte range of the definition in the original file + size_t end = 0; + std::string name; // as given or found, e.g. "Ped::GetX" +}; + +// Names of the functions called in f's body (the last component: "GetX" for +// "p->GetX()" or "Ped::GetX()"). +std::set calledNames(const Func& f); + +// Files reachable from file through #include "..." lines, resolved against +// the including file's directory and then dirs. Missing files are skipped. +std::vector includedFiles(const std::string& file, const std::vector& dirs); + +// Definitions of the functions main calls that VC6 could inline into it: +// those in headers reachable from srcPath, and those in srcPath itself that are +// declared inline. At most maxRegions, in the order the calls appear. +std::vector inlineCallees(const std::string& srcPath, size_t mainStart, size_t mainEnd, + const Func& main, const std::vector& includeDirs, + size_t maxRegions); + +// Replaces each region's range in original with its text. Ranges must not overlap. +std::string spliceRegions(const std::string& original, + std::vector> parts); + +// A directory that mirrors root through symlinks, with real copies of the files +// a candidate changes. +class Mirror { +public: + bool create(const std::string& root, const std::string& dir, + const std::vector& realFiles, std::string& err); + std::string map(const std::string& original) const; // path inside the mirror + const std::string& dir() const { return dir_; } + +private: + std::string root_, dir_; +}; + +} // namespace perm diff --git a/tests/integration/bool_return/base/main.cpp b/tests/integration/bool_return/base/main.cpp new file mode 100644 index 0000000..ddfd3ae --- /dev/null +++ b/tests/integration/bool_return/base/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +bool IsReady(Ped* p) +{ + return p->a <= p->b; +} diff --git a/tests/integration/bool_return/target/main.cpp b/tests/integration/bool_return/target/main.cpp new file mode 100644 index 0000000..2c7f9cc --- /dev/null +++ b/tests/integration/bool_return/target/main.cpp @@ -0,0 +1,11 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +bool IsReady(Ped* p) +{ + if (p->a <= p->b) + { + return true; + } + return false; +} diff --git a/tests/integration/branch_dup/base/main.cpp b/tests/integration/branch_dup/base/main.cpp new file mode 100644 index 0000000..54134b4 --- /dev/null +++ b/tests/integration/branch_dup/base/main.cpp @@ -0,0 +1,15 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void Update(Ped* p) +{ + if (p->state) + { + g(1); + } + else + { + g(2); + } + p->c = 5; +} diff --git a/tests/integration/branch_dup/target/main.cpp b/tests/integration/branch_dup/target/main.cpp new file mode 100644 index 0000000..ed0a9fc --- /dev/null +++ b/tests/integration/branch_dup/target/main.cpp @@ -0,0 +1,16 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void Update(Ped* p) +{ + if (p->state) + { + g(1); + p->c = 5; + } + else + { + g(2); + p->c = 5; + } +} diff --git a/tests/integration/chain_assign/base/main.cpp b/tests/integration/chain_assign/base/main.cpp new file mode 100644 index 0000000..61151b8 --- /dev/null +++ b/tests/integration/chain_assign/base/main.cpp @@ -0,0 +1,8 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Clear(Ped* p) +{ + p->a = 0; + p->b = 0; +} diff --git a/tests/integration/chain_assign/target/main.cpp b/tests/integration/chain_assign/target/main.cpp new file mode 100644 index 0000000..1eabff0 --- /dev/null +++ b/tests/integration/chain_assign/target/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Clear(Ped* p) +{ + p->a = p->b = 0; +} diff --git a/tests/integration/combo/base/main.cpp b/tests/integration/combo/base/main.cpp new file mode 100644 index 0000000..010c48b --- /dev/null +++ b/tests/integration/combo/base/main.cpp @@ -0,0 +1,27 @@ +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int ammo; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; int count; void Swap(int idx); }; + +void Group::Swap(int idx) +{ + Weapon* memberWeapon2 = list[idx]->weapon; + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon = leader->sel; + Weapon* leaderWeapon2 = leader->weapon; + + leader->Copy(list[idx]); + leader->sel = memberWeapon; + leader->weapon = memberWeapon2; + list[idx]->sel = leaderWeapon; + list[idx]->weapon = leaderWeapon2; + + if (idx >= count - 1) + { + list[idx]->ammo = 0; + count--; + } + else + { + list[idx]->ammo = 99; + } +} diff --git a/tests/integration/combo/target/main.cpp b/tests/integration/combo/target/main.cpp new file mode 100644 index 0000000..d8bc8ab --- /dev/null +++ b/tests/integration/combo/target/main.cpp @@ -0,0 +1,27 @@ +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int ammo; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; int count; void Swap(int idx); }; + +void Group::Swap(int idx) +{ + Weapon* leaderWeapon = leader->sel; + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon2 = leader->weapon; + Weapon* memberWeapon2 = list[idx]->weapon; + + leader->Copy(list[idx]); + leader->sel = memberWeapon; + leader->weapon = memberWeapon2; + list[idx]->sel = leaderWeapon; + list[idx]->weapon = leaderWeapon2; + + if (idx < count - 1) + { + list[idx]->ammo = 99; + } + else + { + list[idx]->ammo = 0; + count--; + } +} diff --git a/tests/integration/cond_split/base/main.cpp b/tests/integration/cond_split/base/main.cpp new file mode 100644 index 0000000..f815637 --- /dev/null +++ b/tests/integration/cond_split/base/main.cpp @@ -0,0 +1,8 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void Check(Ped* p) +{ + if (p->a && p->b) + g(1); +} diff --git a/tests/integration/cond_split/target/main.cpp b/tests/integration/cond_split/target/main.cpp new file mode 100644 index 0000000..acd3fcf --- /dev/null +++ b/tests/integration/cond_split/target/main.cpp @@ -0,0 +1,11 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void Check(Ped* p) +{ + if (p->a) + { + if (p->b) + g(1); + } +} diff --git a/tests/integration/gta2_parse.sh b/tests/integration/gta2_parse.sh new file mode 100644 index 0000000..75645bd --- /dev/null +++ b/tests/integration/gta2_parse.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Parses every function in gta2_re (Source/*.cpp and *.hpp) and runs every +# pass on each, checking that: +# - every function parses, +# - every candidate a pass produces parses back (and isn't a no-op), +# - almost every expression and statement is understood: anything the +# parser can't read is a barrier the passes never touch, so a rise here +# means less of gta2_re gets permuted. +# +# Needs GTA2_RE pointing at a gta2_re checkout; skips (exit 77) otherwise. +set -u +HERE=$(cd "$(dirname "$0")" && pwd) +PERMUTER=${PERMUTER:-$HERE/../../build/cpp_permuter} +[ -n "${GTA2_RE:-}" ] && [ -d "$GTA2_RE/Source" ] || { echo "skip: GTA2_RE not set"; exit 77; } + +OUT=$(mktemp) +trap 'rm -f "$OUT"' EXIT +"$PERMUTER" --check-parse "$GTA2_RE"/Source/*.cpp "$GTA2_RE"/Source/*.hpp > "$OUT" 2>&1 +RC=$? +grep -v '^ ' "$OUT" | grep -v '^BAD\|^PARSE' | sed 's/^/ /' +if [ "$RC" -ne 0 ]; then + echo "FAIL: functions that don't parse or bad candidates:" + grep '^BAD\|^PARSE' "$OUT" | head -40 + exit 1 +fi + +# at most 0.1% of expressions and 10 statements not understood +set -- $(sed -n 's/^\([0-9]*\) statements left unclassified.*, \([0-9]*\) of \([0-9]*\) expressions.*/\1 \2 \3/p' "$OUT") +[ $# -eq 3 ] || { echo "FAIL: can't read the summary"; cat "$OUT"; exit 1; } +if [ "$1" -gt 10 ] || [ $(($2 * 1000)) -gt "$3" ]; then + echo "FAIL: too much of gta2_re isn't understood ($1 statements, $2 of $3 expressions)" + exit 1 +fi +echo "ok: every gta2_re function parses, every candidate parses back" diff --git a/tests/integration/inequalities/base/main.cpp b/tests/integration/inequalities/base/main.cpp new file mode 100644 index 0000000..17f792d --- /dev/null +++ b/tests/integration/inequalities/base/main.cpp @@ -0,0 +1,10 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Check(Ped* p) +{ + if (p->a > 4) + { + g(1); + } +} diff --git a/tests/integration/inequalities/target/main.cpp b/tests/integration/inequalities/target/main.cpp new file mode 100644 index 0000000..2cb45d6 --- /dev/null +++ b/tests/integration/inequalities/target/main.cpp @@ -0,0 +1,10 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Check(Ped* p) +{ + if (p->a >= 5) + { + g(1); + } +} diff --git a/tests/integration/inline_callee/base/helper.hpp b/tests/integration/inline_callee/base/helper.hpp new file mode 100644 index 0000000..0716d5d --- /dev/null +++ b/tests/integration/inline_callee/base/helper.hpp @@ -0,0 +1,6 @@ +struct Ped { int a, b, c; }; + +__forceinline int Weight(int x, int y) +{ + return y + x * 4; +} diff --git a/tests/integration/inline_callee/base/main.cpp b/tests/integration/inline_callee/base/main.cpp new file mode 100644 index 0000000..02db8f5 --- /dev/null +++ b/tests/integration/inline_callee/base/main.cpp @@ -0,0 +1,6 @@ +#include "helper.hpp" + +int Score(Ped* p) +{ + return Weight(p->a, p->b) + p->c; +} diff --git a/tests/integration/inline_callee/target/helper.hpp b/tests/integration/inline_callee/target/helper.hpp new file mode 100644 index 0000000..6fdabdf --- /dev/null +++ b/tests/integration/inline_callee/target/helper.hpp @@ -0,0 +1,6 @@ +struct Ped { int a, b, c; }; + +__forceinline int Weight(int x, int y) +{ + return x * 4 + y; +} diff --git a/tests/integration/inline_callee/target/main.cpp b/tests/integration/inline_callee/target/main.cpp new file mode 100644 index 0000000..02db8f5 --- /dev/null +++ b/tests/integration/inline_callee/target/main.cpp @@ -0,0 +1,6 @@ +#include "helper.hpp" + +int Score(Ped* p) +{ + return Weight(p->a, p->b) + p->c; +} diff --git a/tests/integration/inline_local/base/main.cpp b/tests/integration/inline_local/base/main.cpp new file mode 100644 index 0000000..dc97c7f --- /dev/null +++ b/tests/integration/inline_local/base/main.cpp @@ -0,0 +1,10 @@ +struct Car { int speed; int gear; }; +struct Driver { Car* car; int skill; }; + +int Boost(Driver* d, int amount) +{ + int boost = amount * d->skill; + d->car->speed += boost; + d->car->gear = 2; + return d->car->speed; +} diff --git a/tests/integration/inline_local/target/main.cpp b/tests/integration/inline_local/target/main.cpp new file mode 100644 index 0000000..e87e330 --- /dev/null +++ b/tests/integration/inline_local/target/main.cpp @@ -0,0 +1,9 @@ +struct Car { int speed; int gear; }; +struct Driver { Car* car; int skill; }; + +int Boost(Driver* d, int amount) +{ + d->car->speed += amount * d->skill; + d->car->gear = 2; + return d->car->speed; +} diff --git a/tests/integration/local_type/base/main.cpp b/tests/integration/local_type/base/main.cpp new file mode 100644 index 0000000..91fb9c6 --- /dev/null +++ b/tests/integration/local_type/base/main.cpp @@ -0,0 +1,11 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Count(Ped* p) +{ + int x = p->a; + if (x > 3) + { + g(x); + } +} diff --git a/tests/integration/local_type/target/main.cpp b/tests/integration/local_type/target/main.cpp new file mode 100644 index 0000000..e23245f --- /dev/null +++ b/tests/integration/local_type/target/main.cpp @@ -0,0 +1,11 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Count(Ped* p) +{ + unsigned int x = p->a; + if (x > 3) + { + g(x); + } +} diff --git a/tests/integration/locals_to_array/base/main.cpp b/tests/integration/locals_to_array/base/main.cpp new file mode 100644 index 0000000..23cc0b0 --- /dev/null +++ b/tests/integration/locals_to_array/base/main.cpp @@ -0,0 +1,10 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Pair(Ped* p) +{ + int x = p->a; + int y = p->b; + g(x); + g(y); +} diff --git a/tests/integration/locals_to_array/target/main.cpp b/tests/integration/locals_to_array/target/main.cpp new file mode 100644 index 0000000..66dd3af --- /dev/null +++ b/tests/integration/locals_to_array/target/main.cpp @@ -0,0 +1,11 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Pair(Ped* p) +{ + int x_arr[2]; + x_arr[0] = p->a; + x_arr[1] = p->b; + g(x_arr[0]); + g(x_arr[1]); +} diff --git a/tests/integration/negate_const/base/main.cpp b/tests/integration/negate_const/base/main.cpp new file mode 100644 index 0000000..8f49852 --- /dev/null +++ b/tests/integration/negate_const/base/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +int Offset(Ped* p) +{ + return p->a - 256; +} diff --git a/tests/integration/negate_const/target/main.cpp b/tests/integration/negate_const/target/main.cpp new file mode 100644 index 0000000..89766ca --- /dev/null +++ b/tests/integration/negate_const/target/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +int Offset(Ped* p) +{ + return p->a + -256; +} diff --git a/tests/integration/ops/base/main.cpp b/tests/integration/ops/base/main.cpp new file mode 100644 index 0000000..40f41a0 --- /dev/null +++ b/tests/integration/ops/base/main.cpp @@ -0,0 +1,16 @@ +struct V { int x, y, z; }; +int g; + +int Sum(V* v, int k) +{ + int r = v->x + k * v->z; + if (r <= 10) + { + g = v->y; + } + else + { + g = r; + } + return r + v->y; +} diff --git a/tests/integration/ops/target/main.cpp b/tests/integration/ops/target/main.cpp new file mode 100644 index 0000000..de7a492 --- /dev/null +++ b/tests/integration/ops/target/main.cpp @@ -0,0 +1,16 @@ +struct V { int x, y, z; }; +int g; + +int Sum(V* v, int k) +{ + int r = k * v->z + v->x; + if (r > 10) + { + g = r; + } + else + { + g = v->y; + } + return r + v->y; +} diff --git a/tests/integration/perm_macros/base/main.cpp b/tests/integration/perm_macros/base/main.cpp new file mode 100644 index 0000000..5d34f80 --- /dev/null +++ b/tests/integration/perm_macros/base/main.cpp @@ -0,0 +1,12 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Run(Ped* p) +{ + PERM_LINESWAP( + g(p->a); + g(p->b); + g(p->c); + ) + p->state = PERM_GENERAL(1, 2, 3); +} diff --git a/tests/integration/perm_macros/target/main.cpp b/tests/integration/perm_macros/target/main.cpp new file mode 100644 index 0000000..4ac5bc2 --- /dev/null +++ b/tests/integration/perm_macros/target/main.cpp @@ -0,0 +1,10 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Run(Ped* p) +{ + g(p->c); + g(p->a); + g(p->b); + p->state = 2; +} diff --git a/tests/integration/reassociate/base/main.cpp b/tests/integration/reassociate/base/main.cpp new file mode 100644 index 0000000..fa5e5d7 --- /dev/null +++ b/tests/integration/reassociate/base/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +int Sum(Ped* p) +{ + return p->a + p->b + p->c; +} diff --git a/tests/integration/reassociate/target/main.cpp b/tests/integration/reassociate/target/main.cpp new file mode 100644 index 0000000..d6f78b6 --- /dev/null +++ b/tests/integration/reassociate/target/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +int Sum(Ped* p) +{ + return p->a + (p->b + p->c); +} diff --git a/tests/integration/remove_stmt/base/main.cpp b/tests/integration/remove_stmt/base/main.cpp new file mode 100644 index 0000000..d4ec94e --- /dev/null +++ b/tests/integration/remove_stmt/base/main.cpp @@ -0,0 +1,8 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Finish(Ped* p) +{ + g(1); + p->c = 2; +} diff --git a/tests/integration/remove_stmt/target/main.cpp b/tests/integration/remove_stmt/target/main.cpp new file mode 100644 index 0000000..8c6b2a1 --- /dev/null +++ b/tests/integration/remove_stmt/target/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Finish(Ped* p) +{ + g(1); +} diff --git a/tests/integration/reorder_cases/base/main.cpp b/tests/integration/reorder_cases/base/main.cpp new file mode 100644 index 0000000..c565ee8 --- /dev/null +++ b/tests/integration/reorder_cases/base/main.cpp @@ -0,0 +1,15 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +int Pick(Ped* p) +{ + switch (p->state) + { + case 1: + return 5; + case 2: + return 6; + default: + return 0; + } +} diff --git a/tests/integration/reorder_cases/target/main.cpp b/tests/integration/reorder_cases/target/main.cpp new file mode 100644 index 0000000..11c204d --- /dev/null +++ b/tests/integration/reorder_cases/target/main.cpp @@ -0,0 +1,15 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +int Pick(Ped* p) +{ + switch (p->state) + { + case 2: + return 6; + case 1: + return 5; + default: + return 0; + } +} diff --git a/tests/integration/run.sh b/tests/integration/run.sh new file mode 100755 index 0000000..1c3ab32 --- /dev/null +++ b/tests/integration/run.sh @@ -0,0 +1,181 @@ +#!/bin/sh +# Integration tests. Each case directory holds base/ (the "decompiled" +# source, wrong somewhere) and target/ (the source the target object is built +# from); main.cpp holds the function, other files are headers it includes. The +# permuter has to turn base/ into code that compiles to exactly the target, and +# where the case says so, into exactly the target source. +# +# The compiler is clang's i686 MSVC target at -O0, standing in for VC6: at -O0 +# statement order and expression shape show up directly in the code. See +# vc6.sh for the same kind of checks with the real VC6 on gta2_re. +# +# Negative controls run without the pass under test (usually: every other +# pass) and must NOT match, which shows the match comes from that pass. +set -u +HERE=$(cd "$(dirname "$0")" && pwd) +PERMUTER=${PERMUTER:-$HERE/../../build/cpp_permuter} +CXX=${CXX_INTEGRATION:-clang++} +command -v "$CXX" >/dev/null || { echo "skip: $CXX not found"; exit 77; } +command -v llvm-objdump >/dev/null || { echo "skip: llvm-objdump not found"; exit 77; } +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +COMPILE="$CXX --target=i686-pc-windows-msvc -O0 -c {src} -o {obj}" +FAILED=0 +N=0 + +# Every pass except the ones named. +all_but() { + "$PERMUTER" --list-passes | sed -n 's/^ \([a-z_]*\)$/\1/p' | grep -v -x -F "$(printf '%s\n' "$@")" | + paste -sd, - +} + +# run ; sets RC and OUT +run() { + case_=$1 fn=$2 name=$3 + shift 3 + N=$((N + 1)) + OUT="$WORK/out_$N" + mkdir -p "$WORK/src_$N" + cp "$HERE/$case_"/base/* "$WORK/src_$N/" + $CXX --target=i686-pc-windows-msvc -O0 -c "$HERE/$case_/target/main.cpp" -o "$WORK/target_$N.obj" || exit 1 + "$PERMUTER" -s "$WORK/src_$N/main.cpp" -f "$fn" -c "$COMPILE" -t "$WORK/target_$N.obj" \ + -j 2 --seed 1 -o "$OUT" "$@" > "$WORK/log_$N" 2>&1 + RC=$? + LABEL="$case_: $name" + CASE=$case_ +} + +fail() { + echo "FAIL: $LABEL: $1" + tr '\r' '\n' < "$WORK/log_$N" | grep -v '^iterations' | tail -30 + FAILED=$((FAILED + 1)) +} + +base_score() { tr '\r' '\n' < "$WORK/log_$N" | sed -n 's/^base score: //p'; } + +# An exact match (exit 0, score 0) from a base that didn't match. +expect_match() { + if [ "$RC" -ne 0 ]; then fail "expected a match, exit $RC"; return 1; fi + b=$(base_score) + if [ -z "$b" ] || [ "$b" -eq 0 ]; then fail "base already matched (score '$b')"; return 1; fi + if ! ls -d "$OUT"/output-0-* >/dev/null 2>&1; then fail "no output-0-* written"; return 1; fi + echo "ok: $LABEL: base $b -> 0" +} + +# Like expect_match, and the matched sources are the target sources (ignoring +# whitespace): main.cpp, and every header that differs between base and target. +expect_exact_source() { + expect_match || return + m=$(ls -d "$OUT"/output-0-* | head -1) + if ! diff -w -B "$m/source.cpp" "$HERE/$CASE/target/main.cpp" > "$WORK/srcdiff_$N"; then + fail "matched, but main.cpp differs from the target:" + cat "$WORK/srcdiff_$N" + return + fi + for t in "$HERE/$CASE"/target/*; do + h=$(basename "$t") + [ "$h" = main.cpp ] && continue + cmp -s "$t" "$HERE/$CASE/base/$h" && continue + if [ ! -f "$m/$h" ] || ! diff -w "$m/$h" "$t" > "$WORK/srcdiff_$N"; then + fail "matched, but $h isn't the target's:" + cat "$WORK/srcdiff_$N" 2>/dev/null + return + fi + done + echo "ok: $LABEL: matched source == target" +} + +expect_no_match() { + if [ "$RC" -eq 0 ]; then fail "matched without the needed pass"; return; fi + if [ "$RC" -ne 3 ]; then fail "expected 'no match' (exit 3), got exit $RC"; return; fi + echo "ok: $LABEL: no match, as expected" +} + +# --- var_reorder: four local saves in the wrong order ----------------------- +run var_reorder Group::Promote exhaustive -m exhaustive -p reorder_saves +expect_exact_source +run var_reorder Group::Promote random -m random -p reorder_saves -n 200 +expect_exact_source +run var_reorder Group::Promote control -m exhaustive -p "$(all_but reorder_saves move_stmt)" +expect_no_match + +# --- combo: saves in the wrong order AND an inverted if --------------------- +# Neither pass alone gets there; the combination does. +run combo Group::Swap single-passes -m exhaustive -p reorder_saves,invert_if +expect_no_match +run combo Group::Swap combo -m exhaustive -p reorder_saves+invert_if +expect_exact_source +run combo Group::Swap repeated-p-depth2 -m exhaustive -p reorder_saves -p invert_if --depth 2 +expect_exact_source +run combo Group::Swap random-combo -m random -p reorder_saves+invert_if -n 300 +expect_exact_source +run combo Group::Swap combo-plus-noise -m exhaustive -p swap_operands+reorder_saves+flip_compare+invert_if +expect_exact_source + +# --- inline_local: a local the original didn't have ------------------------- +# -j 1: the cast variant compiles to the same code; keep the plain one (tried first) +run inline_local Boost inline -m exhaustive -p inline_local -j 1 +expect_exact_source +run inline_local Boost control -m exhaustive -p "$(all_but inline_local)" +expect_no_match + +# --- ops: swapped operands and an inverted if -------------------------------- +run ops Sum combo -m exhaustive -p swap_operands+invert_if +expect_match +run ops Sum random -m random -p swap_operands,invert_if,flip_compare -n 400 +expect_match + +# --- one case per pass ---------------------------------------------------------- +# chain_assign: at -O0 "a = b = 0;" is "b = 0; a = 0;", which move_stmt reaches too +# bool_return: at clang -O0 the else/1-0 forms compile the same as the target's, +# so any of them is a real match (vc6.sh checks the exact form with VC6). +run bool_return IsReady exhaustive -m exhaustive -p bool_return +expect_match +run bool_return IsReady control -m exhaustive -p "$(all_but bool_return)" +expect_no_match +for c in ternary_arg:Notify switch_if:OnCommand branch_dup:Update \ + cond_split:Check negate_const:Offset reassociate:Sum inequalities:Check \ + chain_assign:Clear:move_stmt local_type:Count locals_to_array:Pair reorder_cases:Pick \ + split_case_labels:Pick use_getter:Next temp_for_expr:Send remove_stmt:Finish; do + # pass:function[:other pass that reaches the same code, left out of the control] + pass=${c%%:*} rest=${c#*:} fn=${rest%%:*} also="" + [ "$rest" != "$fn" ] && also=${rest#*:} + run "$pass" "$fn" exhaustive -m exhaustive -p "$pass" + expect_exact_source + run "$pass" "$fn" random -m random -p "$pass" -n 50 + # where an equivalent form exists, random mode may land on it instead + if [ -n "$also" ]; then expect_match; else expect_exact_source; fi + run "$pass" "$fn" control -m exhaustive -p "$(all_but "$pass" $also)" + expect_no_match +done + +# --- PERM macros: the alternatives written into the function --------------- +# 3! line orders x 3 values; without PERM_RANDOMIZE only the macros are tried +run perm_macros Run exhaustive -m exhaustive +expect_exact_source +grep -q "^17 candidates generated" "$WORK/log_$N" || fail "expected 17 candidates (18 expansions less the base)" +run perm_macros Run random -m random -n 60 +expect_exact_source +run perm_macros Run control -m exhaustive -p none --max-candidates 5 +expect_no_match + +# --- inline_callee: the function is fine, the inline helper it calls isn't -- +run inline_callee Score inline-callees -m exhaustive -p swap_operands --inline-callees +expect_exact_source +run inline_callee Score also -m exhaustive -p swap_operands --also Weight +expect_exact_source +run inline_callee Score also-file -m exhaustive -p swap_operands --also helper.hpp:Weight +expect_exact_source +# random runs can reach an equivalent helper ("4 * x + y" compiles the same at +# -O0), so this one only needs a match +run inline_callee Score random -m random -p swap_operands,flip_compare --inline-callees -n 100 +expect_match +run inline_callee Score control -m exhaustive +expect_no_match + +echo +if [ "$FAILED" -ne 0 ]; then + echo "$FAILED of $N integration runs failed" + exit 1 +fi +echo "all $N integration runs passed" diff --git a/tests/integration/split_case_labels/base/main.cpp b/tests/integration/split_case_labels/base/main.cpp new file mode 100644 index 0000000..506d9e0 --- /dev/null +++ b/tests/integration/split_case_labels/base/main.cpp @@ -0,0 +1,14 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +int Pick(Ped* p) +{ + switch (p->state) + { + case 1: + case 2: + return 5; + default: + return 0; + } +} diff --git a/tests/integration/split_case_labels/target/main.cpp b/tests/integration/split_case_labels/target/main.cpp new file mode 100644 index 0000000..4fc422d --- /dev/null +++ b/tests/integration/split_case_labels/target/main.cpp @@ -0,0 +1,15 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +int Pick(Ped* p) +{ + switch (p->state) + { + case 1: + return 5; + case 2: + return 5; + default: + return 0; + } +} diff --git a/tests/integration/switch_if/base/main.cpp b/tests/integration/switch_if/base/main.cpp new file mode 100644 index 0000000..8447c49 --- /dev/null +++ b/tests/integration/switch_if/base/main.cpp @@ -0,0 +1,10 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void OnCommand(Ped* p, int notify) +{ + if (notify == 1) + { + g(p->a); + } +} diff --git a/tests/integration/switch_if/target/main.cpp b/tests/integration/switch_if/target/main.cpp new file mode 100644 index 0000000..38876fd --- /dev/null +++ b/tests/integration/switch_if/target/main.cpp @@ -0,0 +1,12 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void OnCommand(Ped* p, int notify) +{ + switch (notify) + { + case 1: + g(p->a); + break; + } +} diff --git a/tests/integration/temp_for_expr/base/main.cpp b/tests/integration/temp_for_expr/base/main.cpp new file mode 100644 index 0000000..1d292cd --- /dev/null +++ b/tests/integration/temp_for_expr/base/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Send(Ped* p) +{ + g((char)p->a); +} diff --git a/tests/integration/temp_for_expr/target/main.cpp b/tests/integration/temp_for_expr/target/main.cpp new file mode 100644 index 0000000..6ccf6ae --- /dev/null +++ b/tests/integration/temp_for_expr/target/main.cpp @@ -0,0 +1,8 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +void Send(Ped* p) +{ + char tmp = (char)p->a; + g(tmp); +} diff --git a/tests/integration/ternary_arg/base/main.cpp b/tests/integration/ternary_arg/base/main.cpp new file mode 100644 index 0000000..4f446c7 --- /dev/null +++ b/tests/integration/ternary_arg/base/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void Notify(Ped* p) +{ + g(p->state == 0 ? 1 : 0); +} diff --git a/tests/integration/ternary_arg/target/main.cpp b/tests/integration/ternary_arg/target/main.cpp new file mode 100644 index 0000000..e6341f1 --- /dev/null +++ b/tests/integration/ternary_arg/target/main.cpp @@ -0,0 +1,14 @@ +struct Ped { int a, b, c; int state; }; +void g(int v); + +void Notify(Ped* p) +{ + if (p->state == 0) + { + g(1); + } + else + { + g(0); + } +} diff --git a/tests/integration/use_getter/base/main.cpp b/tests/integration/use_getter/base/main.cpp new file mode 100644 index 0000000..cd7ba12 --- /dev/null +++ b/tests/integration/use_getter/base/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +int Next(Ped* p) +{ + return p->a + 1; +} diff --git a/tests/integration/use_getter/target/main.cpp b/tests/integration/use_getter/target/main.cpp new file mode 100644 index 0000000..1b220d0 --- /dev/null +++ b/tests/integration/use_getter/target/main.cpp @@ -0,0 +1,7 @@ +struct Ped { int a, b, c; int state; int get_a() { return a; } }; +void g(int v); + +int Next(Ped* p) +{ + return p->get_a() + 1; +} diff --git a/tests/integration/var_reorder/base/main.cpp b/tests/integration/var_reorder/base/main.cpp new file mode 100644 index 0000000..2078092 --- /dev/null +++ b/tests/integration/var_reorder/base/main.cpp @@ -0,0 +1,17 @@ +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int pad[4]; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; void Promote(int idx); }; + +void Group::Promote(int idx) +{ + Weapon* leaderWeapon = leader->sel; + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon2 = leader->weapon; + Weapon* memberWeapon2 = list[idx]->weapon; + + leader->Copy(list[idx]); + leader->sel = leaderWeapon; + leader->weapon = leaderWeapon2; + list[idx]->sel = memberWeapon; + list[idx]->weapon = memberWeapon2; +} diff --git a/tests/integration/var_reorder/target/main.cpp b/tests/integration/var_reorder/target/main.cpp new file mode 100644 index 0000000..463ff05 --- /dev/null +++ b/tests/integration/var_reorder/target/main.cpp @@ -0,0 +1,17 @@ +struct Weapon; +struct Ped { Weapon* sel; Weapon* weapon; int pad[4]; void Copy(Ped* other); }; +struct Group { Ped* leader; Ped* list[8]; void Promote(int idx); }; + +void Group::Promote(int idx) +{ + Weapon* memberWeapon = list[idx]->sel; + Weapon* leaderWeapon2 = leader->weapon; + Weapon* leaderWeapon = leader->sel; + Weapon* memberWeapon2 = list[idx]->weapon; + + leader->Copy(list[idx]); + leader->sel = leaderWeapon; + leader->weapon = leaderWeapon2; + list[idx]->sel = memberWeapon; + list[idx]->weapon = memberWeapon2; +} diff --git a/tests/integration/vc6.sh b/tests/integration/vc6.sh new file mode 100644 index 0000000..9c4ead5 --- /dev/null +++ b/tests/integration/vc6.sh @@ -0,0 +1,358 @@ +#!/bin/sh +# Integration tests with the real compiler: MSVC 6 under wine, on gta2_re. +# +# Every function used here is a MATCH_FUNC, so VC6's code for the unmodified +# gta2_re source is the original game's code. Each case compiles that as the +# target, breaks the function (or an inline helper it calls) the way a +# decompiler would, and checks that the permuter finds the way back to exactly +# the original source. A control run without the needed pass must not match. +# +# The work happens in a copy of Source/ made of symlinks, so the gta2_re +# checkout is never modified. +# +# Needs GTA2_RE pointing at a gta2_re checkout with its compile tools +# submodule, and wine. Skips (exit 77) otherwise. +set -u +HERE=$(cd "$(dirname "$0")" && pwd) +ROOT=$HERE/../.. +PERMUTER=${PERMUTER:-$ROOT/build/cpp_permuter} +[ -n "${GTA2_RE:-}" ] || { echo "skip: GTA2_RE not set"; exit 77; } +[ -d "$GTA2_RE/3rdParty/gta2_re_compile_tools/VC98" ] || { echo "skip: no VC6 in $GTA2_RE"; exit 77; } +command -v wine >/dev/null || { echo "skip: wine not found"; exit 77; } +command -v llvm-objdump >/dev/null || { echo "skip: llvm-objdump not found"; exit 77; } +command -v python3 >/dev/null || { echo "skip: python3 not found"; exit 77; } +export WINEDEBUG=-all GTA2_RE +wineserver -p 2>/dev/null || true + +ORIG=$GTA2_RE/Source +COMPILE="$ROOT/examples/gta2/compile.sh {src} {obj}" +WORK=$(mktemp -d) +SRC=$WORK/Source +trap 'rm -rf "$WORK"' EXIT +FAILED=0 +N=0 + +# target : the original TU's object, compiled once +target() { + obj=$WORK/$1.target.obj + if [ ! -f "$obj" ]; then + "$ROOT/examples/gta2/compile.sh" "$ORIG/$1" "$obj" > "$WORK/target.log" 2>&1 || + { echo "FAIL: $1 doesn't compile"; cat "$WORK/target.log"; exit 1; } + fi + echo "$obj" +} + +# fresh: a pristine symlinked copy of Source/ +fresh() { + rm -rf "$SRC" + mkdir -p "$SRC" + for e in "$ORIG"/*; do ln -s "$e" "$SRC/"; done +} + +# mutate : replaces the text before the "====" line on stdin with the +# text after it, in a private copy of Source/ +mutate() { + f=$SRC/$1 + [ -L "$f" ] && cp --remove-destination "$(readlink -f "$f")" "$f" + python3 -c ' +import sys +path = sys.argv[1] +old, new = sys.stdin.read().split("\n====\n") +s = open(path).read() +old, new = old.strip("\n"), new.strip("\n") +assert old in s, "%s: text to replace not found; gta2_re changed, update vc6.sh" % path +open(path, "w").write(s.replace(old, new, 1)) +' "$f" +} + +# run