slipstream: take txs from anyone, get them into the pool's blocks

.github/workflows/build_docker.yaml

@@ -1,4 +1,4 @@
-# Build the three simplepool container images and push them to the GitHub
+# Build the four simplepool container images and push them to the GitHub
 # Container Registry (ghcr.io) — the same registry coinshift-rs publishes to.
 #
 # This job runs only in the canonical LayerTwo-Labs repo (see the `if:` guard
@@ -47,6 +47,8 @@ jobs:
             dockerfile: deploy/docker/Dockerfile.dashboard
           - image: simplepool-payout
             dockerfile: deploy/docker/Dockerfile.payout
+          - image: simplepool-slipstream
+            dockerfile: deploy/docker/Dockerfile.slipstream
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -88,6 +90,6 @@ jobs:
                 github.event.pull_request.head.repo.full_name == github.repository }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          # Per-image GitHub Actions cache so the three legs don't collide.
+          # Per-image GitHub Actions cache so the legs don't collide.
           cache-from: type=gha,scope=${{ matrix.image }}
           cache-to: type=gha,mode=max,scope=${{ matrix.image }}

.github/workflows/integration_tests.yaml

@@ -71,6 +71,13 @@ jobs:
       - name: Run coinbase-direct PPLNS end-to-end regtest test
         run: bash tests/test_pplns_coinbase_regtest.sh
 
+      # Slipstream: a tx submitted to the service is checked and broadcast to
+      # the node, reaches the enforcer's template through its mempool mirror,
+      # and is followed to confirmed. No proxy; the template is mined with
+      # generateblock.
+      - name: Run slipstream end-to-end regtest test
+        run: bash tests/test_slipstream_regtest.sh
+
       - name: Upload logs
         if: failure()
         uses: actions/upload-artifact@v4
@@ -81,6 +88,8 @@ jobs:
             .regtest-e2e/logs/
             .regtest-pplns/logs/
             .regtest-cbwin/logs/
+            .regtest/slipstream-e2e/logs/
+            .regtest/slipstream-e2e/slipstream.log
             /tmp/simplepool-e2e.log
             /tmp/simplepool-e2e.conf
             /tmp/simplepool-int.log
@@ -126,6 +135,14 @@ jobs:
         working-directory: payout
         run: npm test
 
+      - name: Install slipstream dependencies
+        working-directory: slipstream
+        run: npm ci || npm install
+
+      - name: Run slipstream service tests
+        working-directory: slipstream
+        run: npm test
+
   # The payout path the coinbase e2e skips: wallet-enabled enforcer +
   # thunder, a real deposit, and one payout tick broadcasting a Thunder
   # transaction (tests/test_payout_regtest.sh). Separate job purely for

README.md

@@ -579,7 +579,7 @@ you can commit + push from here), use
 ### Docker
 
 An alternative to the bare-metal script: containerized builds of the
-three services (stratum proxy, dashboard, payout worker) under
+four services (stratum proxy, dashboard, payout worker, slipstream) under
 [`deploy/docker/`](deploy/docker/). One `docker compose up -d --build`
 gets the whole app stack running against a Thunder daemon and Bitcoin
 Core that live on the host (or wherever you point them). Shared bind
@@ -592,6 +592,23 @@ Note: the drivechain infrastructure (`bitcoind`, Thunder, the
 those daemons have their own lifecycles and typically run bare-metal on
 the same host.
 
+### Slipstream
+
+[`slipstream/`](slipstream/) is an optional service that takes a raw tx from
+anyone and gets it into the pool's blocks, including txs the network will not
+relay: BIP300/301 txs, or any other consensus-valid tx. It checks each one
+against the pool's own bitcoind (`testmempoolaccept`) and the fee rule, then
+broadcasts it there; the enforcer's template mempool mirrors that node's, so it
+reaches the templates the proxy mines with no enforcer change. Non-standard txs
+need the node to run `-acceptnonstdtxn` (which Core allows only off mainnet).
+The fee rule is Slipstream's: the higher of a 1 sat/vB floor and the current
+mineable rate. Every submission is kept, and each accepted tx is followed from
+template to block. It also serves the pool's `info.json` for pool directories.
+It is not set up by `install.sh` yet: the systemd unit and nginx vhost are
+templates in [`deploy/`](deploy/), and [`slipstream/README.md`](slipstream/README.md)
+walks through installing them. [`docs/simplepool.html`](docs/simplepool.html)
+explains it end to end.
+
 ## Config keys
 
 ```
@@ -748,6 +765,8 @@ scripts/
 dashboard/           # Node/Express read-only stats UI
 payout/              # payout worker: Thunder rail (pps-classic, pplns-thunder)
                      #   and L1 rail via the enforcer wallet (pplns-btc)
+slipstream/          # slipstream service: takes txs from anyone and gets them
+                     #   into the pool's blocks via its bitcoind; serves info.json
 docs/simplepool.html # single-file explainer: every mode, end to end
 ```
 

deploy/docker/Dockerfile.slipstream

@@ -0,0 +1,21 @@
+# -----------------------------------------------------------------------------
+# simplepool slipstream service — private tx submission (Node.js)
+# -----------------------------------------------------------------------------
+
+FROM node:20-bookworm-slim AS deps
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3 make g++ ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY slipstream/package.json slipstream/package-lock.json ./
+RUN npm ci --omit=dev
+
+# -----------------------------------------------------------------------------
+FROM node:20-bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends tini \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY slipstream/ ./
+USER node
+ENTRYPOINT ["/usr/bin/tini", "--", "node", "index.js"]

deploy/docker/docker-compose.yml

@@ -1,7 +1,7 @@
 # -----------------------------------------------------------------------------
 # simplepool — docker-compose orchestration
 # -----------------------------------------------------------------------------
-# Runs the three simplepool services against a Thunder daemon that lives on
+# Runs the four simplepool services against a Thunder daemon that lives on
 # the host (or wherever THUNDER_RPC_URL points). Bitcoin Core is likewise
 # expected to be reachable from the simplepool container.
 #
@@ -102,3 +102,45 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
     stop_grace_period: 30s
+
+  slipstream:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.slipstream
+    image: simplepool-slipstream:latest
+    container_name: simplepool-slipstream
+    restart: unless-stopped
+    environment:
+      # The pool's bitcoind -- the node the enforcer mirrors its mempool
+      # from. Txs are checked and broadcast here. Run it with
+      # -acceptnonstdtxn to take non-standard txs (not allowed on mainnet).
+      BITCOIND_RPC_URL: ${BITCOIND_RPC_URL:-http://host.docker.internal:8332}
+      BITCOIND_RPC_USER: ${BITCOIND_RPC_USER:-}
+      BITCOIND_RPC_PASS: ${BITCOIND_RPC_PASS:-}
+      # The enforcer's block template server -- the proxy's bitcoind_url.
+      # Read only: the template is where the mineable rate comes from.
+      ENFORCER_GBT_URL: ${ENFORCER_GBT_URL:-http://host.docker.internal:8122}
+      SLIPSTREAM_DB_PATH: /data/slipstream.db
+      PROXY_DB_PATH: /data/shares.db
+      SLIPSTREAM_BIND: 0.0.0.0
+      SLIPSTREAM_PORT: "8124"
+      SLIPSTREAM_TRUST_PROXY: ${SLIPSTREAM_TRUST_PROXY:-0}
+      SLIPSTREAM_MIN_FEE_RATE: ${SLIPSTREAM_MIN_FEE_RATE:-1}
+      POOL_NAME: ${POOL_NAME:-}
+      POOL_OPERATOR: ${POOL_OPERATOR:-}
+      POOL_CHAIN: ${POOL_CHAIN:-}
+      POOL_LOGO: ${POOL_LOGO:-}
+      POOL_CONTACT: ${POOL_CONTACT:-}
+      PUBLIC_STRATUM_URL: ${PUBLIC_STRATUM_URL:-}
+      PUBLIC_DASHBOARD_URL: ${PUBLIC_DASHBOARD_URL:-}
+      PUBLIC_SLIPSTREAM_URL: ${PUBLIC_SLIPSTREAM_URL:-}
+    ports:
+      # Loopback on the host: publish through nginx, which is also what
+      # SLIPSTREAM_TRUST_PROXY=1 assumes sets X-Forwarded-For.
+      - "127.0.0.1:${SLIPSTREAM_PORT:-8124}:8124"
+    volumes:
+      # Writes slipstream.db; reads shares.db.
+      - ../../data:/data
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    stop_grace_period: 10s

deploy/nginx/slipstream.conf

@@ -0,0 +1,44 @@
+# slipstream.example — reverse proxy for the simplepool slipstream service.
+#
+# The service is bound to 127.0.0.1:8124 (see
+# deploy/systemd/simplepool-slipstream.service) and keys its per-client
+# submission limit on the X-Forwarded-For hop set here. Replace
+# slipstream.example with your hostname, then:
+#
+#   sudo cp deploy/nginx/slipstream.conf /etc/nginx/sites-available/<host>
+#   sudo ln -s /etc/nginx/sites-available/<host> /etc/nginx/sites-enabled/
+#   sudo nginx -t && sudo systemctl reload nginx
+#   sudo certbot --nginx -d <host>       # TLS, edits this file in place
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name slipstream.example;
+
+    # Reuses the dashboard's zone (deploy/nginx/pool-ratelimit.conf) for
+    # reads. Submissions have their own, stricter limit inside the service.
+    limit_req zone=pool_dash burst=20 nodelay;
+
+    # A raw tx arrives as hex: up to 1MB of tx is 2MB of body.
+    client_max_body_size 3m;
+
+    access_log /var/log/nginx/slipstream.example.access.log;
+    error_log  /var/log/nginx/slipstream.example.error.log warn;
+
+    location / {
+        proxy_pass         http://127.0.0.1:8124;
+        proxy_http_version 1.1;
+        proxy_set_header   Host              $host;
+        # Overwritten, not appended: the service trusts this hop, so the
+        # client must not be able to put its own in front of it.
+        proxy_set_header   X-Forwarded-For   $remote_addr;
+        proxy_set_header   X-Forwarded-Proto $scheme;
+        proxy_read_timeout 30s;
+        proxy_connect_timeout 5s;
+    }
+
+    location = /healthz {
+        proxy_pass http://127.0.0.1:8124/healthz;
+        access_log off;
+    }
+}

deploy/systemd/simplepool-slipstream.service

@@ -0,0 +1,55 @@
+[Unit]
+Description=simplepool slipstream (tx submission)
+Documentation=https://github.com/LayerTwo-Labs/simplepool/blob/main/slipstream/README.md
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=@USER@
+Group=@USER@
+WorkingDirectory=@ROOT@/slipstream
+# The pool's bitcoind: the node the enforcer mirrors its mempool from. Txs
+# are checked (testmempoolaccept) and broadcast (sendrawtransaction) here.
+# For non-standard txs it must run -acceptnonstdtxn, which Core allows only
+# off mainnet. Credentials: user/pass, or BITCOIND_RPC_COOKIE_FILE instead.
+Environment=BITCOIND_RPC_URL=http://127.0.0.1:8332
+# Environment=BITCOIND_RPC_USER=<rpcuser>
+# Environment=BITCOIND_RPC_PASS=<rpcpass>
+# Environment=BITCOIND_RPC_COOKIE_FILE=/var/lib/bitcoind/.cookie
+# The enforcer's block template server -- the same URL as bitcoind_url in
+# proxy.conf. Read only: the template it serves is where the mineable rate
+# comes from and how a tx is seen to be in play.
+Environment=ENFORCER_GBT_URL=http://127.0.0.1:8122
+Environment=SLIPSTREAM_DB_PATH=@ROOT@/data/slipstream.db
+Environment=PROXY_DB_PATH=@ROOT@/data/shares.db
+# Loopback only; nginx publishes it (deploy/nginx/slipstream.conf), and
+# sets the X-Forwarded-For hop the rate limit is keyed on.
+Environment=SLIPSTREAM_BIND=127.0.0.1
+Environment=SLIPSTREAM_PORT=8124
+Environment=SLIPSTREAM_TRUST_PROXY=1
+Environment=SLIPSTREAM_MIN_FEE_RATE=1
+# info.json presentation. Facts about the pool (mode, fee, coinbase tag,
+# addresses) are read from pool_meta and are NOT set here.
+# Environment=PUBLIC_SLIPSTREAM_URL=https://slipstream.example
+# Environment=PUBLIC_STRATUM_URL=stratum+tcp://stratum.example:3334
+# Environment=PUBLIC_DASHBOARD_URL=https://pool.example
+# Environment=POOL_NAME=<name>
+# Environment=POOL_OPERATOR=<operator>
+# Environment=POOL_CHAIN=<e.g. betanet>
+# Environment=POOL_LOGO=<path or URL>
+# Environment=POOL_CONTACT=<contact>
+ExecStart=/usr/bin/node @ROOT@/slipstream/index.js
+Restart=on-failure
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=read-only
+ReadWritePaths=@ROOT@/data
+
+[Install]
+WantedBy=multi-user.target

docs/sequence-diagrams.py

@@ -274,6 +274,35 @@ def build(title, desc, actors, steps, accent="pplns", min_width=0):
 
 
 
+# ---------------------------------------------------------- slipstream -----
+DIAGRAMS['slipstream'] = build(
+    'slipstream: a tx from anyone, into the pool\'s blocks',
+    'A submitter posts a raw tx. The slipstream service checks it against the '
+    'pool\'s bitcoind and the fee rule without broadcasting, then broadcasts it. '
+    'The enforcer mirrors that node\'s mempool, so the tx reaches the template '
+    'the pool mines; the service follows it to a block.',
+    [('sub', 'Submitter', 'any wallet'),
+     ('slip', 'slipstream', ':8124'),
+     ('node', 'bitcoind', '-acceptnonstdtxn'),
+     ('enf', 'enforcer', 'template server'),
+     POOL],
+    [('msg', 'sub', 'slip', 'POST /api/tx  <raw hex>'),
+     ('msg', 'slip', 'node', 'testmempoolaccept'),
+     ('msg', 'node', 'slip', 'allowed? fee, vsize', 'dashed'),
+     ('self', 'slip', 'fee rule: max(floor, mineable)'),
+     ('note', 'Checked BEFORE it is sent: nothing can be taken back out of a mempool, so refusing afterwards would be too late.', 'warn'),
+     ('msg', 'slip', 'node', 'sendrawtransaction'),
+     ('msg', 'node', 'enf', 'mempool mirror (ZMQ)'),
+     ('note', 'From here it is an ordinary mempool tx: relayed to peers, and minable by any pool whose node took it.'),
+     ('msg', 'pool', 'enf', 'getblocktemplate'),
+     ('msg', 'enf', 'pool', 'a template carrying the tx', 'dashed'),
+     ('msg', 'slip', 'enf', 'poll: in the template?'),
+     ('msg', 'slip', 'node', 'poll: mined? how deep?'),
+     ('note', 'Every submission is kept, refusals included. Each accepted tx is followed to confirmed, or to dropped with the node\'s own reason.', 'win'),
+    ], min_width=600)
+
+
+
 # ---- splicing -------------------------------------------------------------
 #
 # Each figure sits between HTML comment markers so a regeneration replaces