Run every x64 Linux test on bazelflare

.bazelrc

@@ -1,6 +1,7 @@
 common --enable_platform_specific_config
 build --verbose_failures
 build --build_tag_filters=-off-by-default,-requires-container-engine
+build --//build/config:container_test_images
 # block network access by default
 build --nosandbox_default_allow_network
 test --test_tag_filters=-off-by-default,-requires-fuzzilli,-requires-container-engine
@@ -628,10 +629,10 @@ common:bazelflare --extra_execution_platforms=//build/bazelflare:linux_x86_64
 common:bazelflare --spawn_strategy=remote --remote_local_fallback=false
 # V8 marks its inspector code generator `local = 1`; it runs fine remotely.
 common:bazelflare --modify_execution_info=Genrule=-local
-# Executors have neither internet nor a container engine, so targets that need them, or that must
-# run on the client, are skipped. Filters don't change action keys.
-common:bazelflare --build_tag_filters=-off-by-default,-no-remote-exec,-requires-network,-requires-container-engine,-requires-fuzzilli
-test:bazelflare --test_tag_filters=-off-by-default,-no-remote-exec,-requires-network,-requires-container-engine,-requires-fuzzilli
+# Targets that must run on the client are skipped; unlike the default filters, these keep the
+# container tests. Filters don't change action keys.
+common:bazelflare --build_tag_filters=-off-by-default,-no-remote-exec,-requires-fuzzilli
+test:bazelflare --test_tag_filters=-off-by-default,-no-remote-exec,-requires-fuzzilli
 common:bazelflare --remote_download_minimal
 common:bazelflare --remote_cache_compression --remote_timeout=600
 # bazelflare sends nothing while an action runs, so only pings notice a connection that died

.github/workflows/_bazel.yml

@@ -205,16 +205,6 @@ jobs:
           sudo mkdir -p /etc/docker
           echo '{"ipv6": true, "fixed-cidr-v6": "fd00::/80"}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
-      - name: Build and load container images
-        if: inputs.build_container_images
-        run: |
-          docker context use default
-          echo "Docker info:"
-          docker info
-          echo "Docker version:"
-          docker version
-          # Its own profile, so that it leaves the build's for the upload below.
-          bazel run ${{ env.BAZEL_REMOTE_ARGS }} ${{ env.DEVELOPER_FLAGS }} --config=ci ${{ env.EXTRA_BAZEL_ARGS }} --profile=images.bazel-profile.gz //images:load_all
       - name: Bazel test
         if: inputs.run_tests
         run: |

.github/workflows/test.yml

@@ -125,7 +125,7 @@ jobs:
       suffix: ''
       bazelflare: true
       developer_flags: '--config=bazelflare'
-      extra_bazel_args: '--config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare --config=ci-bazelflare-runner-tests'
+      extra_bazel_args: '--config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare'
       runner_extra_bazel_args: '--config=ci-test --config=ci-linux --config=wpt-report'
       upload_test_logs: true
       upload_binary: true
@@ -179,7 +179,7 @@ jobs:
       - name: Check CI's flags against a developer's
         run: |
           # test-linux
-          bash build/check_ci_flags.sh '--config=bazelflare' --config=ci --config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare --config=ci-bazelflare-runner-tests
+          bash build/check_ci_flags.sh '--config=bazelflare' --config=ci --config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare
           # test (linux, -asan)
           bash build/check_ci_flags.sh '--config=bazelflare-asan' --config=ci --config=ci-test --config=ci-bazelflare
           # test-tsan

build/bazelflare/exec_properties.bzl

@@ -6,13 +6,24 @@ def test_exec_properties(size, tags = []):
     The platform packs four actions onto each four-vCPU executor. Large and enormous tests get one
     to themselves, as do tests that ask for more than one CPU (a `cpu:` tag) and tests that hold a
     local resource (a `resources:` tag), which Bazel accounts for only on local execution: two tests
-    that listen on the same fixed port could otherwise share an executor's network. The `test.` prefix applies to the test action alone, leaving the target's
-    build actions on the shared executors. Other remote executors don't know the property, so only
-    bazelflare's platforms take it.
+    that listen on the same fixed port could otherwise share an executor's network. Tests that reach
+    the internet (`requires-network`) run on executors with egress, and tests that drive a Docker
+    engine (`requires-container-engine`) get one of their own, on the deployment's image that runs it
+    (bazelflare-workerd's executor/Dockerfile.dind). The `test.` prefix applies to the test action
+    alone, leaving the target's build actions on the shared executors. Other remote executors don't
+    know the properties, so only bazelflare's platforms take them.
     """
+    props = {}
     if size in ("large", "enormous") or [t for t in tags if t.startswith("cpu:") or t.startswith("resources:")]:
-        return select({
-            Label("//build/bazelflare:on_bazelflare"): {"test.slots": "1"},
-            "//conditions:default": {},
-        })
-    return {}
+        props["test.slots"] = "1"
+    if "requires-network" in tags:
+        props["test.network"] = "on"
+    if "requires-container-engine" in tags:
+        props["test.container-image"] = "executor-dind"
+        props["test.slots"] = "1"
+    if not props:
+        return {}
+    return select({
+        Label("//build/bazelflare:on_bazelflare"): props,
+        "//conditions:default": {},
+    })

build/ci.bazelrc

@@ -113,9 +113,7 @@ build:ci-release --remote_download_minimal --remote_download_regex=".*src/worker
 # bazelflare-coverage in .bazelrc) followed by these, which change how Bazel runs and reports the
 # build, never an action, so CI's action keys equal a workstation's and the two share the cache.
 # build/check_ci_flags.sh, which test.yml's check-ci-flags job runs, fails on any other difference.
-# The targets the bazelflare configs skip (see their tag filters) run on the test-linux job's runner
-# (ci-bazelflare-runner-tests below) and in the arm64 Linux jobs. bazelflare queues actions it can't
-# run at once.
+# bazelflare queues actions it can't run at once.
 build:ci-bazelflare --jobs=400
 test:ci-bazelflare --test_size_filters=
 build:ci-bazelflare-tsan --config=ci-bazelflare
@@ -129,9 +127,3 @@ build:ci-download-workerd --remote_download_regex=".*src/workerd/server/workerd.
 # workers-sdk tests; releases publish them, and the type generating Worker.
 build:ci-download-types --remote_download_regex=".*/bin/types/definitions/.*"
 build:ci-download-types-worker --remote_download_regex=".*/bin/types/dist/.*"
-# The test-linux job also runs, on its runner, the tests that bazelflare's executors can't: those
-# that need the network or the container engine, whose images it loads. A developer's bazelflare
-# config skips them, since it builds for Linux from any workstation.
-build:ci-bazelflare-runner-tests --spawn_strategy=remote,worker,sandboxed,local
-build:ci-bazelflare-runner-tests --build_tag_filters=-off-by-default,-requires-fuzzilli
-test:ci-bazelflare-runner-tests --test_tag_filters=-off-by-default,-requires-fuzzilli

build/config/BUILD.bazel

@@ -12,6 +12,19 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+# Whether the container tests get their images (see //images/container-client-test:images.bzl).
+bool_flag(
+    name = "container_test_images",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "container_test_images_enabled",
+    flag_values = {":container_test_images": "true"},
+    visibility = ["//visibility:public"],
+)
+
 # V8 flags to pass to gen-compile-cache when producing the compile caches
 # embedded in the builtin bundles (//src/node, //src/per_isolate, ...).
 # Must match the flags used by the consumer when compiling, otherwise flag

build/wd_test.bzl

@@ -185,7 +185,7 @@ if [ -n "${{COVERAGE_DIR:-}}" ]; then
     export LLVM_PROFILE_FILE="${{COVERAGE_DIR}}/%p-%m.profraw"
     export KJ_CLEAN_SHUTDOWN=1
 fi
-
+{load_images}
 # Run supervisor to start sidecar if specified
 if [ ! -z "{sidecar}" ]; then
     # These environment variables are processed by the supervisor executable. The supervisor and
@@ -265,6 +265,10 @@ def _wd_test_impl(ctx):
         port_bindings = ",".join(ctx.attr.sidecar_port_bindings),
         randomize_ip = "true" if ctx.attr.sidecar_randomize_ip else "false",
         sidecar_port_mode = ctx.attr.sidecar_port_mode,
+        load_images = "".join([
+            'docker load --quiet --input "{}"\n'.format(f.short_path)
+            for f in ctx.files.container_images
+        ]),
     )
 
     ctx.actions.write(
@@ -273,7 +277,7 @@ def _wd_test_impl(ctx):
         is_executable = True,
     )
 
-    runfiles = ctx.runfiles(files = ctx.files.data)
+    runfiles = ctx.runfiles(files = ctx.files.data + ctx.files.container_images)
 
     # Add runfiles for the workerd binary, such as shared libraries (if any)
     workerd_runfiles = ctx.attr.workerd[DefaultInfo].default_runfiles
@@ -354,6 +358,10 @@ _wd_test = rule(
         ),
         # A list of files that this test requires to be present in order to run.
         "data": attr.label_list(allow_files = True),
+        # Image tarballs (an oci_load's `tarball` output group) that the test loads into the Docker
+        # engine at /var/run/docker.sock before workerd starts, so that a container test brings its
+        # images wherever it runs. Linux only.
+        "container_images": attr.label_list(allow_files = [".tar"]),
         # If set, an executable that is run in parallel with the test, and provides some functionality
         # needed for the test. This is usually a backend server, with workerd serving as the client.
         # The sidecar will be killed once the test completes.

docs/development.md

@@ -38,8 +38,8 @@ Cloudflare employees can build and test on bazelflare, which runs every action o
 executors and shares its cache with CI. Log in once with
 `cloudflared access login https://bazelflare-workerd.runtime-playground.workers.dev`, then add
 `--config=bazelflare` (or `bazelflare-asan`, `bazelflare-tsan`) to a `bazel build` or `bazel test`
-command, from macOS or Linux. Tests that need the network or a container engine are skipped; see
-the bazelflare section of `.bazelrc`.
+command, from macOS or Linux. Tests that need the network or a container engine run on executors
+that have them; the container tests are `enormous`, which Bazel skips without `--test_size_filters=`.
 
 ## Formatting and Linting
 

images/container-client-test/BUILD.bazel

@@ -27,7 +27,6 @@ oci_image(
     base = "@node_25_slim",
     cmd = ["/{}/app".format(package_name())],
     tags = [
-        "no-remote-exec",
         "requires-container-engine",
     ],
     target_compatible_with = [
@@ -48,9 +47,7 @@ oci_load(
         # building a separate image.
         "cloudflare/workerd/container-client-test:override",
     ],
-    # Like :image, whose actions run where the container engine is.
     tags = [
-        "no-remote-exec",
         "requires-container-engine",
     ],
     visibility = ["//visibility:public"],
@@ -61,8 +58,25 @@ oci_load(
     image = "@proxy_everything",
     repo_tags = ["cloudflare/proxy-everything:main"],
     tags = [
-        "no-remote-exec",
         "requires-container-engine",
     ],
     visibility = ["//visibility:public"],
 )
+
+# The images as `docker load` tarballs, for the container tests to load into the engine they run
+# beside (images.bzl).
+filegroup(
+    name = "tarball",
+    srcs = [":load"],
+    output_group = "tarball",
+    tags = ["requires-container-engine"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "proxy-everything-tarball",
+    srcs = [":load-proxy-everything"],
+    output_group = "tarball",
+    tags = ["requires-container-engine"],
+    visibility = ["//visibility:public"],
+)

images/container-client-test/images.bzl

@@ -0,0 +1,12 @@
+"""The container tests' images."""
+
+# The image tarballs the container tests load. They need the repositories that workerd's
+# MODULE.bazel declares for them (rules_oci and the images it pulls), so a build that loads workerd
+# as a plain repository, without that file, leaves //build/config:container_test_images off.
+CONTAINER_TEST_IMAGES = select({
+    Label("//build/config:container_test_images_enabled"): [
+        Label("//images/container-client-test:proxy-everything-tarball"),
+        Label("//images/container-client-test:tarball"),
+    ],
+    "//conditions:default": [],
+})

src/workerd/api/node/tests/BUILD.bazel

@@ -427,11 +427,8 @@ wd_test(
     src = "dns-nodejs-test.wd-test",
     args = ["--experimental"],
     data = ["dns-nodejs-test.js"],
-    # Requires network access and certificates not available in RBE
-    tags = [
-        "no-remote-exec",
-        "requires-network",
-    ],
+    # Resolves names on the internet.
+    tags = ["requires-network"],
 )
 
 js_binary(
@@ -561,11 +558,8 @@ wd_test(
         "HELLO_WORLD_SERVER_PORT",
         "HEADER_VALIDATION_SERVER_PORT",
     ],
-    # Looks up names that remote executors can't resolve
-    tags = [
-        "no-remote-exec",
-        "requires-network",
-    ],
+    # Looks up names on the internet.
+    tags = ["requires-network"],
 )
 
 js_binary(

src/workerd/api/tests/BUILD.bazel

@@ -1474,11 +1474,8 @@ wd_test(
     src = "worker-loader-test.wd-test",
     args = ["--experimental"],
     data = ["worker-loader-test.js"],
-    # Needs to fetch pyodide which is not supported under RBE
-    tags = [
-        "no-remote-exec",
-        "requires-network",
-    ],
+    # Fetches Pyodide.
+    tags = ["requires-network"],
 )
 
 wd_test(

src/workerd/server/tests/container-client/BUILD.bazel

@@ -1,12 +1,13 @@
 load("//:build/wd_test.bzl", "wd_test")
+load("//images/container-client-test:images.bzl", "CONTAINER_TEST_IMAGES")
 
 wd_test(
     size = "enormous",
     src = "container-client.wd-test",
     args = ["--experimental"],
+    container_images = CONTAINER_TEST_IMAGES,
     data = ["test.js"],
     tags = [
-        "no-remote-exec",
         "requires-container-engine",
         "requires-network",  # Accesses unix://var/run/docker.sock
     ],

src/workerd/server/tests/container-client/README.md

@@ -26,13 +26,7 @@ To run the tests:
    docker image rm cf-container-client-test
    ```
 
-5. Build Docker images
-
-   ```shell
-   bazel run //images:load_all
-   ```
-
-6. Run the test
+5. Run the test, which loads its images itself
 
    ```shell
    just stream-test //src/workerd/server/tests/container-client:container-client@

src/workerd/server/tests/container-shutdown/BUILD.bazel

@@ -1,4 +1,12 @@
 load("@aspect_rules_js//js:defs.bzl", "js_test")
+load("//build/bazelflare:exec_properties.bzl", "test_exec_properties")
+load("//images/container-client-test:images.bzl", "CONTAINER_TEST_IMAGES")
+
+_TAGS = [
+    "js-test",
+    "requires-container-engine",
+    "requires-network",  # Accesses unix:///var/run/docker.sock
+]
 
 js_test(
     name = "container-shutdown-test",
@@ -8,16 +16,12 @@ js_test(
         ":worker.mjs",
         "//src/workerd/server:workerd",
         "//src/workerd/server/tests:server-harness_js_lib",
-    ],
+    ] + CONTAINER_TEST_IMAGES,
     entry_point = "test.mjs",
     env = {
         "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)",
         "WORKERD_CONFIG": "$(rootpath :config.capnp)",
     },
-    tags = [
-        "js-test",
-        "no-remote-exec",
-        "requires-container-engine",
-        "requires-network",  # Accesses unix:///var/run/docker.sock
-    ],
+    exec_properties = test_exec_properties("enormous", _TAGS),
+    tags = _TAGS,
 )

src/workerd/server/tests/container-shutdown/test.mjs

@@ -2,6 +2,8 @@
 // Licensed under the Apache 2.0 license found in the LICENSE file or at:
 //     https://opensource.org/licenses/Apache-2.0
 import assert from 'node:assert';
+import { execFileSync } from 'node:child_process';
+import { globSync } from 'node:fs';
 import http from 'node:http';
 import { env } from 'node:process';
 import { test } from 'node:test';
@@ -70,6 +72,12 @@ async function removeTestContainers() {
 }
 
 test('graceful shutdown removes application and sidecar containers', async () => {
+  // The images the containers run, as tarballs among the test's runfiles.
+  for (const tarball of globSync('images/**/tarball.tar')) {
+    execFileSync('docker', ['load', '--quiet', '--input', tarball], {
+      stdio: 'inherit',
+    });
+  }
   await removeTestContainers();
 
   const workerd = new WorkerdServerHarness({