diff --git a/.bazelrc b/.bazelrc index 078ea2977b6..bfc19829395 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,7 @@ common --enable_platform_specific_config build --verbose_failures build --build_tag_filters=-off-by-default,-requires-container-engine +build --//build/config:container_test_images # block network access by default build --nosandbox_default_allow_network test --test_tag_filters=-off-by-default,-requires-fuzzilli,-requires-container-engine @@ -628,10 +629,10 @@ common:bazelflare --extra_execution_platforms=//build/bazelflare:linux_x86_64 common:bazelflare --spawn_strategy=remote --remote_local_fallback=false # V8 marks its inspector code generator `local = 1`; it runs fine remotely. common:bazelflare --modify_execution_info=Genrule=-local -# Executors have neither internet nor a container engine, so targets that need them, or that must -# run on the client, are skipped. Filters don't change action keys. -common:bazelflare --build_tag_filters=-off-by-default,-no-remote-exec,-requires-network,-requires-container-engine,-requires-fuzzilli -test:bazelflare --test_tag_filters=-off-by-default,-no-remote-exec,-requires-network,-requires-container-engine,-requires-fuzzilli +# Targets that must run on the client are skipped; unlike the default filters, these keep the +# container tests. Filters don't change action keys. +common:bazelflare --build_tag_filters=-off-by-default,-no-remote-exec,-requires-fuzzilli +test:bazelflare --test_tag_filters=-off-by-default,-no-remote-exec,-requires-fuzzilli common:bazelflare --remote_download_minimal common:bazelflare --remote_cache_compression --remote_timeout=600 # bazelflare sends nothing while an action runs, so only pings notice a connection that died diff --git a/.github/workflows/_bazel.yml b/.github/workflows/_bazel.yml index ec753a7a136..7afb5f22e1a 100644 --- a/.github/workflows/_bazel.yml +++ b/.github/workflows/_bazel.yml @@ -205,16 +205,6 @@ jobs: sudo mkdir -p /etc/docker echo '{"ipv6": true, "fixed-cidr-v6": "fd00::/80"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - - name: Build and load container images - if: inputs.build_container_images - run: | - docker context use default - echo "Docker info:" - docker info - echo "Docker version:" - docker version - # Its own profile, so that it leaves the build's for the upload below. - bazel run ${{ env.BAZEL_REMOTE_ARGS }} ${{ env.DEVELOPER_FLAGS }} --config=ci ${{ env.EXTRA_BAZEL_ARGS }} --profile=images.bazel-profile.gz //images:load_all - name: Bazel test if: inputs.run_tests run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2bf62ba25a8..32a209b8b83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,7 +125,7 @@ jobs: suffix: '' bazelflare: true developer_flags: '--config=bazelflare' - extra_bazel_args: '--config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare --config=ci-bazelflare-runner-tests' + extra_bazel_args: '--config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare' runner_extra_bazel_args: '--config=ci-test --config=ci-linux --config=wpt-report' upload_test_logs: true upload_binary: true @@ -179,7 +179,7 @@ jobs: - name: Check CI's flags against a developer's run: | # test-linux - bash build/check_ci_flags.sh '--config=bazelflare' --config=ci --config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare --config=ci-bazelflare-runner-tests + bash build/check_ci_flags.sh '--config=bazelflare' --config=ci --config=ci-test --config=ci-download-workerd --config=wpt-report --config=ci-bazelflare # test (linux, -asan) bash build/check_ci_flags.sh '--config=bazelflare-asan' --config=ci --config=ci-test --config=ci-bazelflare # test-tsan diff --git a/build/bazelflare/exec_properties.bzl b/build/bazelflare/exec_properties.bzl index 0b758f2a1e9..18491f5ef53 100644 --- a/build/bazelflare/exec_properties.bzl +++ b/build/bazelflare/exec_properties.bzl @@ -6,13 +6,24 @@ def test_exec_properties(size, tags = []): The platform packs four actions onto each four-vCPU executor. Large and enormous tests get one to themselves, as do tests that ask for more than one CPU (a `cpu:` tag) and tests that hold a local resource (a `resources:` tag), which Bazel accounts for only on local execution: two tests - that listen on the same fixed port could otherwise share an executor's network. The `test.` prefix applies to the test action alone, leaving the target's - build actions on the shared executors. Other remote executors don't know the property, so only - bazelflare's platforms take it. + that listen on the same fixed port could otherwise share an executor's network. Tests that reach + the internet (`requires-network`) run on executors with egress, and tests that drive a Docker + engine (`requires-container-engine`) get one of their own, on the deployment's image that runs it + (bazelflare-workerd's executor/Dockerfile.dind). The `test.` prefix applies to the test action + alone, leaving the target's build actions on the shared executors. Other remote executors don't + know the properties, so only bazelflare's platforms take them. """ + props = {} if size in ("large", "enormous") or [t for t in tags if t.startswith("cpu:") or t.startswith("resources:")]: - return select({ - Label("//build/bazelflare:on_bazelflare"): {"test.slots": "1"}, - "//conditions:default": {}, - }) - return {} + props["test.slots"] = "1" + if "requires-network" in tags: + props["test.network"] = "on" + if "requires-container-engine" in tags: + props["test.container-image"] = "executor-dind" + props["test.slots"] = "1" + if not props: + return {} + return select({ + Label("//build/bazelflare:on_bazelflare"): props, + "//conditions:default": {}, + }) diff --git a/build/ci.bazelrc b/build/ci.bazelrc index 1abe3e19231..378d1b428fd 100644 --- a/build/ci.bazelrc +++ b/build/ci.bazelrc @@ -113,9 +113,7 @@ build:ci-release --remote_download_minimal --remote_download_regex=".*src/worker # bazelflare-coverage in .bazelrc) followed by these, which change how Bazel runs and reports the # build, never an action, so CI's action keys equal a workstation's and the two share the cache. # build/check_ci_flags.sh, which test.yml's check-ci-flags job runs, fails on any other difference. -# The targets the bazelflare configs skip (see their tag filters) run on the test-linux job's runner -# (ci-bazelflare-runner-tests below) and in the arm64 Linux jobs. bazelflare queues actions it can't -# run at once. +# bazelflare queues actions it can't run at once. build:ci-bazelflare --jobs=400 test:ci-bazelflare --test_size_filters= build:ci-bazelflare-tsan --config=ci-bazelflare @@ -129,9 +127,3 @@ build:ci-download-workerd --remote_download_regex=".*src/workerd/server/workerd. # workers-sdk tests; releases publish them, and the type generating Worker. build:ci-download-types --remote_download_regex=".*/bin/types/definitions/.*" build:ci-download-types-worker --remote_download_regex=".*/bin/types/dist/.*" -# The test-linux job also runs, on its runner, the tests that bazelflare's executors can't: those -# that need the network or the container engine, whose images it loads. A developer's bazelflare -# config skips them, since it builds for Linux from any workstation. -build:ci-bazelflare-runner-tests --spawn_strategy=remote,worker,sandboxed,local -build:ci-bazelflare-runner-tests --build_tag_filters=-off-by-default,-requires-fuzzilli -test:ci-bazelflare-runner-tests --test_tag_filters=-off-by-default,-requires-fuzzilli diff --git a/build/config/BUILD.bazel b/build/config/BUILD.bazel index 4c5b0ba8ee3..9b0306e7211 100644 --- a/build/config/BUILD.bazel +++ b/build/config/BUILD.bazel @@ -12,6 +12,19 @@ config_setting( visibility = ["//visibility:public"], ) +# Whether the container tests get their images (see //images/container-client-test:images.bzl). +bool_flag( + name = "container_test_images", + build_setting_default = False, + visibility = ["//visibility:public"], +) + +config_setting( + name = "container_test_images_enabled", + flag_values = {":container_test_images": "true"}, + visibility = ["//visibility:public"], +) + # V8 flags to pass to gen-compile-cache when producing the compile caches # embedded in the builtin bundles (//src/node, //src/per_isolate, ...). # Must match the flags used by the consumer when compiling, otherwise flag diff --git a/build/wd_test.bzl b/build/wd_test.bzl index 26e0daf944a..d39c971f480 100644 --- a/build/wd_test.bzl +++ b/build/wd_test.bzl @@ -185,7 +185,7 @@ if [ -n "${{COVERAGE_DIR:-}}" ]; then export LLVM_PROFILE_FILE="${{COVERAGE_DIR}}/%p-%m.profraw" export KJ_CLEAN_SHUTDOWN=1 fi - +{load_images} # Run supervisor to start sidecar if specified if [ ! -z "{sidecar}" ]; then # These environment variables are processed by the supervisor executable. The supervisor and @@ -265,6 +265,10 @@ def _wd_test_impl(ctx): port_bindings = ",".join(ctx.attr.sidecar_port_bindings), randomize_ip = "true" if ctx.attr.sidecar_randomize_ip else "false", sidecar_port_mode = ctx.attr.sidecar_port_mode, + load_images = "".join([ + 'docker load --quiet --input "{}"\n'.format(f.short_path) + for f in ctx.files.container_images + ]), ) ctx.actions.write( @@ -273,7 +277,7 @@ def _wd_test_impl(ctx): is_executable = True, ) - runfiles = ctx.runfiles(files = ctx.files.data) + runfiles = ctx.runfiles(files = ctx.files.data + ctx.files.container_images) # Add runfiles for the workerd binary, such as shared libraries (if any) workerd_runfiles = ctx.attr.workerd[DefaultInfo].default_runfiles @@ -354,6 +358,10 @@ _wd_test = rule( ), # A list of files that this test requires to be present in order to run. "data": attr.label_list(allow_files = True), + # Image tarballs (an oci_load's `tarball` output group) that the test loads into the Docker + # engine at /var/run/docker.sock before workerd starts, so that a container test brings its + # images wherever it runs. Linux only. + "container_images": attr.label_list(allow_files = [".tar"]), # If set, an executable that is run in parallel with the test, and provides some functionality # needed for the test. This is usually a backend server, with workerd serving as the client. # The sidecar will be killed once the test completes. diff --git a/docs/development.md b/docs/development.md index e342db60657..f2056141222 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,8 +38,8 @@ Cloudflare employees can build and test on bazelflare, which runs every action o executors and shares its cache with CI. Log in once with `cloudflared access login https://bazelflare-workerd.runtime-playground.workers.dev`, then add `--config=bazelflare` (or `bazelflare-asan`, `bazelflare-tsan`) to a `bazel build` or `bazel test` -command, from macOS or Linux. Tests that need the network or a container engine are skipped; see -the bazelflare section of `.bazelrc`. +command, from macOS or Linux. Tests that need the network or a container engine run on executors +that have them; the container tests are `enormous`, which Bazel skips without `--test_size_filters=`. ## Formatting and Linting diff --git a/images/container-client-test/BUILD.bazel b/images/container-client-test/BUILD.bazel index 3fc000d7c14..d0fb7e544e4 100644 --- a/images/container-client-test/BUILD.bazel +++ b/images/container-client-test/BUILD.bazel @@ -27,7 +27,6 @@ oci_image( base = "@node_25_slim", cmd = ["/{}/app".format(package_name())], tags = [ - "no-remote-exec", "requires-container-engine", ], target_compatible_with = [ @@ -48,9 +47,7 @@ oci_load( # building a separate image. "cloudflare/workerd/container-client-test:override", ], - # Like :image, whose actions run where the container engine is. tags = [ - "no-remote-exec", "requires-container-engine", ], visibility = ["//visibility:public"], @@ -61,8 +58,25 @@ oci_load( image = "@proxy_everything", repo_tags = ["cloudflare/proxy-everything:main"], tags = [ - "no-remote-exec", "requires-container-engine", ], visibility = ["//visibility:public"], ) + +# The images as `docker load` tarballs, for the container tests to load into the engine they run +# beside (images.bzl). +filegroup( + name = "tarball", + srcs = [":load"], + output_group = "tarball", + tags = ["requires-container-engine"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "proxy-everything-tarball", + srcs = [":load-proxy-everything"], + output_group = "tarball", + tags = ["requires-container-engine"], + visibility = ["//visibility:public"], +) diff --git a/images/container-client-test/images.bzl b/images/container-client-test/images.bzl new file mode 100644 index 00000000000..4f40c7c1736 --- /dev/null +++ b/images/container-client-test/images.bzl @@ -0,0 +1,12 @@ +"""The container tests' images.""" + +# The image tarballs the container tests load. They need the repositories that workerd's +# MODULE.bazel declares for them (rules_oci and the images it pulls), so a build that loads workerd +# as a plain repository, without that file, leaves //build/config:container_test_images off. +CONTAINER_TEST_IMAGES = select({ + Label("//build/config:container_test_images_enabled"): [ + Label("//images/container-client-test:proxy-everything-tarball"), + Label("//images/container-client-test:tarball"), + ], + "//conditions:default": [], +}) diff --git a/src/workerd/api/node/tests/BUILD.bazel b/src/workerd/api/node/tests/BUILD.bazel index d9bf7200081..96721068733 100644 --- a/src/workerd/api/node/tests/BUILD.bazel +++ b/src/workerd/api/node/tests/BUILD.bazel @@ -427,11 +427,8 @@ wd_test( src = "dns-nodejs-test.wd-test", args = ["--experimental"], data = ["dns-nodejs-test.js"], - # Requires network access and certificates not available in RBE - tags = [ - "no-remote-exec", - "requires-network", - ], + # Resolves names on the internet. + tags = ["requires-network"], ) js_binary( @@ -561,11 +558,8 @@ wd_test( "HELLO_WORLD_SERVER_PORT", "HEADER_VALIDATION_SERVER_PORT", ], - # Looks up names that remote executors can't resolve - tags = [ - "no-remote-exec", - "requires-network", - ], + # Looks up names on the internet. + tags = ["requires-network"], ) js_binary( diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index ff1fcf6ccc1..74feb4107fe 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -1474,11 +1474,8 @@ wd_test( src = "worker-loader-test.wd-test", args = ["--experimental"], data = ["worker-loader-test.js"], - # Needs to fetch pyodide which is not supported under RBE - tags = [ - "no-remote-exec", - "requires-network", - ], + # Fetches Pyodide. + tags = ["requires-network"], ) wd_test( diff --git a/src/workerd/server/tests/container-client/BUILD.bazel b/src/workerd/server/tests/container-client/BUILD.bazel index 5ae2299f35a..5420c54d3f0 100644 --- a/src/workerd/server/tests/container-client/BUILD.bazel +++ b/src/workerd/server/tests/container-client/BUILD.bazel @@ -1,12 +1,13 @@ load("//:build/wd_test.bzl", "wd_test") +load("//images/container-client-test:images.bzl", "CONTAINER_TEST_IMAGES") wd_test( size = "enormous", src = "container-client.wd-test", args = ["--experimental"], + container_images = CONTAINER_TEST_IMAGES, data = ["test.js"], tags = [ - "no-remote-exec", "requires-container-engine", "requires-network", # Accesses unix://var/run/docker.sock ], diff --git a/src/workerd/server/tests/container-client/README.md b/src/workerd/server/tests/container-client/README.md index 27eae8c1fe4..426964641ed 100644 --- a/src/workerd/server/tests/container-client/README.md +++ b/src/workerd/server/tests/container-client/README.md @@ -26,13 +26,7 @@ To run the tests: docker image rm cf-container-client-test ``` -5. Build Docker images - - ```shell - bazel run //images:load_all - ``` - -6. Run the test +5. Run the test, which loads its images itself ```shell just stream-test //src/workerd/server/tests/container-client:container-client@ diff --git a/src/workerd/server/tests/container-shutdown/BUILD.bazel b/src/workerd/server/tests/container-shutdown/BUILD.bazel index f6d5360a2a0..54477643391 100644 --- a/src/workerd/server/tests/container-shutdown/BUILD.bazel +++ b/src/workerd/server/tests/container-shutdown/BUILD.bazel @@ -1,4 +1,12 @@ load("@aspect_rules_js//js:defs.bzl", "js_test") +load("//build/bazelflare:exec_properties.bzl", "test_exec_properties") +load("//images/container-client-test:images.bzl", "CONTAINER_TEST_IMAGES") + +_TAGS = [ + "js-test", + "requires-container-engine", + "requires-network", # Accesses unix:///var/run/docker.sock +] js_test( name = "container-shutdown-test", @@ -8,16 +16,12 @@ js_test( ":worker.mjs", "//src/workerd/server:workerd", "//src/workerd/server/tests:server-harness_js_lib", - ], + ] + CONTAINER_TEST_IMAGES, entry_point = "test.mjs", env = { "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", "WORKERD_CONFIG": "$(rootpath :config.capnp)", }, - tags = [ - "js-test", - "no-remote-exec", - "requires-container-engine", - "requires-network", # Accesses unix:///var/run/docker.sock - ], + exec_properties = test_exec_properties("enormous", _TAGS), + tags = _TAGS, ) diff --git a/src/workerd/server/tests/container-shutdown/test.mjs b/src/workerd/server/tests/container-shutdown/test.mjs index 049f52ba979..d8a3cb74d9a 100644 --- a/src/workerd/server/tests/container-shutdown/test.mjs +++ b/src/workerd/server/tests/container-shutdown/test.mjs @@ -2,6 +2,8 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 import assert from 'node:assert'; +import { execFileSync } from 'node:child_process'; +import { globSync } from 'node:fs'; import http from 'node:http'; import { env } from 'node:process'; import { test } from 'node:test'; @@ -70,6 +72,12 @@ async function removeTestContainers() { } test('graceful shutdown removes application and sidecar containers', async () => { + // The images the containers run, as tarballs among the test's runfiles. + for (const tarball of globSync('images/**/tarball.tar')) { + execFileSync('docker', ['load', '--quiet', '--input', tarball], { + stdio: 'inherit', + }); + } await removeTestContainers(); const workerd = new WorkerdServerHarness({