diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 86073d5b56..4e862fb77e 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -92,6 +92,7 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s-ipv6","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful-cdi","installer":"deb","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index 5a8a3942aa..7736feae39 100644 --- a/TESTING.md +++ b/TESTING.md @@ -607,6 +607,12 @@ installer applies it to the gateway's own file: with no overrides leaves it untouched. - `k3s` installers ignore the file. +K3s environments adapt the Helm chart instead: a playbook that runs before the +installer writes values to `/etc/openshell-test/helm-values.yaml`, and the +`k3s` and `k3s-ha` installers apply them after their own values. +`ubuntu-k3s-ipv6` uses this to set the gateway and supervisor `listenAddress` +to `::`. + `tests/tmachine/ansible/roles/openshell_gateway/files/merge-gateway-toml.py` does the work. Each key in the overrides replaces the same key in the table, or is added to it, and a table the file lacks is appended. The rest of the file, diff --git a/tests/tmachine/ansible/playbooks/k3s-ipv6.yaml b/tests/tmachine/ansible/playbooks/k3s-ipv6.yaml new file mode 100644 index 0000000000..6a709430f6 --- /dev/null +++ b/tests/tmachine/ansible/playbooks/k3s-ipv6.yaml @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Turn the K3s base into a dual-stack cluster with IPv6 first, so Pods and +# Services get an IPv6 address first and OpenShell runs over IPv6. +# +# k3s.yaml ran first and reset the cluster, so the next start creates it again +# from this configuration. This playbook starts it once to check it, then +# resets it the same way. It also writes the Helm values the chart needs on an +# IPv6 cluster to /etc/openshell-test/helm-values.yaml, which the k3s +# installers (openshell-k3s.yaml) apply. +--- +- name: Configure IPv6-first dual-stack K3s + hosts: all + become: true + gather_facts: false + vars: + k3s_ipv6_cluster_cidr: fd00:42::/56,10.42.0.0/16 + k3s_ipv6_service_cidr: fd00:43::/112,10.43.0.0/16 + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + # SSH can be up before the guest has its IPv6 address, so wait for it. + - name: Gather the guest addresses + ansible.builtin.setup: + gather_subset: [network] + register: k3s_ipv6_facts + until: (k3s_ipv6_facts.ansible_facts.ansible_default_ipv6.scope | default('')) == "global" + retries: 10 + delay: 2 + failed_when: false + + # K3s takes the node's addresses from the default routes, IPv6 first for + # an IPv6-first cluster. + - name: Require a global IPv6 address + ansible.builtin.assert: + that: + - ansible_facts.default_ipv6.address is defined + - ansible_facts.default_ipv6.scope == "global" + fail_msg: >- + The guest needs a global IPv6 address with a default route, not + {{ ansible_facts.default_ipv6 | default('none') }}; tmachine sets + ipv6-net on QEMU user networking for this. + + - name: Create the K3s configuration directory + ansible.builtin.file: + path: /etc/rancher/k3s/config.yaml.d + state: directory + owner: root + group: root + mode: "0755" + + # The order of each list makes IPv6 the primary family. Flannel + # masquerades IPv6 Pod traffic leaving the cluster, as it does for IPv4. + - name: Write the dual-stack K3s configuration + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml.d/50-ipv6-dual-stack.yaml + owner: root + group: root + mode: "0644" + content: | + cluster-cidr: {{ k3s_ipv6_cluster_cidr }} + service-cidr: {{ k3s_ipv6_service_cidr }} + flannel-ipv6-masq: true + + - name: Create the test configuration directory + ansible.builtin.file: + path: /etc/openshell-test + state: directory + owner: root + group: root + mode: "0755" + + - name: Write the Helm values for an IPv6 cluster + ansible.builtin.copy: + dest: /etc/openshell-test/helm-values.yaml + owner: root + group: root + mode: "0644" + content: | + gateway: + listenAddress: "::" + supervisor: + listenAddress: "::" + + - name: Start K3s + ansible.builtin.systemd: + name: k3s + state: started + + - name: Wait for K3s + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - wait + - --for=condition=Ready + - node + - --all + - --timeout=120s + register: k3s_ready + changed_when: false + until: k3s_ready.rc == 0 + retries: 12 + delay: 5 + + - name: Wait for CoreDNS + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace=kube-system + - rollout + - status + - deployment/coredns + - --timeout=180s + register: k3s_coredns + changed_when: false + until: k3s_coredns.rc == 0 + retries: 30 + delay: 5 + + - name: Read the cluster's address families + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace=kube-system + - get + - service + - kube-dns + - --output=jsonpath={.spec.clusterIPs} + register: k3s_dns_addresses + changed_when: false + + - name: Require an IPv6-first dual-stack Service + ansible.builtin.assert: + that: + - k3s_dns_addresses.stdout | from_json | length == 2 + - (k3s_dns_addresses.stdout | from_json)[0] is search(':') + - (k3s_dns_addresses.stdout | from_json)[1] is match('^[0-9.]+$') + fail_msg: "kube-dns has addresses {{ k3s_dns_addresses.stdout }}" + + # Tests reach the gateway's Service from the node, which fails when the + # node picks a source address that replies can't reach. + - name: Check the node reaches a Pod through an IPv6 Service + ansible.builtin.uri: + url: "http://[{{ (k3s_dns_addresses.stdout | from_json)[0] }}]:9153/metrics" + timeout: 10 + register: k3s_ipv6_service + until: k3s_ipv6_service.status == 200 + retries: 6 + delay: 5 + + - name: Reset the K3s cluster state + ansible.builtin.include_tasks: k3s-reset.yaml diff --git a/tests/tmachine/ansible/playbooks/k3s-reset.yaml b/tests/tmachine/ansible/playbooks/k3s-reset.yaml new file mode 100644 index 0000000000..51bebcab8f --- /dev/null +++ b/tests/tmachine/ansible/playbooks/k3s-reset.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A setup layer is cached across runs, so it must not carry cluster +# credentials: every disk built on it would share one CA, token and admin +# kubeconfig. Stop K3s and delete the state it created, keeping the containerd +# image store (agent/containerd) and the K3s data directory. The next start +# generates fresh credentials and creates the cluster again from the K3s +# configuration. +--- +- name: Stop K3s and its containers + ansible.builtin.command: + argv: [/usr/local/bin/k3s-killall.sh] + changed_when: true + +- name: Remove the K3s credentials and cluster state + ansible.builtin.shell: | + set -eu + cd /var/lib/rancher/k3s + rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token + find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete + rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni + changed_when: true + +- name: Check no K3s credentials remain + ansible.builtin.shell: | + find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \ + -maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \ + -o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print + register: k3s_leftover_credentials + changed_when: false + failed_when: k3s_leftover_credentials.stdout | length > 0 diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index 07b0de85d8..03484cc4b4 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -72,30 +72,5 @@ environment: DESIRED_VERSION: "v{{ helm_version }}" - # This layer is cached across runs, so it must not carry cluster - # credentials: every disk built on it would share one CA, token and admin - # kubeconfig. Stop K3s and delete the state it created on first start, - # keeping the containerd image store (agent/containerd) and the K3s data - # directory. The next start generates fresh credentials. - - name: Stop K3s and its containers - ansible.builtin.command: - argv: [/usr/local/bin/k3s-killall.sh] - changed_when: true - - - name: Remove the K3s credentials and cluster state - ansible.builtin.shell: | - set -eu - cd /var/lib/rancher/k3s - rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token - find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete - rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni - changed_when: true - - - name: Check no K3s credentials remain - ansible.builtin.shell: | - find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \ - -maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \ - -o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print - register: k3s_leftover_credentials - changed_when: false - failed_when: k3s_leftover_credentials.stdout | length > 0 + - name: Reset the K3s cluster state + ansible.builtin.include_tasks: k3s-reset.yaml diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml index efcc736d49..3518cc1a7f 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml @@ -64,7 +64,7 @@ ansible.builtin.lineinfile: path: /etc/hosts regexp: '\sopenshell\.openshell\.svc\.cluster\.local$' - line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local" + line: "{{ k3s_gateway_ip }} openshell.openshell.svc.cluster.local" - name: Read chart client TLS Secret ansible.builtin.command: @@ -103,4 +103,4 @@ apply: become: false vars: - openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" + openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_port }}" diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index 5b68bc0067..a1a43c3f8a 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -66,7 +66,9 @@ # address must be the one of the node the tests run on. QEMU user # networking gives every boot the same guest address (10.0.2.15), so the # value stays valid when the disk is reused. - # The node can also list an IPv6 address after it, so take the IPv4 one. + # A dual-stack node also lists an IPv6 address, first on an IPv6-first + # cluster. Take the IPv4 one: the servers tests start for sandboxes to + # reach (tests/support/host-server) listen on IPv4 only. - name: Discover the K3s node address ansible.builtin.command: argv: @@ -133,6 +135,27 @@ mode: "0600" content: "{{ openshell_k3s_helm_values_overlay | default('{}') }}" + # An environment adapts the chart to its cluster by writing values to + # /etc/openshell-test/helm-values.yaml before the installer runs (for + # example playbooks/k3s-ipv6.yaml). They apply last. Other environments + # get an empty file. + - name: Create the test configuration directory + ansible.builtin.file: + path: /etc/openshell-test + state: directory + owner: root + group: root + mode: "0755" + + - name: Ensure environment Helm values exist + ansible.builtin.copy: + dest: /etc/openshell-test/helm-values.yaml + content: "{}\n" + force: false + owner: root + group: root + mode: "0644" + - name: Install Agent Sandbox ansible.builtin.command: argv: @@ -180,6 +203,8 @@ - /var/lib/openshell/artifacts/values.yaml - --values - /var/lib/openshell/artifacts/values-overlay.yaml + - --values + - /etc/openshell-test/helm-values.yaml - --wait - --timeout=5m environment: @@ -196,17 +221,29 @@ - get - service - openshell - - --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port} + - --output=jsonpath={.spec.clusterIP} {.spec.ports[?(@.name=="grpc")].port} register: k3s_gateway_address changed_when: false # Rejects headless Services (ClusterIP "None") and a missing grpc port. + # The ClusterIP is IPv6 on a cluster whose primary family is IPv6. - name: Require a gateway Service address ansible.builtin.assert: that: - - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') + - k3s_gateway_address.stdout is match('^([0-9.]+|[0-9a-f:]+) [0-9]+$') fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" + # A URL needs brackets around an IPv6 address, for example + # http://[fd00:43::a]:8080; without them the port can't be told apart. + - name: Record the gateway Service address + vars: + gateway_ip: "{{ k3s_gateway_address.stdout.split()[0] }}" + gateway_port: "{{ k3s_gateway_address.stdout.split()[1] }}" + ansible.builtin.set_fact: + k3s_gateway_ip: "{{ gateway_ip }}" + k3s_gateway_port: "{{ gateway_port }}" + k3s_gateway_authority: "{{ ('[' ~ gateway_ip ~ ']') if ':' in gateway_ip else gateway_ip }}:{{ gateway_port }}" + # Deployment readiness doesn't establish that the controller reconciles # Sandboxes. This checks the installed cluster; a booted guest restarts the # controller, so the test runner checks again after it boots @@ -218,6 +255,6 @@ hosts: all gather_facts: false vars: - openshell_client_gateway_endpoint: "http://{{ k3s_gateway_address.stdout }}" + openshell_client_gateway_endpoint: "http://{{ k3s_gateway_authority }}" roles: - openshell_client diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 5a41618bf4..115d717eaa 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -107,6 +107,12 @@ let inputs.keycloak_realm_file = "../../scripts/keycloak-realm.json"; }; }; + # A dual-stack cluster with IPv6 first, and the chart's IPv6 listen + # addresses, for running the k3s suites over IPv6. + ipv6 = { + bases = [ "ubuntu-k3s" ]; + setup.playbooks = [ "ansible/playbooks/k3s-ipv6.yaml" ]; + }; # Hardware-free CDI fixtures, for the cdi suite. cdi = { bases = dockerBases ++ podmanBases; diff --git a/tests/tmachine/src/qemu/vm.rs b/tests/tmachine/src/qemu/vm.rs index 354d04c53a..000d9d1779 100644 --- a/tests/tmachine/src/qemu/vm.rs +++ b/tests/tmachine/src/qemu/vm.rs @@ -225,8 +225,14 @@ fn command( .arg("virtio-blk-pci,drive=rootfs,bootindex=1") .arg("-device") .arg("virtio-rng-pci") + // A unique local IPv6 prefix instead of user networking's default + // site-local fec0::/64. Linux ranks a site-local source below any + // global address, so on a dual-stack K3s guest it picks flannel's + // address to reach an IPv6 Service and never gets the reply. .arg("-netdev") - .arg(format!("user,id=net0,hostfwd=tcp:127.0.0.1:{ssh_port}-:22")) + .arg(format!( + "user,id=net0,ipv6-net=fd00:10::/64,hostfwd=tcp:127.0.0.1:{ssh_port}-:22" + )) .arg("-device") .arg("virtio-net-pci,netdev=net0");