Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/e2e-suites.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ jobs:
{"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s-ipv6","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful-cdi","installer":"deb","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootful-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"},
Expand Down
6 changes: 6 additions & 0 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,12 @@ installer applies it to the gateway's own file:
with no overrides leaves it untouched.
- `k3s` installers ignore the file.

K3s environments adapt the Helm chart instead: a playbook that runs before the
installer writes values to `/etc/openshell-test/helm-values.yaml`, and the
`k3s` and `k3s-ha` installers apply them after their own values.
`ubuntu-k3s-ipv6` uses this to set the gateway and supervisor `listenAddress`
to `::`.

`tests/tmachine/ansible/roles/openshell_gateway/files/merge-gateway-toml.py`
does the work. Each key in the overrides replaces the same key in the table, or
is added to it, and a table the file lacks is appended. The rest of the file,
Expand Down
157 changes: 157 additions & 0 deletions tests/tmachine/ansible/playbooks/k3s-ipv6.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Turn the K3s base into a dual-stack cluster with IPv6 first, so Pods and
# Services get an IPv6 address first and OpenShell runs over IPv6.
#
# k3s.yaml ran first and reset the cluster, so the next start creates it again
# from this configuration. This playbook starts it once to check it, then
# resets it the same way. It also writes the Helm values the chart needs on an
# IPv6 cluster to /etc/openshell-test/helm-values.yaml, which the k3s
# installers (openshell-k3s.yaml) apply.
---
- name: Configure IPv6-first dual-stack K3s
hosts: all
become: true
gather_facts: false
vars:
k3s_ipv6_cluster_cidr: fd00:42::/56,10.42.0.0/16
k3s_ipv6_service_cidr: fd00:43::/112,10.43.0.0/16
tasks:
- name: Wait for SSH
ansible.builtin.wait_for_connection:

# SSH can be up before the guest has its IPv6 address, so wait for it.
- name: Gather the guest addresses
ansible.builtin.setup:
gather_subset: [network]
register: k3s_ipv6_facts
until: (k3s_ipv6_facts.ansible_facts.ansible_default_ipv6.scope | default('')) == "global"
retries: 10
delay: 2
failed_when: false

# K3s takes the node's addresses from the default routes, IPv6 first for
# an IPv6-first cluster.
- name: Require a global IPv6 address
ansible.builtin.assert:
that:
- ansible_facts.default_ipv6.address is defined
- ansible_facts.default_ipv6.scope == "global"
fail_msg: >-
The guest needs a global IPv6 address with a default route, not
{{ ansible_facts.default_ipv6 | default('none') }}; tmachine sets
ipv6-net on QEMU user networking for this.

- name: Create the K3s configuration directory
ansible.builtin.file:
path: /etc/rancher/k3s/config.yaml.d
state: directory
owner: root
group: root
mode: "0755"

# The order of each list makes IPv6 the primary family. Flannel
# masquerades IPv6 Pod traffic leaving the cluster, as it does for IPv4.
- name: Write the dual-stack K3s configuration
ansible.builtin.copy:
dest: /etc/rancher/k3s/config.yaml.d/50-ipv6-dual-stack.yaml
owner: root
group: root
mode: "0644"
content: |
cluster-cidr: {{ k3s_ipv6_cluster_cidr }}
service-cidr: {{ k3s_ipv6_service_cidr }}
flannel-ipv6-masq: true

- name: Create the test configuration directory
ansible.builtin.file:
path: /etc/openshell-test
state: directory
owner: root
group: root
mode: "0755"

- name: Write the Helm values for an IPv6 cluster
ansible.builtin.copy:
dest: /etc/openshell-test/helm-values.yaml
owner: root
group: root
mode: "0644"
content: |
gateway:
listenAddress: "::"
supervisor:
listenAddress: "::"

- name: Start K3s
ansible.builtin.systemd:
name: k3s
state: started

- name: Wait for K3s
ansible.builtin.command:
argv:
- /usr/local/bin/k3s
- kubectl
- wait
- --for=condition=Ready
- node
- --all
- --timeout=120s
register: k3s_ready
changed_when: false
until: k3s_ready.rc == 0
retries: 12
delay: 5

- name: Wait for CoreDNS
ansible.builtin.command:
argv:
- /usr/local/bin/k3s
- kubectl
- --namespace=kube-system
- rollout
- status
- deployment/coredns
- --timeout=180s
register: k3s_coredns
changed_when: false
until: k3s_coredns.rc == 0
retries: 30
delay: 5

- name: Read the cluster's address families
ansible.builtin.command:
argv:
- /usr/local/bin/k3s
- kubectl
- --namespace=kube-system
- get
- service
- kube-dns
- --output=jsonpath={.spec.clusterIPs}
register: k3s_dns_addresses
changed_when: false

- name: Require an IPv6-first dual-stack Service
ansible.builtin.assert:
that:
- k3s_dns_addresses.stdout | from_json | length == 2
- (k3s_dns_addresses.stdout | from_json)[0] is search(':')
- (k3s_dns_addresses.stdout | from_json)[1] is match('^[0-9.]+$')
fail_msg: "kube-dns has addresses {{ k3s_dns_addresses.stdout }}"

# Tests reach the gateway's Service from the node, which fails when the
# node picks a source address that replies can't reach.
- name: Check the node reaches a Pod through an IPv6 Service
ansible.builtin.uri:
url: "http://[{{ (k3s_dns_addresses.stdout | from_json)[0] }}]:9153/metrics"
timeout: 10
register: k3s_ipv6_service
until: k3s_ipv6_service.status == 200
retries: 6
delay: 5

- name: Reset the K3s cluster state
ansible.builtin.include_tasks: k3s-reset.yaml
32 changes: 32 additions & 0 deletions tests/tmachine/ansible/playbooks/k3s-reset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# A setup layer is cached across runs, so it must not carry cluster
# credentials: every disk built on it would share one CA, token and admin
# kubeconfig. Stop K3s and delete the state it created, keeping the containerd
# image store (agent/containerd) and the K3s data directory. The next start
# generates fresh credentials and creates the cluster again from the K3s
# configuration.
---
- name: Stop K3s and its containers
ansible.builtin.command:
argv: [/usr/local/bin/k3s-killall.sh]
changed_when: true

- name: Remove the K3s credentials and cluster state
ansible.builtin.shell: |
set -eu
cd /var/lib/rancher/k3s
rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token
find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete
rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni
changed_when: true

- name: Check no K3s credentials remain
ansible.builtin.shell: |
find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \
-maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \
-o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print
register: k3s_leftover_credentials
changed_when: false
failed_when: k3s_leftover_credentials.stdout | length > 0
29 changes: 2 additions & 27 deletions tests/tmachine/ansible/playbooks/k3s.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,30 +72,5 @@
environment:
DESIRED_VERSION: "v{{ helm_version }}"

# This layer is cached across runs, so it must not carry cluster
# credentials: every disk built on it would share one CA, token and admin
# kubeconfig. Stop K3s and delete the state it created on first start,
# keeping the containerd image store (agent/containerd) and the K3s data
# directory. The next start generates fresh credentials.
- name: Stop K3s and its containers
ansible.builtin.command:
argv: [/usr/local/bin/k3s-killall.sh]
changed_when: true

- name: Remove the K3s credentials and cluster state
ansible.builtin.shell: |
set -eu
cd /var/lib/rancher/k3s
rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token
find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete
rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni
changed_when: true

- name: Check no K3s credentials remain
ansible.builtin.shell: |
find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \
-maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \
-o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print
register: k3s_leftover_credentials
changed_when: false
failed_when: k3s_leftover_credentials.stdout | length > 0
- name: Reset the K3s cluster state
ansible.builtin.include_tasks: k3s-reset.yaml
4 changes: 2 additions & 2 deletions tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
ansible.builtin.lineinfile:
path: /etc/hosts
regexp: '\sopenshell\.openshell\.svc\.cluster\.local$'
line: "{{ k3s_gateway_address.stdout.split(':')[0] }} openshell.openshell.svc.cluster.local"
line: "{{ k3s_gateway_ip }} openshell.openshell.svc.cluster.local"

- name: Read chart client TLS Secret
ansible.builtin.command:
Expand Down Expand Up @@ -103,4 +103,4 @@
apply:
become: false
vars:
openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}"
openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_port }}"
45 changes: 41 additions & 4 deletions tests/tmachine/ansible/playbooks/openshell-k3s.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,9 @@
# address must be the one of the node the tests run on. QEMU user
# networking gives every boot the same guest address (10.0.2.15), so the
# value stays valid when the disk is reused.
# The node can also list an IPv6 address after it, so take the IPv4 one.
# A dual-stack node also lists an IPv6 address, first on an IPv6-first
# cluster. Take the IPv4 one: the servers tests start for sandboxes to
# reach (tests/support/host-server) listen on IPv4 only.
- name: Discover the K3s node address
ansible.builtin.command:
argv:
Expand Down Expand Up @@ -133,6 +135,27 @@
mode: "0600"
content: "{{ openshell_k3s_helm_values_overlay | default('{}') }}"

# An environment adapts the chart to its cluster by writing values to
# /etc/openshell-test/helm-values.yaml before the installer runs (for
# example playbooks/k3s-ipv6.yaml). They apply last. Other environments
# get an empty file.
- name: Create the test configuration directory
ansible.builtin.file:
path: /etc/openshell-test
state: directory
owner: root
group: root
mode: "0755"

- name: Ensure environment Helm values exist
ansible.builtin.copy:
dest: /etc/openshell-test/helm-values.yaml
content: "{}\n"
force: false
owner: root
group: root
mode: "0644"

- name: Install Agent Sandbox
ansible.builtin.command:
argv:
Expand Down Expand Up @@ -180,6 +203,8 @@
- /var/lib/openshell/artifacts/values.yaml
- --values
- /var/lib/openshell/artifacts/values-overlay.yaml
- --values
- /etc/openshell-test/helm-values.yaml
- --wait
- --timeout=5m
environment:
Expand All @@ -196,17 +221,29 @@
- get
- service
- openshell
- --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port}
- --output=jsonpath={.spec.clusterIP} {.spec.ports[?(@.name=="grpc")].port}
register: k3s_gateway_address
changed_when: false

# Rejects headless Services (ClusterIP "None") and a missing grpc port.
# The ClusterIP is IPv6 on a cluster whose primary family is IPv6.
- name: Require a gateway Service address
ansible.builtin.assert:
that:
- k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$')
- k3s_gateway_address.stdout is match('^([0-9.]+|[0-9a-f:]+) [0-9]+$')
fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}"

# A URL needs brackets around an IPv6 address, for example
# http://[fd00:43::a]:8080; without them the port can't be told apart.
- name: Record the gateway Service address
vars:
gateway_ip: "{{ k3s_gateway_address.stdout.split()[0] }}"
gateway_port: "{{ k3s_gateway_address.stdout.split()[1] }}"
ansible.builtin.set_fact:
k3s_gateway_ip: "{{ gateway_ip }}"
k3s_gateway_port: "{{ gateway_port }}"
k3s_gateway_authority: "{{ ('[' ~ gateway_ip ~ ']') if ':' in gateway_ip else gateway_ip }}:{{ gateway_port }}"

# Deployment readiness doesn't establish that the controller reconciles
# Sandboxes. This checks the installed cluster; a booted guest restarts the
# controller, so the test runner checks again after it boots
Expand All @@ -218,6 +255,6 @@
hosts: all
gather_facts: false
vars:
openshell_client_gateway_endpoint: "http://{{ k3s_gateway_address.stdout }}"
openshell_client_gateway_endpoint: "http://{{ k3s_gateway_authority }}"
roles:
- openshell_client
6 changes: 6 additions & 0 deletions tests/tmachine/config.nix
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@ let
inputs.keycloak_realm_file = "../../scripts/keycloak-realm.json";
};
};
# A dual-stack cluster with IPv6 first, and the chart's IPv6 listen
# addresses, for running the k3s suites over IPv6.
ipv6 = {
bases = [ "ubuntu-k3s" ];
setup.playbooks = [ "ansible/playbooks/k3s-ipv6.yaml" ];
};
# Hardware-free CDI fixtures, for the cdi suite.
cdi = {
bases = dockerBases ++ podmanBases;
Expand Down
8 changes: 7 additions & 1 deletion tests/tmachine/src/qemu/vm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,14 @@ fn command(
.arg("virtio-blk-pci,drive=rootfs,bootindex=1")
.arg("-device")
.arg("virtio-rng-pci")
// A unique local IPv6 prefix instead of user networking's default
// site-local fec0::/64. Linux ranks a site-local source below any
// global address, so on a dual-stack K3s guest it picks flannel's
// address to reach an IPv6 Service and never gets the reply.
.arg("-netdev")
.arg(format!("user,id=net0,hostfwd=tcp:127.0.0.1:{ssh_port}-:22"))
.arg(format!(
"user,id=net0,ipv6-net=fd00:10::/64,hostfwd=tcp:127.0.0.1:{ssh_port}-:22"
))
.arg("-device")
.arg("virtio-net-pci,netdev=net0");

Expand Down
Loading