diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index 6e0d0e7d6be5..2ca0e9c8c94c 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -32,7 +32,7 @@ CI wiring: test/integration/plugins/ontap/ ├── ontap.cfg # Environment config (IPs, credentials, zone info) ├── ontap_test_base.py # Shared base class and ONTAP REST client -├── TEST_CASES.md # Full test case reference table (62 tests) +├── TEST_CASES.md # Full test case reference table (72 tests) ├── README.md # This file │ ├── nfs3/ @@ -42,8 +42,10 @@ test/integration/plugins/ontap/ │ │ └── test_zone_scoped_pool.py # Zone-scoped pool (attachZone) │ ├── volume/ │ │ └── test_volume_lifecycle.py # Volume create/delete/negative-delete -│ └── instance/ -│ └── test_vm_volume_attach.py # Pool + volume + VM + attach/detach +│ ├── instance/ +│ │ └── test_vm_volume_attach.py # Pool + volume + VM + attach/detach + template cache +│ └── template/ +│ └── test_template_cache_negative.py # Template-cache boundary cases │ └── iscsi/ ├── pool/ @@ -52,8 +54,10 @@ test/integration/plugins/ontap/ │ └── test_zone_scoped_pool.py # Zone-scoped iSCSI pool ├── volume/ │ └── test_volume_lifecycle.py # LUN create/delete/negative-delete - └── instance/ - └── test_vm_volume_attach.py # Pool + LUN + VM + attach/LUN-map lifecycle + ├── instance/ + │ └── test_vm_volume_attach.py # Pool + LUN + VM + attach/LUN-map lifecycle + template cache + └── template/ + └── test_template_cache_negative.py # Template-cache boundary cases ``` --- @@ -136,29 +140,36 @@ The test classes read `storageIP`, `svmName`, `username`, and `password` from th Run all suites for one protocol in a single batch, then inspect consolidated results: ```bash -# iSCSI only — 5 suites, ~30–45 min +# iSCSI only — 6 suites, ~40–60 min bash test/integration/plugins/ontap/run_tests.sh iscsi -# NFS3 only — 5 suites, ~30–45 min +# NFS3 only — 6 suites, ~40–60 min bash test/integration/plugins/ontap/run_tests.sh nfs3 -# Full plugin validation: iSCSI batch, then NFS3 batch (~60–90 min) +# Full plugin validation: iSCSI batch, then NFS3 batch bash test/integration/plugins/ontap/run_tests.sh both # Default (setup_zone + iscsi + nfs3; excludes cleanup_zone) bash test/integration/plugins/ontap/run_tests.sh bash test/integration/plugins/ontap/run_tests.sh all + +# Template-cache negative / boundary suites only +# (the template-cache happy path runs inside the VM attach suites) +bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache_negative +bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache_negative ``` -Each protocol batch runs suites in this order: pool lifecycle → pool with volumes → volume lifecycle → zone-scoped pool → VM attach (last). +Each protocol batch runs suites in this order: pool lifecycle → pool with volumes → volume lifecycle → zone-scoped pool → VM attach (includes template cache seed / reuse / survive) → template cache negative (last). | Command | What it runs | |---------|--------------| -| `run_tests.sh iscsi` | All 5 iSCSI suites + unified iSCSI report | -| `run_tests.sh nfs3` | All 5 NFS3 suites + unified NFS3 report | +| `run_tests.sh iscsi` | All 6 iSCSI suites + unified iSCSI report | +| `run_tests.sh nfs3` | All 6 NFS3 suites + unified NFS3 report | | `run_tests.sh both` | iSCSI batch, then NFS3 batch + combined report | | `run_tests.sh all` | `setup_zone`, then `both` (iSCSI before NFS3) | | `run_tests.sh nfs3_workflow` | Single suite by tag (unchanged) | +| `run_tests.sh nfs3_template_cache_negative` | NFS3 template-cache boundary/negative suite | +| `run_tests.sh iscsi_template_cache_negative` | iSCSI template-cache boundary/negative suite | | `run_tests.sh setup_zone` | Zone setup only | | `run_tests.sh cleanup_zone` | Zone teardown (manual; destructive) | @@ -308,12 +319,14 @@ self.assertEqual(result.state, "Maintenance") | NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 7 | Same + live volume present; negative delete guard | | NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone scope — all hosts connected via `attachZone` | | NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | -| NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle with hot-plug/detach | +| NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 10 | Full VM lifecycle with hot-plug/detach; ROOT on tagged pool seeds/reuses template cache, which survives VM delete | +| NFS3 Template Cache Negative | `nfs3/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | | iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete + igroups | | iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 7 | Same + live LUN present; negative delete guard | | iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone scope | | iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | -| iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle; LUN-maps on VM start/stop/detach | +| iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 10 | Full VM lifecycle; LUN-maps on VM start/stop/detach; ROOT on tagged pool seeds/reuses `cs_tmpl_*` LUN cache | +| iSCSI Template Cache Negative | `iscsi/template/test_template_cache_negative.py` | 3 | Tag mismatch; undersized pool; out-of-band cache delete | For the goal, dependencies, and exact success criteria of every individual test, see [TEST_CASES.md](TEST_CASES.md). diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md index 73dc1990a5b6..61b96cb1d2e7 100644 --- a/test/integration/plugins/ontap/TEST_CASES.md +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -19,7 +19,7 @@ # ONTAP Integration Test Cases -Complete reference for all 62 test cases across 10 test suites. +Complete reference for all 72 test cases across 12 test suites. Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. --- @@ -114,18 +114,20 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/instance/test_vm_volume_attach.py` **Class:** `TestOntapVMVolumeAttach` **Tag:** `vm_volume_workflow` -**Total:** 8 tests | **Scope:** end-to-end — NFS3 pool, data volume, running VM, attach/detach lifecycle +**Total:** 10 tests | **Scope:** end-to-end — NFS3 pool, data volume, running VM (ROOT on the ONTAP pool via a tagged compute offering), primary template cache seed / reuse / survive VM delete, attach/detach lifecycle | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_nfs3_pool` | Create NFS3 ONTAP primary storage pool | setUpClass (zone, cluster, template) | `pool.state == "Up"` | FlexVol `online`; export policy present | positive | +| 01 | `test_01_create_nfs3_pool` | Create NFS3 ONTAP primary storage pool tagged with `templateCacheTags` | setUpClass (zone, cluster, template, tagged SO) | `pool.state == "Up"` | FlexVol `online`; export policy present | positive | | 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume on the ONTAP pool | test_01 (`pool`) | Volume non-None and listed in `listVolumes` | FlexVol still `online` | positive | -| 03 | `test_03_deploy_vm` | Deploy a VM using the first available ready KVM template | test_02 (`pool`, `volume`) | `vm.state == "Running"`; template auto-selected from `listTemplates` | n/a | positive | +| 03 | `test_03_deploy_vm` | Deploy a VM with the tagged SO — ROOT on ONTAP; seeds template cache | test_02 (`pool`, `volume`) | `vm.state == "Running"`; ROOT `storageid` = pool; `template_spool_ref` Ready/DOWNLOADED | Cache file present at spool `install_path` | positive | +| 03a | `test_03a_deploy_second_vm_reuses_template_cache` | Deploy VM-2 — reuses cache | test_03 | VM-2 Running; ROOT on pool; still exactly one `template_spool_ref` | Same cache file (no second cache) | positive | +| 03b | `test_03b_expunge_second_vm_template_cache_survives` | Expunge VM-2 — cache must remain (lazy GC) | test_03a | spool_ref still Ready | Cache file still present | positive | | 04 | `test_04_attach_volume_to_vm` | Attach the ONTAP data volume to the running VM (hot-plug) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id`; `attachVolume` job succeeds | FlexVol `online`; after attach, a data file matching volume UUID present in FlexVol (`list_files_in_volume`) | positive | | 05 | `test_05_stop_vm_export_retained` | Stop the running VM with volume attached | test_04 | `vm.state == "Stopped"` | FlexVol still `online`; NFS export policy still present | positive | | 06 | `test_06_start_vm_volume_accessible` | Start the stopped VM | test_05 | `vm.state == "Running"` | FlexVol still `online` | positive | | 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM (TDS Detach NFS3) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | -| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM no longer listed; volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | cleanup | +| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, force-delete pool | test_07 | VM no longer listed; spool_ref still Ready after VM expunge; volume no longer listed; pool no longer listed | Cache file present after VM expunge; FlexVol deleted; export policy deleted | cleanup | --- @@ -206,23 +208,55 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/instance/test_vm_volume_attach.py` **Class:** `TestOntapVMVolumeAttachISCSI` **Tag:** `iscsi_vm_workflow` -**Total:** 8 tests | **Scope:** end-to-end — iSCSI pool, data volume (LUN), running VM, attach/stop/start/detach lifecycle +**Total:** 10 tests | **Scope:** end-to-end — iSCSI pool, data volume (LUN), running VM (ROOT on the ONTAP pool via a tagged compute offering), `cs_tmpl_` LUN cache seed / reuse / survive VM delete, attach/stop/start/detach lifecycle | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; igroup per cluster host with host IQN | positive | +| 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool tagged with `templateCacheTags` | setUpClass (tagged SO) | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; igroup per cluster host with host IQN | positive | | 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume (creates a LUN in the FlexVol) | test_01 (`pool`) | Volume non-None | ≥1 LUN in FlexVol | positive | -| 03 | `test_03_deploy_vm` | Deploy VM using first ready KVM template; verify 0 LUN-maps exist before attach | test_02 (`volume`) | `vm.state == "Running"`; 0 LUN-maps on ONTAP | 0 LUN-maps (`list_lun_maps_for_volume` returns empty) | positive | +| 03 | `test_03_deploy_vm` | Deploy VM with the tagged SO — ROOT on ONTAP; seeds `cs_tmpl_*`; verify 0 data-volume LUN-maps before attach | test_02 (`volume`) | `vm.state == "Running"`; ROOT on pool; spool_ref Ready (`local_path` = LUN uuid) | Exactly one `/vol//cs_tmpl_` LUN; 0 data-volume LUN-maps | positive | +| 03a | `test_03a_deploy_second_vm_reuses_template_cache` | Deploy VM-2 — reuse cache | test_03 | VM-2 Running; ROOT on pool; still one spool_ref | Still one `cs_tmpl_*`; non-cache LUN count +1 | positive | +| 03b | `test_03b_expunge_second_vm_template_cache_survives` | Expunge VM-2 — cache LUN remains | test_03a | spool_ref still Ready | VM-2 ROOT LUN gone (non-cache count back to baseline); `cs_tmpl_*` still present | positive | | 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created (TDS SN 27) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | | 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed (TDS VM Stop iSCSI) | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | | 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created (TDS VM Start iSCSI) | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | | 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM (TDS Detach iSCSI) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | -| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; volume gone; pool gone | FlexVol deleted; all LUNs and igroups deleted | cleanup | +| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; spool_ref still Ready after VM expunge; volume gone; pool gone | `cs_tmpl_*` present after VM expunge; FlexVol deleted; all LUNs and igroups deleted | cleanup | > ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. All other 61 tests pass. --- +## Suite 11 — NFS3 Template Cache Negative / Boundary + +**File:** `nfs3/template/test_template_cache_negative.py` +**Class:** `TestOntapNfs3TemplateCacheNegative` +**Tag:** `nfs3_template_cache_negative` +**Total:** 3 tests | **Scope:** Boundary conditions for NFS3 primary template cache (isolated from happy path) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_tag_mismatch_does_not_seed_cache` | SO tags ≠ pool tags | setUpClass | Deploy may succeed elsewhere; ROOT not on ONTAP pool; no `template_spool_ref` for pool | No cache file for template on FlexVol | negative | +| 02 | `test_02_undersized_pool_deploy_fails` | Matching tags but `capacitybytes` ≪ template size | setUpClass | Deploy fails / never Running; spool_ref not Ready/DOWNLOADED | No cache file | negative | +| 03 | `test_03_deleted_cache_blocks_reuse` | Seed cache, delete file out-of-band, redeploy | setUpClass | spool_ref still Ready after ONTAP delete; second deploy fails | Cache file absent after delete | negative | + +--- + +## Suite 12 — iSCSI Template Cache Negative / Boundary + +**File:** `iscsi/template/test_template_cache_negative.py` +**Class:** `TestOntapIscsiTemplateCacheNegative` +**Tag:** `iscsi_template_cache_negative` +**Total:** 3 tests | **Scope:** Boundary conditions for iSCSI primary template cache (isolated from happy path) + +| # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | +|---|-------------|------|------------|-----------------------------|------------------------|------| +| 01 | `test_01_tag_mismatch_does_not_seed_cache` | SO tags ≠ pool tags | setUpClass | ROOT not on ONTAP pool; no `template_spool_ref` | No `cs_tmpl_*` LUN | negative | +| 02 | `test_02_undersized_pool_deploy_fails` | Matching tags but undersized capacity | setUpClass | Deploy fails; spool_ref not Ready/DOWNLOADED | No `cs_tmpl_*` LUN | negative | +| 03 | `test_03_deleted_cache_blocks_reuse` | Seed cache, delete LUN out-of-band, redeploy | setUpClass | spool_ref still Ready; second deploy fails | `cs_tmpl_*` absent after delete | negative | + +--- + ## Cross-suite summary | Suite | Protocol | Scope | Tests | Status | @@ -231,10 +265,12 @@ Each suite is sequential — tests must run in numbered order; each step builds | NFS3 Pool with Volumes | NFS3 | Cluster | 7 | ✅ | | NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | | NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | -| NFS3 VM + Volume Attach | NFS3 | Cluster | 8 | ✅ | +| NFS3 VM + Volume Attach | NFS3 | Cluster | 10 | 🆕 +2 template cache | +| NFS3 Template Cache Negative | NFS3 | Cluster | 3 | 🆕 | | iSCSI Pool Lifecycle | iSCSI | Cluster | 8 | ✅ | | iSCSI Pool with Volumes | iSCSI | Cluster | 7 | ✅ | | iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | | iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | -| iSCSI VM + Volume Attach | iSCSI | Cluster | 8 | ⚠️ 7/8 | -| **Total** | | | **62** | **61 passing** | +| iSCSI VM + Volume Attach | iSCSI | Cluster | 10 | ⚠️ 7/8 + 🆕 2 template cache | +| iSCSI Template Cache Negative | iSCSI | Cluster | 3 | 🆕 | +| **Total** | | | **72** | | diff --git a/test/integration/plugins/ontap/helpers/__init__.py b/test/integration/plugins/ontap/helpers/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/helpers/template_cache_base.py b/test/integration/plugins/ontap/helpers/template_cache_base.py new file mode 100644 index 000000000000..05b479643f3e --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_base.py @@ -0,0 +1,427 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Shared building blocks for ONTAP primary template-cache Marvin checks. + + TemplateCacheAssertionsMixin + ROOT placement, template_spool_ref and ONTAP cache-object assertions. + Mixed into the VM instance suites (happy path: seed / reuse / survive) + and the negative suite. + + OntapTemplateCacheBase + setUpClass for the standalone negative suite (zone, template, guest + network, ONTAP client) without the sequential pool / VM state. +""" + +from __future__ import print_function + +import base64 +import logging +import random +import time +import unittest + +from marvin.cloudstackAPI import ( + createNetwork as createNetworkAPI, + deleteNetwork as deleteNetworkAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + listNetworkOfferings as listNetworkOfferingsAPI, + listNetworks as listNetworksAPI, + listTemplates as listTemplatesAPI, + listVirtualMachines as listVirtualMachinesAPI, + listVolumes as listVolumesAPI, + stopVirtualMachine as stopVirtualMachineAPI, +) +from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config +from helpers import template_cache_util as tcu + +logger = logging.getLogger("TemplateCacheBase") + +# ~50 GiB — room for template cache + ROOT clones (+ a data volume) +DEFAULT_TEMPLATE_CACHE_CAPACITY_BYTES = 50 * 1024 * 1024 * 1024 + + +def _list_vms_cmd(vm_id): + cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + cmd.id = vm_id + cmd.listall = True + return cmd + + +def _list_vols_for_vm(vm_id): + cmd = listVolumesAPI.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.listall = True + return cmd + + +def template_cache_tags(proto_cfg, protocol_cfg_key): + """Dedicated storage tag so ROOT is forced onto the ONTAP pool.""" + return (proto_cfg.get("templateCacheTags") + or "ontap-%s-tmpl-cache" % protocol_cfg_key) + + +def template_cache_capacity_bytes(pool_cfg): + """Pool size large enough to hold the template cache plus ROOT clones.""" + return (pool_cfg.get("templateCacheCapacitybytes") + or pool_cfg.get("capacitybytes") + or DEFAULT_TEMPLATE_CACHE_CAPACITY_BYTES) + + +def tagged_compute_offering_data(tags, name_prefix="OntapTmplCacheSO"): + """Compute offering whose storage tags match the ONTAP pool.""" + name = "%s_%d" % (name_prefix, random.randint(0, 99999)) + return { + "name": name, + "displaytext": "ONTAP template-cache SO (tagged)", + "cpunumber": 1, + "cpuspeed": 500, + "memory": 512, + "storagetype": "shared", + "tags": tags, + } + + +def find_ready_kvm_template(api_client, zone_id): + """Return the first ready, non-SYSTEM KVM template in the zone, or None.""" + tpl_cmd = listTemplatesAPI.listTemplatesCmd() + tpl_cmd.templatefilter = "all" + tpl_cmd.listall = True + tpl_cmd.zoneid = zone_id + templates = api_client.listTemplates(tpl_cmd) or [] + kvm_ready = [ + t for t in templates + if getattr(t, "hypervisor", "").lower() == "kvm" + and getattr(t, "isready", False) + and getattr(t, "templatetype", "").upper() != "SYSTEM" + ] + return kvm_ready[0] if kvm_ready else None + + +class TemplateCacheTestData(object): + account = "account" + ontap = "ontap" + primaryStorage = "primaryStorage" + computeOffering = "computeOffering" + provider = "provider" + scope = "scope" + tags = "tags" + + DETAIL_USERNAME = "username" + DETAIL_PASSWORD = "password" + DETAIL_SVM_NAME = "svmName" + DETAIL_PROTOCOL = "protocol" + DETAIL_STORAGE_IP = "storageIP" + + def __init__(self, storage_ip, svm_name, username, password, + protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", + tags="ontap-tmpl-cache", capacitybytes=None, + pool_name_prefix="OntapTmplCache"): + if capacitybytes is None: + capacitybytes = DEFAULT_TEMPLATE_CACHE_CAPACITY_BYTES + encoded_password = base64.b64encode(password.encode()).decode() + suffix = random.randint(0, 99999) + self.testdata = { + self.ontap: { + self.DETAIL_STORAGE_IP: storage_ip, + self.DETAIL_SVM_NAME: svm_name, + self.DETAIL_USERNAME: username, + self.DETAIL_PASSWORD: password, + }, + self.account: { + "email": "ontap-tmpl-cache@test.com", + "firstname": "ONTAP", + "lastname": "TmplCache", + "username": "ontap_tmpl_%d" % suffix, + "password": "password", + }, + self.primaryStorage: { + "name": "%s_%d" % (pool_name_prefix, suffix), + self.scope: scope, + self.provider: provider, + self.tags: tags, + "capacitybytes": capacitybytes, + "managed": True, + "details": { + self.DETAIL_USERNAME: username, + self.DETAIL_PASSWORD: encoded_password, + self.DETAIL_SVM_NAME: svm_name, + self.DETAIL_PROTOCOL: protocol, + self.DETAIL_STORAGE_IP: storage_ip, + }, + }, + self.computeOffering: tagged_compute_offering_data(tags), + } + + +class TemplateCacheAssertionsMixin(object): + """ + Template-cache assertions shared by the instance and negative suites. + + The host class must provide ``apiClient``, ``dbConnection``, ``ontap``, + ``svm_name``, ``PROTOCOL`` ("NFS3" / "ISCSI") and a class-level + ``template_db_id``. + """ + + PROTOCOL = "NFS3" + template_db_id = None + + def _is_iscsi(self): + return self.PROTOCOL.upper() == "ISCSI" + + def _root_volume_for_vm(self, vm_id): + vols = self.apiClient.listVolumes(_list_vols_for_vm(vm_id)) or [] + roots = [ + v for v in vols + if str(getattr(v, "type", "")).upper() == "ROOT" + ] + self.assertTrue(roots, "No ROOT volume for VM %s" % vm_id) + return roots[0] + + def _assert_root_on_pool(self, vm_id, pool): + root = self._root_volume_for_vm(vm_id) + self.assertEqual( + str(root.storageid), str(pool.id), + "ROOT volume storageid=%s should equal ONTAP pool id=%s " + "(check service-offering / pool storage tags)" + % (root.storageid, pool.id), + ) + return root + + def _wait_for_ready_spool_ref(self, pool_db_id, timeout=600): + spool = tcu.wait_for_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + timeout=timeout, + ) + tcu.assert_spool_ref_ready( + self, spool, expect_local_path=self._is_iscsi(), + ) + return spool + + def _assert_single_ready_spool_ref(self, pool_db_id): + count = tcu.count_template_spool_refs( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + ) + self.assertEqual( + count, 1, "Expected one template_spool_ref, got %s" % count, + ) + spool = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + ) + tcu.assert_spool_ref_ready( + self, spool, expect_local_path=self._is_iscsi(), + ) + return spool + + def _assert_cache_on_ontap(self, pool, spool_ref): + if self._is_iscsi(): + tcu.assert_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + cache_count = tcu.count_iscsi_template_cache_luns( + self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + self.assertEqual( + cache_count, 1, + "Expected exactly one cs_tmpl_%s LUN, found %s" + % (self.__class__.template_db_id, cache_count), + ) + else: + tcu.assert_nfs_template_cache_file( + self, self.ontap, pool.name, spool_ref.get("install_path") + ) + + def _count_non_cache_luns(self, pool): + return tcu.count_luns_excluding_template_cache( + self.ontap, self.svm_name, pool.name + ) + + def _wait_for_non_cache_lun_count(self, pool, expected, timeout=180, + interval=10): + """Poll until the FlexVol holds ``expected`` non-cache LUNs.""" + deadline = time.time() + timeout + current = self._count_non_cache_luns(pool) + while current != expected and time.time() < deadline: + time.sleep(interval) + current = self._count_non_cache_luns(pool) + self.assertEqual( + current, expected, + "Expected %s non-cache LUNs in FlexVol '%s', found %s" + % (expected, pool.name, current), + ) + + +class OntapTemplateCacheBase(TemplateCacheAssertionsMixin, OntapTestBase): + """ + Class-level setup for standalone template-cache suites. + + Subclasses must set: + PROTOCOL - "NFS3" or "ISCSI" + NOSE_TAG - nose attr tag string + PROTOCOL_CFG_KEY - "nfs3" or "iscsi" under storagePool.protocols + POOL_URL_SCHEME - used in createStoragePool url (e.g. nfs / iscsi) + POOL_NAME_PREFIX - prefix for pools created by the suite + """ + + PROTOCOL = "NFS3" + NOSE_TAG = "template_cache" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapTmplCache" + + template_id = None # API UUID + template_db_id = None # numeric DB id + network_id = None + _created_network_id = None + + @classmethod + def setUpClass(cls): + super(OntapTemplateCacheBase, cls).setUpClass() + testclient = super(OntapTemplateCacheBase, cls).getClsTestClient() + + cls.apiClient = testclient.getApiClient() + cls.dbConnection = testclient.getDbConnection() + config = get_datacenter_config(testclient, cls) + + ontap_cfg = config.get("ontap", {}) + pool_cfg = config.get("storagePool", {}) + storage_ip = ontap_cfg.get("storageIP", "") + svm_name = ontap_cfg.get("svmName", "") + username = ontap_cfg.get("username", "") + password = ontap_cfg.get("password", "") + + proto_cfg = pool_cfg.get("protocols", {}).get(cls.PROTOCOL_CFG_KEY, {}) + if not proto_cfg.get("enabled", True): + raise unittest.SkipTest( + "%s tests disabled in ontap.cfg " + "(set protocols.%s.enabled=true to enable)" + % (cls.PROTOCOL, cls.PROTOCOL_CFG_KEY) + ) + + tags = template_cache_tags(proto_cfg, cls.PROTOCOL_CFG_KEY) + scope = pool_cfg.get("storagePoolScope", "CLUSTER") + provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") + + cls.testdata = TemplateCacheTestData( + storage_ip, svm_name, username, password, + protocol=cls.PROTOCOL, scope=scope, provider=provider, + tags=tags, capacitybytes=template_cache_capacity_bytes(pool_cfg), + pool_name_prefix=cls.POOL_NAME_PREFIX, + ).testdata + cls.ontap = OntapRestClient(storage_ip, username, password) + cls.svm_name = svm_name + cls.storage_tags = tags + + cls._setup_cloudstack_resources( + config, cls.testdata[TemplateCacheTestData.account] + ) + + template = find_ready_kvm_template(cls.apiClient, cls.zone.id) + if template is None: + raise unittest.SkipTest( + "No ready user KVM template in zone '%s'" % cls.zone.name + ) + cls.template_id = template.id + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) + + cls.network_id = None + cls._created_network_id = None + zone_type = getattr(cls.zone, "networktype", "Basic") + if zone_type.lower() == "advanced": + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.account = cls.account.name + net_cmd.domainid = cls.domain.id + nets = cls.apiClient.listNetworks(net_cmd) or [] + if nets: + cls.network_id = nets[0].id + else: + no_cmd = listNetworkOfferingsAPI.listNetworkOfferingsCmd() + no_cmd.state = "Enabled" + no_cmd.guestiptype = "Isolated" + no_cmd.supportedservices = "SourceNat" + offerings = cls.apiClient.listNetworkOfferings(no_cmd) or [] + if not offerings: + raise unittest.SkipTest( + "No Isolated network offering for Advanced zone" + ) + cn = createNetworkAPI.createNetworkCmd() + cn.name = "ontap-tmpl-cache-net-%d" % random.randint(0, 9999) + cn.displaytext = cn.name + cn.networkofferingid = offerings[0].id + cn.zoneid = cls.zone.id + cn.account = cls.account.name + cn.domainid = cls.domain.id + created = cls.apiClient.createNetwork(cn) + cls.network_id = created.id + cls._created_network_id = created.id + + @classmethod + def tearDownClass(cls): + if cls._created_network_id is not None: + try: + dn = deleteNetworkAPI.deleteNetworkCmd() + dn.id = cls._created_network_id + cls.apiClient.deleteNetwork(dn) + except Exception as e: + logger.warning( + "tearDownClass: delete network failed: %s" % e + ) + cls._created_network_id = None + + super(OntapTemplateCacheBase, cls).tearDownClass() + + @classmethod + def _destroy_vm_static(cls, vm_id): + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if not vms: + return + state = (vms[0].state or "").lower() + if state not in ("stopped", "destroyed", "expunging", "error"): + stop = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop.id = vm_id + cls.apiClient.stopVirtualMachine(stop) + deadline = time.time() + 300 + while time.time() < deadline: + cur = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if cur and cur[0].state.lower() == "stopped": + break + time.sleep(10) + dest = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest.id = vm_id + dest.expunge = True + cls.apiClient.destroyVirtualMachine(dest) + + def _poll_vm_state(self, vm_id, target_state, timeout=900, interval=10): + deadline = time.time() + timeout + current = "unknown" + while time.time() < deadline: + vms = self.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + if vms: + current = vms[0].state + if current.lower() == target_state.lower(): + return vms[0] + time.sleep(interval) + self.fail( + "VM %s did not reach '%s' within %ds (last='%s')" + % (vm_id, target_state, timeout, current) + ) diff --git a/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py b/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py new file mode 100644 index 000000000000..16ca60c88ea6 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py @@ -0,0 +1,640 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Negative / boundary workflow for ONTAP primary template-cache Marvin suites. + +The happy path (seed / reuse / cache survives VM delete) runs inside the VM +instance suites (``*/instance/test_vm_volume_attach.py``). These cases need +their own pool and / or offering (mismatched tags, undersized capacity, a +deliberately broken cache), so they stay isolated and cannot cascade into the +instance workflow. + +Cases: + 01 Tag mismatch — SO tags do not match ONTAP pool; no spool_ref on pool + 02 Undersized pool — capacity << template size; deploy must fail + 03 Cache deleted out-of-band — Ready spool_ref but missing ONTAP object; + subsequent deploy must fail +""" + +from __future__ import print_function + +import copy +import logging +import random +import time + +from marvin.cloudstackAPI import ( + createStoragePool as createStoragePoolAPI, + deleteServiceOffering as deleteServiceOfferingAPI, + deployVirtualMachine as deployVirtualMachineAPI, + enableStorageMaintenance, +) +from marvin.lib.base import ServiceOffering, StoragePool + +from helpers import template_cache_util as tcu +from helpers.template_cache_base import ( + OntapTemplateCacheBase, + TemplateCacheTestData, +) + +logger = logging.getLogger("TemplateCacheNegativeWorkflow") + + +class OntapTemplateCacheNegativeWorkflow(OntapTemplateCacheBase): + """ + Independent negative boundary tests. + + Reuses setUpClass (zone / template / ONTAP client) from the shared base + but creates per-test pools and offerings so state stays isolated. + """ + + NOSE_TAG = "template_cache_negative" + POOL_NAME_PREFIX = "OntapTmplCacheNeg" + + @classmethod + def setUpClass(cls): + super(OntapTemplateCacheNegativeWorkflow, cls).setUpClass() + # Lab public IP pool is tiny (often only 3 addresses: SSVM + CPVM + + # one SourceNat). Starting leftover VRs from prior runs consumes the + # last IP and then this suite's Allocated guest network cannot be + # implemented — seed deploy fails with "Unable to create a deployment". + cls._reclaim_stale_template_cache_networks() + cls._cleanup_stale_negative_pools() + cls._ensure_guest_network_router_running() + + @classmethod + def _cleanup_stale_negative_pools(cls): + """ + Force-delete leftover OntapTmplCacheNeg* pools from prior runs. + + Shared storage_tags on a leftover Up pool will steal ROOT placement + from the pool created by step_02/step_03 and make assertions fail. + """ + from marvin.cloudstackAPI import listStoragePools as listStoragePoolsAPI + + cmd = listStoragePoolsAPI.listStoragePoolsCmd() + cmd.zoneid = cls.zone.id + pools = cls.apiClient.listStoragePools(cmd) or [] + prefixes = ( + cls.POOL_NAME_PREFIX, + "OntapIscsiTmplNeg", + "OntapNfs3TmplNeg", + "OntapTmplCacheNeg", + ) + for pool in pools: + name = getattr(pool, "name", "") or "" + if not any(name.startswith(prefix) for prefix in prefixes): + continue + logger.info("Cleaning leftover negative-suite pool %s" % name) + try: + # Instance method helpers need a throwaway binder. + binder = cls.__new__(cls) + binder.apiClient = cls.apiClient + binder.ontap = cls.ontap + binder._force_delete_pool(pool) + except Exception as ex: + logger.warning("Could not delete leftover pool %s: %s" % (name, ex)) + + @classmethod + def _reclaim_stale_template_cache_networks(cls): + """ + Destroy leftover ontap-tmpl-cache-net-* networks (and their VRs/VMs) + from prior suite accounts so SourceNat public IPs are freed. + Keeps the network created for this class (cls.network_id). + """ + from marvin.cloudstackAPI import ( + deleteNetwork as deleteNetworkAPI, + destroyRouter as destroyRouterAPI, + destroyVirtualMachine as destroyVirtualMachineAPI, + listNetworks as listNetworksAPI, + listRouters as listRoutersAPI, + listVirtualMachines as listVirtualMachinesAPI, + stopVirtualMachine as stopVirtualMachineAPI, + ) + + net_cmd = listNetworksAPI.listNetworksCmd() + net_cmd.zoneid = cls.zone.id + net_cmd.listall = True + networks = cls.apiClient.listNetworks(net_cmd) or [] + for net in networks: + name = getattr(net, "name", "") or "" + # Match names from positive/negative suite network create + # (ontap-tmpl-cache-net-*). + if "tmpl-cache-net" not in name: + continue + if cls.network_id and str(net.id) == str(cls.network_id): + continue + + logger.info( + "Reclaiming stale template-cache network %s (%s)" + % (name, net.id) + ) + + vm_cmd = listVirtualMachinesAPI.listVirtualMachinesCmd() + vm_cmd.listall = True + vm_cmd.networkid = net.id + for vm in (cls.apiClient.listVirtualMachines(vm_cmd) or []): + try: + state = (vm.state or "").lower() + if state not in ( + "destroyed", "expunging", "error", "stopped" + ): + stop = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop.id = vm.id + cls.apiClient.stopVirtualMachine(stop) + dest = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest.id = vm.id + dest.expunge = True + cls.apiClient.destroyVirtualMachine(dest) + except Exception as ex: + logger.warning( + "Reclaim: destroy VM %s on %s: %s" % (vm.id, name, ex) + ) + + rtr_cmd = listRoutersAPI.listRoutersCmd() + rtr_cmd.zoneid = cls.zone.id + rtr_cmd.networkid = net.id + rtr_cmd.listall = True + for router in (cls.apiClient.listRouters(rtr_cmd) or []): + try: + dr = destroyRouterAPI.destroyRouterCmd() + dr.id = router.id + cls.apiClient.destroyRouter(dr) + logger.info( + "Reclaim: destroyed router %s for %s" + % (router.name, name) + ) + except Exception as ex: + logger.warning( + "Reclaim: destroy router %s: %s" % (router.id, ex) + ) + + try: + dn = deleteNetworkAPI.deleteNetworkCmd() + dn.id = net.id + cls.apiClient.deleteNetwork(dn) + logger.info("Reclaim: deleted network %s" % name) + except Exception as ex: + logger.warning( + "Reclaim: delete network %s: %s" % (net.id, ex) + ) + + @classmethod + def _ensure_guest_network_router_running(cls): + """ + Ensure the VR for *this suite's* guest network is Running. + + Do not start unrelated zone routers — that can exhaust the lab's + public IP range and block implementing cls.network_id. + """ + from marvin.cloudstackAPI import ( + listRouters as listRoutersAPI, + startRouter as startRouterAPI, + ) + if not cls.network_id: + logger.warning( + "No guest network_id — deploy may fail in Advanced zone" + ) + return + + cmd = listRoutersAPI.listRoutersCmd() + cmd.zoneid = cls.zone.id + cmd.networkid = cls.network_id + cmd.listall = True + routers = cls.apiClient.listRouters(cmd) or [] + if not routers: + # Allocated isolated network: VR is created on first deploy. + logger.info( + "No VR yet for network %s — will be created on first deploy" + % cls.network_id + ) + return + + for router in routers: + state = (router.state or "").lower() + if state == "running": + continue + logger.info( + "Starting guest-network router %s (was %s)" + % (router.name, router.state) + ) + start = startRouterAPI.startRouterCmd() + start.id = router.id + cls.apiClient.startRouter(start) + deadline = time.time() + 300 + while time.time() < deadline: + cur = cls.apiClient.listRouters(cmd) or [] + match = [r for r in cur if r.id == router.id] + if match and (match[0].state or "").lower() == "running": + logger.info("Router %s is Running" % router.name) + break + time.sleep(10) + else: + raise RuntimeError( + "Guest-network router %s did not reach Running" + % router.name + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _unique_storage_tags(self): + """Per-test tags so leftover pools with shared tags cannot steal ROOT.""" + return "%s-neg-%d" % (self.storage_tags, random.randint(0, 99999)) + + def _create_pool_with(self, tags=None, capacitybytes=None, name_prefix=None): + """Create an ONTAP pool, optionally overriding tags / capacity.""" + ps = copy.deepcopy(self.testdata[TemplateCacheTestData.primaryStorage]) + if tags is not None: + ps[TemplateCacheTestData.tags] = tags + if capacitybytes is not None: + ps["capacitybytes"] = int(capacitybytes) + + storage_ip = self.testdata[TemplateCacheTestData.ontap][ + TemplateCacheTestData.DETAIL_STORAGE_IP + ] + prefix = name_prefix or self.POOL_NAME_PREFIX + pool_name = "%s_%d" % (prefix, random.randint(0, 99999)) + + cmd = createStoragePoolAPI.createStoragePoolCmd() + cmd.name = pool_name + cmd.url = "%s://%s/ontap" % (self.POOL_URL_SCHEME, storage_ip) + cmd.zoneid = self.zone.id + cmd.clusterid = self.cluster.id + cmd.podid = self.cluster.podid + cmd.scope = ps[TemplateCacheTestData.scope] + cmd.provider = ps[TemplateCacheTestData.provider] + cmd.tags = ps[TemplateCacheTestData.tags] + cmd.capacitybytes = ps["capacitybytes"] + cmd.hypervisor = "KVM" + cmd.managed = True + + count = 1 + for key, value in ps["details"].items(): + setattr(cmd, "details[{}].{}".format(count, key), value) + count += 1 + + response = self.apiClient.createStoragePool(cmd) + pool = StoragePool(response.__dict__) + self.assertEqual( + pool.state, "Up", + "Pool %s should be Up after create, got %s" % (pool.name, pool.state), + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "FlexVol missing for pool %s" % pool.name) + self.assertEqual(ontap_vol.get("state"), "online") + return pool + + def _create_service_offering_with(self, tags): + """Create a compute offering with the given storage tags.""" + so_data = copy.deepcopy( + self.testdata[TemplateCacheTestData.computeOffering] + ) + so_data["tags"] = tags + so_data["name"] = "OntapTmplCacheNegSO_%d" % random.randint(0, 99999) + so_data["displaytext"] = so_data["name"] + so = ServiceOffering.create(self.apiClient, so_data) + self.assertIsNotNone(so.id, "Service offering create failed") + return so + + def _deploy_vm_with_offering(self, name_suffix, service_offering_id): + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-neg-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm, "deployVirtualMachine returned None") + return self._poll_vm_state(vm.id, "Running", timeout=900) + + def _deploy_vm_expect_failure(self, name_suffix, service_offering_id, + timeout=300): + """Assert deploy does not reach Running (API error or Error state).""" + vm = None + try: + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.name = "tmpl-cache-neg-%s-%d" % ( + name_suffix, random.randint(0, 9999) + ) + cmd.displayname = cmd.name + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + vm = self.apiClient.deployVirtualMachine(cmd) + except Exception as ex: + logger.info( + "deployVirtualMachine failed as expected for %s: %s" + % (name_suffix, ex) + ) + return + + self.assertIsNotNone(vm, "deploy returned None without raising") + try: + self._poll_vm_state(vm.id, "Running", timeout=timeout) + self.fail( + "Deploy %s unexpectedly reached Running (boundary should fail)" + % name_suffix + ) + except Exception as ex: + logger.info( + "Deploy %s did not reach Running as expected: %s" + % (name_suffix, ex) + ) + finally: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + + def _force_delete_pool(self, pool): + if pool is None: + return + pool_name = pool.name + try: + mc = enableStorageMaintenance.enableStorageMaintenanceCmd() + mc.id = pool.id + self.apiClient.enableStorageMaintenance(mc) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + except Exception as ex: + logger.warning( + "enableStorageMaintenance for %s: %s" % (pool.id, ex) + ) + try: + self._delete_pool(pool.id, forced=True) + except Exception as ex: + logger.warning("force delete pool %s: %s" % (pool.id, ex)) + deadline = time.time() + 120 + while time.time() < deadline: + if self.ontap.get_volume(pool_name) is None: + break + time.sleep(5) + + def _delete_service_offering(self, offering): + if offering is None: + return + try: + cmd = deleteServiceOfferingAPI.deleteServiceOfferingCmd() + cmd.id = offering.id + self.apiClient.deleteServiceOffering(cmd) + except Exception as ex: + logger.warning("delete SO %s: %s" % (offering.id, ex)) + + def _delete_ontap_cache(self, pool, spool_ref): + """Remove ONTAP cache object while leaving CloudStack spool_ref.""" + if self.PROTOCOL.upper() == "ISCSI": + path = tcu.template_cache_lun_path( + pool.name, self.__class__.template_db_id + ) + deleted = self.ontap.delete_lun(self.svm_name, path) + self.assertTrue( + deleted, + "Failed to delete template cache LUN at %s" % path, + ) + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + else: + install_path = spool_ref.get("install_path") + self.assertTrue(install_path, "spool_ref.install_path required") + deleted = self.ontap.delete_file_in_volume( + pool.name, install_path + ) + self.assertTrue( + deleted, + "Failed to delete NFS cache file %s in %s" + % (install_path, pool.name), + ) + tcu.assert_no_nfs_template_cache_file( + self, self.ontap, pool.name, install_path + ) + + # ------------------------------------------------------------------ + # Negative cases + # ------------------------------------------------------------------ + + def step_01_tag_mismatch_does_not_seed_cache(self): + """ + Pool has template-cache tags; SO uses a different tag. + + Valid outcomes: + a) deploy fails (no alternate tagged pool) — common in lab + b) deploy succeeds elsewhere — ROOT must not be on ONTAP pool + + In both cases there must be no spool_ref / cache on the ONTAP pool. + """ + pool = None + so = None + vm = None + mismatch_tag = "ontap-tmpl-cache-mismatch-%d" % random.randint(0, 99999) + try: + pool = self._create_pool_with(tags=self.storage_tags) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=mismatch_tag) + + try: + vm = self._deploy_vm_with_offering("tag-mismatch", so.id) + except Exception as ex: + logger.info( + "Deploy with mismatched tags failed (acceptable if no " + "alternate pool matches): %s" % ex + ) + vm = None + + if vm is not None: + root = self._root_volume_for_vm(vm.id) + self.assertNotEqual( + str(root.storageid), str(pool.id), + "ROOT should not land on ONTAP pool when SO tags mismatch " + "(storageid=%s pool=%s)" % (root.storageid, pool.id), + ) + + tcu.assert_no_spool_ref( + self, self.dbConnection, pool_db_id, + self.__class__.template_db_id, + ) + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + finally: + if vm is not None: + try: + self._destroy_vm_static(vm.id) + except Exception: + pass + self._delete_service_offering(so) + self._force_delete_pool(pool) + + def step_02_undersized_pool_deploy_fails(self): + """ + Matching tags but capacitybytes far below template virtual size. + + Deploy must fail; no Ready spool_ref / cache object should remain. + """ + pool = None + so = None + tmpl_size = tcu.get_template_size_bytes( + self.dbConnection, self.__class__.template_db_id + ) + if not tmpl_size or tmpl_size <= 0: + self.skipTest( + "Cannot determine vm_template.size for template_db_id=%s" + % self.__class__.template_db_id + ) + + # Lab templates can be small (~100–200 MiB); keep capacity under size. + undersized = max(tmpl_size // 4, 8 * 1024 * 1024) + if undersized >= tmpl_size: + undersized = max(tmpl_size // 2, 1024 * 1024) + self.assertLess( + undersized, tmpl_size, + "undersized capacity %s must be < template size %s" + % (undersized, tmpl_size), + ) + + try: + exclusive_tags = self._unique_storage_tags() + pool = self._create_pool_with( + tags=exclusive_tags, + capacitybytes=undersized, + name_prefix=self.POOL_NAME_PREFIX + "Tiny", + ) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=exclusive_tags) + + self._deploy_vm_expect_failure("undersized", so.id, timeout=300) + + spool = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id + ) + if spool is not None: + self.assertNotEqual( + str(spool.get("state")), "Ready", + "spool_ref must not be Ready after undersized deploy failure", + ) + self.assertNotEqual( + str(spool.get("download_state", "")).upper(), + "DOWNLOADED", + "spool_ref must not be DOWNLOADED after undersized failure", + ) + if self.PROTOCOL.upper() == "ISCSI": + tcu.assert_no_iscsi_template_cache_lun( + self, self.ontap, self.svm_name, pool.name, + self.__class__.template_db_id, + ) + finally: + self._delete_service_offering(so) + self._force_delete_pool(pool) + + def step_03_deleted_cache_blocks_reuse(self): + """ + Seed cache, destroy VM, delete ONTAP cache out-of-band, redeploy. + + CloudStack still has Ready spool_ref but backend object is gone — + second deploy must fail (clone / missing cache path). + """ + self._ensure_guest_network_router_running() + pool = None + so = None + vm1 = None + try: + exclusive_tags = self._unique_storage_tags() + pool = self._create_pool_with(tags=exclusive_tags) + pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) + so = self._create_service_offering_with(tags=exclusive_tags) + + # Seed deploy can flake on lab planner/VR; retry once after + # re-ensuring the guest network router is Running. + last_ex = None + for attempt in range(2): + try: + self._ensure_guest_network_router_running() + vm1 = self._deploy_vm_with_offering("neg-seed", so.id) + last_ex = None + break + except Exception as ex: + last_ex = ex + logger.warning( + "Seed deploy attempt %s failed: %s" + % (attempt + 1, ex) + ) + self._reclaim_stale_template_cache_networks() + time.sleep(15) + if last_ex is not None: + # Seed must succeed — this case is not optional. Failure here + # is usually exhausted public IPs / stale VR from prior runs. + self.fail( + "Could not seed template cache for out-of-band delete " + "case after retries (check public IP capacity / leftover " + "VRs): %s" % last_ex + ) + self._assert_root_on_pool(vm1.id, pool) + spool = tcu.wait_for_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id, + timeout=600, + ) + tcu.assert_spool_ref_ready( + self, spool, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + self._assert_cache_on_ontap(pool, spool) + + self._destroy_vm_static(vm1.id) + vm1 = None + # Allow clone/ROOT LUN cleanup so the cache LUN is no longer a + # FlexClone parent before out-of-band delete. + time.sleep(30) + + self._delete_ontap_cache(pool, spool) + stale = tcu.get_template_spool_ref( + self.dbConnection, pool_db_id, self.__class__.template_db_id + ) + tcu.assert_spool_ref_ready( + self, stale, + expect_local_path=(self.PROTOCOL.upper() == "ISCSI"), + ) + + self._deploy_vm_expect_failure("stale-cache", so.id, timeout=300) + finally: + if vm1 is not None: + try: + self._destroy_vm_static(vm1.id) + except Exception: + pass + self._delete_service_offering(so) + self._force_delete_pool(pool) diff --git a/test/integration/plugins/ontap/helpers/template_cache_util.py b/test/integration/plugins/ontap/helpers/template_cache_util.py new file mode 100644 index 000000000000..e733924ee940 --- /dev/null +++ b/test/integration/plugins/ontap/helpers/template_cache_util.py @@ -0,0 +1,256 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Helpers for ONTAP primary template-cache Marvin assertions. + +Covers CloudStack DB (``template_spool_ref``) and ONTAP REST checks for the +cache LUN (iSCSI: ``/vol//cs_tmpl_``) or cache file (NFS). +""" + +from __future__ import print_function + +import logging +import time + +logger = logging.getLogger("template_cache_util") + +# Must stay in sync with OntapStorageConstants.TEMPLATE_LUN_PREFIX +TEMPLATE_LUN_PREFIX = "cs_tmpl_" + + +def get_db_id(db_connection, table, api_uuid): + """Resolve CloudStack API UUID to numeric DB id.""" + rows = db_connection.execute( + "SELECT id FROM `%s` WHERE uuid = '%s'" % (table, api_uuid) + ) + if not rows: + raise AssertionError( + "No row in %s for uuid=%s" % (table, api_uuid) + ) + return rows[0][0] + + +def get_template_spool_ref(db_connection, pool_db_id, template_db_id): + """ + Return template_spool_ref row as a dict, or None if absent. + + Columns: id, pool_id, template_id, download_state, local_path, + install_path, template_size, marked_for_gc, state + """ + rows = db_connection.execute( + "SELECT id, pool_id, template_id, download_state, local_path, " + "install_path, template_size, marked_for_gc, state " + "FROM template_spool_ref " + "WHERE pool_id = %s AND template_id = %s" + % (int(pool_db_id), int(template_db_id)) + ) + if not rows: + return None + row = rows[0] + return { + "id": row[0], + "pool_id": row[1], + "template_id": row[2], + "download_state": row[3], + "local_path": row[4], + "install_path": row[5], + "template_size": row[6], + "marked_for_gc": row[7], + "state": row[8], + } + + +def count_template_spool_refs(db_connection, pool_db_id, template_db_id): + """Return number of template_spool_ref rows for pool+template.""" + rows = db_connection.execute( + "SELECT COUNT(*) FROM template_spool_ref " + "WHERE pool_id = %s AND template_id = %s" + % (int(pool_db_id), int(template_db_id)) + ) + return int(rows[0][0]) if rows else 0 + + +def assert_spool_ref_ready(testcase, spool_ref, expect_local_path=True): + """Assert spool_ref is DOWNLOADED / Ready after a successful cache seed.""" + testcase.assertIsNotNone(spool_ref, "Expected template_spool_ref row") + testcase.assertEqual( + str(spool_ref["download_state"]).upper(), + "DOWNLOADED", + "download_state should be DOWNLOADED, got %s" + % spool_ref["download_state"], + ) + testcase.assertEqual( + str(spool_ref["state"]), + "Ready", + "state should be Ready, got %s" % spool_ref["state"], + ) + if expect_local_path: + testcase.assertTrue( + spool_ref.get("local_path"), + "local_path (LUN uuid / cache id) should be set", + ) + testcase.assertTrue( + spool_ref.get("install_path"), + "install_path should be set after template is cached", + ) + + +def assert_no_spool_ref(testcase, db_connection, pool_db_id, template_db_id): + """Assert there is no template_spool_ref for pool+template.""" + count = count_template_spool_refs( + db_connection, pool_db_id, template_db_id + ) + testcase.assertEqual( + count, 0, + "Expected no template_spool_ref for pool=%s template=%s, found %s" + % (pool_db_id, template_db_id, count), + ) + + +def template_cache_lun_path(flexvol_name, template_db_id): + """iSCSI cache LUN path: /vol//cs_tmpl_.""" + return "/vol/%s/%s%s" % ( + flexvol_name, TEMPLATE_LUN_PREFIX, int(template_db_id) + ) + + +def assert_iscsi_template_cache_lun(testcase, ontap, svm_name, flexvol_name, + template_db_id): + """Assert the deterministic template-cache LUN exists on ONTAP.""" + path = template_cache_lun_path(flexvol_name, template_db_id) + lun = ontap.get_lun(svm_name, path) + testcase.assertIsNotNone( + lun, + "Expected template cache LUN at %s on SVM %s" % (path, svm_name), + ) + return lun + + +def assert_no_iscsi_template_cache_lun(testcase, ontap, svm_name, flexvol_name, + template_db_id): + """Assert the template-cache LUN is absent.""" + path = template_cache_lun_path(flexvol_name, template_db_id) + lun = ontap.get_lun(svm_name, path) + testcase.assertIsNone( + lun, + "Template cache LUN should be absent at %s" % path, + ) + + +def count_iscsi_template_cache_luns(ontap, svm_name, flexvol_name, + template_db_id=None): + """Count cs_tmpl_* LUNs in the FlexVol (optionally for one template id).""" + luns = ontap.list_luns_in_volume(svm_name, flexvol_name) or [] + prefix = "/vol/%s/%s" % (flexvol_name, TEMPLATE_LUN_PREFIX) + matches = [l for l in luns if l.get("name", "").startswith(prefix)] + if template_db_id is not None: + exact = template_cache_lun_path(flexvol_name, template_db_id) + matches = [l for l in matches if l.get("name") == exact] + return len(matches) + + +def count_luns_excluding_template_cache(ontap, svm_name, flexvol_name): + """Count non-cache LUNs (volume clones / blank volumes) in the FlexVol.""" + luns = ontap.list_luns_in_volume(svm_name, flexvol_name) or [] + prefix = "/vol/%s/%s" % (flexvol_name, TEMPLATE_LUN_PREFIX) + return len([l for l in luns if not l.get("name", "").startswith(prefix)]) + + +def _normalize_nfs_path(install_path): + """Strip leading slash for OntapRestClient.list_files_in_volume paths.""" + if not install_path: + return "/" + path = install_path if install_path.startswith("/") else "/" + install_path + # Parent directory listing: if path is a file, list its parent + if path.endswith("/"): + return path.rstrip("/") or "/" + parent = path.rsplit("/", 1)[0] + return parent or "/" + + +def assert_nfs_template_cache_file(testcase, ontap, flexvol_name, install_path): + """ + Assert the NFS cache file referenced by install_path exists in the FlexVol. + + ``install_path`` comes from ``template_spool_ref.install_path`` (relative or + absolute path inside the FlexVol as reported by the KVM copy). + """ + testcase.assertTrue( + install_path, + "install_path is required to locate the NFS template cache file", + ) + file_name = install_path.rstrip("/").rsplit("/", 1)[-1] + parent = _normalize_nfs_path(install_path) + names = ontap.list_files_in_volume(flexvol_name, path=parent) + # Also try volume root if parent listing is empty (path styles differ) + if file_name not in names: + root_names = ontap.list_files_in_volume(flexvol_name, path="/") + testcase.assertIn( + file_name, root_names + names, + "Expected NFS template cache file '%s' under '%s' or '/' in " + "FlexVol '%s'; listed parent=%s root=%s" + % (file_name, parent, flexvol_name, names, root_names), + ) + return file_name + + +def wait_for_spool_ref(db_connection, pool_db_id, template_db_id, + timeout=600, interval=10): + """ + Poll until template_spool_ref exists and is Ready/DOWNLOADED. + + Returns the spool_ref dict, or None on timeout. + """ + deadline = time.time() + timeout + last = None + while time.time() < deadline: + last = get_template_spool_ref( + db_connection, pool_db_id, template_db_id + ) + if last and str(last.get("download_state", "")).upper() == "DOWNLOADED" \ + and str(last.get("state", "")) == "Ready": + return last + time.sleep(interval) + return last + + +def assert_no_nfs_template_cache_file(testcase, ontap, flexvol_name, install_path): + """Assert the NFS cache file referenced by install_path is absent.""" + testcase.assertTrue( + install_path, + "install_path is required to assert NFS cache file absence", + ) + file_name = install_path.rstrip("/").rsplit("/", 1)[-1] + parent = _normalize_nfs_path(install_path) + names = ontap.list_files_in_volume(flexvol_name, path=parent) or [] + root_names = ontap.list_files_in_volume(flexvol_name, path="/") or [] + testcase.assertNotIn( + file_name, names + root_names, + "NFS template cache file '%s' should be absent under '%s' or '/' in " + "FlexVol '%s'" % (file_name, parent, flexvol_name), + ) + + +def get_template_size_bytes(db_connection, template_db_id): + """Return vm_template.size (bytes) for the given DB id, or None.""" + rows = db_connection.execute( + "SELECT size FROM vm_template WHERE id = %s" % int(template_db_id) + ) + if not rows or rows[0][0] is None: + return None + return int(rows[0][0]) diff --git a/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py index 4c843bfbd90f..ddefa09bfd3b 100644 --- a/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py +++ b/test/integration/plugins/ontap/iscsi/instance/test_vm_volume_attach.py @@ -34,18 +34,28 @@ - startVirtualMachine → LUN-map is re-created - detachVolume → LUN-map is removed -Tests are numbered test_01 ... test_08 and must run in that order. Each step -builds on the shared state established by the previous step. +Tests must run in order (test_01, test_02, test_03, test_03a, test_03b, +test_04 ... test_08). Each step builds on the shared state established by the +previous step. + +The VM is deployed with a compute offering whose storage tags match the pool, +so its ROOT volume lands on the ONTAP pool and exercises the primary +template cache (``cs_tmpl_`` LUN: seed / reuse / survive VM delete) +alongside the data-volume workflow. LUN-map assertions are filtered to the +data volume's LUN, so the ROOT and cache LUNs in the same FlexVol do not +affect them. Workflow: - 01 Create iSCSI primary storage pool on ONTAP + 01 Create iSCSI primary storage pool on ONTAP (tagged with templateCacheTags) 02 Create a CloudStack data volume on the iSCSI pool (LUN on ONTAP) - 03 Deploy a VM using any available KVM template + 03 Deploy a VM — ROOT on ONTAP; seeds the cs_tmpl_* cache LUN + 03a Deploy a second VM — reuses the cache LUN (one more ROOT clone LUN only) + 03b Expunge the second VM — its ROOT LUN is removed; cache LUN survives 04 Attach iSCSI data volume to running VM (LUN-map created) 05 Stop VM — LUN-map for attached volume is removed from ONTAP 06 Start VM — LUN-map is re-created on ONTAP 07 Detach data volume from running VM — LUN-map removed - 08 Destroy VM, delete data volume, delete pool + 08 Destroy VM (cache LUN still survives), delete data volume, delete pool Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -81,17 +91,23 @@ enableStorageMaintenance, listNetworkOfferings as listNetworkOfferingsAPI, listNetworks as listNetworksAPI, - listServiceOfferings as listServiceOfferingsAPI, - listTemplates as listTemplatesAPI, listVirtualMachines as listVirtualMachinesAPI, listVolumes as listVolumesAPI, startVirtualMachine as startVirtualMachineAPI, stopVirtualMachine as stopVirtualMachineAPI, ) -from marvin.lib.base import StoragePool +from marvin.lib.base import ServiceOffering, StoragePool from marvin.lib.common import list_storage_pools from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config +from helpers import template_cache_util as tcu +from helpers.template_cache_base import ( + TemplateCacheAssertionsMixin, + find_ready_kvm_template, + tagged_compute_offering_data, + template_cache_capacity_bytes, + template_cache_tags, +) logger = logging.getLogger("TestOntapVMVolumeAttachISCSI") @@ -181,15 +197,22 @@ def __init__(self, storage_ip, svm_name, username, password, # Sequential workflow test class # --------------------------------------------------------------------------- -class TestOntapVMVolumeAttachISCSI(OntapTestBase): +class TestOntapVMVolumeAttachISCSI(TemplateCacheAssertionsMixin, OntapTestBase): """ - Tests iSCSI ONTAP data volume lifecycle with a running CloudStack VM. + Tests iSCSI ONTAP data volume lifecycle with a running CloudStack VM whose + ROOT volume is on the ONTAP pool (primary template cache). All tests are sequential — state is carried on class attributes. """ + PROTOCOL = "ISCSI" + # ---- extra shared state beyond OntapTestBase ----------------------- vm = None + vm2 = None # second VM used only to verify template-cache reuse + vm2_baseline_luns = None # non-cache LUN count before the second deploy template_id = None + template_db_id = None # numeric vm_template.id (template_spool_ref key) + pool_db_id = None # numeric storage_pool.id (template_spool_ref key) service_offering_id = None network_id = None _created_network_id = None # network created by this suite for Advanced zones @@ -223,8 +246,10 @@ def setUpClass(cls): ) scope = pool_cfg.get("storagePoolScope", "CLUSTER") provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") - tags = iscsi_cfg.get("storagePoolTags", "ontap-iscsi") - capacitybytes = pool_cfg.get("capacitybytes", None) + # Dedicated tag shared by the pool and the compute offering so the + # VM's ROOT volume is placed on this ONTAP pool (template cache path). + tags = template_cache_tags(iscsi_cfg, "iscsi") + capacitybytes = template_cache_capacity_bytes(pool_cfg) cls.testdata = TestData( storage_ip, svm_name, username, password, @@ -237,29 +262,22 @@ def setUpClass(cls): cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) # Discover a ready user KVM template (exclude SYSTEM type) - tpl_cmd = listTemplatesAPI.listTemplatesCmd() - tpl_cmd.templatefilter = "all" - tpl_cmd.listall = True - tpl_cmd.zoneid = cls.zone.id - templates = cls.apiClient.listTemplates(tpl_cmd) or [] - kvm_ready = [ - t for t in templates - if getattr(t, "hypervisor", "").lower() == "kvm" - and getattr(t, "isready", False) - and getattr(t, "templatetype", "").upper() != "SYSTEM" - ] - cls.template_id = kvm_ready[0].id if kvm_ready else None + template = find_ready_kvm_template(cls.apiClient, cls.zone.id) + cls.template_id = template.id if template is not None else None if cls.template_id is None: logger.warning( "No ready KVM user template found — VM tests will be skipped." ) + else: + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) - # Smallest service offering - so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() - offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] - assert offerings, "No service offerings available in CloudStack" - offerings.sort(key=lambda s: getattr(s, "memory", 9999)) - cls.service_offering_id = offerings[0].id + service_offering = ServiceOffering.create( + cls.apiClient, tagged_compute_offering_data(tags, "OntapISCSIVMSO") + ) + cls._cleanup.append(service_offering) + cls.service_offering_id = service_offering.id # Network ID for VM deployment cls.network_id = None @@ -302,34 +320,28 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): """ - Safety-net cleanup: destroy VM if still alive, delete the guest + Safety-net cleanup: destroy VMs if still alive, delete the guest network created for Advanced zones (if not already deleted by test_08), then delegate pool/volume/account cleanup to the base class. """ - if cls.vm is not None: - try: - vms = cls.apiClient.listVirtualMachines( - _list_vms_cmd(cls.vm.id)) - state = vms[0].state if vms else "unknown" - if state.lower() not in ("stopped", "destroyed", - "expunging", "error"): - stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() - stop_cmd.id = cls.vm.id - stop_cmd.forced = True - cls.apiClient.stopVirtualMachine(stop_cmd) - _wait_for_vm_state(cls.apiClient, cls.vm.id, - "Stopped", timeout=120) - except Exception as e: - logger.warning("tearDownClass: could not stop VM %s: %s" - % (cls.vm.id, e)) + for vm in (cls.vm2, cls.vm): + if vm is None: + continue try: - dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() - dest_cmd.id = cls.vm.id - dest_cmd.expunge = True - cls.apiClient.destroyVirtualMachine(dest_cmd) + cls._stop_and_expunge_vm(vm.id) except Exception as e: - logger.warning("tearDownClass: could not destroy VM %s: %s" - % (cls.vm.id, e)) + logger.warning("tearDownClass: could not stop/destroy VM %s: " + "%s" % (vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as de: + logger.warning("tearDownClass: could not destroy VM %s: " + "%s" % (vm.id, de)) + cls.vm2 = None + cls.vm = None # Delete the guest network created for this account in Advanced zones. # test_08 deletes it on the happy path; this is the fallback for @@ -347,6 +359,23 @@ def tearDownClass(cls): super(TestOntapVMVolumeAttachISCSI, cls).tearDownClass() + @classmethod + def _stop_and_expunge_vm(cls, vm_id): + """Force-stop the VM if needed, then destroy it with expunge.""" + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + state = vms[0].state if vms else "unknown" + if state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=120) + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + # ---- helpers ------------------------------------------------------- def _create_pool(self): @@ -460,6 +489,9 @@ def test_01_create_iscsi_pool(self): """ pool = self._create_pool() self.__class__.pool = pool + self.__class__.pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) self.assertEqual(pool.state, "Up", "Pool state should be 'Up', got '%s'" % pool.state) @@ -507,9 +539,14 @@ def test_02_create_ontap_data_volume(self): @attr(tags=["iscsi_vm_workflow"], required_hardware=True) def test_03_deploy_vm(self): """ - Deploy a VM using the first available ready KVM template. + Deploy a VM using the first available ready KVM template and the + compute offering tagged to match the ONTAP pool. Verifies: - VM reaches Running state + - ROOT volume is placed on the ONTAP pool + - template_spool_ref for pool + template is Ready / DOWNLOADED + (local_path = cache LUN uuid) + - ONTAP: exactly one cache LUN /vol//cs_tmpl_ - ONTAP: the iSCSI data volume's LUN is NOT yet mapped (no VM attachment has been performed yet) """ @@ -531,12 +568,17 @@ def test_03_deploy_vm(self): vm = self.apiClient.deployVirtualMachine(cmd) self.__class__.vm = vm - result = self._poll_vm_state(vm.id, "Running", timeout=300) + result = self._poll_vm_state(vm.id, "Running", timeout=900) self.assertEqual( result.state, "Running", "VM should be 'Running' after deploy, got '%s'" % result.state ) + pool = self.__class__.pool + self._assert_root_on_pool(vm.id, pool) + spool = self._wait_for_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + # Data volume LUN-map must not exist yet (volume not yet attached) lun_maps = self._lun_maps() self.assertEqual( @@ -545,6 +587,75 @@ def test_03_deploy_vm(self): % (len(lun_maps), lun_maps) ) + # ------------------------------------------------------------------ + # Step 03a — Deploy a second VM — reuses the template cache LUN + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03a_deploy_second_vm_reuses_template_cache(self): + """ + Deploy a second VM from the same template and offering. + Verifies: + - VM reaches Running; its ROOT volume is on the ONTAP pool + - Still exactly one template_spool_ref (cache reused, not re-seeded) + - ONTAP: still exactly one cs_tmpl_* cache LUN + - ONTAP: exactly one additional non-cache LUN (the VM-2 ROOT clone) + """ + self.assertIsNotNone(self.__class__.vm, + "VM absent — test_03 must pass first") + pool = self.__class__.pool + baseline = self._count_non_cache_luns(pool) + self.__class__.vm2_baseline_luns = baseline + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm2 = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm2, "deployVirtualMachine returned None") + self.__class__.vm2 = vm2 + self._poll_vm_state(vm2.id, "Running", timeout=900) + + self._assert_root_on_pool(vm2.id, pool) + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + self.assertEqual( + self._count_non_cache_luns(pool), baseline + 1, + "Expected one additional non-cache LUN for the VM-2 ROOT clone", + ) + + # ------------------------------------------------------------------ + # Step 03b — Expunge the second VM — template cache LUN survives + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_vm_workflow"], required_hardware=True) + def test_03b_expunge_second_vm_template_cache_survives(self): + """ + Destroy (expunge) the second VM. The primary template cache is only + removed by storage GC / pool deletion, never by VM lifecycle. + Verifies: + - ONTAP: the VM-2 ROOT LUN is removed (non-cache LUN count back to + the pre-deploy baseline) + - template_spool_ref is still Ready / DOWNLOADED + - ONTAP: the cs_tmpl_* cache LUN is still present + """ + self.assertIsNotNone(self.__class__.vm2, + "Second VM absent — test_03a must pass first") + pool = self.__class__.pool + + self._stop_and_expunge_vm(self.__class__.vm2.id) + self.__class__.vm2 = None + + self._wait_for_non_cache_lun_count( + pool, self.__class__.vm2_baseline_luns) + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + # ------------------------------------------------------------------ # Step 04 — Attach iSCSI volume to running VM (TDS SN 27 iSCSI) # ------------------------------------------------------------------ @@ -741,6 +852,8 @@ def test_08_destroy_vm_and_cleanup(self): This test leaves no entities behind in either CloudStack or ONTAP. Verifies: - VM is destroyed and expunged from CloudStack + - Template cache survives VM delete: template_spool_ref still Ready + and the cs_tmpl_* cache LUN is still in the FlexVol - deleteVolume removes the LUN from the ONTAP FlexVol - deleteStoragePool(forced=True) removes the pool from CS - ONTAP: FlexVol deleted @@ -769,6 +882,11 @@ def test_08_destroy_vm_and_cleanup(self): self.apiClient.destroyVirtualMachine(dest_cmd) self.__class__.vm = None + if pool is not None: + spool = self._assert_single_ready_spool_ref( + self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + if vol is not None and pool is not None: pool_name = pool.name diff --git a/test/integration/plugins/ontap/iscsi/template/__init__.py b/test/integration/plugins/ontap/iscsi/template/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py b/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py new file mode 100644 index 000000000000..33ee6f7a5a38 --- /dev/null +++ b/test/integration/plugins/ontap/iscsi/template/test_template_cache_negative.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +iSCSI primary template-cache negative / boundary suite (Marvin). + +Independent of the sequential happy-path suite. + + 01 Tag mismatch — no spool_ref / cache on ONTAP pool + 02 Undersized pool — deploy fails + 03 Out-of-band cache delete — reuse deploy fails + +Running: + bash test/integration/plugins/ontap/run_tests.sh iscsi_template_cache_negative +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_negative_workflow import ( + OntapTemplateCacheNegativeWorkflow, +) + + +class TestOntapIscsiTemplateCacheNegative(OntapTemplateCacheNegativeWorkflow): + PROTOCOL = "ISCSI" + NOSE_TAG = "iscsi_template_cache_negative" + PROTOCOL_CFG_KEY = "iscsi" + POOL_URL_SCHEME = "iscsi" + POOL_NAME_PREFIX = "OntapIscsiTmplNeg" + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_01_tag_mismatch_does_not_seed_cache(self): + self.step_01_tag_mismatch_does_not_seed_cache() + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_02_undersized_pool_deploy_fails(self): + self.step_02_undersized_pool_deploy_fails() + + @attr(tags=["iscsi_template_cache_negative"], required_hardware=True) + def test_03_deleted_cache_blocks_reuse(self): + self.step_03_deleted_cache_blocks_reuse() diff --git a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py index 48158c682bd0..475ff87ea860 100644 --- a/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py +++ b/test/integration/plugins/ontap/nfs3/instance/test_vm_volume_attach.py @@ -19,18 +19,28 @@ Sequential workflow integration tests for NetApp ONTAP data volume lifecycle with a running virtual machine. -Tests are numbered test_01 ... test_08 and must run in that order. Each step -builds on the shared state established by the previous step. +Tests must run in order (test_01, test_02, test_03, test_03a, test_03b, +test_04 ... test_08). Each step builds on the shared state established by the +previous step. + +The VM is deployed with a compute offering whose storage tags match the pool, +so its ROOT volume lands on the ONTAP pool and exercises the primary +template cache (seed / reuse / survive VM delete) alongside the data-volume +workflow. Workflow: - 01 Create NFS3 primary storage pool on ONTAP + 01 Create NFS3 primary storage pool on ONTAP (tagged with templateCacheTags) 02 Create a CloudStack data volume on the ONTAP pool - 03 Deploy a VM (template and service offering discovered at setup time) + 03 Deploy a VM — ROOT on ONTAP; seeds template cache (template_spool_ref + Ready + cache file on the FlexVol) + 03a Deploy a second VM — reuses the cache (still one template_spool_ref) + 03b Expunge the second VM — cache and template_spool_ref survive 04 Attach the ONTAP data volume to the running VM 05 Stop the VM — export policy stays; volume remains attached in CS 06 Start the VM — VM Running; volume still attached; FlexVol online 07 Detach the ONTAP data volume from the VM - 08 Destroy VM; delete ONTAP volume; enter maintenance; delete pool + 08 Destroy VM (cache still survives); delete ONTAP volume; enter + maintenance; force-delete pool Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -66,18 +76,24 @@ enableStorageMaintenance, listNetworkOfferings as listNetworkOfferingsAPI, listNetworks as listNetworksAPI, - listServiceOfferings as listServiceOfferingsAPI, - listTemplates as listTemplatesAPI, listVirtualMachines as listVirtualMachinesAPI, listVolumes as listVolumesAPI, startVirtualMachine as startVirtualMachineAPI, stopVirtualMachine as stopVirtualMachineAPI, updateStoragePool as updateStoragePoolAPI, ) -from marvin.lib.base import StoragePool +from marvin.lib.base import ServiceOffering, StoragePool from marvin.lib.common import list_storage_pools from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config +from helpers import template_cache_util as tcu +from helpers.template_cache_base import ( + TemplateCacheAssertionsMixin, + find_ready_kvm_template, + tagged_compute_offering_data, + template_cache_capacity_bytes, + template_cache_tags, +) logger = logging.getLogger("TestOntapVMVolumeAttach") @@ -144,16 +160,22 @@ def __init__(self, storage_ip, svm_name, username, password, # Sequential workflow test class # --------------------------------------------------------------------------- -class TestOntapVMVolumeAttach(OntapTestBase): +class TestOntapVMVolumeAttach(TemplateCacheAssertionsMixin, OntapTestBase): """ - Tests ONTAP data volume lifecycle with a running CloudStack VM. + Tests ONTAP data volume lifecycle with a running CloudStack VM whose ROOT + volume is on the ONTAP pool (primary template cache). All tests are sequential — state is carried on class attributes. """ + PROTOCOL = "NFS3" + # ---- extra shared state beyond OntapTestBase ----------------------- vm = None # running VirtualMachine + vm2 = None # second VM used only to verify template-cache reuse template_id = None # KVM template ID discovered at setup + template_db_id = None # numeric vm_template.id (template_spool_ref key) + pool_db_id = None # numeric storage_pool.id (template_spool_ref key) service_offering_id = None network_id = None # None for Basic zones _created_network_id = None # network created by this suite for Advanced zones @@ -188,8 +210,10 @@ def setUpClass(cls): protocol = "NFS3" scope = pool_cfg.get("storagePoolScope", "CLUSTER") provider = pool_cfg.get("storagePoolProvider", "NetApp ONTAP") - tags = nfs3_cfg.get("storagePoolTags", "ontap-nfs3") - capacitybytes = pool_cfg.get("capacitybytes", None) + # Dedicated tag shared by the pool and the compute offering so the + # VM's ROOT volume is placed on this ONTAP pool (template cache path). + tags = template_cache_tags(nfs3_cfg, "nfs3") + capacitybytes = template_cache_capacity_bytes(pool_cfg) cls.testdata = TestData( storage_ip, svm_name, username, password, @@ -204,19 +228,12 @@ def setUpClass(cls): # Discover a suitable user KVM template in the zone (must be fully # downloaded; system-type templates are excluded as they cannot be # deployed as user VMs). - tpl_cmd = listTemplatesAPI.listTemplatesCmd() - tpl_cmd.templatefilter = "all" - tpl_cmd.listall = True - tpl_cmd.zoneid = cls.zone.id - templates = cls.apiClient.listTemplates(tpl_cmd) or [] - kvm_ready = [ - t for t in templates - if getattr(t, "hypervisor", "").lower() == "kvm" - and getattr(t, "isready", False) - and getattr(t, "templatetype", "").upper() != "SYSTEM" - ] - if kvm_ready: - cls.template_id = kvm_ready[0].id + template = find_ready_kvm_template(cls.apiClient, cls.zone.id) + if template is not None: + cls.template_id = template.id + cls.template_db_id = tcu.get_db_id( + cls.dbConnection, "vm_template", cls.template_id + ) else: logger.warning( "No ready user KVM template found in zone '%s'. " @@ -225,12 +242,11 @@ def setUpClass(cls): ) cls.template_id = None - # Discover the smallest service offering - so_cmd = listServiceOfferingsAPI.listServiceOfferingsCmd() - offerings = cls.apiClient.listServiceOfferings(so_cmd) or [] - assert offerings, "No service offerings available in CloudStack" - offerings.sort(key=lambda s: getattr(s, "memory", 9999)) - cls.service_offering_id = offerings[0].id + service_offering = ServiceOffering.create( + cls.apiClient, tagged_compute_offering_data(tags, "OntapNFS3VMSO") + ) + cls._cleanup.append(service_offering) + cls.service_offering_id = service_offering.id # Detect zone type; resolve network ID for Advanced zones cls.network_id = None @@ -272,32 +288,25 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - """Destroy the VM first, then delegate pool/volume cleanup to super.""" - if cls.vm is not None: + """Destroy the VMs first, then delegate pool/volume cleanup to super.""" + for vm in (cls.vm2, cls.vm): + if vm is None: + continue try: - # Ensure VM is stopped before destroying - vms = cls.apiClient.listVirtualMachines( - _list_vms_cmd(cls.vm.id)) - current_state = vms[0].state if vms else "unknown" - if current_state.lower() not in ("stopped", "destroyed", - "expunging", "error"): - stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() - stop_cmd.id = cls.vm.id - stop_cmd.forced = True - cls.apiClient.stopVirtualMachine(stop_cmd) - _wait_for_vm_state(cls.apiClient, cls.vm.id, "Stopped", - timeout=120) + cls._stop_and_expunge_vm(vm.id) except Exception as e: - logger.warning("tearDownClass: could not stop VM %s: %s" - % (cls.vm.id, e)) - try: - dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() - dest_cmd.id = cls.vm.id - dest_cmd.expunge = True - cls.apiClient.destroyVirtualMachine(dest_cmd) - except Exception as e: - logger.warning("tearDownClass: could not destroy VM %s: %s" - % (cls.vm.id, e)) + logger.warning("tearDownClass: could not stop/destroy VM %s: " + "%s" % (vm.id, e)) + try: + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm.id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + except Exception as de: + logger.warning("tearDownClass: could not destroy VM %s: " + "%s" % (vm.id, de)) + cls.vm2 = None + cls.vm = None # Delete the guest network created for this account in Advanced zones. if cls._created_network_id is not None: @@ -313,6 +322,23 @@ def tearDownClass(cls): super(TestOntapVMVolumeAttach, cls).tearDownClass() + @classmethod + def _stop_and_expunge_vm(cls, vm_id): + """Force-stop the VM if needed, then destroy it with expunge.""" + vms = cls.apiClient.listVirtualMachines(_list_vms_cmd(vm_id)) + current_state = vms[0].state if vms else "unknown" + if current_state.lower() not in ("stopped", "destroyed", + "expunging", "error"): + stop_cmd = stopVirtualMachineAPI.stopVirtualMachineCmd() + stop_cmd.id = vm_id + stop_cmd.forced = True + cls.apiClient.stopVirtualMachine(stop_cmd) + _wait_for_vm_state(cls.apiClient, vm_id, "Stopped", timeout=120) + dest_cmd = destroyVirtualMachineAPI.destroyVirtualMachineCmd() + dest_cmd.id = vm_id + dest_cmd.expunge = True + cls.apiClient.destroyVirtualMachine(dest_cmd) + # ---- pool creation helper ----------------------------------------- def _create_pool(self): @@ -385,6 +411,9 @@ def test_01_create_nfs3_pool(self): """ pool = self._create_pool() self.__class__.pool = pool + self.__class__.pool_db_id = tcu.get_db_id( + self.dbConnection, "storage_pool", pool.id + ) self.assertEqual( pool.state, "Up", @@ -445,10 +474,13 @@ def test_02_create_ontap_data_volume(self): @attr(tags=["vm_volume_workflow"], required_hardware=True) def test_03_deploy_vm(self): """ - Deploy a VM using the first available KVM template and smallest - service offering discovered at setup time. + Deploy a VM using the first available KVM template and the compute + offering tagged to match the ONTAP pool. Verifies: - VM reaches 'Running' state in CloudStack + - ROOT volume is placed on the ONTAP pool + - template_spool_ref for pool + template is Ready / DOWNLOADED + - ONTAP: template cache file exists at the spool install_path """ self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") @@ -473,12 +505,75 @@ def test_03_deploy_vm(self): self.assertIsNotNone(vm, "deployVirtualMachine returned None") self.__class__.vm = vm - vm_obj = self._poll_vm_state(vm.id, "Running", timeout=600) + vm_obj = self._poll_vm_state(vm.id, "Running", timeout=900) self.assertEqual( vm_obj.state, "Running", "VM should be 'Running', got '%s'" % vm_obj.state ) + pool = self.__class__.pool + self._assert_root_on_pool(vm.id, pool) + spool = self._wait_for_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # ------------------------------------------------------------------ + # Step 03a - Deploy a second VM — reuses the template cache + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03a_deploy_second_vm_reuses_template_cache(self): + """ + Deploy a second VM from the same template and offering. + Verifies: + - VM reaches 'Running'; its ROOT volume is on the ONTAP pool + - Still exactly one template_spool_ref (cache reused, not re-seeded) + - ONTAP: the same template cache file is still present + """ + self.assertIsNotNone(self.__class__.vm, + "VM absent — test_03 must pass first") + + cmd = deployVirtualMachineAPI.deployVirtualMachineCmd() + cmd.zoneid = self.zone.id + cmd.templateid = self.__class__.template_id + cmd.serviceofferingid = self.__class__.service_offering_id + cmd.account = self.account.name + cmd.domainid = self.domain.id + if self.__class__.network_id: + cmd.networkids = self.__class__.network_id + + vm2 = self.apiClient.deployVirtualMachine(cmd) + self.assertIsNotNone(vm2, "deployVirtualMachine returned None") + self.__class__.vm2 = vm2 + self._poll_vm_state(vm2.id, "Running", timeout=900) + + pool = self.__class__.pool + self._assert_root_on_pool(vm2.id, pool) + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + + # ------------------------------------------------------------------ + # Step 03b - Expunge the second VM — template cache survives + # ------------------------------------------------------------------ + + @attr(tags=["vm_volume_workflow"], required_hardware=True) + def test_03b_expunge_second_vm_template_cache_survives(self): + """ + Destroy (expunge) the second VM. The primary template cache is only + removed by storage GC / pool deletion, never by VM lifecycle. + Verifies: + - template_spool_ref is still Ready / DOWNLOADED + - ONTAP: template cache file is still present + """ + self.assertIsNotNone(self.__class__.vm2, + "Second VM absent — test_03a must pass first") + + self._stop_and_expunge_vm(self.__class__.vm2.id) + self.__class__.vm2 = None + + pool = self.__class__.pool + spool = self._assert_single_ready_spool_ref(self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + # ------------------------------------------------------------------ # Step 04 - Attach ONTAP data volume to the running VM # ------------------------------------------------------------------ @@ -761,12 +856,14 @@ def test_07_detach_volume_from_vm(self): def test_08_destroy_vm_and_cleanup(self): """ Destroy the VM, delete the ONTAP data volume, enter maintenance, - then delete the pool. + then force-delete the pool. Verifies: - VM is destroyed/expunged from CloudStack + - Template cache survives VM delete: template_spool_ref still Ready + and the cache file is still on the FlexVol - Volume is deleted from CloudStack - ONTAP: NFS3 volume data file removed from FlexVol after deleteVolume - - Pool is removed from CloudStack + - Pool (still holding the template cache) is removed from CloudStack - ONTAP: FlexVol is deleted after pool removal - ONTAP: Export policy is removed after pool removal """ @@ -795,6 +892,10 @@ def test_08_destroy_vm_and_cleanup(self): self.apiClient.destroyVirtualMachine(dest_cmd) self.__class__.vm = None + spool = self._assert_single_ready_spool_ref( + self.__class__.pool_db_id) + self._assert_cache_on_ontap(pool, spool) + # Delete the ONTAP data volume if vol is not None: vol_id = vol.id @@ -821,7 +922,9 @@ def test_08_destroy_vm_and_cleanup(self): self.apiClient.enableStorageMaintenance(maint_cmd) self._poll_pool_state(pool.id, "Maintenance", timeout=120) - self._delete_pool(pool.id) + # forced=True: the template cache (template_spool_ref) is still on + # the pool — it is only reclaimed by storage GC or pool deletion. + self._delete_pool(pool.id, forced=True) self.__class__.pool = None # CloudStack: pool must be gone diff --git a/test/integration/plugins/ontap/nfs3/template/__init__.py b/test/integration/plugins/ontap/nfs3/template/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py b/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py new file mode 100644 index 000000000000..184b8ac84def --- /dev/null +++ b/test/integration/plugins/ontap/nfs3/template/test_template_cache_negative.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +NFS3 primary template-cache negative / boundary suite (Marvin). + +Independent of the sequential happy-path suite. + + 01 Tag mismatch — no spool_ref / cache on ONTAP pool + 02 Undersized pool — deploy fails + 03 Out-of-band cache delete — reuse deploy fails + +Running: + bash test/integration/plugins/ontap/run_tests.sh nfs3_template_cache_negative +""" + +from nose.plugins.attrib import attr + +from helpers.template_cache_negative_workflow import ( + OntapTemplateCacheNegativeWorkflow, +) + + +class TestOntapNfs3TemplateCacheNegative(OntapTemplateCacheNegativeWorkflow): + PROTOCOL = "NFS3" + NOSE_TAG = "nfs3_template_cache_negative" + PROTOCOL_CFG_KEY = "nfs3" + POOL_URL_SCHEME = "nfs" + POOL_NAME_PREFIX = "OntapNfsTmplNeg" + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_01_tag_mismatch_does_not_seed_cache(self): + self.step_01_tag_mismatch_does_not_seed_cache() + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_02_undersized_pool_deploy_fails(self): + self.step_02_undersized_pool_deploy_fails() + + @attr(tags=["nfs3_template_cache_negative"], required_hardware=True) + def test_03_deleted_cache_blocks_reuse(self): + self.step_03_deleted_cache_blocks_reuse() diff --git a/test/integration/plugins/ontap/ontap.cfg b/test/integration/plugins/ontap/ontap.cfg index 64ee5174d379..41e3d6f42110 100644 --- a/test/integration/plugins/ontap/ontap.cfg +++ b/test/integration/plugins/ontap/ontap.cfg @@ -108,11 +108,13 @@ "protocols": { "iscsi": { "enabled": true, - "storagePoolTags": "ontap-iscsi" + "storagePoolTags": "ontap-iscsi", + "templateCacheTags": "ontap-iscsi-tmpl-cache" }, "nfs3": { "enabled": true, - "storagePoolTags": "ontap-nfs3" + "storagePoolTags": "ontap-nfs3", + "templateCacheTags": "ontap-nfs3-tmpl-cache" } } }, diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 4f60dbf9433f..73681b9eb701 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -144,7 +144,24 @@ def _delete(self, path, params=None): url = self._base + path resp = requests.delete(url, auth=self._auth, params=params, verify=False, timeout=30) + if not resp.ok: + raise requests.HTTPError( + "%s for url: %s body=%s" + % (resp.status_code, resp.url, resp.text), + response=resp, + ) + + def _patch(self, path, payload=None, params=None): + url = self._base + path + resp = requests.patch(url, auth=self._auth, params=params, + json=payload or {}, verify=False, timeout=30) resp.raise_for_status() + if resp.content: + try: + return resp.json() + except ValueError: + return None + return None def delete_volume(self, name): """Delete the ONTAP FlexVol with the given name. No-op if not found.""" @@ -274,6 +291,126 @@ def list_files_in_volume(self, vol_name, path="/"): return [r.get("name", "") for r in resp.get("records", []) if r.get("name") not in (".", "..")] + def _unmap_lun(self, svm_name, lun_uuid): + """Best-effort removal of all lun-maps for a LUN UUID.""" + try: + maps = self._get( + "/protocols/san/lun-maps", + params={ + "svm.name": svm_name, + "lun.uuid": lun_uuid, + "fields": "lun.uuid,igroup.uuid", + }, + ) + for lun_map in maps.get("records", []): + mapped_lun = lun_map.get("lun", {}).get("uuid") or lun_uuid + igroup_uuid = lun_map.get("igroup", {}).get("uuid") + if not igroup_uuid: + continue + try: + self._delete( + "/protocols/san/lun-maps/%s/%s" + % (mapped_lun, igroup_uuid) + ) + except Exception: + pass + except Exception: + pass + + def _offline_lun(self, svm_name, lun_path, lun_uuid): + """Take LUN offline. Lab ONTAP rejects allow_delete_online.""" + try: + self._patch( + "/storage/luns/%s" % lun_uuid, + payload={"enabled": False, "status": {"state": "offline"}}, + ) + except Exception: + try: + self._patch( + "/storage/luns/%s" % lun_uuid, + payload={"enabled": False}, + ) + except Exception: + return + deadline = time.time() + 60 + while time.time() < deadline: + cur = self.get_lun(svm_name, lun_path) + if not cur: + return + state = ( + (cur.get("status") or {}).get("state") + or ("online" if cur.get("enabled") else "offline") + ) + if str(state).lower() == "offline" or cur.get("enabled") is False: + return + time.sleep(2) + + def _delete_lun_clones(self, svm_name, parent_uuid): + """Delete FlexClone child LUNs so the parent cache LUN can be removed.""" + try: + data = self._get( + "/storage/luns", + params={ + "svm.name": svm_name, + "fields": "name,uuid,clone", + }, + ) + except Exception: + return + for child in data.get("records", []): + clone = child.get("clone") or {} + source = clone.get("source") or {} + if source.get("uuid") != parent_uuid: + continue + child_path = child.get("name") + child_uuid = child.get("uuid") + if not child_path or not child_uuid: + continue + self._unmap_lun(svm_name, child_uuid) + self._offline_lun(svm_name, child_path, child_uuid) + try: + self._delete("/storage/luns/%s" % child_uuid) + except Exception: + pass + + def delete_lun(self, svm_name, lun_path): + """Unmap + offline + delete a LUN by full path. No-op if missing.""" + lun = self.get_lun(svm_name, lun_path) + if not lun: + return False + uuid = lun.get("uuid") + if not uuid: + return False + + # Mapped LUNs / FlexClone parents cannot be deleted until dependents go. + self._unmap_lun(svm_name, uuid) + self._delete_lun_clones(svm_name, uuid) + self._offline_lun(svm_name, lun_path, uuid) + if not self.get_lun(svm_name, lun_path): + return True + # Do not pass allow_delete_online — rejected on this ONTAP build. + self._delete("/storage/luns/%s" % uuid) + return True + + def delete_file_in_volume(self, vol_name, file_path): + """ + Delete a file inside a FlexVol via ONTAP files API. + + ``file_path`` may be absolute (``/foo/bar``) or relative to volume root. + Returns True if a delete was attempted on an existing volume. + """ + vol = self.get_volume(vol_name) + if not vol: + return False + vol_uuid = vol.get("uuid", "") + if not vol_uuid: + return False + from urllib.parse import quote + path = file_path if file_path.startswith("/") else "/" + file_path + encoded_path = quote(path, safe="") + self._delete("/storage/volumes/%s/files/%s" % (vol_uuid, encoded_path)) + return True + # --------------------------------------------------------------------------- # Base test class diff --git a/test/integration/plugins/ontap/run_tests.sh b/test/integration/plugins/ontap/run_tests.sh index 8063fc3238e3..84d77a407a02 100755 --- a/test/integration/plugins/ontap/run_tests.sh +++ b/test/integration/plugins/ontap/run_tests.sh @@ -65,6 +65,7 @@ ISCSI_SUITES=( "iSCSI volume lifecycle|iscsi_volume|${ONTAP_DIR}/iscsi/volume/test_volume_lifecycle.py" "iSCSI zone-scoped pool|iscsi_zone_pool|${ONTAP_DIR}/iscsi/pool/test_zone_scoped_pool.py" "iSCSI VM volume workflow|iscsi_vm_workflow|${ONTAP_DIR}/iscsi/instance/test_vm_volume_attach.py" + "iSCSI template cache negative|iscsi_template_cache_negative|${ONTAP_DIR}/iscsi/template/test_template_cache_negative.py" ) NFS3_SUITES=( @@ -73,6 +74,7 @@ NFS3_SUITES=( "NFS3 volume lifecycle|nfs3_volume|${ONTAP_DIR}/nfs3/volume/test_volume_lifecycle.py" "NFS3 zone-scoped pool|zone_pool|${ONTAP_DIR}/nfs3/pool/test_zone_scoped_pool.py" "NFS3 VM volume attach|vm_volume_workflow|${ONTAP_DIR}/nfs3/instance/test_vm_volume_attach.py" + "NFS3 template cache negative|nfs3_template_cache_negative|${ONTAP_DIR}/nfs3/template/test_template_cache_negative.py" ) record_results() {