deal with excessive attempts on failing snapshots

api/src/main/java/com/cloud/event/EventTypes.java

@@ -370,6 +370,8 @@ public class EventTypes {
     public static final String EVENT_SNAPSHOT_DELETE = "SNAPSHOT.DELETE";
     public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT";
     public static final String EVENT_SNAPSHOT_EXTRACT = "SNAPSHOT.EXTRACT";
+    public static final String EVENT_SNAPSHOT_SKIPPED = "SNAPSHOT.SKIPPED";
+    public static final String EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED = "SNAPSHOT.RECURRING.FAILURE.LIMIT.REACHED";
     public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE";
     public static final String EVENT_SNAPSHOT_POLICY_UPDATE = "SNAPSHOTPOLICY.UPDATE";
     public static final String EVENT_SNAPSHOT_POLICY_DELETE = "SNAPSHOTPOLICY.DELETE";
@@ -1074,6 +1076,8 @@ public class EventTypes {
 
         // Snapshots
         entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class);
+        entityEventDetails.put(EVENT_SNAPSHOT_SKIPPED, Snapshot.class);
+        entityEventDetails.put(EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_EXTRACT, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class);

engine/schema/src/main/java/com/cloud/event/dao/EventDao.java

@@ -48,4 +48,10 @@ public interface EventDao extends GenericDao<EventVO, Long> {
 
     public void archiveEvents(List<EventVO> events);
 
+    /**
+     * Returns the most recent events of the given type for a resource, newest first.
+     * Used to derive how many consecutive attempts have failed for that resource without a dedicated counter column.
+     */
+    List<EventVO> listLatestEventsByResource(long resourceId, String resourceType, String type, int limit);
+
 }

engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java

@@ -35,9 +35,13 @@
 
 @Component
 public class EventDaoImpl extends GenericDaoBase<EventVO, Long> implements EventDao {
+    private static final String FIELD_RESOURCE_ID = "resourceId";
+    private static final String FIELD_RESOURCE_TYPE = "resourceType";
+
     protected final SearchBuilder<EventVO> CompletedEventSearch;
     protected final SearchBuilder<EventVO> ToArchiveOrDeleteEventSearch;
     protected final SearchBuilder<EventVO> ArchiveByIdsSearch;
+    protected final SearchBuilder<EventVO> LatestEventsByResourceSearch;
     protected final SearchBuilder<EventVO> LastStartEventSearch;
 
     public EventDaoImpl() {
@@ -47,6 +51,13 @@
         CompletedEventSearch.and("archived", CompletedEventSearch.entity().getArchived(), Op.EQ);
         CompletedEventSearch.done();
 
+        LatestEventsByResourceSearch = createSearchBuilder();
+        LatestEventsByResourceSearch.and(FIELD_RESOURCE_ID, LatestEventsByResourceSearch.entity().getResourceId(), Op.EQ);
+        LatestEventsByResourceSearch.and(FIELD_RESOURCE_TYPE, LatestEventsByResourceSearch.entity().getResourceType(), Op.EQ);
+        LatestEventsByResourceSearch.and("type", LatestEventsByResourceSearch.entity().getType(), Op.EQ);
+        LatestEventsByResourceSearch.and("archived", LatestEventsByResourceSearch.entity().getArchived(), Op.EQ);
+        LatestEventsByResourceSearch.done();
+
         ToArchiveOrDeleteEventSearch = createSearchBuilder();
         ToArchiveOrDeleteEventSearch.and("id", ToArchiveOrDeleteEventSearch.entity().getId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("type", ToArchiveOrDeleteEventSearch.entity().getType(), Op.EQ);
@@ -63,8 +74,8 @@
         LastStartEventSearch = createSearchBuilder();
         LastStartEventSearch.and("type", LastStartEventSearch.entity().getType(), Op.EQ);
         LastStartEventSearch.and("state", LastStartEventSearch.entity().getState(), Op.EQ);
-        LastStartEventSearch.and("resourceId", LastStartEventSearch.entity().getResourceId(), Op.EQ);
-        LastStartEventSearch.and("resourceType", LastStartEventSearch.entity().getResourceType(), Op.EQ);
+        LastStartEventSearch.and(FIELD_RESOURCE_ID, LastStartEventSearch.entity().getResourceId(), Op.EQ);
+        LastStartEventSearch.and(FIELD_RESOURCE_TYPE, LastStartEventSearch.entity().getResourceType(), Op.EQ);
         LastStartEventSearch.and("archived", LastStartEventSearch.entity().getArchived(), Op.EQ);
         LastStartEventSearch.done();
     }
@@ -98,8 +109,8 @@
         SearchCriteria<EventVO> sc = LastStartEventSearch.create();
         sc.setParameters("type", type);
         sc.setParameters("state", state);
-        sc.setParameters("resourceId", resourceId);
-        sc.setParameters("resourceType", resourceType);
+        sc.setParameters(FIELD_RESOURCE_ID, resourceId);
+        sc.setParameters(FIELD_RESOURCE_TYPE, resourceType);
         sc.setParameters("archived", false);
         return findLastOneBy(sc);
     }
@@ -125,6 +136,17 @@
         return search(sc, null);
     }
 
+    @Override
+    public List<EventVO> listLatestEventsByResource(long resourceId, String resourceType, String type, int limit) {
+        SearchCriteria<EventVO> sc = LatestEventsByResourceSearch.create();
+        sc.setParameters(FIELD_RESOURCE_ID, resourceId);
+        sc.setParameters(FIELD_RESOURCE_TYPE, resourceType);
+        sc.setParameters("type", type);
+        sc.setParameters("archived", false);
+        Filter filter = new Filter(EventVO.class, "createDate", false, 0L, (long) limit);
+        return listBy(sc, filter);
+    }
+
     @Override
     public void archiveEvents(List<EventVO> events) {
         if (CollectionUtils.isEmpty(events)) {

server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.storage.snapshot;
 
+import java.util.List;
+
 import com.cloud.user.Account;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.StoragePool;
@@ -53,6 +55,18 @@ public interface SnapshotManager extends Configurable {
     public static final ConfigKey<Integer> BackupRetryInterval = new ConfigKey<Integer>(Integer.class, "backup.retry.interval", "Advanced", "300",
             "Time in seconds between retries in backing up snapshot to secondary", false, ConfigKey.Scope.Global, null);
 
+    ConfigKey<Integer> SnapshotRecurringMaxFailures = new ConfigKey<>(Integer.class, "snapshot.recurring.max.failures", "Snapshots", "3",
+            "Maximum number of consecutive failed attempts allowed for a recurring snapshot before it is left until its next regularly scheduled run and a failure event is logged. Set to 0 to retry indefinitely.",
+            true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null);
+
+    ConfigKey<Integer> SnapshotRecurringRetryInterval = new ConfigKey<>(Integer.class, "snapshot.recurring.retry.interval", "Snapshots", "300",
+            "Time in seconds to wait before retrying a failed recurring snapshot attempt.",
+            true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null);
+
+    ConfigKey<Boolean> SnapshotSkipIfVmNotRunning = new ConfigKey<>(Boolean.class, "snapshot.skip.if.vm.not.running", "Snapshots", "false",
+            "For recurring snapshots, skip taking a new snapshot of a volume (and log a skipped event instead) when the VM it is attached to has not been running since the last snapshot was taken, since nothing on the volume could have changed.",
+            true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null);
+
     public static final ConfigKey<Boolean> VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "true", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requires qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null);
 
     ConfigKey<Boolean> KVMSnapshotEnabled = new ConfigKey<>(Boolean.class, "kvm.snapshot.enabled", "Snapshots", "true", "Whether volume snapshot is enabled on running instances " +

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -310,7 +310,8 @@ public String getConfigComponentName() {
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {BackupRetryAttempts, BackupRetryInterval, SnapshotHourlyMax, SnapshotDailyMax, SnapshotMonthlyMax, SnapshotWeeklyMax, usageSnapshotSelection,
-                SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication, KVMSnapshotEnabled};
+                SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication, KVMSnapshotEnabled,
+                SnapshotRecurringMaxFailures, SnapshotRecurringRetryInterval, SnapshotSkipIfVmNotRunning};
     }
 
     @Override