diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index f7d13343d469..acc14e7df35b 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -370,6 +370,8 @@ public class EventTypes { public static final String EVENT_SNAPSHOT_DELETE = "SNAPSHOT.DELETE"; public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT"; public static final String EVENT_SNAPSHOT_EXTRACT = "SNAPSHOT.EXTRACT"; + public static final String EVENT_SNAPSHOT_SKIPPED = "SNAPSHOT.SKIPPED"; + public static final String EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED = "SNAPSHOT.RECURRING.FAILURE.LIMIT.REACHED"; public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE"; public static final String EVENT_SNAPSHOT_POLICY_UPDATE = "SNAPSHOTPOLICY.UPDATE"; public static final String EVENT_SNAPSHOT_POLICY_DELETE = "SNAPSHOTPOLICY.DELETE"; @@ -1074,6 +1076,8 @@ public class EventTypes { // Snapshots entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class); + entityEventDetails.put(EVENT_SNAPSHOT_SKIPPED, Snapshot.class); + entityEventDetails.put(EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED, Snapshot.class); entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class); entityEventDetails.put(EVENT_SNAPSHOT_EXTRACT, Snapshot.class); entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class); diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java index 03716237cdb0..c5440d23d8cb 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java @@ -48,4 +48,10 @@ public interface EventDao extends GenericDao { public void archiveEvents(List events); + /** + * Returns the most recent events of the given type for a resource, newest first. + * Used to derive how many consecutive attempts have failed for that resource without a dedicated counter column. + */ + List listLatestEventsByResource(long resourceId, String resourceType, String type, int limit); + } diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java index 94a7d6a1679b..b2530c8817f9 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java @@ -35,9 +35,13 @@ @Component public class EventDaoImpl extends GenericDaoBase implements EventDao { + private static final String FIELD_RESOURCE_ID = "resourceId"; + private static final String FIELD_RESOURCE_TYPE = "resourceType"; + protected final SearchBuilder CompletedEventSearch; protected final SearchBuilder ToArchiveOrDeleteEventSearch; protected final SearchBuilder ArchiveByIdsSearch; + protected final SearchBuilder LatestEventsByResourceSearch; protected final SearchBuilder LastStartEventSearch; public EventDaoImpl() { @@ -47,6 +51,13 @@ public EventDaoImpl() { CompletedEventSearch.and("archived", CompletedEventSearch.entity().getArchived(), Op.EQ); CompletedEventSearch.done(); + LatestEventsByResourceSearch = createSearchBuilder(); + LatestEventsByResourceSearch.and(FIELD_RESOURCE_ID, LatestEventsByResourceSearch.entity().getResourceId(), Op.EQ); + LatestEventsByResourceSearch.and(FIELD_RESOURCE_TYPE, LatestEventsByResourceSearch.entity().getResourceType(), Op.EQ); + LatestEventsByResourceSearch.and("type", LatestEventsByResourceSearch.entity().getType(), Op.EQ); + LatestEventsByResourceSearch.and("archived", LatestEventsByResourceSearch.entity().getArchived(), Op.EQ); + LatestEventsByResourceSearch.done(); + ToArchiveOrDeleteEventSearch = createSearchBuilder(); ToArchiveOrDeleteEventSearch.and("id", ToArchiveOrDeleteEventSearch.entity().getId(), Op.IN); ToArchiveOrDeleteEventSearch.and("type", ToArchiveOrDeleteEventSearch.entity().getType(), Op.EQ); @@ -63,8 +74,8 @@ public EventDaoImpl() { LastStartEventSearch = createSearchBuilder(); LastStartEventSearch.and("type", LastStartEventSearch.entity().getType(), Op.EQ); LastStartEventSearch.and("state", LastStartEventSearch.entity().getState(), Op.EQ); - LastStartEventSearch.and("resourceId", LastStartEventSearch.entity().getResourceId(), Op.EQ); - LastStartEventSearch.and("resourceType", LastStartEventSearch.entity().getResourceType(), Op.EQ); + LastStartEventSearch.and(FIELD_RESOURCE_ID, LastStartEventSearch.entity().getResourceId(), Op.EQ); + LastStartEventSearch.and(FIELD_RESOURCE_TYPE, LastStartEventSearch.entity().getResourceType(), Op.EQ); LastStartEventSearch.and("archived", LastStartEventSearch.entity().getArchived(), Op.EQ); LastStartEventSearch.done(); } @@ -98,8 +109,8 @@ public EventVO findLastEvent(String type, State state, Long resourceId, String r SearchCriteria sc = LastStartEventSearch.create(); sc.setParameters("type", type); sc.setParameters("state", state); - sc.setParameters("resourceId", resourceId); - sc.setParameters("resourceType", resourceType); + sc.setParameters(FIELD_RESOURCE_ID, resourceId); + sc.setParameters(FIELD_RESOURCE_TYPE, resourceType); sc.setParameters("archived", false); return findLastOneBy(sc); } @@ -125,6 +136,17 @@ public List listToArchiveOrDeleteEvents(List ids, String type, Da return search(sc, null); } + @Override + public List listLatestEventsByResource(long resourceId, String resourceType, String type, int limit) { + SearchCriteria sc = LatestEventsByResourceSearch.create(); + sc.setParameters(FIELD_RESOURCE_ID, resourceId); + sc.setParameters(FIELD_RESOURCE_TYPE, resourceType); + sc.setParameters("type", type); + sc.setParameters("archived", false); + Filter filter = new Filter(EventVO.class, "createDate", false, 0L, (long) limit); + return listBy(sc, filter); + } + @Override public void archiveEvents(List events) { if (CollectionUtils.isEmpty(events)) { diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java index 10dcc2683de8..6ad50e3df4bb 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage.snapshot; +import java.util.List; + import com.cloud.user.Account; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePool; @@ -53,6 +55,18 @@ public interface SnapshotManager extends Configurable { public static final ConfigKey BackupRetryInterval = new ConfigKey(Integer.class, "backup.retry.interval", "Advanced", "300", "Time in seconds between retries in backing up snapshot to secondary", false, ConfigKey.Scope.Global, null); + ConfigKey SnapshotRecurringMaxFailures = new ConfigKey<>(Integer.class, "snapshot.recurring.max.failures", "Snapshots", "3", + "Maximum number of consecutive failed attempts allowed for a recurring snapshot before it is left until its next regularly scheduled run and a failure event is logged. Set to 0 to retry indefinitely.", + true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null); + + ConfigKey SnapshotRecurringRetryInterval = new ConfigKey<>(Integer.class, "snapshot.recurring.retry.interval", "Snapshots", "300", + "Time in seconds to wait before retrying a failed recurring snapshot attempt.", + true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null); + + ConfigKey SnapshotSkipIfVmNotRunning = new ConfigKey<>(Boolean.class, "snapshot.skip.if.vm.not.running", "Snapshots", "false", + "For recurring snapshots, skip taking a new snapshot of a volume (and log a skipped event instead) when the VM it is attached to has not been running since the last snapshot was taken, since nothing on the volume could have changed.", + true, List.of(ConfigKey.Scope.Account, ConfigKey.Scope.Domain, ConfigKey.Scope.Zone, ConfigKey.Scope.Global), null); + public static final ConfigKey VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "true", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requires qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null); ConfigKey KVMSnapshotEnabled = new ConfigKey<>(Boolean.class, "kvm.snapshot.enabled", "Snapshots", "true", "Whether volume snapshot is enabled on running instances " + diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index dc33a4442a33..250f85ca5237 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -310,7 +310,8 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] {BackupRetryAttempts, BackupRetryInterval, SnapshotHourlyMax, SnapshotDailyMax, SnapshotMonthlyMax, SnapshotWeeklyMax, usageSnapshotSelection, - SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication, KVMSnapshotEnabled}; + SnapshotInfo.BackupSnapshotAfterTakingSnapshot, VmStorageSnapshotKvm, kvmIncrementalSnapshot, snapshotDeltaMax, snapshotShowChainSize, UseStorageReplication, KVMSnapshotEnabled, + SnapshotRecurringMaxFailures, SnapshotRecurringRetryInterval, SnapshotSkipIfVmNotRunning}; } @Override diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java index ddaa96100bd1..eab87f91dbff 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher; import org.apache.cloudstack.framework.jobs.AsyncJobManager; @@ -43,11 +44,14 @@ import com.cloud.api.ApiGsonHelper; import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; +import com.cloud.event.dao.EventDao; import com.cloud.server.ResourceTag; import com.cloud.server.TaggedResourceService; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotScheduleVO; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotPolicyDao; @@ -63,8 +67,12 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.TestClock; import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; @@ -98,6 +106,10 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu protected VMSnapshotManager _vmSnaphostManager; @Inject public TaggedResourceService taggedResourceService; + @Inject + protected EventDao eventDao; + @Inject + protected VMInstanceDao vmInstanceDao; protected AsyncJobDispatcher _asyncDispatcher; @@ -196,8 +208,10 @@ protected void scheduleNextSnapshotJobIfNecessary(SnapshotScheduleVO snapshotSch if (JobInfo.Status.SUCCEEDED.equals(status)) { logger.debug("Last job of schedule [{}] succeeded; scheduling the next snapshot job.", snapshotSchedule); + recordSnapshotAttemptOutcome(snapshotSchedule, true, null); } else if (JobInfo.Status.FAILED.equals(status)) { logger.debug("Last job of schedule [{}] failed with [{}]; scheduling a new snapshot job.", snapshotSchedule, asyncJob.getResult()); + recordSnapshotAttemptOutcome(snapshotSchedule, false, asyncJob.getResult()); } else { logger.debug("Schedule [{}] is still in progress, skipping next job scheduling.", snapshotSchedule); return; @@ -206,6 +220,50 @@ protected void scheduleNextSnapshotJobIfNecessary(SnapshotScheduleVO snapshotSch scheduleNextSnapshotJob(snapshotSchedule); } + /** + * Logs an event for the outcome of a recurring snapshot job (keyed by the volume, since a fresh snapshot entity + * ID is minted on every attempt) so that consecutive failures can be counted from event history, and raises a + * WARN notification once {@link SnapshotManager#SnapshotRecurringMaxFailures} consecutive failures are reached. + */ + protected void recordSnapshotAttemptOutcome(final SnapshotScheduleVO snapshotSchedule, final boolean succeeded, final String failureResult) { + final VolumeVO volume = _volsDao.findByIdIncludingRemoved(snapshotSchedule.getVolumeId()); + if (volume == null) { + return; + } + + if (succeeded) { + ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_SNAPSHOT_CREATE, true, + String.format("Scheduled snapshot creation job for volume [%s] succeeded.", volume), + volume.getId(), ApiCommandResourceType.Volume.toString()); + return; + } + + final Account account = _acctDao.findById(volume.getAccountId()); + final int maxFailures = getScopedConfigValue(SnapshotManager.SnapshotRecurringMaxFailures, volume, account); + final int totalFailures = countConsecutiveFailedAttempts(volume.getId(), maxFailures) + 1; + + ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_SNAPSHOT_CREATE, true, + String.format("Scheduled snapshot creation job for volume [%s] failed: %s", volume, failureResult), + volume.getId(), ApiCommandResourceType.Volume.toString()); + + if (maxFailures > 0 && totalFailures >= maxFailures) { + logger.warn("Snapshot schedule [{}] for volume [{}] has failed [{}] consecutive times.", snapshotSchedule, volume, totalFailures); + raiseFailureLimitReachedEvent(volume, String.format("Recurring snapshot for volume [%s] has failed %d consecutive times.", volume, totalFailures)); + } + } + + /** + * Raises the WARN notification event for a volume's recurring snapshot having reached + * {@link SnapshotManager#SnapshotRecurringMaxFailures} consecutive failures. Kept as a single call site (used by + * both {@link #recordSnapshotAttemptOutcome} and {@link #handleFailedSnapshotDispatch}) so it can't drift onto + * {@link EventTypes#EVENT_SNAPSHOT_CREATE} again, which would make it part of the event stream that + * {@link #countConsecutiveFailedAttempts} scans and silently reset the count. + */ + private void raiseFailureLimitReachedEvent(final VolumeVO volume, final String message) { + ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_WARN, EventTypes.EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED, true, + message, volume.getId(), ApiCommandResourceType.Volume.toString()); + } + @DB protected void deleteExpiredVMSnapshots() { Date now = new Date(); @@ -237,58 +295,21 @@ protected void scheduleSnapshots() { for (final SnapshotScheduleVO snapshotToBeExecuted : snapshotsToBeExecuted) { SnapshotScheduleVO tmpSnapshotScheduleVO = null; + Long eventId = null; final long snapshotScheId = snapshotToBeExecuted.getId(); - final long policyId = snapshotToBeExecuted.getPolicyId(); - final long volumeId = snapshotToBeExecuted.getVolumeId(); final VolumeVO volume = _volsDao.findByIdIncludingRemoved(snapshotToBeExecuted.getVolumeId()); try { - if (!canSnapshotBeScheduled(snapshotToBeExecuted, volume)) { + if (shouldSkipSchedule(snapshotToBeExecuted, volume)) { continue; } tmpSnapshotScheduleVO = _snapshotScheduleDao.acquireInLockTable(snapshotScheId); - final Long eventId = - ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventTypes.EVENT_SNAPSHOT_CREATE, "creating snapshot for volume Id:" + - volume.getUuid(), volumeId, ApiCommandResourceType.Volume.toString(), true, 0); - - logger.trace("Mapping parameters required to generate a CreateSnapshotCmd for snapshot [{}].", snapshotToBeExecuted); - final Map params = new HashMap(); - params.put(ApiConstants.VOLUME_ID, "" + volumeId); - params.put(ApiConstants.POLICY_ID, "" + policyId); - params.put("ctxUserId", "1"); - params.put("ctxAccountId", "" + volume.getAccountId()); - params.put("ctxStartEventId", String.valueOf(eventId)); - List resourceTags = taggedResourceService.listByResourceTypeAndId(ResourceTag.ResourceObjectType.SnapshotPolicy, policyId); - if (resourceTags != null && !resourceTags.isEmpty()) { - int tagNumber = 0; - for (ResourceTag resourceTag : resourceTags) { - params.put("tags[" + tagNumber + "].key", resourceTag.getKey()); - params.put("tags[" + tagNumber + "].value", resourceTag.getValue()); - tagNumber++; - } - } - - logger.trace("Generating a CreateSnapshotCmd for snapshot [{}] with parameters: [{}].", snapshotToBeExecuted, params.toString()); - final CreateSnapshotCmd cmd = new CreateSnapshotCmd(); - ComponentContext.inject(cmd); - _dispatcher.dispatchCreateCmd(cmd, params); - params.put("id", "" + cmd.getEntityId()); - params.put("ctxStartEventId", "1"); - - final Date scheduledTimestamp = snapshotToBeExecuted.getScheduledTimestamp(); - displayTime = DateUtil.displayDateInTimezone(DateUtil.GMT_TIMEZONE, scheduledTimestamp); - logger.debug("Scheduling snapshot [{}] for volume [{}] at [{}].", snapshotToBeExecuted, volume, displayTime); - AsyncJobVO job = new AsyncJobVO("", User.UID_SYSTEM, volume.getAccountId(), CreateSnapshotCmd.class.getName(), - ApiGsonHelper.getBuilder().create().toJson(params), cmd.getEntityId(), - cmd.getApiResourceType() != null ? cmd.getApiResourceType().toString() : null, null); - job.setDispatcher(_asyncDispatcher.getName()); - final long jobId = _asyncMgr.submitAsyncJob(job); - logger.debug("Scheduled snapshot [{}] for volume [{}] as job [{}].", snapshotToBeExecuted, volume, job); - - tmpSnapshotScheduleVO.setAsyncJobId(jobId); - _snapshotScheduleDao.update(snapshotScheId, tmpSnapshotScheduleVO); + eventId = dispatchSnapshotCreateJob(snapshotToBeExecuted, volume, tmpSnapshotScheduleVO); } catch (final Exception e) { logger.error("The scheduling of snapshot [{}] for volume [{}] failed due to [{}].", snapshotToBeExecuted, volume, e.toString(), e); + if (tmpSnapshotScheduleVO != null) { + handleFailedSnapshotDispatch(snapshotToBeExecuted, volume, tmpSnapshotScheduleVO, eventId, e); + } } finally { if (tmpSnapshotScheduleVO != null) { _snapshotScheduleDao.releaseFromLockTable(snapshotScheId); @@ -297,6 +318,224 @@ protected void scheduleSnapshots() { } } + /** + * Returns true (after rescheduling as needed) when this iteration's snapshot should not be dispatched: either + * because it can't be scheduled at all, or because it's a redundant snapshot of an unchanged volume that gets + * skipped and rescheduled to its next regular run instead. Kept as a single decision point so the caller only + * needs one {@code continue}. + */ + private boolean shouldSkipSchedule(final SnapshotScheduleVO snapshotToBeExecuted, final VolumeVO volume) { + if (!canSnapshotBeScheduled(snapshotToBeExecuted, volume)) { + return true; + } + if (shouldSkipUnchangedVolumeSnapshot(volume)) { + skipAndRescheduleSnapshot(snapshotToBeExecuted, volume); + return true; + } + return false; + } + + /** + * Builds and dispatches the CreateSnapshotCmd async job for a scheduled snapshot, returning the "scheduled" + * action event id so the caller can complete it if dispatch subsequently fails. + */ + private Long dispatchSnapshotCreateJob(final SnapshotScheduleVO snapshotToBeExecuted, final VolumeVO volume, final SnapshotScheduleVO tmpSnapshotScheduleVO) throws Exception { + final long snapshotScheId = snapshotToBeExecuted.getId(); + final long policyId = snapshotToBeExecuted.getPolicyId(); + final long volumeId = snapshotToBeExecuted.getVolumeId(); + + final Long eventId = + ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventTypes.EVENT_SNAPSHOT_CREATE, "creating snapshot for volume Id:" + + volume.getUuid(), volumeId, ApiCommandResourceType.Volume.toString(), true, 0); + + logger.trace("Mapping parameters required to generate a CreateSnapshotCmd for snapshot [{}].", snapshotToBeExecuted); + final Map params = new HashMap(); + params.put(ApiConstants.VOLUME_ID, "" + volumeId); + params.put(ApiConstants.POLICY_ID, "" + policyId); + params.put("ctxUserId", "1"); + params.put("ctxAccountId", "" + volume.getAccountId()); + params.put("ctxStartEventId", String.valueOf(eventId)); + List resourceTags = taggedResourceService.listByResourceTypeAndId(ResourceTag.ResourceObjectType.SnapshotPolicy, policyId); + if (resourceTags != null && !resourceTags.isEmpty()) { + int tagNumber = 0; + for (ResourceTag resourceTag : resourceTags) { + params.put("tags[" + tagNumber + "].key", resourceTag.getKey()); + params.put("tags[" + tagNumber + "].value", resourceTag.getValue()); + tagNumber++; + } + } + + logger.trace("Generating a CreateSnapshotCmd for snapshot [{}] with parameters: [{}].", snapshotToBeExecuted, params.toString()); + final CreateSnapshotCmd cmd = new CreateSnapshotCmd(); + ComponentContext.inject(cmd); + _dispatcher.dispatchCreateCmd(cmd, params); + params.put("id", "" + cmd.getEntityId()); + params.put("ctxStartEventId", "1"); + + final Date scheduledTimestamp = snapshotToBeExecuted.getScheduledTimestamp(); + final String displayTime = DateUtil.displayDateInTimezone(DateUtil.GMT_TIMEZONE, scheduledTimestamp); + logger.debug("Scheduling snapshot [{}] for volume [{}] at [{}].", snapshotToBeExecuted, volume, displayTime); + AsyncJobVO job = new AsyncJobVO("", User.UID_SYSTEM, volume.getAccountId(), CreateSnapshotCmd.class.getName(), + ApiGsonHelper.getBuilder().create().toJson(params), cmd.getEntityId(), + cmd.getApiResourceType() != null ? cmd.getApiResourceType().toString() : null, null); + job.setDispatcher(_asyncDispatcher.getName()); + final long jobId = _asyncMgr.submitAsyncJob(job); + logger.debug("Scheduled snapshot [{}] for volume [{}] as job [{}].", snapshotToBeExecuted, volume, job); + + tmpSnapshotScheduleVO.setAsyncJobId(jobId); + _snapshotScheduleDao.update(snapshotScheId, tmpSnapshotScheduleVO); + + return eventId; + } + + /** + * Handles a synchronous failure to dispatch the CreateSnapshotCmd (e.g. an allocation error) for a recurring + * snapshot. Without this, the schedule's {@code scheduledTimestamp} is never advanced, so it gets retried on + * every poll (every {@code snapshot.poll.interval} seconds) forever. Instead: log a failure event keyed by the + * volume (so consecutive failures can be counted from event history), and either back off by the configured + * retry interval, or - once the configured maximum consecutive failures is reached - give up until the next + * regularly scheduled run and raise a WARN notification event. + */ + protected void handleFailedSnapshotDispatch(final SnapshotScheduleVO snapshotToBeExecuted, final VolumeVO volume, + final SnapshotScheduleVO lockedSchedule, final Long eventId, final Exception cause) { + final long volumeId = volume.getId(); + final Account account = _acctDao.findById(volume.getAccountId()); + final int maxFailures = getScopedConfigValue(SnapshotManager.SnapshotRecurringMaxFailures, volume, account); + final int retryInterval = getScopedConfigValue(SnapshotManager.SnapshotRecurringRetryInterval, volume, account); + final int totalFailures = countConsecutiveFailedAttempts(volumeId, maxFailures) + 1; + + final String failureMessage = String.format("Failed to create scheduled snapshot for volume [%s]: %s", volume, cause.getMessage()); + if (eventId != null) { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_ERROR, + EventTypes.EVENT_SNAPSHOT_CREATE, failureMessage, volumeId, ApiCommandResourceType.Volume.toString(), eventId); + } else { + ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_ERROR, + EventTypes.EVENT_SNAPSHOT_CREATE, true, failureMessage, volumeId, ApiCommandResourceType.Volume.toString()); + } + + if (maxFailures > 0 && totalFailures >= maxFailures) { + final Date nextRegularRun = getNextScheduledTime(snapshotToBeExecuted.getPolicyId(), _currentTimestamp); + lockedSchedule.setScheduledTimestamp(nextRegularRun); + logger.warn("Snapshot schedule [{}] for volume [{}] has failed [{}] consecutive times; it will not be retried until its next regularly scheduled run at [{}].", + snapshotToBeExecuted, volume, totalFailures, nextRegularRun); + raiseFailureLimitReachedEvent(volume, String.format( + "Recurring snapshot for volume [%s] has failed %d consecutive times and will not be retried until its next regularly scheduled run.", volume, totalFailures)); + } else { + final Date nextRetry = new Date(_currentTimestamp.getTime() + retryInterval * 1000L); + lockedSchedule.setScheduledTimestamp(nextRetry); + logger.debug("Snapshot schedule [{}] for volume [{}] failed [{}] time(s); retrying at [{}].", + snapshotToBeExecuted, volume, totalFailures, nextRetry); + } + _snapshotScheduleDao.update(lockedSchedule.getId(), lockedSchedule); + } + + /** + * Counts how many of the most recent {@code EVENT_SNAPSHOT_CREATE} events logged for this volume are + * consecutive failures (level ERROR), starting from the most recent event and stopping at the first + * non-failure (or absent) event. This derives the "number of failed attempts" from event history instead of a + * dedicated counter column. + */ + protected int countConsecutiveFailedAttempts(final long volumeId, final int limit) { + if (limit <= 0) { + return 0; + } + final List recentEvents = eventDao.listLatestEventsByResource(volumeId, ApiCommandResourceType.Volume.toString(), + EventTypes.EVENT_SNAPSHOT_CREATE, limit); + int count = 0; + for (final EventVO event : recentEvents) { + if (!EventVO.LEVEL_ERROR.equals(event.getLevel())) { + break; + } + count++; + } + return count; + } + + /** + * Resolves a config value in order of most to least specific scope: account, domain, zone, then global. A + * {@link ConfigKey} can only walk a single scope-parent chain automatically (Account->Domain->Global, or + * Zone->Global), so the four scopes are resolved manually here. + */ + protected T getScopedConfigValue(final ConfigKey key, final VolumeVO volume, final Account account) { + T value = key.valueInScope(ConfigKey.Scope.Account, volume.getAccountId(), true); + if (value == null && account != null) { + value = key.valueInScope(ConfigKey.Scope.Domain, account.getDomainId(), true); + } + if (value == null) { + value = key.valueInScope(ConfigKey.Scope.Zone, volume.getDataCenterId(), true); + } + if (value == null) { + value = key.value(); + } + return value; + } + + /** + * Implements https://github.com/apache/cloudstack/issues/6827: a recurring snapshot is redundant when nothing + * could have changed on the volume since the last one was taken, i.e. when the attached VM has not been running + * at any point since then. Volumes with no snapshot yet, or that are not attached to a VM, are never skipped. + */ + protected boolean shouldSkipUnchangedVolumeSnapshot(final VolumeVO volume) { + final Account account = _acctDao.findById(volume.getAccountId()); + if (!Boolean.TRUE.equals(getScopedConfigValue(SnapshotManager.SnapshotSkipIfVmNotRunning, volume, account))) { + return false; + } + + final Long instanceId = volume.getInstanceId(); + if (instanceId == null) { + return false; + } + + final SnapshotVO lastSnapshot = findLastSnapshot(volume.getId()); + if (lastSnapshot == null) { + return false; + } + + final VMInstanceVO vm = vmInstanceDao.findById(instanceId); + if (vm == null || vm.getPowerState() == VirtualMachine.PowerState.PowerOn) { + return false; + } + + final Date poweredOffSince = vm.getPowerStateUpdateTime(); + // If the VM's power state changed at or after the last snapshot, it may have been running (and the volume + // may have changed) at some point since; only skip when it has been off since strictly before that snapshot. + return poweredOffSince != null && poweredOffSince.before(lastSnapshot.getCreated()); + } + + protected SnapshotVO findLastSnapshot(final long volumeId) { + final Filter filter = new Filter(SnapshotVO.class, "created", false, 0L, 1L); + final List snapshots = _snapshotDao.listByVolumeId(filter, volumeId); + return (snapshots == null || snapshots.isEmpty()) ? null : snapshots.get(0); + } + + /** + * Advances a schedule that was skipped (see {@link #shouldSkipUnchangedVolumeSnapshot}) to its next regularly + * scheduled run, and logs an informational event so the skip is visible and not mistaken for a missed snapshot. + */ + @DB + protected void skipAndRescheduleSnapshot(final SnapshotScheduleVO snapshotToBeExecuted, final VolumeVO volume) { + SnapshotScheduleVO lockedSchedule = null; + final long snapshotScheId = snapshotToBeExecuted.getId(); + try { + lockedSchedule = _snapshotScheduleDao.acquireInLockTable(snapshotScheId); + if (lockedSchedule == null) { + return; + } + final Date nextRegularRun = getNextScheduledTime(snapshotToBeExecuted.getPolicyId(), _currentTimestamp); + lockedSchedule.setScheduledTimestamp(nextRegularRun); + _snapshotScheduleDao.update(snapshotScheId, lockedSchedule); + logger.info("Skipped scheduled snapshot [{}] for volume [{}] because its instance has not been running since the last snapshot; next run at [{}].", + snapshotToBeExecuted, volume, nextRegularRun); + ActionEventUtils.onCreatedActionEvent(User.UID_SYSTEM, volume.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_SNAPSHOT_SKIPPED, true, + String.format("Skipped scheduled snapshot for volume [%s] because its instance has not been running since the last snapshot.", volume), + volume.getId(), ApiCommandResourceType.Volume.toString()); + } finally { + if (lockedSchedule != null) { + _snapshotScheduleDao.releaseFromLockTable(snapshotScheId); + } + } + } + /** * Verifies if a snapshot for a volume can be scheduled or not based on volume and account status, and removes it from the snapshot scheduler if its policy was removed. * diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotSchedulerImplTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotSchedulerImplTest.java index 3827531891fd..39dc63b58b9a 100644 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotSchedulerImplTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotSchedulerImplTest.java @@ -16,9 +16,14 @@ // under the License. package com.cloud.storage.snapshot; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; +import com.cloud.event.dao.EventDao; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotScheduleVO; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.SnapshotScheduleDao; @@ -26,6 +31,9 @@ import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.jobs.JobInfo; @@ -34,11 +42,21 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.List; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) public class SnapshotSchedulerImplTest { @@ -77,6 +95,18 @@ public class SnapshotSchedulerImplTest { @Mock private AsyncJobVO asyncJobVoMock; + @Mock + private EventDao eventDaoMock; + + @Mock + private VMInstanceDao vmInstanceDaoMock; + + @Mock + private VMInstanceVO vmInstanceVoMock; + + @Mock + private SnapshotVO snapshotVoMock; + @Test public void scheduleNextSnapshotJobTestParameterIsNullReturnNull() { @@ -103,13 +133,13 @@ public void scheduleNextSnapshotJobTestPolicyIsNotNullDoNotCallExpunge() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); snapshotScheduleVO.setPolicyId(1l); - Mockito.doReturn(snapshotPolicyVoMock).when(snapshotPolicyDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(expected).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotPolicyVO.class)); + doReturn(snapshotPolicyVoMock).when(snapshotPolicyDaoMock).findById(Mockito.anyLong()); + doReturn(expected).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotPolicyVO.class)); Date result = snapshotSchedulerImplSpy.scheduleNextSnapshotJob(snapshotScheduleVO); Assert.assertEquals(expected, result); - Mockito.verify(snapshotScheduleDaoMock, Mockito.never()).expunge(Mockito.anyLong()); + verify(snapshotScheduleDaoMock, never()).expunge(Mockito.anyLong()); } @Test @@ -119,21 +149,21 @@ public void scheduleNextSnapshotJobTestPolicyIsNullCallExpunge() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); snapshotScheduleVO.setPolicyId(1l); - Mockito.doReturn(snapshotPolicyVO).when(snapshotPolicyDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(true).when(snapshotScheduleDaoMock).expunge(Mockito.anyLong()); - Mockito.doReturn(expected).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(snapshotPolicyVO); + doReturn(snapshotPolicyVO).when(snapshotPolicyDaoMock).findById(Mockito.anyLong()); + doReturn(true).when(snapshotScheduleDaoMock).expunge(Mockito.anyLong()); + doReturn(expected).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(snapshotPolicyVO); Date result = snapshotSchedulerImplSpy.scheduleNextSnapshotJob(snapshotScheduleVO); Assert.assertEquals(expected, result); - Mockito.verify(snapshotScheduleDaoMock).expunge(Mockito.anyLong()); + verify(snapshotScheduleDaoMock).expunge(Mockito.anyLong()); } @Test public void isAccountRemovedOrDisabledTestVolumeAccountIsNullReturnTrue() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(null).when(accountDaoMock).findById(Mockito.anyLong()); + doReturn(null).when(accountDaoMock).findById(Mockito.anyLong()); boolean result = snapshotSchedulerImplSpy.isAccountRemovedOrDisabled(snapshotScheduleVO, volumeVoMock); @@ -144,8 +174,8 @@ public void isAccountRemovedOrDisabledTestVolumeAccountIsNullReturnTrue() { public void isAccountRemovedOrDisabledTestVolumeAccountStateIsDisabledReturnTrue() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(Account.State.DISABLED).when(accountVoMock).getState(); + doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); + doReturn(Account.State.DISABLED).when(accountVoMock).getState(); boolean result = snapshotSchedulerImplSpy.isAccountRemovedOrDisabled(snapshotScheduleVO, volumeVoMock); @@ -156,8 +186,8 @@ public void isAccountRemovedOrDisabledTestVolumeAccountStateIsDisabledReturnTrue public void isAccountRemovedOrDisabledTestVolumeAccountStateIsNotNullNorDisabledReturnFalse() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(Account.State.ENABLED).when(accountVoMock).getState(); + doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); + doReturn(Account.State.ENABLED).when(accountVoMock).getState(); boolean result = snapshotSchedulerImplSpy.isAccountRemovedOrDisabled(snapshotScheduleVO, volumeVoMock); @@ -168,7 +198,7 @@ public void isAccountRemovedOrDisabledTestVolumeAccountStateIsNotNullNorDisabled public void canSnapshotBeScheduledTestVolumeIsRemovedReturnFalse() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(new Date()).when(volumeVoMock).getRemoved(); + doReturn(new Date()).when(volumeVoMock).getRemoved(); boolean result = snapshotSchedulerImplSpy.canSnapshotBeScheduled(snapshotScheduleVO, volumeVoMock); @@ -179,7 +209,7 @@ public void canSnapshotBeScheduledTestVolumeIsRemovedReturnFalse() { public void canSnapshotBeScheduledTestVolumeIsNotAttachedToStoragePoolReturnFalse() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(null).when(volumeVoMock).getPoolId(); + doReturn(null).when(volumeVoMock).getPoolId(); boolean result = snapshotSchedulerImplSpy.canSnapshotBeScheduled(snapshotScheduleVO, volumeVoMock); @@ -190,8 +220,8 @@ public void canSnapshotBeScheduledTestVolumeIsNotAttachedToStoragePoolReturnFals public void canSnapshotBeScheduledTestAccountIsRemovedOrDisabledReturnFalse() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(1l).when(volumeVoMock).getPoolId(); - Mockito.doReturn(true).when(snapshotSchedulerImplSpy).isAccountRemovedOrDisabled(Mockito.any(), Mockito.any()); + doReturn(1l).when(volumeVoMock).getPoolId(); + doReturn(true).when(snapshotSchedulerImplSpy).isAccountRemovedOrDisabled(Mockito.any(), Mockito.any()); boolean result = snapshotSchedulerImplSpy.canSnapshotBeScheduled(snapshotScheduleVO, volumeVoMock); @@ -202,15 +232,15 @@ public void canSnapshotBeScheduledTestAccountIsRemovedOrDisabledReturnFalse() { public void canSnapshotBeScheduledTestSnapshotPolicyIsRemovedCallRemove() { SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); - Mockito.doReturn(1l).when(volumeVoMock).getPoolId(); - Mockito.doReturn(false).when(snapshotSchedulerImplSpy).isAccountRemovedOrDisabled(Mockito.any(), Mockito.any()); - Mockito.doReturn(null).when(snapshotPolicyDaoMock).findById(Mockito.any()); + doReturn(1l).when(volumeVoMock).getPoolId(); + doReturn(false).when(snapshotSchedulerImplSpy).isAccountRemovedOrDisabled(Mockito.any(), Mockito.any()); + doReturn(null).when(snapshotPolicyDaoMock).findById(Mockito.any()); boolean result = snapshotSchedulerImplSpy.canSnapshotBeScheduled(snapshotScheduleVO, volumeVoMock); Assert.assertTrue(result); - Mockito.verify(snapshotScheduleDaoMock).remove(Mockito.anyLong()); + verify(snapshotScheduleDaoMock).remove(Mockito.anyLong()); } @Test @@ -218,60 +248,294 @@ public void canSnapshotBeScheduledTestSnapshotPolicyIsNotRemovedDoNotCallRemove( SnapshotScheduleVO snapshotScheduleVO = new SnapshotScheduleVO(); SnapshotPolicyVO snapshotPolicyVO = new SnapshotPolicyVO(); - Mockito.doReturn(1l).when(volumeVoMock).getPoolId(); - Mockito.doReturn(false).when(snapshotSchedulerImplSpy).isAccountRemovedOrDisabled(Mockito.any(), Mockito.any()); - Mockito.doReturn(snapshotPolicyVO).when(snapshotPolicyDaoMock).findById(Mockito.any()); + doReturn(1l).when(volumeVoMock).getPoolId(); + doReturn(false).when(snapshotSchedulerImplSpy).isAccountRemovedOrDisabled(Mockito.any(), Mockito.any()); + doReturn(snapshotPolicyVO).when(snapshotPolicyDaoMock).findById(Mockito.any()); boolean result = snapshotSchedulerImplSpy.canSnapshotBeScheduled(snapshotScheduleVO, volumeVoMock); Assert.assertTrue(result); - Mockito.verify(snapshotScheduleDaoMock, Mockito.never()).remove(Mockito.anyLong()); + verify(snapshotScheduleDaoMock, never()).remove(Mockito.anyLong()); } @Test public void scheduleNextSnapshotJobIfNecessaryTestAsyncJobIsNullThenScheduleNextSnapshot() { - Mockito.doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); - Mockito.doReturn(null).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); - Mockito.doReturn(new Date()).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); + doReturn(null).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); + doReturn(new Date()).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); snapshotSchedulerImplSpy.scheduleNextSnapshotJobIfNecessary(snapshotScheduleVoMock); - Mockito.verify(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + verify(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); } @Test public void scheduleNextSnapshotJobIfNecessaryTestAsyncJobSucceededThenScheduleNextSnapshot() { - Mockito.doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); - Mockito.doReturn(asyncJobVoMock).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); - Mockito.doReturn(JobInfo.Status.SUCCEEDED).when(asyncJobVoMock).getStatus(); - Mockito.doReturn(new Date()).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); + doReturn(asyncJobVoMock).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); + doReturn(JobInfo.Status.SUCCEEDED).when(asyncJobVoMock).getStatus(); + doReturn(new Date()).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); snapshotSchedulerImplSpy.scheduleNextSnapshotJobIfNecessary(snapshotScheduleVoMock); - Mockito.verify(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + verify(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); } @Test public void scheduleNextSnapshotJobIfNecessaryTestAsyncJobFailedThenScheduleNextSnapshot() { - Mockito.doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); - Mockito.doReturn(asyncJobVoMock).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); - Mockito.doReturn(JobInfo.Status.FAILED).when(asyncJobVoMock).getStatus(); - Mockito.doReturn(new Date()).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); + doReturn(asyncJobVoMock).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); + doReturn(JobInfo.Status.FAILED).when(asyncJobVoMock).getStatus(); + doReturn(new Date()).when(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); snapshotSchedulerImplSpy.scheduleNextSnapshotJobIfNecessary(snapshotScheduleVoMock); - Mockito.verify(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + verify(snapshotSchedulerImplSpy).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); } @Test public void scheduleNextSnapshotJobIfNecessaryTestAsyncJobInProgressThenDoNothing() { - Mockito.doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); - Mockito.doReturn(asyncJobVoMock).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); - Mockito.doReturn(JobInfo.Status.IN_PROGRESS).when(asyncJobVoMock).getStatus(); + doReturn(1L).when(snapshotScheduleVoMock).getAsyncJobId(); + doReturn(asyncJobVoMock).when(asyncJobDaoMock).findByIdIncludingRemoved(Mockito.any()); + doReturn(JobInfo.Status.IN_PROGRESS).when(asyncJobVoMock).getStatus(); snapshotSchedulerImplSpy.scheduleNextSnapshotJobIfNecessary(snapshotScheduleVoMock); - Mockito.verify(snapshotSchedulerImplSpy, Mockito.never()).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + verify(snapshotSchedulerImplSpy, never()).scheduleNextSnapshotJob(Mockito.any(SnapshotScheduleVO.class)); + } + + // --- countConsecutiveFailedAttempts (#13454) --- + + @Test + public void countConsecutiveFailedAttemptsTestLimitZeroReturnsZeroWithoutQuerying() { + int result = snapshotSchedulerImplSpy.countConsecutiveFailedAttempts(1L, 0); + + Assert.assertEquals(0, result); + verify(eventDaoMock, never()).listLatestEventsByResource(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); + } + + @Test + public void countConsecutiveFailedAttemptsTestStopsAtFirstNonErrorEvent() { + EventVO error1 = mock(EventVO.class); + doReturn(EventVO.LEVEL_ERROR).when(error1).getLevel(); + EventVO error2 = mock(EventVO.class); + doReturn(EventVO.LEVEL_ERROR).when(error2).getLevel(); + EventVO success = mock(EventVO.class); + doReturn(EventVO.LEVEL_INFO).when(success).getLevel(); + + doReturn(List.of(error1, error2, success)).when(eventDaoMock).listLatestEventsByResource(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); + + int result = snapshotSchedulerImplSpy.countConsecutiveFailedAttempts(1L, 3); + + Assert.assertEquals(2, result); + } + + @Test + public void countConsecutiveFailedAttemptsTestNoEventsReturnsZero() { + doReturn(Collections.emptyList()).when(eventDaoMock).listLatestEventsByResource(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); + + int result = snapshotSchedulerImplSpy.countConsecutiveFailedAttempts(1L, 3); + + Assert.assertEquals(0, result); + } + + // --- getScopedConfigValue (#13454) --- + + @Test + public void getScopedConfigValueTestFallsBackToGlobalDefaultWhenNoDepotConfigured() { + doReturn(1L).when(volumeVoMock).getAccountId(); + doReturn(1L).when(volumeVoMock).getDataCenterId(); + doReturn(1L).when(accountVoMock).getDomainId(); + + Integer result = snapshotSchedulerImplSpy.getScopedConfigValue(SnapshotManager.SnapshotRecurringMaxFailures, volumeVoMock, accountVoMock); + + Assert.assertEquals(Integer.valueOf(3), result); + } + + // --- handleFailedSnapshotDispatch (#13454) --- + + private void stubVolumeAndAccount() { + doReturn(1L).when(volumeVoMock).getId(); + doReturn(1L).when(volumeVoMock).getAccountId(); + doReturn(1L).when(volumeVoMock).getDataCenterId(); + doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); + } + + private void stubConsecutiveFailureEvents(int count) { + List events = new ArrayList<>(); + for (int i = 0; i < count; i++) { + EventVO event = mock(EventVO.class); + doReturn(EventVO.LEVEL_ERROR).when(event).getLevel(); + events.add(event); + } + doReturn(events).when(eventDaoMock).listLatestEventsByResource(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); + } + + // Must use a distinct event type from EVENT_SNAPSHOT_CREATE, otherwise it becomes the "latest event" scanned + // by countConsecutiveFailedAttempts and silently resets the consecutive-failure count on the next attempt. + private void verifyFailureThresholdEventsRaised(MockedStatic actionEventUtilsMocked) { + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCreatedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_ERROR), Mockito.eq(EventTypes.EVENT_SNAPSHOT_CREATE), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString())); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCreatedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_WARN), Mockito.eq(EventTypes.EVENT_SNAPSHOT_RECURRING_FAILURE_LIMIT_REACHED), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString())); + } + + @Test + public void handleFailedSnapshotDispatchTestUnderMaxReschedulesWithRetryIntervalOnly() { + stubVolumeAndAccount(); + doReturn(Collections.emptyList()).when(eventDaoMock).listLatestEventsByResource(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); + ReflectionTestUtils.setField(snapshotSchedulerImplSpy, "_currentTimestamp", new Date()); + + try (MockedStatic actionEventUtilsMocked = Mockito.mockStatic(ActionEventUtils.class)) { + snapshotSchedulerImplSpy.handleFailedSnapshotDispatch(snapshotScheduleVoMock, volumeVoMock, snapshotScheduleVoMock, 5L, new Exception("boom")); + + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCompletedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_ERROR), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), Mockito.eq(5L))); + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCreatedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()), never()); + } + + verify(snapshotScheduleVoMock).setScheduledTimestamp(Mockito.any(Date.class)); + verify(snapshotScheduleDaoMock).update(Mockito.anyLong(), Mockito.eq(snapshotScheduleVoMock)); + } + + @Test + public void handleFailedSnapshotDispatchTestAtMaxGivesUpAndNotifies() { + stubVolumeAndAccount(); + stubConsecutiveFailureEvents(2); + + doReturn(1L).when(snapshotScheduleVoMock).getPolicyId(); + doReturn(null).when(snapshotPolicyDaoMock).findById(Mockito.anyLong()); + + try (MockedStatic actionEventUtilsMocked = Mockito.mockStatic(ActionEventUtils.class)) { + snapshotSchedulerImplSpy.handleFailedSnapshotDispatch(snapshotScheduleVoMock, volumeVoMock, snapshotScheduleVoMock, null, new Exception("boom")); + + verifyFailureThresholdEventsRaised(actionEventUtilsMocked); + } + + verify(snapshotScheduleDaoMock).update(Mockito.anyLong(), Mockito.eq(snapshotScheduleVoMock)); + } + + @Test + public void recordSnapshotAttemptOutcomeTestAtMaxNotifiesWithDistinctEventType() { + doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + stubVolumeAndAccount(); + stubConsecutiveFailureEvents(2); + + try (MockedStatic actionEventUtilsMocked = Mockito.mockStatic(ActionEventUtils.class)) { + snapshotSchedulerImplSpy.recordSnapshotAttemptOutcome(snapshotScheduleVoMock, false, "boom"); + + verifyFailureThresholdEventsRaised(actionEventUtilsMocked); + } + } + + // --- shouldSkipUnchangedVolumeSnapshot (#6827) --- + + @Test + public void shouldSkipUnchangedVolumeSnapshotTestConfigDisabledByDefaultReturnsFalse() { + doReturn(1L).when(volumeVoMock).getAccountId(); + doReturn(1L).when(volumeVoMock).getDataCenterId(); + doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); + + boolean result = snapshotSchedulerImplSpy.shouldSkipUnchangedVolumeSnapshot(volumeVoMock); + + Assert.assertFalse(result); + Mockito.verifyNoInteractions(vmInstanceDaoMock); + } + + @Test + public void shouldSkipUnchangedVolumeSnapshotTestNoInstanceReturnsFalse() { + stubSkipConfigEnabled(); + doReturn(null).when(volumeVoMock).getInstanceId(); + + boolean result = snapshotSchedulerImplSpy.shouldSkipUnchangedVolumeSnapshot(volumeVoMock); + + Assert.assertFalse(result); + } + + @Test + public void shouldSkipUnchangedVolumeSnapshotTestNoPriorSnapshotReturnsFalse() { + stubSkipConfigEnabled(); + doReturn(5L).when(volumeVoMock).getInstanceId(); + doReturn(1L).when(volumeVoMock).getId(); + doReturn(null).when(snapshotSchedulerImplSpy).findLastSnapshot(Mockito.anyLong()); + + boolean result = snapshotSchedulerImplSpy.shouldSkipUnchangedVolumeSnapshot(volumeVoMock); + + Assert.assertFalse(result); + } + + @Test + public void shouldSkipUnchangedVolumeSnapshotTestVmRunningReturnsFalse() { + stubSkipConfigEnabled(); + doReturn(5L).when(volumeVoMock).getInstanceId(); + doReturn(1L).when(volumeVoMock).getId(); + doReturn(snapshotVoMock).when(snapshotSchedulerImplSpy).findLastSnapshot(Mockito.anyLong()); + doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findById(5L); + doReturn(VirtualMachine.PowerState.PowerOn).when(vmInstanceVoMock).getPowerState(); + + boolean result = snapshotSchedulerImplSpy.shouldSkipUnchangedVolumeSnapshot(volumeVoMock); + + Assert.assertFalse(result); + } + + @Test + public void shouldSkipUnchangedVolumeSnapshotTestVmOffSinceBeforeLastSnapshotReturnsTrue() { + stubSkipConfigEnabled(); + doReturn(5L).when(volumeVoMock).getInstanceId(); + doReturn(1L).when(volumeVoMock).getId(); + doReturn(new Date(2000L)).when(snapshotVoMock).getCreated(); + doReturn(snapshotVoMock).when(snapshotSchedulerImplSpy).findLastSnapshot(Mockito.anyLong()); + doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findById(5L); + doReturn(VirtualMachine.PowerState.PowerOff).when(vmInstanceVoMock).getPowerState(); + doReturn(new Date(1000L)).when(vmInstanceVoMock).getPowerStateUpdateTime(); + + boolean result = snapshotSchedulerImplSpy.shouldSkipUnchangedVolumeSnapshot(volumeVoMock); + + Assert.assertTrue(result); + } + + @Test + public void shouldSkipUnchangedVolumeSnapshotTestVmOffSinceAfterLastSnapshotReturnsFalse() { + stubSkipConfigEnabled(); + doReturn(5L).when(volumeVoMock).getInstanceId(); + doReturn(1L).when(volumeVoMock).getId(); + doReturn(new Date(1000L)).when(snapshotVoMock).getCreated(); + doReturn(snapshotVoMock).when(snapshotSchedulerImplSpy).findLastSnapshot(Mockito.anyLong()); + doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findById(5L); + doReturn(VirtualMachine.PowerState.PowerOff).when(vmInstanceVoMock).getPowerState(); + doReturn(new Date(2000L)).when(vmInstanceVoMock).getPowerStateUpdateTime(); + + boolean result = snapshotSchedulerImplSpy.shouldSkipUnchangedVolumeSnapshot(volumeVoMock); + + Assert.assertFalse(result); + } + + private void stubSkipConfigEnabled() { + doReturn(true).when(snapshotSchedulerImplSpy).getScopedConfigValue(Mockito.eq(SnapshotManager.SnapshotSkipIfVmNotRunning), Mockito.any(), Mockito.any()); + doReturn(1L).when(volumeVoMock).getAccountId(); + doReturn(accountVoMock).when(accountDaoMock).findById(Mockito.anyLong()); + } + + @Test + public void skipAndRescheduleSnapshotTestUpdatesScheduleAndLogsEvent() { + doReturn(snapshotScheduleVoMock).when(snapshotScheduleDaoMock).acquireInLockTable(Mockito.anyLong()); + doReturn(1L).when(snapshotScheduleVoMock).getId(); + doReturn(1L).when(snapshotScheduleVoMock).getPolicyId(); + doReturn(null).when(snapshotPolicyDaoMock).findById(Mockito.anyLong()); + doReturn(1L).when(volumeVoMock).getAccountId(); + + try (MockedStatic actionEventUtilsMocked = Mockito.mockStatic(ActionEventUtils.class)) { + snapshotSchedulerImplSpy.skipAndRescheduleSnapshot(snapshotScheduleVoMock, volumeVoMock); + + actionEventUtilsMocked.verify(() -> ActionEventUtils.onCreatedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.eq(EventVO.LEVEL_INFO), Mockito.eq(com.cloud.event.EventTypes.EVENT_SNAPSHOT_SKIPPED), + Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString())); + } + + verify(snapshotScheduleVoMock).setScheduledTimestamp(Mockito.any()); + verify(snapshotScheduleDaoMock).update(1L, snapshotScheduleVoMock); + verify(snapshotScheduleDaoMock).releaseFromLockTable(1L); } }