From 5d20c01d914759b22cebd1796911e67461fd6828 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 28 Nov 2025 11:54:47 +0530 Subject: [PATCH 01/29] List async jobs by the parameters below. - status (0 - Pending, 1 - Success, 2 - Failed, 3 - Cancelled), use comma-seperated list for multiple status - end date (jobs created on or before this date) - duration (in hours) up to now --- .../apache/cloudstack/api/ApiConstants.java | 1 - .../command/user/job/ListAsyncJobsCmd.java | 48 +++++++++++++++++-- .../org/apache/cloudstack/jobs/JobInfo.java | 19 +++++++- .../com/cloud/api/query/QueryManagerImpl.java | 44 +++++++++++------ 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index f74c46161180..dc66e2fcaf22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -339,7 +339,6 @@ public class ApiConstants { public static final String IP_ADDRESS_ID = "ipaddressid"; public static final String IS_2FA_ENABLED = "is2faenabled"; public static final String IS_2FA_VERIFIED = "is2faverified"; - public static final String IS_2FA_MANDATED = "is2famandated"; public static final String IS_ACTIVE = "isactive"; public static final String IS_ASYNC = "isasync"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index 2c8401831132..7c3a27bf1e8b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.api.command.user.job; import java.util.Date; +import java.util.List; + +import com.cloud.exception.InvalidParameterValueException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; @@ -26,8 +29,9 @@ import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.jobs.JobInfo; -@APICommand(name = "listAsyncJobs", description = "Lists all pending asynchronous jobs for the Account.", responseObject = AsyncJobResponse.class, +@APICommand(name = "listAsyncJobs", description = "Lists asynchronous jobs for the Account.", responseObject = AsyncJobResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @@ -35,9 +39,12 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date of the async job (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date from which the async jobs should be listed. Only jobs created on or after this date will be included. (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")") private Date startDate; + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date up to which the async jobs should be listed. Only jobs created on or before this date will be included. (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")") + private Date endDate; + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "The id of the management server", since="4.19") private Long managementServerId; @@ -47,6 +54,13 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") private String resourceType; + @Parameter(name = ApiConstants.JOB_STATUS, type = CommandType.LIST, collectionType = CommandType.LONG, description = "Comma-seperated list of job statuses (0 - Pending, 1 - Success, 2 - Failed, 3 - Cancelled) to list the async jobs. " + + "Only pending jobs are listed by default.", since="4.23") + private List jobStatuses; + + @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the async jobs started or completed within that period up to now.") + private Integer duration; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -55,6 +69,10 @@ public Date getStartDate() { return startDate; } + public Date getEndDate() { + return endDate; + } + public Long getManagementServerId() { return managementServerId; } @@ -67,15 +85,37 @@ public String getResourceType() { return resourceType; } + public List getJobStatuses() { + if (jobStatuses == null) { + return null; + } + + if (jobStatuses.isEmpty()) { + throw new InvalidParameterValueException("Empty job status"); + } + + for (Long status : jobStatuses) { + try { + JobInfo.Status.fromValue(Math.toIntExact(status)); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException("Invalid job status: " + status); + } + } + + return jobStatuses; + } + + public Integer getDuration() { + return duration; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override public void execute() { - ListResponse response = _queryService.searchForAsyncJobs(this); response.setResponseName(getCommandName()); this.setResponseObject(response); - } } diff --git a/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java b/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java index 5b63e627d622..998f8a20a130 100644 --- a/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java +++ b/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java @@ -23,17 +23,32 @@ public interface JobInfo extends Identity, InternalIdentity { public enum Status { - IN_PROGRESS(false), SUCCEEDED(true), FAILED(true), CANCELLED(true); + IN_PROGRESS(0, false), SUCCEEDED(1, true), FAILED(2, true), CANCELLED(3, true); + private final int value; private final boolean done; - private Status(boolean done) { + private Status(int value, boolean done) { + this.value = value; this.done = done; } + public int value() { + return value; + } + public boolean done() { return done; } + + public static Status fromValue(int value) { + for (Status status : Status.values()) { + if (status.value() == value) { + return status; + } + } + throw new IllegalArgumentException("Invalid status value: " + value); + } } String getType(); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index d700fba5a787..0f49908109f0 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -20,6 +20,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -3208,7 +3211,6 @@ public ListResponse searchForAsyncJobs(ListAsyncJobsCmd cmd) { } private Pair, Integer> searchForAsyncJobsInternal(ListAsyncJobsCmd cmd) { - Account caller = CallContext.current().getCallingAccount(); List permittedAccounts = new ArrayList<>(); @@ -3219,8 +3221,14 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync Boolean isRecursive = domainIdRecursiveListProject.second(); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + boolean pendingJobsOnly = cmd.getJobStatuses() == null; + Filter searchFilter = new Filter(AsyncJobJoinVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); SearchBuilder sb = _jobJoinDao.createSearchBuilder(); + + if (!pendingJobsOnly) { + sb.and("statuses", sb.entity().getStatus(), SearchCriteria.Op.IN); + } sb.and("instanceTypeNEQ", sb.entity().getInstanceType(), SearchCriteria.Op.NEQ); sb.and("accountIdIN", sb.entity().getAccountId(), SearchCriteria.Op.IN); boolean accountJoinIsDone = false; @@ -3231,7 +3239,6 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync } if (listProjectResourcesCriteria != null) { - if (listProjectResourcesCriteria == Project.ListProjectResourcesCriteria.ListProjectResourcesOnly) { sb.and("type", sb.entity().getAccountType(), SearchCriteria.Op.EQ); } else if (listProjectResourcesCriteria == Project.ListProjectResourcesCriteria.SkipProjectResources) { @@ -3250,8 +3257,12 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync Object keyword = cmd.getKeyword(); Object startDate = cmd.getStartDate(); + Object endDate = cmd.getEndDate(); SearchCriteria sc = sb.create(); + if (!pendingJobsOnly) { + sc.setParameters("statuses", cmd.getJobStatuses().toArray()); + } sc.setParameters("instanceTypeNEQ", AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE); if (listProjectResourcesCriteria != null) { sc.setParameters("type", Account.Type.PROJECT); @@ -3276,21 +3287,26 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync sc.addAnd("created", SearchCriteria.Op.GTEQ, startDate); } - if (cmd.getManagementServerId() != null) { - ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); - sc.setParameters("executingMsid", msHost.getMsid()); + if (endDate != null) { + sc.addAnd("created", SearchCriteria.Op.LTEQ, endDate); } - if (cmd.getResourceType() != null) { - ApiCommandResourceType resourceType = getResourceType(cmd.getResourceType()); - sc.addAnd("instanceType", SearchCriteria.Op.EQ, resourceType.toString()); + if (cmd.getDuration() != null) { + ZoneId systemZoneId = ZoneId.systemDefault(); + LocalDateTime nowLocal = LocalDateTime.now(); + ZonedDateTime nowZoned = nowLocal.atZone(systemZoneId); + ZonedDateTime lastDateTimeZoned = nowZoned.minusHours(cmd.getDuration()); + Date lastDate = Date.from(lastDateTimeZoned.toInstant()); - final String resourceId = getResourceUuid(cmd.getResourceId()); - if (resourceId != null) { - sc.addAnd("instanceUuid", SearchCriteria.Op.EQ, resourceId); - } - } else if (cmd.getResourceId() != null) { - throw new InvalidParameterValueException(String.format("%s parameter must be used with %s parameter", ApiConstants.RESOURCE_ID, ApiConstants.RESOURCE_TYPE)); + SearchCriteria scc = _jobJoinDao.createSearchCriteria(); + scc.addOr("created", SearchCriteria.Op.GTEQ, lastDate); + scc.addOr("removed", SearchCriteria.Op.GTEQ, lastDate); + sc.addAnd("created", SearchCriteria.Op.SC, scc); + } + + if (cmd.getManagementServerId() != null) { + ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); + sc.setParameters("executingMsid", msHost.getMsid()); } return _jobJoinDao.searchAndCount(sc, searchFilter); From 76e9c90aa4ca6b442672fb8f08bb7c872cd8a6d4 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 3 Dec 2025 13:25:18 +0530 Subject: [PATCH 02/29] Add support to list usage server jobs, by the params below - usage server: name / ip of the host where usage server is running - start date: usage jobs started on or after this date - end date: usage jobs started on or before this date - duration in hours up to now: usage jobs started or completed within that period up to now --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../command/admin/usage/ListUsageJobsCmd.java | 77 +++++++++++++++ .../api/response/UsageJobResponse.java | 93 +++++++++++++++++++ .../apache/cloudstack/usage/UsageService.java | 5 + .../cloud/server/ManagementServerImpl.java | 2 + .../com/cloud/usage/UsageServiceImpl.java | 76 +++++++++++++++ 6 files changed, 254 insertions(+) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/UsageJobResponse.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index dc66e2fcaf22..28ff3ebe9881 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -674,6 +674,7 @@ public class ApiConstants { public static final String USAGE_ID = "usageid"; public static final String USAGE_NAME = "usagename"; public static final String USAGE_TYPE = "usagetype"; + public static final String USAGE_SERVER = "usageserver"; public static final String INCLUDE_TAGS = "includetags"; public static final String VLAN = "vlan"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java new file mode 100644 index 000000000000..f72f884863e3 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.usage; + +import java.util.Date; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UsageJobResponse; + +@APICommand(name = "listUsageJobs", description = "Lists the usage jobs.", responseObject = UsageJobResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.23") +public class ListUsageJobsCmd extends BaseListCmd { + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date from which the usage jobs should be listed. Only jobs started on or after this date will be included. (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")") + private Date startDate; + + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date up to which the usage jobs should be listed. Only jobs started on or before this date will be included. (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")") + private Date endDate; + + @Parameter(name = ApiConstants.USAGE_SERVER, type = CommandType.STRING, description = "The usage server host name or ip") + private String usageServer; + + @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the usage jobs started or completed within that period up to now.") + private Integer duration; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Date getStartDate() { + return startDate; + } + + public Date getEndDate() { + return endDate; + } + + public String getUsageServer() { + return usageServer; + } + + public Integer getDuration() { + return duration; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() { + ListResponse response = _usageService.getUsageJobs(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UsageJobResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UsageJobResponse.java new file mode 100644 index 000000000000..40b13c472516 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/UsageJobResponse.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import java.util.Date; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class UsageJobResponse extends BaseResponse { + + @SerializedName(ApiConstants.USAGE_SERVER) + @Param(description = "the usage server host") + private String usageServer; + + @SerializedName("jobtype") + @Param(description = "the job type (0 - Recurring, 1 - Single)") + private Integer jobType; + + @SerializedName("scheduled") + @Param(description = "the job is scheduled or not") + private Integer scheduled; + + @SerializedName(ApiConstants.START_DATE) + @Param(description = " the start date of the job") + private Date startDate; + + @SerializedName(ApiConstants.END_DATE) + @Param(description = " the end date of the job") + private Date endDate; + + @SerializedName("executiontime") + @Param(description = " the execution time of the job") + private Long executionTime; + + @SerializedName(ApiConstants.SUCCESS) + @Param(description = "the job is success or not") + private Boolean success; + + @SerializedName("heartbeat") + @Param(description = "the job heartbeat") + private Date heartbeat; + + public void setUsageServer(String usageServer) { + this.usageServer = usageServer; + } + + public void setJobType(Integer jobType) { + this.jobType = jobType; + } + + public void setScheduled(Integer scheduled) { + this.scheduled = scheduled; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public void setEndDate(final Date endDate) { + this.endDate = endDate; + } + + public void setExecutionTime(Long executionTime) { + this.executionTime = executionTime; + } + + public void setSuccess(Boolean success) { + this.success = success; + } + + public void setHeartbeat(final Date heartbeat) { + this.heartbeat = heartbeat; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/usage/UsageService.java b/api/src/main/java/org/apache/cloudstack/usage/UsageService.java index 00e8b431f8fe..32256c538fc3 100644 --- a/api/src/main/java/org/apache/cloudstack/usage/UsageService.java +++ b/api/src/main/java/org/apache/cloudstack/usage/UsageService.java @@ -18,8 +18,11 @@ import com.cloud.utils.Pair; import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd; +import org.apache.cloudstack.api.command.admin.usage.ListUsageJobsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UsageJobResponse; import java.util.List; import java.util.TimeZone; @@ -61,4 +64,6 @@ public interface UsageService { TimeZone getUsageTimezone(); boolean removeRawUsageRecords(RemoveRawUsageRecordsCmd cmd); + + ListResponse getUsageJobs(ListUsageJobsCmd cmd); } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index f32857d7cf04..cd37d7897d9c 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -275,6 +275,7 @@ import org.apache.cloudstack.api.command.admin.usage.ListTrafficMonitorsCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypesCmd; +import org.apache.cloudstack.api.command.admin.usage.ListUsageJobsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageTypesCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; @@ -3973,6 +3974,7 @@ public List> getCommands() { cmdList.add(DeleteTrafficTypeCmd.class); cmdList.add(GenerateUsageRecordsCmd.class); cmdList.add(ListUsageRecordsCmd.class); + cmdList.add(ListUsageJobsCmd.class); cmdList.add(RemoveRawUsageRecordsCmd.class); cmdList.add(ListTrafficMonitorsCmd.class); cmdList.add(ListTrafficTypeImplementorsCmd.class); diff --git a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java index de8d4633d224..204974d582b3 100644 --- a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java +++ b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.usage; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -27,8 +30,11 @@ import com.cloud.configuration.ConfigurationManagerImpl; import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd; +import org.apache.cloudstack.api.command.admin.usage.ListUsageJobsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UsageJobResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.usage.Usage; @@ -500,4 +506,74 @@ public boolean removeRawUsageRecords(RemoveRawUsageRecordsCmd cmd) throws Invali _usageDao.expungeAllOlderThan(interval, ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE.value()); return true; } + + @Override + public ListResponse getUsageJobs(ListUsageJobsCmd cmd) { + Filter usageJobFilter = new Filter(UsageJobVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + SearchCriteria sc = _usageJobDao.createSearchCriteria(); + + if (StringUtils.isNotBlank(cmd.getUsageServer())) { + sc.addAnd("host", SearchCriteria.Op.EQ, cmd.getUsageServer().trim()); + } + + Object startDate = cmd.getStartDate(); + if (startDate != null) { + sc.addAnd("startDate", SearchCriteria.Op.GTEQ, startDate); + } + + Object endDate = cmd.getEndDate(); + if (endDate != null) { + sc.addAnd("startDate", SearchCriteria.Op.LTEQ, endDate); + } + + if (cmd.getDuration() != null) { + ZoneId systemZoneId = ZoneId.systemDefault(); + LocalDateTime nowLocal = LocalDateTime.now(); + ZonedDateTime nowZoned = nowLocal.atZone(systemZoneId); + ZonedDateTime lastDateTimeZoned = nowZoned.minusHours(cmd.getDuration()); + Date lastDate = Date.from(lastDateTimeZoned.toInstant()); + + SearchCriteria scc = _usageJobDao.createSearchCriteria(); + scc.addOr("startDate", SearchCriteria.Op.GTEQ, lastDate); + scc.addOr("endDate", SearchCriteria.Op.GTEQ, lastDate); + sc.addAnd("startDate", SearchCriteria.Op.SC, scc); + } + + Pair, Integer> usageJobs = null; + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + usageJobs = _usageJobDao.searchAndCount(sc, usageJobFilter); + } finally { + txn.close(); + + // switch back to VMOPS_DB + TransactionLegacy swap = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); + swap.close(); + } + + ListResponse response = new ListResponse<>(); + if (usageJobs != null) { + List responses = new ArrayList<>(); + for (UsageJobVO job : usageJobs.first()) { + UsageJobResponse jobResponse = createUsageJobResponse(job); + responses.add(jobResponse); + } + response.setResponses(responses, usageJobs.second()); + } + + return response; + } + + private UsageJobResponse createUsageJobResponse(UsageJobVO job) { + UsageJobResponse jobResponse = new UsageJobResponse(); + jobResponse.setUsageServer(job.getHost()); + jobResponse.setJobType(job.getJobType()); + jobResponse.setScheduled(job.getScheduled()); + jobResponse.setStartDate(job.getStartDate()); + jobResponse.setEndDate(job.getEndDate()); + jobResponse.setSuccess(job.getSuccess()); + jobResponse.setHeartbeat(job.getHeartbeat()); + jobResponse.setObjectName("usagejobs"); + return jobResponse; + } } From 3030ce457ef5cd1b30b07846e1a60205bff88086 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 15 Dec 2025 18:39:24 +0530 Subject: [PATCH 03/29] List async jobs by resource type and resource id --- .../command/user/job/ListAsyncJobsCmd.java | 27 ++- .../main/java/com/cloud/api/ApiDBUtils.java | 215 ++++++++++++++++++ .../com/cloud/api/query/QueryManagerImpl.java | 18 ++ 3 files changed, 258 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index 7c3a27bf1e8b..e2348d659b49 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; @@ -55,12 +56,18 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { private String resourceType; @Parameter(name = ApiConstants.JOB_STATUS, type = CommandType.LIST, collectionType = CommandType.LONG, description = "Comma-seperated list of job statuses (0 - Pending, 1 - Success, 2 - Failed, 3 - Cancelled) to list the async jobs. " + - "Only pending jobs are listed by default.", since="4.23") + "Only pending jobs are listed by default.", since = "4.23") private List jobStatuses; - @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the async jobs started or completed within that period up to now.") + @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the async jobs started or completed within that period up to now.", since = "4.23") private Integer duration; + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource.", since = "4.23") + private String resourceType; + + @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING, description = "the id of the resource", since = "4.23") + private String resourceId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -109,6 +116,22 @@ public Integer getDuration() { return duration; } + public ApiCommandResourceType getResourceType() { + if (resourceType != null) { + ApiCommandResourceType jobInstanceType = com.cloud.utils.EnumUtils.getEnumIgnoreCase(ApiCommandResourceType.class, resourceType, ApiCommandResourceType.None); + if (jobInstanceType == ApiCommandResourceType.None) { + throw new InvalidParameterValueException("Not a valid resource type"); + } + return jobInstanceType; + } + + return null; + } + + public String getResourceId() { + return resourceId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 934600eb2b61..caa83fd3a277 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -32,6 +32,8 @@ import com.cloud.cpu.CPU; import com.cloud.storage.GuestOSVO; +import com.cloud.utils.StringUtils; +import com.cloud.utils.UuidUtils; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.affinity.AffinityGroup; @@ -938,6 +940,10 @@ public static VMInstanceVO findVMInstanceById(long vmId) { return s_vmDao.findByIdIncludingRemoved(vmId); } + public static VMInstanceVO findVMInstanceByUuid(String vmUuid) { + return s_vmDao.findByUuidIncludingRemoved(vmUuid); + } + public static long getStorageCapacitybyPool(Long poolId, short capacityType) { // TODO: This method is for the API only, but it has configuration values (ramSize for system vms) // so if this Utils class can have some kind of config rather than a static initializer (maybe from @@ -1106,6 +1112,10 @@ public static Account findAccountByIdIncludingRemoved(Long accountId) { return s_accountDao.findByIdIncludingRemoved(accountId); } + public static Account findAccountByUuid(String accountUuid) { + return s_accountDao.findByUuidIncludingRemoved(accountUuid); + } + public static Account findAccountByNameDomain(String accountName, Long domainId) { return s_accountDao.findActiveAccount(accountName, domainId); } @@ -1186,6 +1196,10 @@ public static HostVO findHostById(Long hostId) { return s_hostDao.findByIdIncludingRemoved(hostId); } + public static HostVO findHostByUuid(String hostUuid) { + return s_hostDao.findByUuidIncludingRemoved(hostUuid); + } + public static HostVO findHostByTypeNameAndZoneId(Long zoneId, String name, Host.Type type) { return s_hostDao.findByTypeNameAndZoneId(zoneId, name, type); } @@ -1194,6 +1208,10 @@ public static IPAddressVO findIpAddressById(long addressId) { return s_ipAddressDao.findById(addressId); } + public static IPAddressVO findIpAddressByUuid(String addressUuid) { + return s_ipAddressDao.findByUuidIncludingRemoved(addressUuid); + } + public static GuestOSCategoryVO getHostGuestOSCategory(long hostId) { Long guestOSCategoryID = s_resourceMgr.getGuestOSCategoryId(hostId); @@ -1220,6 +1238,10 @@ public static SecurityGroup findSecurityGroupById(Long groupId) { return s_securityGroupDao.findById(groupId); } + public static SecurityGroup findSecurityGroupByUuid(String groupUuid) { + return s_securityGroupDao.findByUuidIncludingRemoved(groupUuid); + } + public static HostPodVO findPodById(Long podId) { return s_podDao.findById(podId); } @@ -1249,10 +1271,18 @@ public static Snapshot findSnapshotById(long snapshotId) { return s_snapshotDao.findByIdIncludingRemoved(snapshotId); } + public static Snapshot findSnapshotByUuid(String snapshotUuid) { + return s_snapshotDao.findByUuidIncludingRemoved(snapshotUuid); + } + public static StoragePoolVO findStoragePoolById(Long storagePoolId) { return s_storagePoolDao.findByIdIncludingRemoved(storagePoolId); } + public static StoragePoolVO findStoragePoolByUuid(String storagePoolUuid) { + return s_storagePoolDao.findByUuidIncludingRemoved(storagePoolUuid); + } + public static VMTemplateVO findTemplateById(Long templateId) { VMTemplateVO template = s_templateDao.findByIdIncludingRemoved(templateId); if (template != null) { @@ -1264,6 +1294,10 @@ public static VMTemplateVO findTemplateById(Long templateId) { return template; } + public static VMTemplateVO findTemplateByUuid(String templateUuid) { + return s_templateDao.findByUuidIncludingRemoved(templateUuid); + } + public static UploadVO findUploadById(Long id) { return s_uploadDao.findById(id); } @@ -1272,6 +1306,10 @@ public static User findUserById(Long userId) { return s_userDao.findById(userId); } + public static User findUserByUuid(String userUuid) { + return s_userDao.findByUuidIncludingRemoved(userUuid); + } + public static UserAccountJoinVO findUserAccountById(Long id) { return s_userAccountJoinDao.findById(id); } @@ -1288,6 +1326,10 @@ public static VolumeVO findVolumeById(Long volumeId) { return s_volumeDao.findByIdIncludingRemoved(volumeId); } + public static VolumeVO findVolumeByUuid(String volumeUuid) { + return s_volumeDao.findByUuidIncludingRemoved(volumeUuid); + } + public static Site2SiteVpnGatewayVO findVpnGatewayById(Long vpnGatewayId) { return s_site2SiteVpnGatewayDao.findById(vpnGatewayId); } @@ -1445,14 +1487,26 @@ public static PhysicalNetworkVO findPhysicalNetworkById(long id) { return s_physicalNetworkDao.findById(id); } + public static PhysicalNetworkVO findPhysicalNetworkByUuid(String uuid) { + return s_physicalNetworkDao.findByUuidIncludingRemoved(uuid); + } + public static PhysicalNetworkTrafficTypeVO findPhysicalNetworkTrafficTypeById(long id) { return s_physicalNetworkTrafficTypeDao.findById(id); } + public static PhysicalNetworkTrafficTypeVO findPhysicalNetworkTrafficTypeByUuid(String uuid) { + return s_physicalNetworkTrafficTypeDao.findByUuidIncludingRemoved(uuid); + } + public static NetworkVO findNetworkById(long id) { return s_networkDao.findByIdIncludingRemoved(id); } + public static NetworkVO findNetworkByUuid(String uuid) { + return s_networkDao.findByUuidIncludingRemoved(uuid); + } + public static Map> getNetworkCapabilities(long networkId, long zoneId) { return s_networkModel.getNetworkCapabilities(networkId); } @@ -1682,38 +1736,74 @@ public static CounterVO getCounter(long counterId) { return s_counterDao.findById(counterId); } + public static CounterVO getCounterByUuid(String counterUuid) { + return s_counterDao.findByUuidIncludingRemoved(counterUuid); + } + public static ConditionVO findConditionById(long conditionId) { return s_asConditionDao.findById(conditionId); } + public static ConditionVO findConditionByUuid(String conditionUuid) { + return s_asConditionDao.findByUuidIncludingRemoved(conditionUuid); + } + public static PhysicalNetworkServiceProviderVO findPhysicalNetworkServiceProviderById(long providerId) { return s_physicalNetworkServiceProviderDao.findById(providerId); } + public static PhysicalNetworkServiceProviderVO findPhysicalNetworkServiceProviderByUuid(String providerUuid) { + return s_physicalNetworkServiceProviderDao.findByUuidIncludingRemoved(providerUuid); + } + public static FirewallRuleVO findFirewallRuleById(long ruleId) { return s_firewallRuleDao.findById(ruleId); } + public static FirewallRuleVO findFirewallRuleByUuid(String ruleUuid) { + return s_firewallRuleDao.findByUuidIncludingRemoved(ruleUuid); + } + public static StaticRouteVO findStaticRouteById(long routeId) { return s_staticRouteDao.findById(routeId); } + public static StaticRouteVO findStaticRouteByUuid(String routeUuid) { + return s_staticRouteDao.findByUuidIncludingRemoved(routeUuid); + } + public static VpcGatewayVO findVpcGatewayById(long gatewayId) { return s_vpcGatewayDao.findById(gatewayId); } + public static VpcGatewayVO findVpcGatewayByUuid(String gatewayUUid) { + return s_vpcGatewayDao.findByUuidIncludingRemoved(gatewayUUid); + } + public static AutoScalePolicyVO findAutoScalePolicyById(long policyId) { return s_asPolicyDao.findById(policyId); } + public static AutoScalePolicyVO findAutoScalePolicyByUuid(String policyUuid) { + return s_asPolicyDao.findByUuidIncludingRemoved(policyUuid); + } + public static AutoScaleVmProfileVO findAutoScaleVmProfileById(long profileId) { return s_asVmProfileDao.findById(profileId); } + public static AutoScaleVmProfileVO findAutoScaleVmProfileByUuid(String profileUuid) { + return s_asVmProfileDao.findByUuidIncludingRemoved(profileUuid); + } + public static AutoScaleVmGroupVO findAutoScaleVmGroupById(long groupId) { return s_asVmGroupDao.findById(groupId); } + public static AutoScaleVmGroupVO findAutoScaleVmGroupByUuid(String groupUuid) { + return s_asVmGroupDao.findByUuidIncludingRemoved(groupUuid); + } + public static int countAvailableVmsByGroupId(long groupId) { return s_autoScaleVmGroupVmMapDao.countAvailableVmsByGroup(groupId); } @@ -1915,6 +2005,131 @@ public static String findJobInstanceUuid(AsyncJob job) { return jobInstanceId; } + public static Long findResourceId(ApiCommandResourceType resourceType, String resourceUuid) { + if (StringUtils.isBlank(resourceUuid) + || !UuidUtils.isUuid(resourceUuid) + || resourceType == ApiCommandResourceType.None) { + return null; + } + + Long resourceId = null; + + if (resourceType == ApiCommandResourceType.Volume) { + VolumeVO volume = ApiDBUtils.findVolumeByUuid(resourceUuid); + if (volume != null) { + resourceId = volume.getId(); + } + } else if (resourceType == ApiCommandResourceType.Template || resourceType == ApiCommandResourceType.Iso) { + VMTemplateVO template = ApiDBUtils.findTemplateByUuid(resourceUuid); + if (template != null) { + resourceId = template.getId(); + } + } else if (resourceType == ApiCommandResourceType.VirtualMachine || resourceType == ApiCommandResourceType.ConsoleProxy || + resourceType == ApiCommandResourceType.SystemVm || resourceType == ApiCommandResourceType.DomainRouter) { + VMInstanceVO vm = ApiDBUtils.findVMInstanceByUuid(resourceUuid); + if (vm != null) { + resourceId = vm.getId(); + } + } else if (resourceType == ApiCommandResourceType.Snapshot) { + Snapshot snapshot = ApiDBUtils.findSnapshotByUuid(resourceUuid); + if (snapshot != null) { + resourceId = snapshot.getId(); + } + } else if (resourceType == ApiCommandResourceType.Host) { + Host host = ApiDBUtils.findHostByUuid(resourceUuid); + if (host != null) { + resourceId = host.getId(); + } + } else if (resourceType == ApiCommandResourceType.StoragePool) { + StoragePoolVO spool = ApiDBUtils.findStoragePoolByUuid(resourceUuid); + if (spool != null) { + resourceId = spool.getId(); + } + } else if (resourceType == ApiCommandResourceType.IpAddress) { + IPAddressVO ip = ApiDBUtils.findIpAddressByUuid(resourceUuid); + if (ip != null) { + resourceId = ip.getId(); + } + } else if (resourceType == ApiCommandResourceType.SecurityGroup) { + SecurityGroup sg = ApiDBUtils.findSecurityGroupByUuid(resourceUuid); + if (sg != null) { + resourceId = sg.getId(); + } + } else if (resourceType == ApiCommandResourceType.PhysicalNetwork) { + PhysicalNetworkVO pnet = ApiDBUtils.findPhysicalNetworkByUuid(resourceUuid); + if (pnet != null) { + resourceId = pnet.getId(); + } + } else if (resourceType == ApiCommandResourceType.TrafficType) { + PhysicalNetworkTrafficTypeVO trafficType = ApiDBUtils.findPhysicalNetworkTrafficTypeByUuid(resourceUuid); + if (trafficType != null) { + resourceId = trafficType.getId(); + } + } else if (resourceType == ApiCommandResourceType.PhysicalNetworkServiceProvider) { + PhysicalNetworkServiceProvider sp = ApiDBUtils.findPhysicalNetworkServiceProviderByUuid(resourceUuid); + if (sp != null) { + resourceId = sp.getId(); + } + } else if (resourceType == ApiCommandResourceType.FirewallRule) { + FirewallRuleVO fw = ApiDBUtils.findFirewallRuleByUuid(resourceUuid); + if (fw != null) { + resourceId = fw.getId(); + } + } else if (resourceType == ApiCommandResourceType.Account) { + Account acct = ApiDBUtils.findAccountByUuid(resourceUuid); + if (acct != null) { + resourceId = acct.getId(); + } + } else if (resourceType == ApiCommandResourceType.User) { + User usr = ApiDBUtils.findUserByUuid(resourceUuid); + if (usr != null) { + resourceId = usr.getId(); + } + } else if (resourceType == ApiCommandResourceType.StaticRoute) { + StaticRouteVO route = ApiDBUtils.findStaticRouteByUuid(resourceUuid); + if (route != null) { + resourceId = route.getId(); + } + } else if (resourceType == ApiCommandResourceType.PrivateGateway) { + VpcGatewayVO gateway = ApiDBUtils.findVpcGatewayByUuid(resourceUuid); + if (gateway != null) { + resourceId = gateway.getId(); + } + } else if (resourceType == ApiCommandResourceType.Counter) { + CounterVO counter = ApiDBUtils.getCounterByUuid(resourceUuid); + if (counter != null) { + resourceId = counter.getId(); + } + } else if (resourceType == ApiCommandResourceType.Condition) { + ConditionVO condition = ApiDBUtils.findConditionByUuid(resourceUuid); + if (condition != null) { + resourceId = condition.getId(); + } + } else if (resourceType == ApiCommandResourceType.AutoScalePolicy) { + AutoScalePolicyVO policy = ApiDBUtils.findAutoScalePolicyByUuid(resourceUuid); + if (policy != null) { + resourceId = policy.getId(); + } + } else if (resourceType == ApiCommandResourceType.AutoScaleVmProfile) { + AutoScaleVmProfileVO profile = ApiDBUtils.findAutoScaleVmProfileByUuid(resourceUuid); + if (profile != null) { + resourceId = profile.getId(); + } + } else if (resourceType == ApiCommandResourceType.AutoScaleVmGroup) { + AutoScaleVmGroupVO group = ApiDBUtils.findAutoScaleVmGroupByUuid(resourceUuid); + if (group != null) { + resourceId = group.getId(); + } + } else if (resourceType == ApiCommandResourceType.Network) { + NetworkVO networkVO = ApiDBUtils.findNetworkByUuid(resourceUuid); + if(networkVO != null) { + resourceId = networkVO.getId(); + } + } + + return resourceId; + } + /////////////////////////////////////////////////////////////////////// // Newly Added Utility Methods for List API refactoring // /////////////////////////////////////////////////////////////////////// diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 0f49908109f0..0a4cfee03727 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -39,6 +39,7 @@ import javax.inject.Inject; +import com.cloud.api.ApiDBUtils; import com.cloud.network.PublicIpQuarantine; import com.cloud.network.dao.PublicIpQuarantineDao; import com.cloud.network.vo.PublicIpQuarantineVO; @@ -3304,6 +3305,23 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync sc.addAnd("created", SearchCriteria.Op.SC, scc); } + ApiCommandResourceType resourceType = cmd.getResourceType(); + if (resourceType != null) { + sc.addAnd("instanceType", SearchCriteria.Op.EQ, resourceType); + + if (cmd.getResourceId() != null) { + Long resourceId = ApiDBUtils.findResourceId(resourceType, cmd.getResourceId()); + if (resourceId == null) { + throw new InvalidParameterValueException("Invalid resource id for the resource type " + resourceType); + } + + sc.addAnd("instanceUuid", SearchCriteria.Op.EQ, cmd.getResourceId()); +// sc.addAnd("instanceId", SearchCriteria.Op.EQ, resourceId); + } + } else if (cmd.getResourceId() != null) { + throw new InvalidParameterValueException("Resource type must be specified for the resource id"); + } + if (cmd.getManagementServerId() != null) { ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); sc.setParameters("executingMsid", msHost.getMsid()); From b07f76e0e71f4a91516fb9d324299165d9fa28e1 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 24 Dec 2025 16:05:49 +0530 Subject: [PATCH 04/29] Support to cancel the async jobs --- .../main/java/com/cloud/event/EventTypes.java | 2 + .../exception/AgentUnavailableException.java | 11 + .../OperationCancelledException.java | 61 +++++ .../exception/OperationTimedoutException.java | 9 +- .../java/com/cloud/vm/VirtualMachine.java | 2 + .../api/ApiCommandResourceType.java | 5 +- .../apache/cloudstack/api/CancellableCmd.java | 20 ++ .../cloudstack/api/ResponseGenerator.java | 3 + .../command/admin/vm/DestroyVMCmdByAdmin.java | 3 +- .../command/admin/vm/RebootVMCmdByAdmin.java | 3 +- .../command/admin/vm/StartVMCmdByAdmin.java | 3 +- .../command/admin/vm/StopVMCmdByAdmin.java | 3 +- .../command/user/job/CancelAsyncJobCmd.java | 101 ++++++++ .../api/command/user/vm/BaseDeployVMCmd.java | 3 +- .../api/command/user/vm/DestroyVMCmd.java | 3 +- .../api/command/user/vm/RebootVMCmd.java | 3 +- .../api/command/user/vm/StartVMCmd.java | 3 +- .../api/command/user/vm/StopVMCmd.java | 3 +- .../org/apache/cloudstack/ca/CAManager.java | 5 +- .../cloudstack/jobs/AsyncJobService.java | 27 ++ .../com/cloud/vm/VirtualMachineManager.java | 5 +- .../java/com/cloud/agent/AgentManager.java | 9 +- .../com/cloud/agent/manager/AgentAttache.java | 30 ++- .../cloud/agent/manager/AgentManagerImpl.java | 91 ++++++- .../manager/ClusteredAgentManagerImpl.java | 5 +- .../cloud/vm/VirtualMachineManagerImpl.java | 74 ++++-- .../vm/VirtualMachineManagerImplTest.java | 25 +- .../db/views/cloud.async_job_view.sql | 1 + .../jobs/AsyncJobExecutionContext.java | 2 +- .../framework/jobs/dao/AsyncJobDao.java | 4 + .../framework/jobs/dao/AsyncJobDaoImpl.java | 19 ++ .../framework/jobs/dao/SyncQueueItemDao.java | 2 + .../jobs/dao/SyncQueueItemDaoImpl.java | 15 ++ .../jobs/impl/AsyncJobManagerImpl.java | 235 +++++++++--------- .../com/cloud/ovm/hypervisor/OvmFencer.java | 6 + .../ovm3/resources/Ovm3FenceBuilder.java | 7 + .../java/com/cloud/ha/SimulatorFencer.java | 6 + .../java/com/cloud/ha/XenServerFencer.java | 6 + .../discoverer/XcpServerDiscoverer.java | 3 + .../XenServerStorageMotionStrategy.java | 7 + .../com/cloud/api/ApiAsyncJobDispatcher.java | 40 +-- .../java/com/cloud/api/ApiResponseHelper.java | 7 + .../cloud/api/query/vo/AsyncJobJoinVO.java | 7 + .../cloud/ha/AbstractInvestigatorImpl.java | 6 + .../cloud/ha/CheckOnAgentInvestigator.java | 4 + .../src/main/java/com/cloud/ha/KVMFencer.java | 5 + .../discoverer/LibvirtServerDiscoverer.java | 5 +- .../network/router/NetworkHelperImpl.java | 4 + ...VpcVirtualNetworkApplianceManagerImpl.java | 3 +- .../security/SecurityGroupManagerImpl.java | 9 +- .../RollingMaintenanceManagerImpl.java | 5 +- .../cloud/server/ManagementServerImpl.java | 7 +- .../com/cloud/storage/StorageManagerImpl.java | 5 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 9 + .../apache/cloudstack/ca/CAManagerImpl.java | 7 +- .../ConsoleAccessManagerImpl.java | 3 +- .../download/DirectDownloadManagerImpl.java | 3 +- .../vm/UnmanagedVMsManagerImpl.java | 7 +- .../test/java/com/cloud/ha/KVMFencerTest.java | 7 +- .../network/as/AutoScaleManagerImplTest.java | 3 +- .../network/router/NetworkHelperImplTest.java | 9 +- .../cloud/storage/StorageManagerImplTest.java | 7 +- .../vm/UnmanagedVMsManagerImplTest.java | 19 +- .../com/cloud/utils/SerialVersionUID.java | 1 + .../utils/exception/CSExceptionErrorCode.java | 1 + 65 files changed, 760 insertions(+), 248 deletions(-) create mode 100644 api/src/main/java/com/cloud/exception/OperationCancelledException.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index f7d13343d469..334da23ef289 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -899,6 +899,8 @@ public class EventTypes { public static final String EVENT_DNS_RECORD_DELETE = "DNS.RECORD.DELETE"; public static final String EVENT_DNS_NAME_COLLISION = "DNS.NAME.COLLISION"; + public static final String EVENT_JOB_CANCEL = "JOB.CANCEL"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking diff --git a/api/src/main/java/com/cloud/exception/AgentUnavailableException.java b/api/src/main/java/com/cloud/exception/AgentUnavailableException.java index 89193243e1d1..2adba254e153 100644 --- a/api/src/main/java/com/cloud/exception/AgentUnavailableException.java +++ b/api/src/main/java/com/cloud/exception/AgentUnavailableException.java @@ -27,6 +27,8 @@ public class AgentUnavailableException extends ResourceUnavailableException { private static final long serialVersionUID = SerialVersionUID.AgentUnavailableException; + private boolean isCancelled; + public AgentUnavailableException(String msg, long agentId) { this(msg, agentId, null); } @@ -38,4 +40,13 @@ public AgentUnavailableException(long agentId) { public AgentUnavailableException(String msg, long agentId, Throwable cause) { super("Host " + agentId + ": " + msg, Host.class, agentId, cause); } + + public AgentUnavailableException(String msg, long agentId, boolean isCancelled) { + this(msg, agentId, null); + this.isCancelled = isCancelled; + } + + public boolean isCancelled() { + return isCancelled; + } } diff --git a/api/src/main/java/com/cloud/exception/OperationCancelledException.java b/api/src/main/java/com/cloud/exception/OperationCancelledException.java new file mode 100644 index 000000000000..444d652a5a0c --- /dev/null +++ b/api/src/main/java/com/cloud/exception/OperationCancelledException.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.exception; + +import com.cloud.agent.api.Command; +import com.cloud.utils.SerialVersionUID; + +import java.util.Arrays; + +public class OperationCancelledException extends CloudException { + private static final long serialVersionUID = SerialVersionUID.OperationCancelledException; + long _agentId; + long _seqId; + int _time; + + transient Command[] _cmds; + boolean _isActive; + + public OperationCancelledException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) { + super("Commands: " + Arrays.toString(cmds) + " to Host " + agentId + " with seqId " + seqId + " cancelled after " + time + " secs"); + _agentId = agentId; + _seqId = seqId; + _time = time; + _cmds = cmds; + _isActive = isActive; + } + + public long getAgentId() { + return _agentId; + } + + public long getSequenceId() { + return _seqId; + } + + public int getWaitTime() { + return _time; + } + + public Command[] getCommands() { + return _cmds; + } + + public boolean isActive() { + return _isActive; + } +} diff --git a/api/src/main/java/com/cloud/exception/OperationTimedoutException.java b/api/src/main/java/com/cloud/exception/OperationTimedoutException.java index 66b607100d97..ec16cea045a2 100644 --- a/api/src/main/java/com/cloud/exception/OperationTimedoutException.java +++ b/api/src/main/java/com/cloud/exception/OperationTimedoutException.java @@ -19,6 +19,8 @@ import com.cloud.agent.api.Command; import com.cloud.utils.SerialVersionUID; +import java.util.Arrays; + /** * wait timeout. */ @@ -38,9 +40,10 @@ public class OperationTimedoutException extends CloudException { // transient Command[] _cmds; boolean _isActive; + boolean _isCancelled; public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) { - super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs"); + super("Commands: " + Arrays.toString(cmds) + " to Host " + agentId + " with seqId " + seqId + " timed out after " + time + " secs"); _agentId = agentId; _seqId = seqId; _time = time; @@ -67,4 +70,8 @@ public Command[] getCommands() { public boolean isActive() { return _isActive; } + + public boolean isCancelled() { + return _isCancelled; + } } diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index 3adcc85d28a1..46adb9032195 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -105,6 +105,8 @@ public static StateMachine2 getStat s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.AgentReportRunning, State.Running, Arrays.asList(new Impact[]{Impact.USAGE}))); s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.AgentReportStopped, State.Stopped, null)); s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.AgentReportShutdowned, State.Stopped, null)); + s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.StopRequested, State.Stopping, null)); +// s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.OperationCancelled, State.Stopping, null)); s_fsm.addTransition(new Transition(State.Destroyed, VirtualMachine.Event.RecoveryRequested, State.Stopped, Arrays.asList(new Impact[]{Impact.USAGE}))); s_fsm.addTransition(new Transition(State.Destroyed, VirtualMachine.Event.ExpungeOperation, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Running, VirtualMachine.Event.MigrationRequested, State.Migrating, null)); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 2aa97b65a3d5..165a662c2ad1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -91,7 +91,10 @@ public enum ApiCommandResourceType { Extension(org.apache.cloudstack.extension.Extension.class), ExtensionCustomAction(org.apache.cloudstack.extension.ExtensionCustomAction.class), KmsKey(org.apache.cloudstack.kms.KMSKey.class), - HsmProfile(org.apache.cloudstack.kms.HSMProfile.class); + HsmProfile(org.apache.cloudstack.kms.HSMProfile.class), + Job(org.apache.cloudstack.jobs.JobInfo.class); // org.apache.cloudstack.framework.jobs.AsyncJob.class + // Job(null); + private final Class clazz; diff --git a/api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java b/api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java new file mode 100644 index 000000000000..3dc390151ad7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api; + +public interface CancellableCmd { +} diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 6e880c89432f..e4b0e5d97184 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.command.user.job.CancelAsyncJobCmd; import org.apache.cloudstack.api.response.ConsoleSessionResponse; import org.apache.cloudstack.consoleproxy.ConsoleSession; import org.apache.cloudstack.acl.apikeypair.ApiKeyPair; @@ -397,6 +398,8 @@ TemplatePermissionsResponse createTemplatePermissionsResponse(ResponseView view, AsyncJobResponse queryJobResult(QueryAsyncJobResultCmd cmd); + AsyncJobResponse cancelJobResponse(CancelAsyncJobCmd cmd); + NetworkOfferingResponse createNetworkOfferingResponse(NetworkOffering offering); NetworkResponse createNetworkResponse(ResponseView view, Network network); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java index cbe6494d4004..b09a112ce6ba 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java @@ -18,6 +18,7 @@ import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; @@ -29,7 +30,7 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance. Once destroyed, only the administrator can recover it.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd { +public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd, CancellableCmd { @Parameter( name = ApiConstants.FORCED, type = CommandType.BOOLEAN, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java index 36fdc9d8ec18..fcbf19842cec 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; @@ -26,4 +27,4 @@ @APICommand(name = "rebootVirtualMachine", description = "Reboots an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class RebootVMCmdByAdmin extends RebootVMCmd implements AdminCmd {} +public class RebootVMCmdByAdmin extends RebootVMCmd implements AdminCmd, CancellableCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java index 30fab3b9382f..087abd468948 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.StartVMCmd; @@ -26,4 +27,4 @@ @APICommand(name = "startVirtualMachine", responseObject = UserVmResponse.class, description = "Starts an Instance.", responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StartVMCmdByAdmin extends StartVMCmd implements AdminCmd {} +public class StartVMCmdByAdmin extends StartVMCmd implements AdminCmd, CancellableCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java index 6dc1947712d6..685c232bcfec 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.StopVMCmd; @@ -26,4 +27,4 @@ @APICommand(name = "stopVirtualMachine", responseObject = UserVmResponse.class, description = "Stops an Instance.", responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StopVMCmdByAdmin extends StopVMCmd implements AdminCmd {} +public class StopVMCmdByAdmin extends StopVMCmd implements AdminCmd, CancellableCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java new file mode 100644 index 000000000000..6fbdce4dbaa8 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.job; + + +import com.cloud.event.EventTypes; +import com.cloud.utils.StringUtils; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.AsyncJobResponse; + +import com.cloud.user.Account; +import org.apache.cloudstack.jobs.AsyncJobService; + +import javax.inject.Inject; + +@APICommand(name = CancelAsyncJobCmd.APINAME, description = "Cancels the asynchronous job.", responseObject = AsyncJobResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}, since = "4.23") +public class CancelAsyncJobCmd extends BaseCmd { + public static final String APINAME = "cancelAsyncJob"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, required = true, description = "the ID of the asynchronous job") + private Long id; + + @Inject + private AsyncJobService asyncJobService; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + public String getEventType() { + return EventTypes.EVENT_JOB_CANCEL; + } + + public String getEventDescription() { + return "Cancelling job with id: " + id; + } + + public ApiCommandResourceType getInstanceType() { + return ApiCommandResourceType.Job; + } + + public Long getInstanceId() { + return getId(); + } + + @Override + public void execute() { + String status = asyncJobService.cancelAsyncJob(id, "cancel request by user using cancelAsyncJob api"); + if (StringUtils.isBlank(status)) { + AsyncJobResponse response = _responseGenerator.cancelJobResponse(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, status); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index 8d79ac4af45f..e9621fa484a1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.api.BaseAsyncCreateCustomIdCmd; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.DiskOfferingResponse; @@ -68,7 +69,7 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; -public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { +public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd, CancellableCmd { private static final String s_name = "deployvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index 0a3e510d3d53..7ac12851cf9c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -44,7 +45,7 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class DestroyVMCmd extends BaseAsyncCmd implements UserCmd { +public class DestroyVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { private static final String s_name = "destroyvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java index 6f4431547848..65ff55e674e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java @@ -25,6 +25,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -41,7 +42,7 @@ @APICommand(name = "rebootVirtualMachine", description = "Reboots an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class RebootVMCmd extends BaseAsyncCmd implements UserCmd { +public class RebootVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { private static final String s_name = "rebootvirtualmachineresponse"; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java index 40ae91d4c264..53d93d3d65fe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -49,7 +50,7 @@ @APICommand(name = "startVirtualMachine", responseObject = UserVmResponse.class, description = "Starts an Instance.", responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StartVMCmd extends BaseAsyncCmd implements UserCmd { +public class StartVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { private static final String s_name = "startvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java index 232eeebd34be..f383a9d6c41e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -39,7 +40,7 @@ @APICommand(name = "stopVirtualMachine", responseObject = UserVmResponse.class, description = "Stops an Instance.", responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StopVMCmd extends BaseAsyncCmd implements UserCmd { +public class StopVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { private static final String s_name = "stopvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java index d2ebdc25f1bd..607c1c28a521 100644 --- a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java +++ b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import com.cloud.exception.OperationCancelledException; import com.trilead.ssh2.Connection; import org.apache.cloudstack.framework.ca.CAProvider; @@ -170,7 +171,7 @@ public interface CAManager extends CAService, Configurable, PluggableService { * @throws AgentUnavailableException * @throws OperationTimedoutException */ - String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException; + String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; /** * Deploys a Certificate payload to a provided host @@ -182,7 +183,7 @@ public interface CAManager extends CAService, Configurable, PluggableService { * @throws AgentUnavailableException * @throws OperationTimedoutException */ - boolean deployCertificate(final Host host, final Certificate certificate, final Boolean reconnect, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException; + boolean deployCertificate(final Host host, final Certificate certificate, final Boolean reconnect, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; /** * Removes the host from an internal active client/certificate map diff --git a/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java b/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java new file mode 100644 index 000000000000..df35dfe73145 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.jobs; + +import com.cloud.utils.component.PluggableService; +import org.apache.cloudstack.framework.config.ConfigKey; + +public interface AsyncJobService extends PluggableService { + ConfigKey CancelJobInterval = new ConfigKey("Advanced", Integer.class, "ping.interval", "30", + "Interval in seconds to check the jobs cancelled", false); + + String cancelAsyncJob(long jobId, String reason); +} diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index c7238a513693..4df7345bc7f1 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.vm; +import com.cloud.exception.OperationCancelledException; import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import java.net.URI; @@ -166,11 +167,11 @@ void advanceStart(String vmUuid, Map params ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; void orchestrateStart(String vmUuid, Map params, DeploymentPlan planToDeploy, DeploymentPlanner planner) throws InsufficientCapacityException, - ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; + ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; void advanceStop(String vmUuid, boolean cleanupEvenIfUnableToStop) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException; - void advanceExpunge(String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException; + void advanceExpunge(String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException, OperationCancelledException; void destroy(String vmUuid, boolean expunge) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException; diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index 4d63fae33560..6bf97758dfc8 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -18,6 +18,7 @@ import java.util.Map; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Answer; @@ -85,7 +86,7 @@ enum TapAgentsAction { * @return an Answer */ - Answer send(Long hostId, Command cmd) throws AgentUnavailableException, OperationTimedoutException; + Answer send(Long hostId, Command cmd) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; /** * Synchronous sending a list of commands to the agent. @@ -100,9 +101,9 @@ enum TapAgentsAction { * should the agent stop execution on the first error. * @return an array of Answer */ - Answer[] send(Long hostId, Commands cmds) throws AgentUnavailableException, OperationTimedoutException; + Answer[] send(Long hostId, Commands cmds) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; - Answer[] send(Long hostId, Commands cmds, int timeout) throws AgentUnavailableException, OperationTimedoutException; + Answer[] send(Long hostId, Commands cmds, int timeout) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; /** * Asynchronous sending of a command to the agent. @@ -178,4 +179,6 @@ enum TapAgentsAction { boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance); int getHostSshPort(HostVO host); + + Long getAsyncJobId(); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 402bd2b6b9b9..d181fd06dac3 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -33,11 +33,14 @@ import java.util.concurrent.TimeUnit; import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand; +import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.agent.lb.SetupMSListCommand; import org.apache.cloudstack.command.ReconcileAnswer; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.logging.log4j.Logger; @@ -409,15 +412,18 @@ public void send(final Request req, final Listener listener) throws AgentUnavail } } - public Answer[] send(final Request req, final int wait) throws AgentUnavailableException, OperationTimedoutException { + public Answer[] send(final Request req, final int wait) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { SynchronousListener sl = new SynchronousListener(null); - + Long jobId = _agentMgr.getAsyncJobId(); long seq = req.getSequence(); send(req, sl); try { for (int i = 0; i < 2; i++) { Answer[] answers = null; + if (_agentMgr._asyncJobDao.isJobCancelled(jobId)) { + throw new OperationCancelledException(req.getCommands(), _id, seq, wait, false); + } Command[] cmds = req.getCommands(); if (cmds != null && cmds.length == 1 && (cmds[0] != null) && cmds[0].isReconcile() && !sl.isDisconnected() && _agentMgr.isReconcileCommandsEnabled(_hypervisorType)) { @@ -428,6 +434,9 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE answers = sl.waitFor(wait); } catch (final InterruptedException e) { logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted"); + if (_agentMgr._asyncJobDao.isJobCancelled(jobId)) { + throw new OperationCancelledException(req.getCommands(), _id, seq, wait, false); + } } } if (answers != null) { @@ -471,12 +480,29 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE sendNext(seq); } _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); + if (e instanceof OperationCancelledException) { + throw e; + } throw new OperationTimedoutException(req.getCommands(), _id, seq, wait, false); } finally { unregisterListener(seq); } } + private Long getAsyncJobId() { + Long jobId = null; + final AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrent(); + if (context != null && context.getJob() != null) { + AsyncJob job = context.getJob(); + if (job.getRelated() != null && !job.getRelated().isEmpty()) { + jobId = Long.parseLong(job.getRelated()); + } else { + jobId = job.getId(); + } + } + return jobId; + } + private Answer[] waitForAnswerOfReconcileCommand(SynchronousListener sl, final long seq, final Command command, final int wait) { Answer[] answers = null; int waitTimeLeft = wait; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index d3225f2fda5e..34ddce6ebb3a 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -42,6 +42,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import com.cloud.utils.StringUtils; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; @@ -55,6 +56,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.maintenance.ManagementServerMaintenanceListener; import org.apache.cloudstack.maintenance.ManagementServerMaintenanceManager; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -143,6 +146,8 @@ import com.cloud.utils.nio.Task; import com.cloud.utils.time.InaccurateClock; +import static org.apache.cloudstack.jobs.AsyncJobService.CancelJobInterval; + /** * Implementation of the Agent Manager. This class controls the connection to the agents. **/ @@ -159,6 +164,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected List _loadingAgents = new ArrayList<>(); protected Map _commandTimeouts = new HashMap<>(); private int _monitorId = 0; + protected Map> _sequenceMap = new HashMap<>(); @Inject protected CAManager caService; @@ -182,6 +188,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected ConfigurationDao _configDao = null; @Inject protected ClusterDao _clusterDao = null; + @Inject + public AsyncJobDao _asyncJobDao = null; @Inject protected HighAvailabilityManager _haMgr = null; @@ -210,6 +218,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected ScheduledExecutorService _directAgentExecutor; protected ScheduledExecutorService _cronJobExecutor; protected ScheduledExecutorService _monitorExecutor; + protected ScheduledExecutorService _cancelJobExecutor; private int _directAgentThreadCap; @@ -390,6 +399,7 @@ public void onManagementServerCancelPreparingForMaintenance() { public void onManagementServerMaintenance() { logger.debug("Management server maintenance enabled"); _monitorExecutor.shutdownNow(); + _cancelJobExecutor.shutdownNow(); newAgentConnectionsMonitor.shutdownNow(); if (_connection != null) { _connection.stop(); @@ -421,6 +431,9 @@ public void onManagementServerCancelMaintenance() { if (_monitorExecutor.isShutdown()) { initAndScheduleMonitorExecutor(); } + if (_connectExecutor.isShutdown()) { + initAndScheduleCancelJobExecutor(); + } if (newAgentConnectionsMonitor.isShutdown()) { initAndScheduleAgentConnectionsMonitor(); } @@ -437,6 +450,11 @@ private void initAndScheduleMonitorExecutor() { _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS); } + private void initAndScheduleCancelJobExecutor() { + _cancelJobExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("CancelJob")); + _cancelJobExecutor.scheduleWithFixedDelay(new CancelJobTask(), CancelJobInterval.value(), CancelJobInterval.value(), TimeUnit.SECONDS); + } + private void initAndScheduleAgentConnectionsMonitor() { final int cleanupTimeInSecs = Wait.value(); newAgentConnectionsMonitor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NewAgentConnectionsMonitor")); @@ -509,7 +527,7 @@ public Answer sendTo(final Long dcId, final HypervisorType type, final Command c } @Override - public Answer send(final Long hostId, final Command cmd) throws AgentUnavailableException, OperationTimedoutException { + public Answer send(final Long hostId, final Command cmd) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { final Commands cmds = new Commands(Command.OnError.Stop); cmds.addCommand(cmd); send(hostId, cmds, cmd.getWait()); @@ -631,12 +649,18 @@ private Map getCommandTimeoutsMap(String commandWaits) { } @Override - public Answer[] send(final Long hostId, final Commands commands, int timeout) throws AgentUnavailableException, OperationTimedoutException { + public Answer[] send(final Long hostId, final Commands commands, int timeout) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { assert hostId != null : "Who's not checking the agent id before sending? ... (finger wagging)"; if (hostId == null) { throw new AgentUnavailableException(-1); } + Long jobId = getAsyncJobId(); + if (jobId != null && _asyncJobDao.isJobCancelled(jobId)) { + logger.debug("job-{} for host: {}, with commands: {} is cancelled", jobId, hostId, commands.toString()); + throw new OperationCancelledException(commands.toCommands(), hostId, 0, 0, false); + } + int wait = getTimeout(commands, timeout); logger.debug("Wait time setting on {} is {} seconds", commands, wait); for (Command cmd : commands) { @@ -664,18 +688,27 @@ public Answer[] send(final Long hostId, final Commands commands, int timeout) th throw new AgentUnavailableException("agent not logged into this management server", hostId); } - final Request req = new Request(hostId, agent.getName(), _nodeId, cmds, commands.stopOnError(), true); - req.setSequence(agent.getNextSequence()); + try { + final Request req = new Request(hostId, agent.getName(), _nodeId, cmds, commands.stopOnError(), true); + req.setSequence(agent.getNextSequence()); + if (jobId != null) { + _sequenceMap.put(jobId, new Pair<>(hostId, req.getSequence())); + } - reconcileCommandService.persistReconcileCommands(hostId, req.getSequence(), cmds); + reconcileCommandService.persistReconcileCommands(hostId, req.getSequence(), cmds); - final Answer[] answers = agent.send(req, wait); + final Answer[] answers = agent.send(req, wait); - reconcileCommandService.processAnswers(req.getSequence(), cmds, answers); + reconcileCommandService.processAnswers(req.getSequence(), cmds, answers); - notifyAnswersToMonitors(hostId, req.getSequence(), answers); - commands.setAnswers(answers); - return answers; + notifyAnswersToMonitors(hostId, req.getSequence(), answers); + commands.setAnswers(answers); + return answers; + } finally { + if (jobId != null) { + _sequenceMap.remove(jobId); + } + } } protected Status investigate(final AgentAttache agent) { @@ -891,6 +924,7 @@ public boolean start() { ManagementServerHostVO msHost = _mshostDao.findByMsid(_nodeId); if (msHost != null && (ManagementServerHost.State.Maintenance.equals(msHost.getState()) || ManagementServerHost.State.PreparingForMaintenance.equals(msHost.getState()))) { _monitorExecutor.shutdownNow(); + _cancelJobExecutor.shutdownNow(); newAgentConnectionsMonitor.shutdownNow(); return true; } @@ -907,6 +941,7 @@ public boolean start() { } initAndScheduleMonitorExecutor(); + initAndScheduleCancelJobExecutor(); initAndScheduleAgentConnectionsMonitor(); return true; } @@ -1069,6 +1104,7 @@ public boolean stop() { _connectExecutor.shutdownNow(); _monitorExecutor.shutdownNow(); + _cancelJobExecutor.shutdownNow(); newAgentConnectionsMonitor.shutdownNow(); return true; } @@ -1297,7 +1333,7 @@ public Answer easySend(final Long hostId, final Command cmd) { } @Override - public Answer[] send(final Long hostId, final Commands cmds) throws AgentUnavailableException, OperationTimedoutException { + public Answer[] send(final Long hostId, final Commands cmds) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { int wait = 0; if (cmds.size() > 1) { logger.debug("Checking the wait time in seconds to be used for the following commands : {}. If there are multiple commands sent at once," + @@ -2064,6 +2100,24 @@ protected List findAgentsBehindOnPing() { } } + protected class CancelJobTask extends ManagedContextRunnable { + @Override + protected void runInContext() { + final List jobs = _asyncJobDao.getCancelledJobs(); + for (final AsyncJobVO job : jobs) { + Pair sequence = _sequenceMap.get(job.getId()); + if (sequence != null) { + try { + final AgentAttache agent = getAttache(sequence.first()); + agent.cancel(sequence.second()); + } catch (AgentUnavailableException e) { + logger.debug("Agent {} not found", sequence.first()); + } + } + } + } + } + protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable { @Override protected void runInContext() { @@ -2306,6 +2360,21 @@ public int getHostSshPort(HostVO host) { return Integer.parseInt(hostPort); } + @Override + public Long getAsyncJobId() { + Long jobId = null; + final AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrent(); + if (context != null && context.getJob() != null) { + AsyncJob job = context.getJob(); + if (StringUtils.isNotEmpty(job.getRelated())) { + jobId = Long.parseLong(job.getRelated()); + } else { + jobId = job.getId(); + } + } + return jobId; + } + private GlobalLock getHostJoinLock(Long hostId) { return GlobalLock.getInternLock(String.format("%s-%s", "Host-Join", hostId)); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 38a198b73040..f61b3328a758 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -42,6 +42,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import com.cloud.exception.OperationCancelledException; import com.cloud.resource.ResourceState; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.config.ConfigDepot; @@ -1236,7 +1237,7 @@ private String handleScheduleHostScanTaskCommand(final ScheduleHostScanTaskComma return _gson.toJson(answers); } - public Answer[] sendToAgent(final Long hostId, final Command[] cmds, final boolean stopOnError) throws AgentUnavailableException, OperationTimedoutException { + public Answer[] sendToAgent(final Long hostId, final Command[] cmds, final boolean stopOnError) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { final Commands commands = new Commands(stopOnError ? Command.OnError.Stop : Command.OnError.Continue); for (final Command cmd : cmds) { commands.addCommand(cmd); @@ -1355,6 +1356,8 @@ public String dispatch(final ClusterServicePdu pdu) { logger.warn("Agent is unavailable", e); } catch (final OperationTimedoutException e) { logger.warn("Timed Out", e); + } catch (OperationCancelledException e) { + logger.warn("Operation Cancelled", e); } return null; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 8c9a1653a260..a7525c0a7145 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -51,6 +51,7 @@ import javax.persistence.EntityExistsException; import com.cloud.agent.api.PostMigrationCommand; +import com.cloud.exception.OperationCancelledException; import com.cloud.storage.clvm.ClvmPoolManager; import com.cloud.hypervisor.KVMGuru; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -674,13 +675,15 @@ public void expunge(final String vmUuid) throws ResourceUnavailableException { advanceExpunge(vmUuid); } catch (final OperationTimedoutException e) { throw new CloudRuntimeException("Operation timed out", e); + } catch (final OperationCancelledException e) { + throw new CloudRuntimeException("Operation cancelled", e); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Concurrent operation ", e); } } @Override - public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { + public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException, OperationCancelledException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); advanceExpunge(vm); } @@ -698,7 +701,7 @@ private boolean isVmDestroyed(VMInstanceVO vm) { return false; } - protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { + protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException, OperationCancelledException { if (isVmDestroyed(vm)) { return; } @@ -784,7 +787,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti } private void handleUnsuccessfulExpungeOperation(List finalizeExpungeCommands, List nicExpungeCommands, - VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException { + VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { if ((CollectionUtils.isNotEmpty(finalizeExpungeCommands) || CollectionUtils.isNotEmpty(nicExpungeCommands)) && hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); addAllExpungeCommandsFromList(finalizeExpungeCommands, cmds, vm); @@ -1139,7 +1142,7 @@ public void advanceStart(final String vmUuid, final Map sshAccessDetails, final VirtualMachine vm) throws AgentUnavailableException, OperationTimedoutException { + private void setupAgentSecurity(final Host vmHost, final Map sshAccessDetails, final VirtualMachine vm) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { final String csr = caManager.generateKeyStoreAndCsr(vmHost, sshAccessDetails); if (org.apache.commons.lang3.StringUtils.isNotEmpty(csr)) { final Map ipAddressDetails = new HashMap<>(sshAccessDetails); @@ -1331,7 +1334,7 @@ protected void processPrepareExternalProvisioning(boolean firstStart, Host host, CloudRuntimeException cre = new CloudRuntimeException("Failed to prepare VM"); try { answer = _agentMgr.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Failed PrepareExternalProvisioningCommand due to : {}", e.getMessage(), e); throw cre; } @@ -1677,6 +1680,9 @@ public void orchestrateStart(final String vmUuid, final Map sendStop(final VirtualMachineGuru guru, final Vi return new Pair<>(false, errorMsg); } - } catch (final AgentUnavailableException | OperationTimedoutException e) { + } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String errorMsg = String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()); logger.warn(errorMsg, e); if (!force) { @@ -2490,7 +2496,7 @@ private Pair getVMNetworkDetails(NetworkVO networkVO, boolean i } private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, - ConcurrentOperationException { + ConcurrentOperationException { final State state = vm.getState(); if (state == State.Stopped) { logger.debug("VM is already stopped: {}", vm); @@ -2612,7 +2618,7 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl throw new CloudRuntimeException("Invalid answer received in response to a StopCommand on " + vm.instanceName); } - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { agentExceptionDetail = e.getMessage(); logger.warn("Unable to stop {} due to [{}].", profile.toString(), e.toString(), e); } finally { @@ -2781,7 +2787,7 @@ private void deleteVMSnapshots(VMInstanceVO vm, boolean expunge) { } } - protected boolean checkVmOnHost(final VirtualMachine vm, final long hostId) throws AgentUnavailableException, OperationTimedoutException { + protected boolean checkVmOnHost(final VirtualMachine vm, final long hostId) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { final Answer answer = _agentMgr.send(hostId, new CheckVirtualMachineCommand(vm.getInstanceName())); if (answer == null || !answer.getResult()) { return false; @@ -2878,7 +2884,7 @@ private Answer[] attemptHypervisorMigration(VMInstanceVO vm, Map()).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1415,7 +1416,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfVolumesDoNotHave } @Test (expected = CloudRuntimeException.class) - public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1428,7 +1429,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCl } @Test (expected = CloudRuntimeException.class) - public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExceptionThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExceptionThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1441,7 +1442,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExcepti } @Test - public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws OperationTimedoutException, AgentUnavailableException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1454,7 +1455,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws } @Test - public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationSucceeds() throws OperationTimedoutException, AgentUnavailableException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationSucceeds() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1667,7 +1668,7 @@ public void updateExternalVmNicsFromPrepareAnswer_emptyNics_noAction() { } @Test - public void processPrepareExternalProvisioning_nonExternalHypervisor_noAction() throws OperationTimedoutException, AgentUnavailableException { + public void processPrepareExternalProvisioning_nonExternalHypervisor_noAction() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Host host = mock(Host.class); VirtualMachineProfile vmProfile = mock(VirtualMachineProfile.class); VirtualMachineTemplate template = mock(VirtualMachineTemplate.class); @@ -1678,7 +1679,7 @@ public void processPrepareExternalProvisioning_nonExternalHypervisor_noAction() } @Test - public void processPrepareExternalProvisioning_externalHypervisor_sendsCommand() throws OperationTimedoutException, AgentUnavailableException { + public void processPrepareExternalProvisioning_externalHypervisor_sendsCommand() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Host host = mock(Host.class); VirtualMachineProfile vmProfile = mock(VirtualMachineProfile.class); VirtualMachineTemplate template = mock(VirtualMachineTemplate.class); @@ -1787,7 +1788,7 @@ public void testPrepVMSpecForUnmanageInstance() { } @Test - public void testPersistDomainForKvmForRunningVmSuccess() throws AgentUnavailableException, OperationTimedoutException { + public void testPersistDomainForKvmForRunningVmSuccess() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); UnmanageInstanceAnswer successAnswer = new UnmanageInstanceAnswer(null, true, "success"); @@ -1800,7 +1801,7 @@ public void testPersistDomainForKvmForRunningVmSuccess() throws AgentUnavailable } @Test - public void testPersistDomainForKvmForStoppedVmSuccess() throws AgentUnavailableException, OperationTimedoutException { + public void testPersistDomainForKvmForStoppedVmSuccess() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Stopped); VirtualMachineTO vmTO = new VirtualMachineTO() {}; vmTO.setName(vmName); @@ -1832,7 +1833,7 @@ public void testPersistDomainForKvmForStoppedVmNoHost() { } @Test - public void testPersistDomainForKvmForRunningVmAgentFailure() throws AgentUnavailableException, OperationTimedoutException { + public void testPersistDomainForKvmForRunningVmAgentFailure() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); UnmanageInstanceAnswer failureAnswer = new UnmanageInstanceAnswer(null, false, "failure"); @@ -1842,7 +1843,7 @@ public void testPersistDomainForKvmForRunningVmAgentFailure() throws AgentUnavai } @Test - public void testPersistDomainForKvmAgentUnavailable() throws AgentUnavailableException, OperationTimedoutException { + public void testPersistDomainForKvmAgentUnavailable() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); doThrow(new AgentUnavailableException("Agent down", hostMockId)).when(agentManagerMock).send(anyLong(), any(UnmanageInstanceCommand.class)); @@ -1859,7 +1860,7 @@ public void testUnmanagePendingHaWork() { } @Test - public void testPersistDomainForKvmOperationTimedOut() throws AgentUnavailableException, OperationTimedoutException { + public void testPersistDomainForKvmOperationTimedOut() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); doThrow(new OperationTimedoutException(null, hostMockId, 123L, 60, false)).when(agentManagerMock).send(anyLong(), any(UnmanageInstanceCommand.class)); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql index 8e941a04e8ea..6369cb21fbfa 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql @@ -42,6 +42,7 @@ select async_job.removed, async_job.instance_type, async_job.instance_id, + async_job.related, async_job.job_executing_msid, CASE WHEN async_job.instance_type = 'Volume' THEN volumes.uuid diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java index 465a80b62c7f..2d39e18edea3 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java @@ -128,7 +128,7 @@ public void disjoinJob(long joinedJobId) throws InsufficientCapacityException, AsyncJobJoinMapVO record = s_joinMapDao.getJoinRecord(_job.getId(), joinedJobId); s_jobMgr.disjoinJob(_job.getId(), joinedJobId); - if (record.getJoinStatus() == JobInfo.Status.FAILED) { + if (record != null && record.getJoinStatus() == JobInfo.Status.FAILED) { if (record.getJoinResult() != null) { Object exception = JobSerializerHelper.fromObjectSerializedString(record.getJoinResult()); if (exception != null && exception instanceof Exception) { diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index c334e91feb84..f4f74a56eb7f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -70,4 +70,8 @@ public interface AsyncJobDao extends GenericDao { long countPendingNonPseudoJobs(Long... msIds); List listPendingJobIdsForAccount(long accountId); + + List getCancelledJobs(); + + boolean isJobCancelled(Long jobId); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index d8385e9aecd1..e680a5f7520c 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -50,6 +50,7 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements private final SearchBuilder byIdResourceIdResourceTypeSearch; private final GenericSearchBuilder asyncJobTypeSearch; private final GenericSearchBuilder pendingNonPseudoAsyncJobsSearch; + private final SearchBuilder cancelledAsyncJobSearch; public AsyncJobDaoImpl() { pendingAsyncJobSearch = createSearchBuilder(); @@ -116,6 +117,11 @@ public AsyncJobDaoImpl() { pendingNonPseudoAsyncJobsSearch.and("instanceTypeNEQ", pendingNonPseudoAsyncJobsSearch.entity().getInstanceType(), SearchCriteria.Op.NEQ); pendingNonPseudoAsyncJobsSearch.and("jobStatusEQ", pendingNonPseudoAsyncJobsSearch.entity().getStatus(), SearchCriteria.Op.EQ); pendingNonPseudoAsyncJobsSearch.and("executingMsidIN", pendingNonPseudoAsyncJobsSearch.entity().getExecutingMsid(), SearchCriteria.Op.IN); + + cancelledAsyncJobSearch = createSearchBuilder(); + cancelledAsyncJobSearch.and("status", cancelledAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ); + cancelledAsyncJobSearch.and("completeMsId", cancelledAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NULL); + cancelledAsyncJobSearch.done(); } @Override @@ -309,4 +315,17 @@ public List listPendingJobIdsForAccount(long accountId) { sc.setParameters("accountId", accountId); return customSearch(sc, null); } + + @Override + public List getCancelledJobs() { + SearchCriteria sc = cancelledAsyncJobSearch.create(); + sc.setParameters("status", JobInfo.Status.CANCELLED); + return listBy(sc); + } + + @Override + public boolean isJobCancelled(Long jobId) { + AsyncJobVO job = findById(jobId); + return job != null && job.getStatus() == JobInfo.Status.CANCELLED; + } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java index 7b6eed75a008..c87d88457c75 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java @@ -30,6 +30,8 @@ public interface SyncQueueItemDao extends GenericDao { public List getActiveQueueItems(Long msid, boolean exclusive); + public List getQueuedItems(long queueId); + public List getBlockedQueueItems(long thresholdMs, boolean exclusive); public Long getQueueItemIdByContentIdAndType(long contentId, String contentType); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java index 756cbb7efb0f..50e10e75e439 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java @@ -43,6 +43,7 @@ public class SyncQueueItemDaoImpl extends GenericDaoBase implements SyncQueueItemDao { final GenericSearchBuilder queueIdSearch; final GenericSearchBuilder queueActiveItemSearch; + final SearchBuilder queuedItemsSearch; public SyncQueueItemDaoImpl() { super(); @@ -57,6 +58,10 @@ public SyncQueueItemDaoImpl() { queueActiveItemSearch.and("processNumber", queueActiveItemSearch.entity().getLastProcessNumber(), Op.NNULL); queueActiveItemSearch.select(null, Func.COUNT, queueActiveItemSearch.entity().getId()); queueActiveItemSearch.done(); + + queuedItemsSearch = createSearchBuilder(); + queuedItemsSearch.and("queueId", queuedItemsSearch.entity().getQueueId(), SearchCriteria.Op.EQ); + queuedItemsSearch.done(); } @Override @@ -138,6 +143,16 @@ public List getActiveQueueItems(Long msid, boolean exclusive) { return listBy(sc, filter); } + @Override + public List getQueuedItems(long queueId) { + SearchCriteria sc = queuedItemsSearch.create(); + sc.setParameters("queueId", queueId); + + Filter filter = new Filter(SyncQueueItemVO.class, "created", true, null, null); + List queueItems = search(sc, filter); + return queueItems; + } + @Override public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { Date cutTime = DateUtil.currentGMTTime(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 1cb1cb4e309f..5e55a2f2594f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -35,6 +35,11 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.SnapshotVO; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotService; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.command.ReconcileCommandService; @@ -65,17 +70,15 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.logging.log4j.ThreadContext; import com.cloud.cluster.ClusterManagerListener; import com.cloud.network.Network; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; import com.cloud.storage.Volume; -import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; @@ -109,13 +112,10 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; -import com.cloud.vm.snapshot.VMSnapshot; -import com.cloud.vm.snapshot.VMSnapshotService; -import com.cloud.vm.snapshot.VMSnapshotVO; -import com.cloud.vm.snapshot.dao.VMSnapshotDao; -public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable { +import org.apache.logging.log4j.ThreadContext; +public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable, AsyncJobService { // Advanced public static final ConfigKey JobExpireMinutes = new ConfigKey("Advanced", Long.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", true, ConfigKey.Scope.Global); @@ -184,7 +184,6 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private volatile long _executionRunNumber = 1; private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat")); - private final ExecutorService _eventBusPublisher = Executors.newSingleThreadExecutor(new NamedThreadFactory("AsyncJobMgr-EventBus")); private ExecutorService _apiJobExecutor; private ExecutorService _workerJobExecutor; @@ -364,7 +363,7 @@ public void completeAsyncJob(final long jobId, final Status jobStatus, final int } if (resultObject != null) { - job.updateResultWithEncryptionIfNeeded(resultObject); + job.setResult(resultObject); } if (logger.isDebugEnabled()) { @@ -385,9 +384,9 @@ public List doInTransaction(final TransactionStatus status) { job.setResultCode(resultCode); if (resultObject != null) { - job.updateResultWithEncryptionIfNeeded(resultObject); + job.setResult(resultObject); } else { - job.updateResultWithEncryptionIfNeeded(null); + job.setResult(null); } final Date currentGMTTime = DateUtil.currentGMTTime(); @@ -456,7 +455,7 @@ public void updateAsyncJobStatus(final long jobId, final int processStatus, fina public void doInTransactionWithoutResult(TransactionStatus status) { job.setProcessStatus(processStatus); if (resultObject != null) { - job.updateResultWithEncryptionIfNeeded(resultObject); + job.setResult(resultObject); } job.setLastUpdated(DateUtil.currentGMTTime()); _jobDao.update(jobId, job); @@ -554,11 +553,23 @@ public AsyncJob queryJob(final long jobId, final boolean updatePollTime) { return job; } - public String obfuscatePassword(String result, boolean hidePassword) { - if (!hidePassword) { - return result; + public String obfuscatePassword(String result, boolean hidePassword) { + if (hidePassword) { + String pattern = "\"password\":"; + if (result != null) { + if (result.contains(pattern)) { + String[] resp = result.split(pattern); + String psswd = resp[1].toString().split(",")[0]; + if (psswd.endsWith("}")) { + psswd = psswd.substring(0, psswd.length() - 1); + result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "}," + resp[1].split(",", 2)[1]; + } else { + result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1]; + } + } + } } - return StringUtils.obfuscatePasswordInJsonLikeString(result); + return result; } private void scheduleExecution(final AsyncJobVO job) { @@ -995,9 +1006,6 @@ public void reallyRun() { } logger.trace("End cleanup expired async-jobs"); - - cleanupNetworksStuckInImplementing(); - } catch (Throwable e) { logger.error("Unexpected exception when trying to execute queue item, ", e); } @@ -1005,15 +1013,6 @@ public void reallyRun() { }; } - private void cleanupNetworksStuckInImplementing() { - // Cleanup orphaned networks stuck in Implementing state without async jobs - try { - cleanupOrphanedNetworks(); - } catch (Throwable e) { - logger.error("Unexpected exception when trying to cleanup orphaned networks", e); - } - } - @DB protected void expungeAsyncJob(final AsyncJobVO job) { Transaction.execute(new TransactionCallbackNoReturn() { @@ -1159,7 +1158,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { cleanupResources(job); job.setStatus(JobInfo.Status.FAILED); job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode()); - job.updateResultWithEncryptionIfNeeded("job cancelled because of management server restart or shutdown"); + job.setResult("job cancelled because of management server restart or shutdown"); job.setCompleteMsid(msid); final Date currentGMTTime = DateUtil.currentGMTTime(); job.setLastUpdated(currentGMTTime); @@ -1351,74 +1350,6 @@ private void cleanupFailedSnapshotsCreatedWithDefaultStrategy(final long msid) { } } - /** - * Cleanup networks that are stuck in Implementing state without associated async jobs. - * This only processes networks that have been stuck for longer than the job expiration threshold. - */ - private void cleanupOrphanedNetworks() { - try { - SearchCriteria sc = networkDao.createSearchCriteria(); - sc.addAnd("state", SearchCriteria.Op.EQ, Network.State.Implementing); - sc.addAnd("removed", SearchCriteria.Op.NULL); - List implementingNetworks = networkDao.search(sc, null); - - if (implementingNetworks == null || implementingNetworks.isEmpty()) { - return; - } - - logger.debug("Found {} networks in Implementing state, checking for orphaned networks", implementingNetworks.size()); - - final long expireMinutes = JobExpireMinutes.value(); - final Date cutoffTime = new Date(System.currentTimeMillis() - (expireMinutes * 60 * 1000)); - - for (NetworkVO network : implementingNetworks) { - if (network.getCreated().after(cutoffTime)) { - logger.trace("Network {} in Implementing state is only {} minutes old (threshold: {} minutes), skipping cleanup", - network.getId(), - (System.currentTimeMillis() - network.getCreated().getTime()) / 60000, - expireMinutes); - continue; - } - - List jobs = _jobDao.findInstancePendingAsyncJobs("Network", network.getAccountId()); - boolean hasActiveJob = false; - for (AsyncJobVO job : jobs) { - if (job.getInstanceId() != null && job.getInstanceId().equals(network.getId())) { - hasActiveJob = true; - break; - } - } - - if (hasActiveJob) { - logger.debug("Network {} in Implementing state has active async job, skipping cleanup", network.getId()); - continue; - } - - logger.warn("Found orphaned network {} in Implementing state without async job. " + - "Network created: {}, age: {} minutes, expiration threshold: {} minutes. Transitioning to Shutdown state.", - network.getId(), network.getCreated(), - (System.currentTimeMillis() - network.getCreated().getTime()) / 60000, - expireMinutes); - updateNetworkState(network); - - } - } catch (Exception e) { - logger.error("Error while cleaning up orphaned networks", e); - } - } - - private void updateNetworkState(NetworkVO network) { - try { - networkOrchestrationService.stateTransitTo(network, Network.Event.OperationFailed); - logger.info("Successfully transitioned orphaned network {} to Shutdown state using state machine", network.getId()); - } catch (final NoTransitionException e) { - logger.debug("State transition failed for orphaned network {}, forcing state update", network.getId()); - network.setState(Network.State.Shutdown); - networkDao.update(network.getId(), network); - logger.info("Successfully forced orphaned network {} to Shutdown state", network.getId()); - } - } - @Override public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @@ -1446,6 +1377,9 @@ public boolean start() { @Override public boolean stop() { + cancelPendingJobs(); + shutdownAndAwaitTermination(_workerJobExecutor); + shutdownAndAwaitTermination(_apiJobExecutor); _heartbeatScheduler.shutdown(); _eventBusPublisher.shutdown(); _apiJobExecutor.shutdown(); @@ -1453,6 +1387,89 @@ public boolean stop() { return true; } + private void cancelPendingJobs() { + List jobs = _syncQueueItemDao.getActiveQueueItems(getMsid(), false); + for (SyncQueueItemVO job : jobs) { + AsyncJobVO childJob = _jobDao.findById(job.getContentId()); + if (childJob != null && StringUtils.isNotBlank(childJob.getRelated())) { + long parentJobId = Long.parseLong(childJob.getRelated()); + try { + logger.debug("Cancel left-over async. job-{}", parentJobId); + cancelAsyncJob(parentJobId, "Management Server shutdown"); + } catch (Exception e) { + logger.error("Exception while cancelling job-{}", parentJobId, e); + } + } + _queueMgr.purgeItem(job.getId()); + } + } + + private static void shutdownAndAwaitTermination(ExecutorService pool) { + pool.shutdown(); + try { + if (!pool.awaitTermination(60, TimeUnit.SECONDS)) { + pool.shutdownNow(); + } + } catch (InterruptedException ignore) { + } + } + + @Override + public String cancelAsyncJob(long jobId, String reason) { + final AsyncJobVO job = _jobDao.findById(jobId); + String errMessage; + if (job == null) { + errMessage = "Cannot cancel, job-" + jobId + " no longer exists."; + logger.debug(errMessage); + // purge queue items (if any) to avoid blocking + _queueMgr.purgeAsyncJobQueueItemId(jobId); + return errMessage; + } + + try { + Class cmdClass = Class.forName(job.getCmd()); + if (!CancellableCmd.class.isAssignableFrom(cmdClass)) { + errMessage = "Cannot cancel, job-" + jobId + " as it is not cancellable."; + logger.debug(errMessage); + return errMessage; + } + } catch (ClassNotFoundException e) { + errMessage = "Command " + job.getCmd() + " of jobid-" + jobId + " not found."; + logger.error(errMessage, e); + return errMessage; + } + + if (job.getStatus() != Status.IN_PROGRESS) { + errMessage = "Cannot cancel, job-" + jobId + " is not running. Current job status is " + job.getStatus() + "."; + logger.debug(errMessage); + _queueMgr.purgeAsyncJobQueueItemId(jobId); + return errMessage; + } + + logger.debug("Cancelling job-{} which is in IN_PROGRESS state.", jobId); + + try { + completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason); + _jobMonitor.unregisterByJobId(jobId); + + // purge the item and resume queue processing + _queueMgr.purgeItem(jobId); + return ""; + } catch (Throwable t) { + errMessage = "Unexpected exception when cancelling async job with id: " + jobId; + logger.error(errMessage, t); + } + + return errMessage; + } + + @Override + public List> getCommands() { + final List> cmdList = new ArrayList>(); + cmdList.add(CancelAsyncJobCmd.class); + return cmdList; + } + private GenericSearchBuilder ContentIdsSearch; private GenericSearchBuilder JoinJobSearch; private SearchBuilder JobIdsSearch; @@ -1467,26 +1484,8 @@ protected AsyncJobManagerImpl() { } private void publishOnEventBus(AsyncJob job, String jobEvent) { - try { - _eventBusPublisher.submit(new ManagedContextRunnable() { - @Override - protected void runInContext() { - publishJobEvent(job, jobEvent); - } - }); - } catch (RejectedExecutionException e) { - logger.warn("Failed to publish async job event, event bus publisher is shut down", e); - } - } - - private void publishJobEvent(AsyncJob job, String jobEvent) { - try { - _messageBus.publish(null, AsyncJob.Topics.JOB_EVENT_PUBLISH, PublishScope.LOCAL, - new Pair<>(job, jobEvent)); - } catch (Throwable t) { - logger.warn("Failed to publish async job event on message bus. jobId={}, jobEvent={}", - job != null ? job.getId() : null, jobEvent, t); - } + _messageBus.publish(null, AsyncJob.Topics.JOB_EVENT_PUBLISH, PublishScope.LOCAL, + new Pair(job, jobEvent)); } @Override diff --git a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java index 60792f0ddb7d..9c95d821454d 100644 --- a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java +++ b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java @@ -27,6 +27,7 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.ha.FenceBuilder; import com.cloud.host.Host; @@ -100,6 +101,11 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { logger.debug("Moving on to the next host because " + h.toString() + " is unavailable", e); } continue; + } catch (OperationCancelledException e) { + if (logger.isDebugEnabled()) { + logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); + } + break; } if (answer != null && answer.getResult()) { diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java index eb83572fc2b8..7a2047e8d534 100755 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.ha.FenceBuilder; import com.cloud.host.Host; @@ -97,6 +98,12 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { + h.toString() + " is unavailable", e); } continue; + } catch (OperationCancelledException e) { + if (logger.isDebugEnabled()) { + logger.debug("Not moving on to the next host because " + + h.toString() + " is unavailable, and operation cancelled", e); + } + break; } if (answer != null && answer.getResult()) { return true; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java index b37960e26ab9..1d905bdd4823 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java @@ -27,6 +27,7 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -96,6 +97,11 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { logger.debug("Moving on to the next host because " + h.toString() + " is unavailable", e); } continue; + } catch (OperationCancelledException e) { + if (logger.isDebugEnabled()) { + logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); + } + break; } if (answer != null && answer.getResult()) { return true; diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java index a29ac2aee76b..9373dfb04599 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -83,6 +84,11 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { logger.debug("Moving on to the next host because " + h.toString() + " is unavailable", e); } continue; + } catch (OperationCancelledException e) { + if (logger.isDebugEnabled()) { + logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); + } + break; } if (answer != null && answer.getResult()) { return true; diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java index 6d298629e7ef..3f555b4e9ec6 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java @@ -30,6 +30,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.xenserver.resource.XcpServer83Resource; import com.cloud.hypervisor.xenserver.resource.Xenserver84Resource; import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs; @@ -634,6 +635,8 @@ public void processConnect(com.cloud.host.Host agent, StartupCommand cmd, boolea logger.warn("Unable to setup agent " + agentId + " because it became unavailable.", e); } catch (OperationTimedoutException e) { logger.warn("Unable to setup agent " + agentId + " because it timed out", e); + } catch (OperationCancelledException e) { + logger.warn("Unable to setup agent " + agentId + " because it is cancelled", e); } throw new ConnectionException(true, "Reinitialize agent after setup."); } diff --git a/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java b/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java index 76807b89d0f7..68585580efda 100644 --- a/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java +++ b/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java @@ -25,6 +25,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -382,6 +383,9 @@ private Answer migrateVmWithVolumesAcrossCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); + } catch (OperationCancelledException e) { + logger.error("Error while migrating vm " + vm + " to host " + destHost, e); + throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } @@ -415,6 +419,9 @@ private Answer migrateVmWithVolumesWithinCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); + } catch (OperationCancelledException e) { + logger.error("Error while migrating vm " + vm + " to host " + destHost, e); + throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } diff --git a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java index e70a6b4da639..1407ff3a4b37 100644 --- a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher; import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.jobs.JobInfo; import com.cloud.exception.InvalidParameterValueException; @@ -117,23 +118,30 @@ public void runJob(final AsyncJob job) { CallContext.unregister(); } } catch (Throwable e) { - String errorMsg = null; - int errorCode = ApiErrorCode.INTERNAL_ERROR.getHttpCode(); - if (!(e instanceof ServerApiException)) { - logger.error("Unexpected exception while executing " + job.getCmd(), e); - errorMsg = e.getMessage(); - } else { - ServerApiException sApiEx = (ServerApiException)e; - errorMsg = sApiEx.getDescription(); - errorCode = sApiEx.getErrorCode().getHttpCode(); + //Get the latest job status from DB to check if it has been cancelled during execution + AsyncJobVO jobFromDb = _asyncJobMgr.getAsyncJob(job.getId()); + if (!jobFromDb.getStatus().done()) { + String errorMsg = null; + int errorCode = ApiErrorCode.INTERNAL_ERROR.getHttpCode(); + if (!(e instanceof ServerApiException)) { + logger.error("Unexpected exception while executing {}", job.getCmd(), e); + errorMsg = e.getMessage(); + } else { + ServerApiException sApiEx = (ServerApiException) e; + errorMsg = sApiEx.getDescription(); + errorCode = sApiEx.getErrorCode().getHttpCode(); + } + + ExceptionResponse response = new ExceptionResponse(); + response.setErrorCode(errorCode); + response.setErrorText(errorMsg); + response.setResponseName((cmdObj == null) ? "unknowncommandresponse" : cmdObj.getCommandName()); + + // FIXME: setting resultCode to ApiErrorCode.INTERNAL_ERROR is not right, usually executors have their exception handling + // and we need to preserve that as much as possible here + + _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), ApiSerializerHelper.toSerializedString(response)); } - - ExceptionResponse response = new ExceptionResponse(); - response.setErrorCode(errorCode); - response.setErrorText(errorMsg); - response.setResponseName((cmdObj == null) ? "unknowncommandresponse" : cmdObj.getCommandName()); - - _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, errorCode, ApiSerializerHelper.toSerializedString(response)); } } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index f56cda6e557a..b457c37f9e4c 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -58,6 +58,7 @@ import org.apache.cloudstack.api.BaseResponseWithAssociatedNetwork; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.job.CancelAsyncJobCmd; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.response.ASNRangeResponse; import org.apache.cloudstack.api.response.ASNumberResponse; @@ -2377,6 +2378,12 @@ public AsyncJobResponse queryJobResult(final QueryAsyncJobResultCmd cmd) { return createAsyncJobResponse(_jobMgr.queryJob(jobId, true)); } + @Override + public AsyncJobResponse cancelJobResponse(CancelAsyncJobCmd cmd) { + AsyncJob job = _jobMgr.queryJob(cmd.getId(), true); + return createAsyncJobResponse(job); + } + public AsyncJobResponse createAsyncJobResponse(AsyncJob job) { AsyncJobJoinVO vJob = ApiDBUtils.newAsyncJobView(job); return ApiDBUtils.newAsyncJobResponse(vJob); diff --git a/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java index b4714c75def4..cb6e949b88c4 100644 --- a/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java @@ -107,6 +107,9 @@ public class AsyncJobJoinVO extends BaseViewVO implements ControlledViewEntity { @Column(name = "instance_uuid") private String instanceUuid; + @Column(name = "related") + private String related; + public AsyncJobJoinVO() { } @@ -208,6 +211,10 @@ public String getInstanceUuid() { return instanceUuid; } + public String getRelated() { + return related; + } + @Override public Class getEntityType() { return AsyncJob.class; diff --git a/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java b/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java index 4ee6adeab603..a25451e57b21 100644 --- a/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java +++ b/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.PingTestCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; @@ -115,6 +116,11 @@ protected Status testIpAddress(Long hostId, String testHostIp) { logger.debug("host (" + testHostIp + "): " + e.getLocalizedMessage() + ", trapped OperationTimedoutException returning Unknown state"); } return Status.Unknown; + } catch (OperationCancelledException e) { + if (logger.isDebugEnabled()) { + logger.debug("host (" + testHostIp + "): " + e.getLocalizedMessage() + ", trapped OperationCancelledException returning Unknown state"); + } + return Status.Unknown; } } } diff --git a/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java b/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java index e0dbc8fcbe11..4ccc43c3c4fb 100644 --- a/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java +++ b/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java @@ -23,6 +23,7 @@ import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.Status; @@ -60,6 +61,9 @@ public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM { } catch (OperationTimedoutException e) { logger.debug("Operation timed out for " + vm.toString() + ": " + e.getMessage()); throw new UnknownVM(); + } catch (OperationCancelledException e) { + logger.debug("Operation cancelled for " + vm.toString() + ": " + e.getMessage()); + throw new UnknownVM(); } } } diff --git a/server/src/main/java/com/cloud/ha/KVMFencer.java b/server/src/main/java/com/cloud/ha/KVMFencer.java index 4a6606b09cc3..b071338cf137 100644 --- a/server/src/main/java/com/cloud/ha/KVMFencer.java +++ b/server/src/main/java/com/cloud/ha/KVMFencer.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -100,6 +101,10 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { } catch (AgentUnavailableException | OperationTimedoutException e) { logger.info("Moving on to the next host because {} is unavailable", h.toString(), e); continue; + } catch(OperationCancelledException e) { + logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); + break; + } if (answer != null && answer.getResult()) { return true; diff --git a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index b9fa3f0ebae7..4b57d5a38b8f 100644 --- a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -29,6 +29,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.direct.download.DirectDownloadManager; @@ -486,9 +487,7 @@ public DeleteHostAnswer deleteHost(HostVO host, boolean isForced, boolean isForc try { ShutdownCommand cmd = new ShutdownCommand(ShutdownCommand.DeleteHost, null, !ADD_HOST_ON_SERVICE_RESTART_KVM.value()); agentMgr.send(host.getId(), cmd); - } catch (AgentUnavailableException e) { - logger.warn("Sending ShutdownCommand failed: ", e); - } catch (OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.warn("Sending ShutdownCommand failed: ", e); } diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index 56534a8ee7df..406c5f507b1f 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -28,6 +28,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.utils.validation.ChecksumUtil; @@ -208,6 +209,9 @@ public boolean sendCommandsToRouter(final VirtualRouter router, final Commands c } catch (final OperationTimedoutException e) { logger.warn("Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); + } catch (final OperationCancelledException e) { + logger.warn("Operation Cancelled", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } if (answers == null || answers.length != cmds.size()) { diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index 49c2243d1ecf..ee9cc02e4b5c 100644 --- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -28,6 +28,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import com.cloud.network.dao.NetworkDao; import com.cloud.network.vpc.dao.VpcDao; import org.apache.cloudstack.agent.routing.ManageServiceCommand; @@ -930,7 +931,7 @@ public boolean startRemoteAccessVpn(final RemoteAccessVpn vpn, final VirtualRout try { _agentMgr.send(router.getHostId(), cmds); - } catch (final OperationTimedoutException e) { + } catch (final OperationTimedoutException | OperationCancelledException e) { logger.debug("Failed to start remote access VPN: ", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index 585b65aa4d94..e621dd3b3a13 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -37,6 +37,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import com.cloud.host.dao.HostDao; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.command.user.securitygroup.AuthorizeSecurityGroupEgressCmd; @@ -572,9 +573,7 @@ protected void handleVmMigrated(VMInstanceVO vm) { cmds = new Commands(nrc); try { _agentMgr.send(vm.getHostId(), cmds); - } catch (AgentUnavailableException e) { - logger.debug(e.toString()); - } catch (OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.debug(e.toString()); } @@ -1480,9 +1479,7 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool cmds = new Commands(cmd); try { _agentMgr.send(vm.getHostId(), cmds); - } catch (AgentUnavailableException e) { - logger.debug(e.toString()); - } catch (OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.debug(e.toString()); } diff --git a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java index 1e6ab9d0ffa2..a93a13e6104f 100644 --- a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java @@ -29,6 +29,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.command.admin.cluster.UpdateClusterCmd; @@ -499,7 +500,7 @@ private boolean isMaintenanceScriptDefinedOnHost(Host host, List ho try { RollingMaintenanceAnswer answer = (RollingMaintenanceAnswer) agentManager.send(host.getId(), new RollingMaintenanceCommand(true)); return answer.isMaintenaceScriptDefined(); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String msg = String.format("Could not check for maintenance script on %s due to: %s", host, e.getMessage()); logger.error(msg, e); return false; @@ -544,7 +545,7 @@ private Ternary sendRollingMaintenanceCommandToHost(Ho while (!completed && timeSpent < timeout * 1000L) { try { answer = agentManager.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { // Agent may be restarted on the scripts - continue polling until it is up String msg = String.format("Cannot send command to %s, waiting %sms - %s", host, pingInterval, e.getMessage()); logger.warn(msg, e); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index cd37d7897d9c..d3428311e3fc 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -45,6 +45,7 @@ import org.apache.cloudstack.acl.ApiKeyPairVO; import com.cloud.api.query.MutualExclusiveIdsManagerBase; +import com.cloud.exception.OperationCancelledException; import com.cloud.network.vpc.VpcVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; @@ -3469,7 +3470,7 @@ public Pair setConsoleAccessForVm(long vmId, String sessionUuid Answer answer; try { answer = _agentMgr.send(hostVO.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String errorMsg = "Could not send allow session command to CPVM: " + e.getMessage(); logger.error(errorMsg, e); return new Pair<>(false, errorMsg); @@ -5961,6 +5962,10 @@ private Pair patchSystemVm(VMInstanceVO systemVM, boolean force String errMsg = "SystemVM live patch failed"; logger.error(errMsg, e); return new Pair<>(false, String.format("%s due to: %s", errMsg, e.getMessage())); + } catch (OperationCancelledException e) { + String errMsg = "SystemVM live patch cancelled"; + logger.error(errMsg, e); + return new Pair<>(false, String.format("%s due to: %s", errMsg, e.getMessage())); } logger.info(String.format("Successfully patched system VM %s", systemVM.getInstanceName())); List routerTypes = new ArrayList<>(); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 69f04393174c..d7b32e77316d 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -200,6 +200,7 @@ import com.cloud.exception.DiscoveryException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceInUseException; @@ -2063,7 +2064,7 @@ public Pair sendToPool(StoragePool pool, long[] hostIdsToTryFirs answers.add(_agentMgr.send(targetHostId, cmd)); } return new Pair<>(hostId, answers.toArray(new Answer[answers.size()])); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.debug("Unable to send storage pool command to {} via {}", pool::toString, () -> _hostDao.findById(hostId), () -> e); } } @@ -3663,7 +3664,7 @@ protected Answer getCheckDatastorePolicyComplianceAnswer(String storagePolicyId, } catch (AgentUnavailableException e) { logger.debug("Unable to send storage pool command to " + pool + " via " + hostIds.get(0), e); throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId()); - } catch (OperationTimedoutException e) { + } catch (OperationTimedoutException | OperationCancelledException e) { logger.debug("Failed to process storage pool command to " + pool + " via " + hostIds.get(0), e); throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId()); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 9bbbdc503508..a0c5774e8757 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -67,6 +67,7 @@ import javax.xml.parsers.ParserConfigurationException; import com.cloud.agent.api.StartCommand; +import com.cloud.exception.OperationCancelledException; import com.cloud.network.NetworkService; import com.cloud.vm.snapshot.dao.VMSnapshotDetailsDao; import org.apache.cloudstack.acl.ControlledEntity; @@ -855,6 +856,8 @@ protected void runInContext() { } } catch (OperationTimedoutException e) { logger.warn("Timed Out", e); + } catch (OperationCancelledException e) { + logger.warn("Operation Cancelled", e); } catch (AgentUnavailableException e) { logger.warn("Agent Unavailable ", e); } finally { @@ -2740,6 +2743,9 @@ public boolean expunge(UserVmVO vm) { } catch (OperationTimedoutException e) { logger.warn("Operation time out on expunging " + vm, e); return false; + } catch (OperationCancelledException e) { + logger.warn("Operation cancelled on expunging " + vm, e); + return false; } catch (ConcurrentOperationException e) { logger.warn("Concurrent operations on expunging " + vm, e); return false; @@ -5720,6 +5726,9 @@ public boolean setupVmForPvlan(boolean add, Long hostId, NicProfile nic) { } catch (OperationTimedoutException e) { logger.warn("Timed Out", e); return false; + } catch (OperationCancelledException e) { + logger.warn("Operation Cancelled", e); + return false; } catch (AgentUnavailableException e) { logger.warn("Agent Unavailable ", e); return false; diff --git a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java index 73ff79301fb7..5d094f2f7785 100644 --- a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java @@ -57,6 +57,7 @@ import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.lang3.math.NumberUtils; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.ca.IssueCertificateCmd; @@ -223,7 +224,7 @@ public boolean provisionCertificate(final Host host, final Boolean reconnect, fi } final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); - } catch (final AgentUnavailableException | OperationTimedoutException e) { + } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent {}, due to: ", host, e); throw new CloudRuntimeException(String.format("Failed to generate keystore and get CSR from the host/agent %s", host)); } @@ -365,7 +366,7 @@ private boolean provisionSystemVmViaSsh(Host host, Boolean reconnect, String caP } @Override - public String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException { + public String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { final SetupKeyStoreCommand cmd = new SetupKeyStoreCommand(CertValidityPeriod.value()); if (sshAccessDetails != null && !sshAccessDetails.isEmpty()) { cmd.setAccessDetail(sshAccessDetails); @@ -377,7 +378,7 @@ public String generateKeyStoreAndCsr(final Host host, final Map @Override public boolean deployCertificate(final Host host, final Certificate certificate, final Boolean reconnect, final Map sshAccessDetails) - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { final SetupCertificateCommand cmd = new SetupCertificateCommand(certificate); if (sshAccessDetails != null && !sshAccessDetails.isEmpty()) { cmd.setAccessDetail(sshAccessDetails); diff --git a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java index 237135b5176b..5aa322fe8f90 100644 --- a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.command.user.consoleproxy.ConsoleEndpoint; @@ -743,7 +744,7 @@ private String acquireVncTicketForVmwareVm(VirtualMachine vm) { logger.info("VNC ticket could not be acquired correctly: " + ans.getDetails()); } return ans.getTicket(); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Error acquiring ticket", e); return null; } diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index 3c9d355cd89f..3cdd474e0fc4 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -42,6 +42,7 @@ import javax.naming.ConfigurationException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationCancelledException; import com.cloud.user.Account; import com.cloud.utils.Pair; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; @@ -763,7 +764,7 @@ protected Pair revokeCertificateAliasFromHost(String alias, Hos try { Answer answer = agentManager.send(host.getId(), cmd); return new Pair<>(answer != null && answer.getResult(), answer != null ? answer.getDetails() : ""); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Error revoking certificate {} from host {}", alias, host, e); return new Pair<>(false, e.getMessage()); } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 840a1de45962..0542ff9301c8 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -61,6 +61,7 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientVirtualNetworkCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; @@ -2111,7 +2112,7 @@ private CheckConvertInstanceAnswer checkConversionSupportOnHost(HostVO convertHo CheckConvertInstanceAnswer checkConvertInstanceAnswer; try { checkConvertInstanceAnswer = (CheckConvertInstanceAnswer) agentManager.send(convertHost.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String err = String.format("Failed to check %s conversion support on the host %s for converting instance %s from VMware to KVM due to: %s", checkWindowsGuestConversionSupport? "windows guest" : "", convertHost, sourceVM, e.getMessage()); logger.error(err); @@ -2202,7 +2203,7 @@ private UnmanagedInstanceTO convertAndImportToKVM(ConvertInstanceCommand convert Answer convertAnswer; try { convertAnswer = agentManager.send(convertHost.getId(), convertInstanceCommand); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String err = String.format("Could not send the convert instance command to host %s due to: %s", convertHost, e.getMessage()); logger.error(err, e); @@ -2234,7 +2235,7 @@ private UnmanagedInstanceTO convertAndImportToKVM(ConvertInstanceCommand convert logger.error(err); throw new CloudRuntimeException(err); } - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { cleanupConvertedDisks = true; String err = String.format( "Could not send the import converted instance command to host %s due to: %s", diff --git a/server/src/test/java/com/cloud/ha/KVMFencerTest.java b/server/src/test/java/com/cloud/ha/KVMFencerTest.java index c4b5666c0206..75fdc1a51111 100644 --- a/server/src/test/java/com/cloud/ha/KVMFencerTest.java +++ b/server/src/test/java/com/cloud/ha/KVMFencerTest.java @@ -23,6 +23,7 @@ import com.cloud.agent.api.FenceCommand; import com.cloud.alert.AlertManager; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -96,7 +97,7 @@ public void testWithSingleHostDown() { } @Test - public void testWithHosts() throws AgentUnavailableException, OperationTimedoutException { + public void testWithHosts() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); @@ -124,7 +125,7 @@ public void testWithHosts() throws AgentUnavailableException, OperationTimedoutE } @Test - public void testWithFailingFence() throws AgentUnavailableException, OperationTimedoutException { + public void testWithFailingFence() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); @@ -151,7 +152,7 @@ public void testWithFailingFence() throws AgentUnavailableException, OperationTi } @Test - public void testWithTimeoutingFence() throws AgentUnavailableException, OperationTimedoutException { + public void testWithTimeoutingFence() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); diff --git a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java index ff1a2f284666..ad7899cd12f2 100644 --- a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java @@ -43,6 +43,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.affinity.AffinityGroupVO; import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.annotation.AnnotationService; @@ -1932,7 +1933,7 @@ public void getPublicIpAndPortFail2() { } @Test - public void checkNetScalerAsGroup() throws OperationTimedoutException, AgentUnavailableException { + public void checkNetScalerAsGroup() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { when(asVmGroupMock.getId()).thenReturn(vmGroupId); AutoScaleVmGroupTO groupTO = Mockito.mock(AutoScaleVmGroupTO.class); when(lbRulesMgr.toAutoScaleVmGroupTO(asVmGroupMock)).thenReturn(groupTO); diff --git a/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java b/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java index e4773d01de92..22bcdf04bbc0 100644 --- a/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java +++ b/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java @@ -30,6 +30,7 @@ import com.cloud.deploy.DeployDestination; import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.OperationCancelledException; import com.cloud.network.Ipv6Service; import com.cloud.network.Network; import com.cloud.network.Networks; @@ -148,7 +149,7 @@ public void setUp() { @Test(expected=ResourceUnavailableException.class) public void testSendCommandsToRouterWrongRouterVersion() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); @@ -163,7 +164,7 @@ public void testSendCommandsToRouterWrongRouterVersion() @Test public void testSendCommandsToRouter() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); @@ -201,7 +202,7 @@ public void testSendCommandsToRouter() */ @Test public void testSendCommandsToRouterWithTrueResult() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); @@ -239,7 +240,7 @@ public void testSendCommandsToRouterWithTrueResult() */ @Test public void testSendCommandsToRouterWithNoAnswers() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index f46699f88cfb..35dfde864d94 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -26,6 +26,7 @@ import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.HostPodDao; +import com.cloud.exception.OperationCancelledException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.resource.ResourceManager; @@ -506,14 +507,14 @@ public void testGetCheckDatastorePolicyComplianceAnswerAgentException() throws S try { Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(Command.class))) .thenThrow(AgentUnavailableException.class); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { Assert.fail(e.getMessage()); } storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool); try { Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(Command.class))) .thenThrow(OperationTimedoutException.class); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { Assert.fail(e.getMessage()); } storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool); @@ -535,7 +536,7 @@ public void testGetCheckDatastorePolicyComplianceAnswerSuccess() throws StorageU Mockito.any(CheckDataStoreStoragePolicyComplianceCommand.class))) .thenReturn(new com.cloud.agent.api.Answer( Mockito.mock(CheckDataStoreStoragePolicyComplianceCommand.class))); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { Assert.fail(e.getMessage()); } com.cloud.agent.api.Answer answer = diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index dbc05ec99f39..82fece462295 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -109,6 +109,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.PermissionDeniedException; @@ -745,7 +746,7 @@ private enum VcenterParameter { } private void baseTestImportVmFromVmwareToKvm(VcenterParameter vcenterParameter, boolean selectConvertHost, - boolean selectTemporaryStorage) throws OperationTimedoutException, AgentUnavailableException { + boolean selectTemporaryStorage) throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { long clusterId = 1L; long zoneId = 1L; long existingDatacenterId = 1L; @@ -993,42 +994,42 @@ private void importFromDisk(String source) throws InsufficientServerCapacityExce } @Test - public void testImportVmFromVmwareToKvmExistingVcenter() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenter() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING, false, false); } @Test - public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHost() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHost() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING, true, false); } @Test - public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHostAndTemporaryStorage() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHostAndTemporaryStorage() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING, true, true); } @Test(expected = ServerApiException.class) - public void testImportVmFromVmwareToKvmExistingVcenterExclusiveParameters() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterExclusiveParameters() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.BOTH, false, false); } @Test(expected = ServerApiException.class) - public void testImportVmFromVmwareToKvmExistingVcenterMissingParameters() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterMissingParameters() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.NONE, false, false); } @Test(expected = CloudRuntimeException.class) - public void testImportVmFromVmwareToKvmExistingVcenterInvalid() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterInvalid() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING_INVALID, false, false); } @Test(expected = CloudRuntimeException.class) - public void testImportVmFromVmwareToKvmExistingVcenterAgentUnavailable() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterAgentUnavailable() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.AGENT_UNAVAILABLE, false, false); } @Test(expected = CloudRuntimeException.class) - public void testImportVmFromVmwareToKvmExistingVcenterConvertFailure() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmExistingVcenterConvertFailure() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { baseTestImportVmFromVmwareToKvm(VcenterParameter.CONVERT_FAILURE, false, false); } diff --git a/utils/src/main/java/com/cloud/utils/SerialVersionUID.java b/utils/src/main/java/com/cloud/utils/SerialVersionUID.java index 30844953e1aa..bf689a243d13 100644 --- a/utils/src/main/java/com/cloud/utils/SerialVersionUID.java +++ b/utils/src/main/java/com/cloud/utils/SerialVersionUID.java @@ -72,4 +72,5 @@ public interface SerialVersionUID { public static final long OriginDeniedException = Base | 0x30; public static final long StorageAccessException = Base | 0x31; public static final long EncryptionException = Base | 0x32; + public static final long OperationCancelledException = Base | 0x33; } diff --git a/utils/src/main/java/com/cloud/utils/exception/CSExceptionErrorCode.java b/utils/src/main/java/com/cloud/utils/exception/CSExceptionErrorCode.java index 8b25d77e7f3e..f458f7de0738 100644 --- a/utils/src/main/java/com/cloud/utils/exception/CSExceptionErrorCode.java +++ b/utils/src/main/java/com/cloud/utils/exception/CSExceptionErrorCode.java @@ -75,6 +75,7 @@ public class CSExceptionErrorCode { ExceptionErrorCodeMap.put("com.cloud.exception.UnavailableCommandException", 4555); ExceptionErrorCodeMap.put("com.cloud.exception.OperationTimedoutException", 4560); ExceptionErrorCodeMap.put("org.apache.cloudstack.framework.kms.KMSException", 4561); + ExceptionErrorCodeMap.put("com.cloud.exception.OperationCancelledException", 4565); // Have a special error code for ServerApiException when it is // thrown in a standalone manner when failing to detect any of the above From 7d30171b25b04b42fffe14ea9a333a0abb5887cc Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 22 Jan 2026 18:23:54 +0530 Subject: [PATCH 05/29] Additional cancel job and VMware resource changes --- .../OperationCancelledException.java | 22 +- .../org/apache/cloudstack/api/APICommand.java | 2 + .../apache/cloudstack/api/BaseAsyncCmd.java | 3 + .../apache/cloudstack/api/Cancellable.java | 29 +++ .../api/command/user/vm/BaseDeployVMCmd.java | 4 + .../api/command/user/vm/DestroyVMCmd.java | 4 + .../api/command/user/vm/RebootVMCmd.java | 4 + .../api/command/user/vm/StartVMCmd.java | 4 + .../api/command/user/vm/StopVMCmd.java | 6 + .../com/cloud/agent/transport/Request.java | 10 + .../com/cloud/agent/manager/AgentAttache.java | 27 ++- .../agent/manager/DirectAgentAttache.java | 81 ++++++- ...vmNonManagedStorageDataMotionStrategy.java | 3 +- .../StorageSystemDataMotionStrategy.java | 13 +- ...NonManagedStorageSystemDataMotionTest.java | 9 +- .../vmsnapshot/DefaultVMSnapshotStrategy.java | 7 +- .../vmsnapshot/StorageVMSnapshotStrategy.java | 8 +- .../vmsnapshot/VMSnapshotStrategyTest.java | 7 +- .../storage/RemoteHostEndPoint.java | 6 +- .../storage/helper/HypervisorHelperImpl.java | 9 +- .../image/BaseImageStoreDriverImpl.java | 3 +- .../manager/ExtensionsManagerImpl.java | 5 + .../manager/ExtensionsManagerImplTest.java | 9 +- .../jobs/impl/AsyncJobManagerImpl.java | 36 +++- .../cloudstack/backup/NASBackupProvider.java | 14 ++ .../backup/NASBackupProviderTest.java | 7 +- .../BaremetalKickStartServiceImpl.java | 3 +- .../discoverer/HypervServerDiscoverer.java | 3 + .../motion/HypervStorageMotionStrategy.java | 4 + .../vmware/manager/VmwareManagerImpl.java | 3 +- .../vmware/resource/VmwareResource.java | 197 +++++++++++++++++- .../motion/VmwareStorageMotionStrategy.java | 7 + .../lb/ElasticLoadBalancerManagerImpl.java | 4 + .../lb/InternalLoadBalancerVMManagerImpl.java | 4 + .../InternalLBVMManagerTest.java | 6 +- .../management/ManagementServerMock.java | 3 + .../network/ovs/OvsTunnelManagerImpl.java | 9 +- .../manager/ScaleIOSDCManagerImpl.java | 5 +- .../motion/StorPoolDataMotionStrategy.java | 6 +- .../vmware/mo/VirtualMachineMO.java | 79 +++++-- .../hypervisor/vmware/util/VmwareClient.java | 90 +++++++- .../hypervisor/vmware/util/VmwareContext.java | 9 + .../hypervisor/vmware/util/VmwareHelper.java | 143 +++++++++++++ 43 files changed, 823 insertions(+), 84 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/Cancellable.java diff --git a/api/src/main/java/com/cloud/exception/OperationCancelledException.java b/api/src/main/java/com/cloud/exception/OperationCancelledException.java index 444d652a5a0c..8e6393b28fad 100644 --- a/api/src/main/java/com/cloud/exception/OperationCancelledException.java +++ b/api/src/main/java/com/cloud/exception/OperationCancelledException.java @@ -26,17 +26,23 @@ public class OperationCancelledException extends CloudException { long _agentId; long _seqId; int _time; + boolean _isActive; + String _reason; transient Command[] _cmds; - boolean _isActive; - public OperationCancelledException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) { + public OperationCancelledException(Command[] cmds, long agentId, long seqId, int time, boolean isActive, String reason) { super("Commands: " + Arrays.toString(cmds) + " to Host " + agentId + " with seqId " + seqId + " cancelled after " + time + " secs"); _agentId = agentId; _seqId = seqId; _time = time; _cmds = cmds; _isActive = isActive; + _reason = reason; + } + + public OperationCancelledException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) { + this(cmds, agentId, seqId, time, isActive, null); } public long getAgentId() { @@ -51,11 +57,15 @@ public int getWaitTime() { return _time; } - public Command[] getCommands() { - return _cmds; - } - public boolean isActive() { return _isActive; } + + public String getReason() { + return _reason; + } + + public Command[] getCommands() { + return _cmds; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/APICommand.java b/api/src/main/java/org/apache/cloudstack/api/APICommand.java index b77649046ca9..ee841b4e0ffa 100644 --- a/api/src/main/java/org/apache/cloudstack/api/APICommand.java +++ b/api/src/main/java/org/apache/cloudstack/api/APICommand.java @@ -52,4 +52,6 @@ Class[] entityType() default {}; String httpMethod() default ""; + + boolean cancellable() default false; } diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java index 6859b0a7f406..64ba8d390415 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java @@ -84,4 +84,7 @@ public Object getJob() { return job; } + public boolean isCancellable() { + return false; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/Cancellable.java b/api/src/main/java/org/apache/cloudstack/api/Cancellable.java new file mode 100644 index 000000000000..d12b4fa9184b --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/Cancellable.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.TYPE; + +@Retention(RetentionPolicy.RUNTIME) +@Target({TYPE}) +public @interface Cancellable { + boolean value() default true; +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index e9621fa484a1..e331078ae778 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -876,4 +876,8 @@ public String getEventDescription() { public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.VirtualMachine; } + + public boolean isCancellable() { + return true; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index 7ac12851cf9c..debe9f309d83 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -142,6 +142,10 @@ public Long getApiResourceId() { return getId(); } + public boolean isCancellable() { + return true; + } + @Override public void execute() throws ResourceUnavailableException, ConcurrentOperationException { CallContext.current().setEventDetails("Instance ID: " + getResourceUuid(ApiConstants.ID)); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java index 65ff55e674e3..c681fb8c2174 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java @@ -114,6 +114,10 @@ public Long getApiResourceId() { return getId(); } + public boolean isCancellable() { + return true; + } + @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { CallContext.current().setEventDetails("Instance ID: " + getResourceUuid(ApiConstants.ID)); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java index 53d93d3d65fe..d3de77a3ffd2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java @@ -175,6 +175,10 @@ public Long getApiResourceId() { return getId(); } + public boolean isCancellable() { + return true; + } + @Override public void execute() { try { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java index f383a9d6c41e..eb3478b6b063 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Cancellable; import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; @@ -38,6 +39,7 @@ import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; +@Cancellable @APICommand(name = "stopVirtualMachine", responseObject = UserVmResponse.class, description = "Stops an Instance.", responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) public class StopVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { @@ -114,6 +116,10 @@ public boolean isForced() { return (forced != null) ? forced : false; } + public boolean isCancellable() { + return true; + } + @Override public void execute() throws ServerApiException, ConcurrentOperationException { CallContext.current().setEventDetails("Instance ID: " + getResourceUuid(ApiConstants.ID)); diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index 3769dbbd612c..199a7e8b9865 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -116,6 +116,8 @@ public static Version get(final byte ver) throws UnsupportedVersionException { protected String _content; protected String _agentName; + private volatile boolean _cancelled = false; + protected Request() { } @@ -262,6 +264,14 @@ public Command[] getCommands() { return _cmds; } + public void cancel() { + _cancelled = true; + } + + public boolean isCancelled() { + return _cancelled; + } + protected String getType() { return "Cmd "; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index d181fd06dac3..3da343e6235a 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -433,13 +433,22 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE try { answers = sl.waitFor(wait); } catch (final InterruptedException e) { - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted"); + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted while waiting for job commands processing"); + Thread.currentThread().interrupt(); if (_agentMgr._asyncJobDao.isJobCancelled(jobId)) { - throw new OperationCancelledException(req.getCommands(), _id, seq, wait, false); + throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, "Cancelled during waiting for job commands processing"); } } } if (answers != null) { + for (Answer answer : answers) { + if (answer != null && !answer.getResult() && + answer.getDetails() != null && + answer.getDetails().contains("cancelled")) { + throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, answer.getDetails()); + } + } + new Response(req, answers).logD("Received: ", false); return answers; } @@ -472,6 +481,14 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE sendNext(seq); } throw e; + } catch (OperationCancelledException e) { + logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Operation cancelled: " + req.toString()); + cancel(seq); + final Long current = _currentSequence; + if (req.executeInSequence() && (current != null && current == seq)) { + sendNext(seq); + } + throw e; } catch (Exception e) { logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Exception while waiting for answer", e); cancel(seq); @@ -480,9 +497,9 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE sendNext(seq); } _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); - if (e instanceof OperationCancelledException) { - throw e; - } +// if (e instanceof OperationCancelledException) { +// throw e; +// } throw new OperationTimedoutException(req.getCommands(), _id, seq, wait, false); } finally { unregisterListener(seq); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index 2edc9ad19bc3..5f28994c889a 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -19,6 +19,9 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -48,6 +51,7 @@ public class DirectAgentAttache extends AgentAttache { "Interval to wait before retrying a host ping while waiting for check results", true); ServerResource _resource; List> _futures = new ArrayList>(); + private final Map> _taskFutures = new ConcurrentHashMap<>(); long _seq = 0; LinkedList tasks = new LinkedList(); AtomicInteger _outstandingTaskCount; @@ -68,12 +72,24 @@ public void disconnect(Status state) { future.cancel(false); } + synchronized (this) { + tasks.clear(); + } + + for (Future future : _taskFutures.values()) { + boolean cancelled = future.cancel(true); + logger.debug("Running task {} for [id: {}, uuid: {}, name: {}]", cancelled ? "cancelled" : "not cancelled", _id, _uuid, _name); + } + _taskFutures.clear(); + synchronized (this) { if (_resource != null) { _resource.disconnected(); _resource = null; } } + + cleanup(state); } @Override @@ -146,7 +162,9 @@ private synchronized void scheduleFromQueue() { _id, _uuid, _name, tasks.size(), _outstandingTaskCount.get()); while (!tasks.isEmpty() && _outstandingTaskCount.get() < _agentMgr.getDirectAgentThreadCap()) { _outstandingTaskCount.incrementAndGet(); - _agentMgr.getDirectAgentPool().execute(tasks.remove()); + Task task = tasks.remove(); + Future future = _agentMgr.getDirectAgentPool().submit(task); + _taskFutures.put(task._req.getSequence(), future); } } @@ -294,13 +312,34 @@ public Task(Request req) { protected void runInContext() { long seq = _req.getSequence(); try { - ServerResource resource = _resource; Command[] cmds = _req.getCommands(); + if (Thread.currentThread().isInterrupted() || _req.isCancelled()) { + throw new InterruptedException("Task execution cancelled"); +// Answer[] answers = new Answer[cmds.length]; +// for (int i = 0; i < cmds.length; i++) { +// answers[i] = new Answer(cmds[i], false, "Task cancelled"); +// } +// Response resp = new Response(_req, answers); +// processAnswers(seq, resp); + +// handleCancellation(seq, false, "Request cancelled before execution"); +// return; + } + + ServerResource resource = _resource; boolean stopOnError = _req.stopOnError(); logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Executing request"); ArrayList answers = new ArrayList(cmds.length); for (int i = 0; i < cmds.length; i++) { + if (Thread.currentThread().isInterrupted() || _req.isCancelled()) { +// throw new InterruptedException("Task execution cancelled"); + for (int j = i; j < cmds.length; j++) { + answers.add(new Answer(cmds[j], false, "Command cancelled")); + } + break; + } + Answer answer = null; Command currentCmd = cmds[i]; if (currentCmd.getContextParam("logid") != null) { @@ -316,6 +355,24 @@ protected void runInContext() { } else { answer = new Answer(cmds[i], false, "Agent is disconnected"); } +// } catch (InterruptedException e) { +// logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Command interrupted: " + cmds[i].getClass().getSimpleName()); +// Thread.currentThread().interrupt(); + +// boolean cancelled = resource.cancelActiveVmTask(cmds[i]); //cmdSequence +// if (cancelled) { +// logger.info("Successfully cancelled task: " + cmds[i]); +// } +// int cancelled = resource.cancelActiveVmTasks(vmName); +// logger.info("Cancelled " + cancelled + " tasks for VM: " + vmName); + +// answer = new Answer(cmds[i], false, "Command execution cancelled: " + e.getMessage()); +// answers.add(answer); +// +// for (int j = i + 1; j < cmds.length; j++) { +// answers.add(new Answer(cmds[j], false, "Command cancelled - previous command was interrupted")); +// } +// break; } catch (Throwable t) { // Catch Throwable as all exceptions will otherwise be eaten by the executor framework logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Throwable caught while executing command", t); @@ -334,13 +391,33 @@ protected void runInContext() { logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Response Received: "); processAnswers(seq, resp); + } catch (InterruptedException e) { + logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Task interrupted"); + Thread.currentThread().interrupt(); + // Send cancellation response + handleCancellation(seq, true, "Task interrupted: " + e.getMessage()); } catch (Throwable t) { // This is pretty serious as processAnswers might not be called and the calling process is stuck waiting for the full timeout logger.error(LOG_SEQ_FORMATTED_STRING, seq, "Throwable caught in runInContext, this will cause the management to become unpredictable", t); } finally { + _taskFutures.remove(seq); _outstandingTaskCount.decrementAndGet(); scheduleFromQueue(); } } + + private void handleCancellation(long seq, boolean isActive, String reason) { + try { + Command[] cmds = _req.getCommands(); + ArrayList answers = new ArrayList<>(cmds.length); + for (Command cmd : cmds) { + answers.add(new Answer(cmd, false, reason != null ? reason : "Task cancelled")); + } + Response resp = new Response(_req, answers.toArray(new Answer[answers.size()])); + processAnswers(seq, resp); + } catch (Exception e) { + logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Exception while handling cancellation", e); + } + } } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index 867470dac040..a6198b26870c 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; @@ -277,7 +278,7 @@ protected Answer sendCopyCommand(Host destHost, TemplateObjectTO sourceTemplate, Answer copyCommandAnswer = agentManager.send(destHost.getId(), copyCommand); logInCaseOfTemplateCopyFailure(copyCommandAnswer, sourceTemplate, destDataStore); return copyCommandAnswer; - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException(generateFailToCopyTemplateMessage(sourceTemplate, destDataStore), e); } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 7674f1ce25a1..86f628891128 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -35,6 +35,7 @@ import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.PrepareForMigrationAnswer; +import com.cloud.exception.OperationCancelledException; import com.cloud.resource.ResourceManager; import com.cloud.storage.clvm.ClvmPoolManager; import org.apache.cloudstack.backup.InternalBackupService; @@ -1884,7 +1885,7 @@ private CopyCmdAnswer copyImageToVolume(DataObject srcDataObject, VolumeInfo des copyCmdAnswer = (CopyCmdAnswer)agentManager.send(hostVO.getId(), copyCommand); } - catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException | OperationCancelledException ex) { String msg = "Failed to copy image : "; logger.warn(msg, ex); @@ -2159,6 +2160,8 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Operation timed out", destHost.getId()); + } catch (final OperationCancelledException e) { + throw new AgentUnavailableException("Operation cancelled", destHost.getId()); } for (VolumeInfo vol : samePoolClvmVolumes) { @@ -2206,7 +2209,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach try { migrateAnswer = (MigrateAnswer)agentManager.send(srcHost.getId(), migrateCommand); success = migrateAnswer != null && migrateAnswer.getResult(); - } catch (OperationTimedoutException ex) { + } catch (OperationTimedoutException | OperationCancelledException ex) { if (HypervisorType.KVM.equals(vm.getHypervisorType())) { final Answer answer = agentManager.send(destHost.getId(), new CheckVirtualMachineCommand(vm.getInstanceName())); if (answer != null && answer.getResult() && answer instanceof CheckVirtualMachineAnswer) { @@ -2235,7 +2238,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException(errMsg); } } - } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException | CloudRuntimeException ex) { String volumesAndStorages = volumeDataStoreMap.entrySet().stream().map(entry -> formatEntryOfVolumesAndStoragesAsJsonToDisplayOnLog(entry)).collect(Collectors.joining(",")); errMsg = String.format("Copy volume(s) to storage(s) [%s] and VM to host [%s] failed in StorageSystemDataMotionStrategy.copyAsync. Error message: [%s].", volumesAndStorages, formatMigrationElementsAsJsonToDisplayOnLog("vm", vmTO.getId(), srcHost.getId(), destHost.getId()), ex.getMessage()); @@ -3119,7 +3122,7 @@ private CopyCmdAnswer performResignature(DataObject dataObj, HostVO hostVO, Map< answer = (ResignatureAnswer)agentManager.send(hostVO.getId(), command); } - catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException | OperationCancelledException ex) { keepGrantedAccess = false; String msg = "Failed to resign the DataObject with the following ID: " + dataObj.getId(); @@ -3328,7 +3331,7 @@ private CopyCmdAnswer performCopyOfVdi(VolumeInfo volumeInfo, SnapshotInfo snaps copyCmdAnswer = (CopyCmdAnswer)agentManager.send(hostVO.getId(), copyCommand); } - catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException | OperationCancelledException ex) { String msg = "Failed to perform VDI copy : "; logger.warn(msg, ex); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 6f0776b27c8c..9af60b18f85a 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.Map; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -281,21 +282,21 @@ public void shouldMigrateVolumeTest() { } @Test - public void sendCopyCommandTest() throws AgentUnavailableException, OperationTimedoutException { + public void sendCopyCommandTest() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { configureAndTestSendCommandTest(null); } @Test(expected = CloudRuntimeException.class) - public void sendCopyCommandTestThrowAgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException { + public void sendCopyCommandTestThrowAgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { configureAndTestSendCommandTest(AgentUnavailableException.class); } @Test(expected = CloudRuntimeException.class) - public void sendCopyCommandTestThrowOperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException { + public void sendCopyCommandTestThrowOperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { configureAndTestSendCommandTest(OperationTimedoutException.class); } - private void configureAndTestSendCommandTest(Class exception) throws AgentUnavailableException, OperationTimedoutException { + private void configureAndTestSendCommandTest(Class exception) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Host destHost = new HostVO("guid"); TemplateObjectTO sourceTemplate = new TemplateObjectTO(); sourceTemplate.setName("name"); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index 665c3a4659ca..5f0e05637e64 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Snapshot; import com.cloud.storage.Storage; @@ -208,6 +209,10 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { } catch (OperationTimedoutException e) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); + } catch (OperationCancelledException e) { + logger.debug("Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); + throw new CloudRuntimeException("Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); + } catch (AgentUnavailableException e) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed", e); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); @@ -449,7 +454,7 @@ public boolean revertVMSnapshot(VMSnapshot vmSnapshot) { logger.error(errMsg); throw new CloudRuntimeException(String.format("Unable to revert VM %s to snapshot %s.", userVm.getInstanceName(), vmSnapshotVO.getName())); } - } catch (OperationTimedoutException e) { + } catch (OperationTimedoutException | OperationCancelledException e) { logger.debug("Failed to revert Instance Snapshot", e); throw new CloudRuntimeException(e.getMessage()); } catch (AgentUnavailableException e) { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java index fd306414ad49..76e88cb4a02d 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java @@ -26,6 +26,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; @@ -210,6 +211,11 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); throw new CloudRuntimeException( "Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); + } catch (OperationCancelledException e) { + logger.debug("Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); + throw new CloudRuntimeException( + "Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); + } catch (AgentUnavailableException e) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed", e); throw new CloudRuntimeException( @@ -222,7 +228,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { .convert(elapsedTime(startFreeze), TimeUnit.NANOSECONDS))); try { thawAnswer = (FreezeThawVMAnswer) agentMgr.send(hostId, thawCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.debug("Could not unfreeze the VM due to " + e); } } diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java index a20b52fac2dd..6e1b837bc3e5 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java @@ -25,6 +25,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import com.cloud.storage.dao.SnapshotDao; import com.cloud.vm.snapshot.dao.VMSnapshotDetailsDao; import org.apache.cloudstack.backup.BackupManager; @@ -107,7 +108,7 @@ public void setUp() { } @Test - public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTimedoutException { + public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long hostId = 1L; Long vmId = 1L; Long guestOsId = 1L; @@ -151,7 +152,7 @@ public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTi } @Test - public void testRevertSnapshot() throws AgentUnavailableException, OperationTimedoutException { + public void testRevertSnapshot() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long hostId = 1L; Long vmId = 1L; Long guestOsId = 1L; @@ -199,7 +200,7 @@ public void testRevertSnapshot() throws AgentUnavailableException, OperationTime } @Test - public void testDeleteVMSnapshot() throws AgentUnavailableException, OperationTimedoutException { + public void testDeleteVMSnapshot() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long hostId = 1L; Long vmId = 1L; Long guestOsId = 1L; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java index bd4bce29b0a0..2b5c293c832c 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -131,10 +132,7 @@ public Answer sendMessage(Command cmd) { setId(newHostId); } return agentMgr.send(newHostId, cmd); - } catch (AgentUnavailableException e) { - errMsg = e.toString(); - logger.debug("Failed to send command, due to Agent:" + getId() + ", " + e.toString()); - } catch (OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { errMsg = e.toString(); logger.debug("Failed to send command, due to Agent:" + getId() + ", " + e.toString()); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java index 10af5d55d619..2b17deb2453c 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -139,9 +140,7 @@ public VMSnapshotTO quiesceVm(VirtualMachine virtualMachine) { String errMsg = (answer != null) ? answer.getDetails() : null; throw new CloudRuntimeException("Failed to quiesce vm, due to " + errMsg); } - } catch (AgentUnavailableException e) { - throw new CloudRuntimeException("Failed to quiesce vm", e); - } catch (OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException("Failed to quiesce vm", e); } return vmSnapshotTO; @@ -162,9 +161,7 @@ public boolean unquiesceVM(VirtualMachine virtualMachine, VMSnapshotTO vmSnapsho String errMsg = (answer != null) ? answer.getDetails() : null; throw new CloudRuntimeException("Failed to unquiesce vm, due to " + errMsg); } - } catch (AgentUnavailableException e) { - throw new CloudRuntimeException("Failed to unquiesce vm", e); - } catch (OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException("Failed to unquiesce vm", e); } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 26b39e30776f..81f43380099a 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -29,6 +29,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -447,7 +448,7 @@ private Answer sendToLeastBusyEndpoint(List eps, CopyCommand cmd) { answer = agentMgr.send(endPoint.getId(), cmd); answer.setContextParam("cmd", cmdExecId.toString()); return answer; - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { errMsg = e.toString(); logger.debug("Failed to send command, due to Agent [id: {}, uuid: {}]: {}", endPoint.getId(), endPoint.getUuid(), e.toString()); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index a209acf6442e..0d02b8de1508 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -48,6 +48,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; @@ -1958,6 +1959,10 @@ private CustomActionResultResponse runVirtualMachineCustomAction(VirtualMachine String msg = "Running custom action timed out, please try again"; logger.error(msg, e); result.put(ApiConstants.DETAILS, msg); + } catch (OperationCancelledException e) { + String msg = "Running custom action cancelled"; + logger.error(msg, e); + result.put(ApiConstants.DETAILS, msg); } response.setResult(result); return response; diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java index fa8c97742697..c16dbf407d74 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java @@ -121,6 +121,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; @@ -486,7 +487,7 @@ public void disableExtensionChangesStateToDisabled() { } @Test - public void updateAllExtensionHostsRemovesHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException { + public void updateAllExtensionHostsRemovesHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Extension extension = mock(Extension.class); when(extension.getId()).thenReturn(1L); Long clusterId = 100L; @@ -497,7 +498,7 @@ public void updateAllExtensionHostsRemovesHostsSuccessfully() throws OperationTi } @Test - public void updateAllExtensionHostsAddsHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException { + public void updateAllExtensionHostsAddsHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Extension extension = mock(Extension.class); when(extension.getId()).thenReturn(1L); Long clusterId = 100L; @@ -508,7 +509,7 @@ public void updateAllExtensionHostsAddsHostsSuccessfully() throws OperationTimed } @Test - public void updateAllExtensionHostsHandlesEmptyHostListGracefully() throws OperationTimedoutException, AgentUnavailableException { + public void updateAllExtensionHostsHandlesEmptyHostListGracefully() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Extension extension = mock(Extension.class); Long clusterId = 100L; when(hostDao.listIdsByClusterId(clusterId)).thenReturn(Collections.emptyList()); @@ -517,7 +518,7 @@ public void updateAllExtensionHostsHandlesEmptyHostListGracefully() throws Opera } @Test - public void updateAllExtensionHostsHandlesNullClusterId() throws OperationTimedoutException, AgentUnavailableException { + public void updateAllExtensionHostsHandlesNullClusterId() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Extension extension = mock(Extension.class); when(extension.getId()).thenReturn(1L); when(extensionResourceMapDao.listResourceIdsByExtensionIdAndType(eq(1L), any())).thenReturn(Collections.emptyList()); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 5e55a2f2594f..7c83ab9f61b7 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -20,6 +20,7 @@ import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -40,8 +41,13 @@ import com.cloud.vm.snapshot.VMSnapshotService; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Cancellable; +import org.apache.cloudstack.api.CancellableCmd; +import org.apache.cloudstack.api.command.user.job.CancelAsyncJobCmd; import org.apache.cloudstack.command.ReconcileCommandService; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -341,7 +347,6 @@ public void completeAsyncJob(final long jobId, final Status jobStatus, final int logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus + ", resultCode: " + resultCode + ", result: " + resultObj); } - final AsyncJobVO job = _jobDao.findById(jobId); if (job == null) { if (logger.isDebugEnabled()) { @@ -1428,7 +1433,26 @@ public String cancelAsyncJob(long jobId, String reason) { try { Class cmdClass = Class.forName(job.getCmd()); - if (!CancellableCmd.class.isAssignableFrom(cmdClass)) { + if (BaseAsyncCmd.class.isAssignableFrom(cmdClass)) { + logger.info(cmdClass.getName() + " is a subclass of BaseAsyncCmd"); + try { + BaseAsyncCmd cmd = (BaseAsyncCmd) cmdClass.getDeclaredConstructor().newInstance(); + boolean cmdCancellable = cmd.isCancellable(); + if (cmdCancellable) { + logger.info("Cmd: " + cmdClass.getName() + " is cancellable"); + } + } catch (NoSuchMethodException e) { + logger.warn("No declared constructor for Cmd: " + cmdClass.getName() + ", not able to check if it is cancellable or not"); + } + } + + boolean cancellable = cmdClass.isAnnotationPresent(Cancellable.class); + if (cancellable) { + logger.info(cmdClass.getName() + " is cancellable"); + } + + APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); + if (apiCommand == null || !apiCommand.cancellable() || !CancellableCmd.class.isAssignableFrom(cmdClass)) { errMessage = "Cannot cancel, job-" + jobId + " as it is not cancellable."; logger.debug(errMessage); return errMessage; @@ -1437,6 +1461,12 @@ public String cancelAsyncJob(long jobId, String reason) { errMessage = "Command " + job.getCmd() + " of jobid-" + jobId + " not found."; logger.error(errMessage, e); return errMessage; + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); } if (job.getStatus() != Status.IN_PROGRESS) { @@ -1453,7 +1483,7 @@ public String cancelAsyncJob(long jobId, String reason) { _jobMonitor.unregisterByJobId(jobId); // purge the item and resume queue processing - _queueMgr.purgeItem(jobId); + _queueMgr.purgeAsyncJobQueueItemId(jobId); return ""; } catch (Throwable t) { errMessage = "Unexpected exception when cancelling async job with id: " + jobId; diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index d08b4775c8bd..d498ecac213e 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -18,6 +18,7 @@ import com.cloud.agent.AgentManager; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.configuration.Resource; import com.cloud.host.Host; @@ -609,6 +610,11 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce backupVO.setStatus(Backup.Status.Failed); backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } catch (OperationCancelledException e) { + logger.error("Operation to initiate backup cancelled for VM {}", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + throw new CloudRuntimeException("Operation to initiate backup cancelled"); } if (answer != null && answer.getResult()) { @@ -730,6 +736,8 @@ private Pair restoreVMBackup(VirtualMachine vm, Backup backup) throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); + } catch (OperationCancelledException e) { + throw new CloudRuntimeException("Operation to restore backup cancelled"); } if (answer == null) { throw new CloudRuntimeException(String.format("No answer received from the backend control plane while restoring VM %s from backup %s", vm.getInstanceName(), backup.getUuid())); @@ -856,6 +864,8 @@ public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeI throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backed up volume timed out, please try again"); + } catch (OperationCancelledException e) { + throw new CloudRuntimeException("Operation to restore backed up volume cancelled"); } if (answer.getResult()) { @@ -992,6 +1002,8 @@ private boolean deleteBackupFileAndRow(Backup backup, BackupRepository repo, Hos throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to delete backup timed out, please try again"); + } catch (OperationCancelledException e) { + throw new CloudRuntimeException("Operation to delete backup cancelled"); } if (answer == null || !answer.getResult()) { logger.error("Failed to delete backup file for {} ({}); leaving DB row intact", @@ -1232,6 +1244,8 @@ public void syncBackupStorageStats(Long zoneId) { logger.warn("Unable to contact backend control plane to get backup stats for repository: {}", repository.getName()); } catch (OperationTimedoutException e) { logger.warn("Operation to get backup stats timed out for the repository: " + repository.getName()); + } catch (OperationCancelledException e) { + logger.warn("Operation to get backup stats cancelled for the repository: " + repository.getName()); } } } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index f1d5613ab7fc..4eba917aa5ed 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -24,6 +24,7 @@ import java.util.Objects; import java.util.Optional; +import com.cloud.exception.OperationCancelledException; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.junit.Assert; import org.junit.Test; @@ -127,7 +128,7 @@ public class NASBackupProviderTest { private ResourceLimitService resourceLimitMgr; @Test - public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { + public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { Long hostId = 1L; BackupVO backup = new BackupVO(); backup.setBackupOfferingId(1L); @@ -153,7 +154,7 @@ public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailab } @Test - public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { + public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", "nfs", "address", "sync", 1024L, null); @@ -204,7 +205,7 @@ public void testGetBackupStorageStats() { } @Test - public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException { + public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long vmId = 1L; Long hostId = 2L; Long backupOfferingId = 3L; diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java index 6e1f422526d9..b9e9d3a9f059 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java @@ -29,6 +29,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.AddBaremetalKickStartPxeCmd; import org.apache.cloudstack.api.AddBaremetalPxeCmd; import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; @@ -173,7 +174,7 @@ public File getSystemVMKeyFile() { return keyFile; } - private boolean preparePxeInBasicZone(VirtualMachineProfile profile, NicProfile nic, DeployDestination dest, ReservationContext context) throws AgentUnavailableException, OperationTimedoutException { + private boolean preparePxeInBasicZone(VirtualMachineProfile profile, NicProfile nic, DeployDestination dest, ReservationContext context) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { NetworkVO nwVO = _nwDao.findById(nic.getNetworkId()); QueryBuilder sc = QueryBuilder.create(BaremetalPxeVO.class); sc.and(sc.entity().getDeviceType(), Op.EQ, BaremetalPxeType.KICK_START.toString()); diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java index a5fe1442e56e..f23958ee1d1c 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java @@ -49,6 +49,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConnectionException; import com.cloud.exception.DiscoveryException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostEnvironment; @@ -169,6 +170,8 @@ public final void processConnect(final Host agent, final StartupCommand cmd, fin logger.warn(String.format("Unable to setup agent %s because it became unavailable.", agent), e); } catch (OperationTimedoutException e) { logger.warn(String.format("Unable to setup agent %s because it timed out", agent), e); + } catch (OperationCancelledException e) { + logger.warn(String.format("Unable to setup agent %s because it is cancelled", agent), e); } throw new ConnectionException(true, "Reinitialize agent after setup."); } diff --git a/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java b/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java index 55944a082427..3565b2032c54 100644 --- a/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java +++ b/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -140,6 +141,9 @@ private Answer migrateVmWithVolumes(VMInstanceVO vm, VirtualMachineTO to, Host s } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); + } catch (OperationCancelledException e) { + logger.error("Error while migrating vm " + vm + " to host " + destHost, e); + throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 8052317817ad..bdc55d792e55 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -43,6 +43,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; import com.cloud.hypervisor.vmware.util.VmwareClient; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; @@ -1582,7 +1583,7 @@ public List listVsphereStoragePolicyCompatibleStoragePools(ListVsph if (result) { compatiblePools.add(pool); } - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Could not verify if storage policy " + storagePolicy.getName() + " is compatible with storage pool " + pool.getName()); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 89b5b111e14f..1872c1afb1b1 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -43,6 +43,7 @@ import java.util.Set; import java.util.TimeZone; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import com.cloud.agent.api.CleanupVMCommand; @@ -434,6 +435,10 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes protected VirtualRoutingResource _vrResource; + private final ConcurrentHashMap _activeVmTasks = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> _vmNameToTaskSequences = new ConcurrentHashMap<>(); + private static final ThreadLocal currentTaskContext = new ThreadLocal<>(); + protected final static HashMap s_powerStatesTable = new HashMap<>(); static { @@ -451,6 +456,148 @@ public VmwareResource() { _gson = GsonHelper.getGsonLogger(); } + public static class ActiveVmTaskInfo { + final long cmdSequence; + final String vmName; + final String commandType; + final ManagedObjectReference taskMor; + final VmwareContext context; + final long startTime; + + ActiveVmTaskInfo(long cmdSequence, String vmName, String commandType, + ManagedObjectReference taskMor, VmwareContext context) { + this.cmdSequence = cmdSequence; + this.vmName = vmName; + this.commandType = commandType; + this.taskMor = taskMor; + this.context = context; + this.startTime = System.currentTimeMillis(); + } + } + + void registerActiveVmTask(long cmdSequence, String vmName, String commandType, + ManagedObjectReference taskMor, VmwareContext context) { + ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, taskMor, context); + _activeVmTasks.put(cmdSequence, taskInfo); + + _vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); + + logger.debug("Registered active VM task: sequence={}, vm={}, command={}, task={}", + cmdSequence, vmName, commandType, taskMor.getValue()); + } + + private void unregisterActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = _activeVmTasks.remove(cmdSequence); + if (taskInfo != null) { + Set sequences = _vmNameToTaskSequences.get(taskInfo.vmName); + if (sequences != null) { + sequences.remove(cmdSequence); + if (sequences.isEmpty()) { + _vmNameToTaskSequences.remove(taskInfo.vmName); + } + } + logger.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={}ms", + cmdSequence, taskInfo.vmName, taskInfo.commandType, + System.currentTimeMillis() - taskInfo.startTime); + } + } + + public int cancelActiveVmTasks(String vmName) { + Set sequences = _vmNameToTaskSequences.get(vmName); + if (sequences == null || sequences.isEmpty()) { + return 0; + } + + int cancelled = 0; + for (Long seq : sequences) { + if (cancelActiveVmTask(seq)) { + cancelled++; + } + } + return cancelled; + } + + public boolean cancelActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = _activeVmTasks.get(cmdSequence); + if (taskInfo == null) { + return false; + } + + try { + // Cancel the vCenter task + TaskMO taskMo = new TaskMO(taskInfo.context, taskInfo.taskMor); + taskMo.cancelTask(); + logger.info("Cancelled vCenter task: sequence={}, vm={}, command={}, task={}", + cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.taskMor.getValue()); + + // Unregister the task + unregisterActiveVmTask(cmdSequence); + return true; + } catch (Exception e) { + logger.warn("Failed to cancel vCenter task: sequence={}, vm={}, error={}", + cmdSequence, taskInfo.vmName, e.getMessage(), e); + return false; + } + } + + public List getActiveVmTasks(String vmName) { + Set sequences = _vmNameToTaskSequences.get(vmName); + if (sequences == null || sequences.isEmpty()) { + return Collections.emptyList(); + } + + List tasks = new ArrayList<>(); + for (Long seq : sequences) { + ActiveVmTaskInfo taskInfo = _activeVmTasks.get(seq); + if (taskInfo != null) { + tasks.add(taskInfo); + } + } + return tasks; + } + + public List getAllActiveVmTasks() { + return new ArrayList<>(_activeVmTasks.values()); + } + + public static class TaskRegistrationContext { + private final VmwareResource resource; + private final long cmdSequence; + private final String vmName; + private final String commandType; + + public TaskRegistrationContext(VmwareResource resource, long cmdSequence, String vmName, String commandType) { + this.resource = resource; + this.cmdSequence = cmdSequence; + this.vmName = vmName; + this.commandType = commandType; + } + + public void registerTask(ManagedObjectReference taskMor, VmwareContext context) { + if (resource != null) { + resource.registerActiveVmTask(cmdSequence, vmName, commandType, taskMor, context); + } + } + + public void unregisterTask() { + if (resource != null) { + resource.unregisterActiveVmTask(cmdSequence); + } + } + } + + private void setTaskRegistrationContext(long cmdSequence, String vmName, String commandType) { + currentTaskContext.set(new TaskRegistrationContext(this, cmdSequence, vmName, commandType)); + } + + private void clearTaskRegistrationContext() { + currentTaskContext.remove(); + } + + public static TaskRegistrationContext getCurrentTaskContext() { + return currentTaskContext.get(); + } + private String getCommandLogTitle(Command cmd) { StringBuffer sb = new StringBuffer(); if (_hostName != null) { @@ -467,10 +614,20 @@ private String getCommandLogTitle(Command cmd) { @Override public Answer executeRequest(Command cmd) { + if (Thread.currentThread().isInterrupted()) { + String msg = "Command " + cmd.getClass().getSimpleName() + " was cancelled before execution"; + logger.warn(msg); + return new Answer(cmd, false, msg); + } + logCommand(cmd); Answer answer; ThreadContext.push(getCommandLogTitle(cmd)); try { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Command execution cancelled"); + } + long cmdSequence = _cmdSequence++; Date startTime = DateUtil.currentGMTTime(); PropertyMapDynamicBean mbean = new PropertyMapDynamicBean(); @@ -499,6 +656,11 @@ public Answer executeRequest(Command cmd) { } else if (clz == CheckHealthCommand.class) { answer = execute((CheckHealthCommand) cmd); } else if (clz == StopCommand.class) { + String vmName = null; + String commandType = cmd.getClass().getSimpleName(); + vmName = ((StopCommand) cmd).getVmName(); + setTaskRegistrationContext(cmdSequence, vmName, commandType); + VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); answer = execute((StopCommand) cmd); } else if (clz == RebootRouterCommand.class) { answer = execute((RebootRouterCommand) cmd); @@ -509,6 +671,11 @@ public Answer executeRequest(Command cmd) { } else if (clz == PrepareForMigrationCommand.class) { answer = execute((PrepareForMigrationCommand) cmd); } else if (clz == MigrateCommand.class) { + String vmName = null; + String commandType = cmd.getClass().getSimpleName(); + vmName = ((MigrateCommand) cmd).getVmName(); + setTaskRegistrationContext(cmdSequence, vmName, commandType); + VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); answer = execute((MigrateCommand) cmd); } else if (clz == MigrateVmToPoolCommand.class) { answer = execute((MigrateVmToPoolCommand) cmd); @@ -565,6 +732,11 @@ public Answer executeRequest(Command cmd) { } else if (clz == NetworkUsageCommand.class) { answer = execute((NetworkUsageCommand) cmd); } else if (clz == StartCommand.class) { + String vmName = null; + String commandType = cmd.getClass().getSimpleName(); + vmName = ((StartCommand) cmd).getVirtualMachine().getName(); + setTaskRegistrationContext(cmdSequence, vmName, commandType); + VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); answer = execute((StartCommand) cmd); } else if (clz == CheckSshCommand.class) { answer = execute((CheckSshCommand) cmd); @@ -647,8 +819,13 @@ public Answer executeRequest(Command cmd) { logger.trace("Unable to register JMX monitoring due to exception " + ExceptionUtil.toString(e)); } } - + } catch (InterruptedException e) { + logger.warn("Command execution interrupted: " + cmd.getClass().getSimpleName()); + Thread.currentThread().interrupt(); // Restore interrupt status + return new Answer(cmd, false, "Command execution was cancelled: " + e.getMessage()); } finally { + clearTaskRegistrationContext(); + VmwareHelper.clearTaskContext(); recycleServiceContext(); ThreadContext.pop(); } @@ -2653,6 +2830,12 @@ protected StartAnswer execute(StartCommand cmd) { return startAnswer; } catch (Throwable e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + logger.warn("StartCommand interrupted"); + return new StartAnswer(cmd, "StartCommand cancelled: " + e.getMessage()); + } + StartAnswer startAnswer = new StartAnswer(cmd, createLogMessageException(e, cmd)); if (vmAlreadyExistsInVcenter) { startAnswer.setContextParam("stopRetry", "true"); @@ -2673,6 +2856,12 @@ protected StartAnswer execute(StartCommand cmd) { } } + private void checkCancellation() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Command cancelled"); + } + } + private Pair getVolumeDatastoreDetails(DiskTO vol, HashMap> dataStoresDetails) throws Exception { boolean managed = false; String iScsiName = null; @@ -6052,6 +6241,12 @@ public CopyVolumeAnswer execute(CopyVolumeCommand cmd) { @Override public void disconnected() { + List tasks = getAllActiveVmTasks(); + for (ActiveVmTaskInfo task : tasks) { + cancelActiveVmTask(task.cmdSequence); + } + _activeVmTasks.clear(); + _vmNameToTaskSequences.clear(); } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index d2d319ed9d03..540169151f97 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -27,6 +27,7 @@ import javax.inject.Inject; import com.cloud.agent.api.to.DiskTO; +import com.cloud.exception.OperationCancelledException; import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; @@ -380,6 +381,9 @@ private Answer migrateVmWithVolumesAcrossCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); + } catch (OperationCancelledException e) { + logger.error("Error while migrating vm " + vm + " to host " + destHost, e); + throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } @@ -413,6 +417,9 @@ private Answer migrateVmWithVolumesWithinCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); + } catch (OperationCancelledException e) { + logger.error("Error while migrating vm " + vm + " to host " + destHost, e); + throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } diff --git a/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index 9dc083015c3b..e2cdea6dad7f 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -30,6 +30,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -170,6 +171,9 @@ private boolean sendCommandsToRouter(final DomainRouterVO elbVm, Commands cmds) } catch (OperationTimedoutException e) { logger.warn("ELB: Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual elbVm ", elbVm.getHostId(), e); + } catch (OperationCancelledException e) { + logger.warn("ELB: Cancelled", e); + throw new AgentUnavailableException("Unable to send commands to virtual elbVm ", elbVm.getHostId(), e); } if (answers == null) { diff --git a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index cda139b5a62b..29df6ea5df6e 100644 --- a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -36,6 +36,7 @@ import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -974,6 +975,9 @@ protected boolean sendCommandsToInternalLbVm(final VirtualRouter internalLbVm, f } catch (final OperationTimedoutException e) { logger.warn("Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", internalLbVm.getHostId(), e); + } catch (final OperationCancelledException e) { + logger.warn("Cancelled", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", internalLbVm.getHostId(), e); } if (answers == null) { diff --git a/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java b/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java index 1550649f847a..9b80008004e6 100644 --- a/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java +++ b/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java @@ -26,6 +26,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; import org.apache.cloudstack.network.lb.InternalLoadBalancerVMManager; import org.junit.Before; @@ -155,10 +156,7 @@ public void setUp() { try { Mockito.when(_agentMgr.send(nullable(Long.class), nullable(Commands.class))).thenReturn(answers); - } catch (final AgentUnavailableException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (final OperationTimedoutException e) { + } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java index 2107850c36be..9b899b7d39dc 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java @@ -53,6 +53,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.OperationCancelledException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -247,6 +248,8 @@ private void sendCommands(Answer callback) { logger.warn("no agent running", e); } catch (OperationTimedoutException e) { logger.warn("agent not responding (in time)", e); + } catch (OperationCancelledException e) { + logger.warn("cancelled", e); } } diff --git a/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java b/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java index 804f29c01b1e..67b7ff00566f 100644 --- a/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java +++ b/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java @@ -26,6 +26,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageSubscriber; @@ -231,7 +232,7 @@ private void handleCreateTunnelAnswer(Answer[] answers) { } private String getGreEndpointIP(Host host, Network nw) - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { String endpointIp = null; // Fetch fefault name for network label from configuration String physNetLabel = _configDao.getValue(Config.OvsTunnelNetworkDefaultLabel.key()); @@ -383,6 +384,8 @@ protected void checkAndCreateTunnel(Network nw, Host host) { } catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) { // I really thing we should do a better handling of these exceptions logger.warn("Ovs Tunnel network created tunnel failed", e); + } catch (OperationCancelledException e) { + logger.warn("Ovs Tunnel network created tunnel cancelled", e); } } @@ -561,6 +564,8 @@ protected void checkAndCreateVpcTunnelNetworks(Host host, long vpcId) { handleSetupBridgeAnswer(answers); } catch (OperationTimedoutException | AgentUnavailableException e) { logger.warn("Ovs Tunnel network created bridge failed", e); + } catch (OperationCancelledException e) { + logger.warn("Ovs Tunnel network created bridge cancelled", e); } // now that bridge is setup, populate network acl's before the VM gets created @@ -644,6 +649,8 @@ protected void checkAndCreateVpcTunnelNetworks(Host host, long vpcId) { } catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) { // I really thing we should do a better handling of these exceptions logger.warn("Ovs Tunnel network created tunnel failed", e); + } catch (OperationCancelledException e) { + logger.warn("Ovs Tunnel network created tunnel cancelled", e); } } } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java index cb111e9ccac5..6b48b04e99ef 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.framework.config.ConfigKey; @@ -216,7 +217,7 @@ private String prepareSDCOnHost(Host host, DataStore dataStore, String systemId, PrepareStorageClientAnswer prepareStorageClientAnswer; try { prepareStorageClientAnswer = (PrepareStorageClientAnswer) agentManager.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String err = String.format("Failed to prepare SDC on the host %s, due to: %s", host, e.getMessage()); logger.error(err); throw new CloudRuntimeException(err); @@ -333,7 +334,7 @@ private boolean unprepareSDCOnHost(Host host, DataStore dataStore, String mdms) Answer unprepareStorageClientAnswer; try { unprepareStorageClientAnswer = agentManager.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Failed to unprepare SDC on the host {} due to: {}", host, e.getMessage()); return false; } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java index 8d3dc868276c..88d87136aafb 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java @@ -33,6 +33,7 @@ import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -354,6 +355,9 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } catch (final OperationTimedoutException e) { errMsg = String.format("Operation timed out due to %s", e.getMessage()); throw new AgentUnavailableException(errMsg, destHost.getId()); + } catch (final OperationCancelledException e) { + errMsg = String.format("Operation cancelled due to %s", e.getMessage()); + throw new AgentUnavailableException(errMsg, destHost.getId()); } VMInstanceVO vm = _vmDao.findById(vmTO.getId()); @@ -386,7 +390,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException(errMsg); } - } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException | CloudRuntimeException ex) { errMsg = String.format( "Copy volume(s) of VM [%s] to storage(s) [%s] and VM to host [%s] failed in StorPoolDataMotionStrategy.copyAsync. Error message: [%s].", diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 288a108249e1..4fbbf9c1d889 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -226,6 +226,17 @@ public boolean powerOn() throws Exception { return true; ManagedObjectReference morTask = _context.getService().powerOnVMTask(_mor, null); + +// VmwareResource.TaskRegistrationContext taskCtx = VmwareResource.getCurrentTaskContext(); +// if (taskCtx != null) { +// taskCtx.registerTask(morTask, _context); +// } + + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); + } + // Monitor VM questions final Boolean[] flags = {false}; final VirtualMachineMO vmMo = this; @@ -293,9 +304,27 @@ public void run() { } else { logger.error("VMware powerOnVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("powerOn interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel powerOn task: " + ex.getMessage()); + } + throw e; } finally { +// if (taskCtx != null) { +// taskCtx.unregisterTask(); +// } + if (taskCtx != null) { + taskCtx.unregisterTask(); + } // make sure to let VM question monitor exit flags[0] = true; + if (future != null) { + future.cancel(true); + } } return false; @@ -346,9 +375,10 @@ public boolean safePowerOff(int shutdownWaitMs) throws Exception { private boolean powerOffNoCheck() throws Exception { ManagedObjectReference morTask = _context.getService().powerOffVMTask(_mor); - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); + try { + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); // It seems that even if a power-off task is returned done, VM state may still not be marked, // wait up to 5 seconds to make sure to avoid race conditioning for immediate following on operations @@ -369,10 +399,20 @@ private boolean powerOffNoCheck() throws Exception { return true; } - logger.error("VMware powerOffVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } + logger.error("VMware powerOffVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } - return false; + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("powerOff interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel powerOff task: " + ex.getMessage()); + } + throw e; + } } public VirtualMachinePowerState getResetSafePowerState() throws Exception { @@ -443,15 +483,26 @@ public boolean isTemplate() throws Exception { public boolean migrate(ManagedObjectReference morRp, ManagedObjectReference morTargetHost) throws Exception { ManagedObjectReference morTask = _context.getService().migrateVMTask(_mor, morRp, morTargetHost, VirtualMachineMovePriority.DEFAULT_PRIORITY, null); - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware migrateVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } + try { + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware migrateVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } - return false; + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("migrate interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel migrate task: " + ex.getMessage()); + } + throw e; + } } public boolean changeDatastore(VirtualMachineRelocateSpec relocateSpec) throws Exception { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 0e46f2e03cd6..eb47e8f14c3a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -37,6 +37,7 @@ import javax.xml.ws.handler.PortInfo; +import com.cloud.utils.Pair; import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; @@ -415,10 +416,39 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa boolean retVal = false; + if (Thread.currentThread().isInterrupted()) { + Thread.currentThread().interrupt(); // Restore interrupt status + try { + cancelTask(task); + } catch (Exception e) { + LOGGER.warn("Failed to cancel task on interruption: " + task.getValue(), e); + } + throw new InterruptedException("Task wait cancelled for task: " + task.getValue()); +// boolean cancellable = false; +// try { +// cancellable = isTaskCancelable(task); +// } catch (Exception e) { +// LOGGER.warn("Failed to check if task is cancelable: " + task.getValue(), e); +// } +// +// if (cancellable) { +// try { +// cancelTask(task); +// LOGGER.info("Cancelled cancelable task on interruption: " + task.getValue()); +// } catch (Exception e) { +// LOGGER.warn("Failed to cancel task on interruption: " + task.getValue(), e); +// } +// throw new InterruptedException("Task wait cancelled for task: " + task.getValue()); +// } else { +// LOGGER.warn("Task " + task.getValue() + " is not cancelable, continuing execution despite interruption"); +// Thread.interrupted(); +// } + } + try { // info has a property - state for state of the task Object[] result = waitForValues(task, new String[] { "info.state", "info.error" }, new String[] { "state" }, new Object[][] { new Object[] { - TaskInfoState.SUCCESS, TaskInfoState.ERROR } }); + TaskInfoState.SUCCESS, TaskInfoState.ERROR }}, false); if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { @@ -442,7 +472,7 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa // Since task cancellation is asynchronous, wait for the task to be cancelled Object[] result = waitForValues(task, new String[] {"info.state", "info.error"}, new String[] {"state"}, - new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}); + new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}, false); if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { @@ -482,8 +512,8 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa * @throws InvalidPropertyFaultMsg * @throws InvalidCollectorVersionFaultMsg */ - private synchronized Object[] waitForValues(ManagedObjectReference objmor, String[] filterProps, String[] endWaitProps, Object[][] expectedVals) throws InvalidPropertyFaultMsg, - RuntimeFaultFaultMsg, InvalidCollectorVersionFaultMsg { + private synchronized Object[] waitForValues(ManagedObjectReference objmor, String[] filterProps, String[] endWaitProps, Object[][] expectedVals, boolean isCancelTask) throws InvalidPropertyFaultMsg, + RuntimeFaultFaultMsg, InvalidCollectorVersionFaultMsg, InterruptedException { // version string is initially null String version = ""; Object[] endVals = new Object[endWaitProps.length]; @@ -511,6 +541,22 @@ private synchronized Object[] waitForValues(ManagedObjectReference objmor, Strin List objupary = null; List propchgary = null; while (!reached) { + if (!isCancelTask && Thread.currentThread().isInterrupted()) { + try { + vimPort.destroyPropertyFilter(filterSpecRef); + } catch (Exception e) { + LOGGER.warn("Failed to destroy property filter on interruption: " + e.getMessage()); + } + Thread.currentThread().interrupt(); + if (objmor.getType().equals("Task")) { + try { + cancelTask(objmor); + } catch (Exception e) { + LOGGER.warn("Failed to cancel task on interruption: " + objmor.getValue(), e); + } + } + throw new InterruptedException("Task wait cancelled for object: " + objmor.getValue()); + } updateset = vimPort.waitForUpdatesEx(propertyCollector, version, new WaitOptions()); if (updateset == null || updateset.getFilterSet() == null) { continue; @@ -780,11 +826,32 @@ public int getVcenterSessionTimeout() { return vCenterSessionTimeout; } - public void cancelTask(ManagedObjectReference task) throws Exception { + public boolean isTaskCancelable(ManagedObjectReference task) throws Exception { + TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); + if (info == null) { + LOGGER.warn("Unable to get the task info for task: " + task.getValue()); + return false; + } + + // Task must be in a pending state and be cancelable + TaskInfoState state = info.getState(); + if (state == null) { + return false; + } + + // Only pending tasks can be cancelled + if (state.equals(TaskInfoState.SUCCESS) || state.equals(TaskInfoState.ERROR)) { + return false; + } + + return info.isCancelable(); + } + + public Pair cancelTask(ManagedObjectReference task) throws Exception { TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); if (info == null) { LOGGER.warn("Unable to get the task info, so couldn't cancel the task"); - return; + return new Pair<>(false, "Unable to get the task info, so couldn't cancel the task"); } String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown"; @@ -794,18 +861,18 @@ public void cancelTask(ManagedObjectReference task) throws Exception { if (info.getState().equals(TaskInfoState.SUCCESS)) { LOGGER.debug(taskName + " task successfully completed for the entity " + entityName + ", can't cancel it"); - return; + return new Pair<>(false, "Task successfully completed for the entity " + entityName); } if (info.getState().equals(TaskInfoState.ERROR)) { LOGGER.debug(taskName + " task execution failed for the entity " + entityName + ", can't cancel it"); - return; + return new Pair<>(false, "Task execution failed for the entity " + entityName); } LOGGER.debug(taskName + " task pending for the entity " + entityName + ", trying to cancel"); if (!info.isCancelable()) { LOGGER.warn(taskName + " task will continue to run on vCenter because it can't be cancelled"); - return; + return new Pair<>(false, "Task will continue to run because it can't be cancelled" + entityName); } LOGGER.debug("Cancelling task " + taskName + " of the entity " + entityName); @@ -813,21 +880,24 @@ public void cancelTask(ManagedObjectReference task) throws Exception { // Since task cancellation is asynchronous, wait for the task to be cancelled Object[] result = waitForValues(task, new String[] {"info.state", "info.error"}, new String[] {"state"}, - new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}); + new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}, true); if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { LOGGER.warn("Failed to cancel" + taskName + " task of the entity " + entityName + ", the task successfully completed"); + return new Pair<>(false, "Task successfully completed for the entity " + entityName); } if (result[1] instanceof LocalizedMethodFault) { MethodFault fault = ((LocalizedMethodFault)result[1]).getFault(); if (fault instanceof RequestCanceled) { LOGGER.debug(taskName + " task of the entity " + entityName + " was successfully cancelled"); + return new Pair<>(true, "Successfully cancelled"); } } else { LOGGER.warn("Couldn't cancel " + taskName + " task of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage()); } } + return new Pair<>(false, "Unable to cancel " + taskName + " of the entity " + entityName); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index c2b34aea5d93..c2b42894c459 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -345,6 +345,11 @@ public String getDatastoreNameFromPath(String inventoryPath) throws Exception { public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Exception { while (true) { + if (Thread.currentThread().isInterrupted()) { + Thread.currentThread().interrupt(); + throw new InterruptedException("Task progress wait cancelled for task: " + morTask.getValue()); + } + TaskInfo tinfo = (TaskInfo)_vimClient.getDynamicProperty(morTask, "info"); Integer progress = tinfo.getProgress(); if (progress == null) @@ -353,6 +358,10 @@ public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Excep if (progress.intValue() >= 100) break; + if (Thread.currentThread().isInterrupted()) { + Thread.currentThread().interrupt(); + throw new InterruptedException("Task progress wait cancelled for task: " + morTask.getValue()); + } Thread.sleep(1000); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 89f6d7abd7d0..419769522049 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -33,7 +33,9 @@ import java.util.HashMap; import java.util.List; import java.util.Random; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; import javax.xml.datatype.DatatypeConfigurationException; @@ -46,6 +48,7 @@ import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.hypervisor.vmware.mo.DistributedVirtualSwitchMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; +import com.cloud.hypervisor.vmware.mo.TaskMO; import com.cloud.serializer.GsonHelper; import com.cloud.storage.Volume; import com.cloud.utils.exception.CloudRuntimeException; @@ -135,6 +138,146 @@ public class VmwareHelper { public static final String MIN_VERSION_UEFI_LEGACY = "5.5"; public static final String MIN_VERSION_VMFS6 = "6.5"; + private static final ConcurrentHashMap activeVmTasks = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> vmNameToTaskSequences = new ConcurrentHashMap<>(); + private static final ThreadLocal currentTaskContext = new ThreadLocal<>(); + + public static class ActiveVmTaskInfo { + final long cmdSequence; + final String vmName; + final String commandType; + final ManagedObjectReference taskMor; + final VmwareContext context; + final long startTime; + + ActiveVmTaskInfo(long cmdSequence, String vmName, String commandType, + ManagedObjectReference taskMor, VmwareContext context) { + this.cmdSequence = cmdSequence; + this.vmName = vmName; + this.commandType = commandType; + this.taskMor = taskMor; + this.context = context; + this.startTime = System.currentTimeMillis(); + } + } + + private static void registerActiveVmTask(long cmdSequence, String vmName, String commandType, + ManagedObjectReference taskMor, VmwareContext context) { + ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, taskMor, context); + activeVmTasks.put(cmdSequence, taskInfo); + + vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); + + LOGGER.debug("Registered active VM task: sequence={}, vm={}, command={}, task={}", + cmdSequence, vmName, commandType, taskMor.getValue()); + } + + private static void unregisterActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = activeVmTasks.remove(cmdSequence); + if (taskInfo != null) { + Set sequences = vmNameToTaskSequences.get(taskInfo.vmName); + if (sequences != null) { + sequences.remove(cmdSequence); + if (sequences.isEmpty()) { + vmNameToTaskSequences.remove(taskInfo.vmName); + } + } + LOGGER.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={}ms", + cmdSequence, taskInfo.vmName, taskInfo.commandType, + System.currentTimeMillis() - taskInfo.startTime); + } + } + + public static int cancelActiveVmTasks(String vmName) { + Set sequences = vmNameToTaskSequences.get(vmName); + if (sequences == null || sequences.isEmpty()) { + return 0; + } + + int cancelled = 0; + for (Long seq : sequences) { + if (cancelActiveVmTask(seq)) { + cancelled++; + } + } + return cancelled; + } + + public static boolean cancelActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); + if (taskInfo == null) { + return false; + } + + try { + // Cancel the vCenter task + TaskMO taskMo = new TaskMO(taskInfo.context, taskInfo.taskMor); + taskMo.cancelTask(); + LOGGER.info("Cancelled vCenter task: sequence={}, vm={}, command={}, task={}", + cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.taskMor.getValue()); + + // Unregister the task + unregisterActiveVmTask(cmdSequence); + return true; + } catch (Exception e) { + LOGGER.warn("Failed to cancel vCenter task: sequence={}, vm={}, error={}", + cmdSequence, taskInfo.vmName, e.getMessage(), e); + return false; + } + } + + public static List getActiveVmTasks(String vmName) { + Set sequences = vmNameToTaskSequences.get(vmName); + if (sequences == null || sequences.isEmpty()) { + return Collections.emptyList(); + } + + List tasks = new ArrayList<>(); + for (Long seq : sequences) { + ActiveVmTaskInfo taskInfo = activeVmTasks.get(seq); + if (taskInfo != null) { + tasks.add(taskInfo); + } + } + return tasks; + } + + public static List getAllActiveVmTasks() { + return new ArrayList<>(activeVmTasks.values()); + } + + public static class TaskContext { + private final long cmdSequence; + private final String vmName; + private final String commandType; + + public TaskContext(long cmdSequence, String vmName, String commandType) { + this.cmdSequence = cmdSequence; + this.vmName = vmName; + this.commandType = commandType; + } + + public void registerTask(ManagedObjectReference taskMor, VmwareContext context) { + registerActiveVmTask(cmdSequence, vmName, commandType, taskMor, context); + } + + public void unregisterTask() { + unregisterActiveVmTask(cmdSequence); + } + } + + public static void setTaskContext(long cmdSequence, String vmName, String commandType) { + currentTaskContext.set(new TaskContext(cmdSequence, vmName, commandType)); + } + + public static void clearTaskContext() { + currentTaskContext.remove(); + } + + public static TaskContext getCurrentTaskContext() { + return currentTaskContext.get(); + } + public static boolean isReservedScsiDeviceNumber(int deviceNumber) { // The SCSI controller is assigned to virtual device node (z:7), so that device node is unavailable for hard disks or SCSI devices. return (deviceNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER) == 7; From 227864d70ca87927b23e7f01a9ae0caeeaaaf9bf Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 23 Jan 2026 15:39:27 +0530 Subject: [PATCH 06/29] changes --- .../apache/cloudstack/api/BaseAsyncCmd.java | 4 - .../apache/cloudstack/api/Cancellable.java | 29 --- .../apache/cloudstack/api/CancellableCmd.java | 20 -- .../command/admin/vm/DestroyVMCmdByAdmin.java | 5 +- .../api/command/admin/vm/MigrateVMCmd.java | 3 +- .../MigrateVirtualMachineWithVolumeCmd.java | 5 +- .../command/admin/vm/RebootVMCmdByAdmin.java | 5 +- .../command/admin/vm/StartVMCmdByAdmin.java | 5 +- .../command/admin/vm/StopVMCmdByAdmin.java | 5 +- .../command/user/job/CancelAsyncJobCmd.java | 3 +- .../user/snapshot/CreateSnapshotCmd.java | 2 +- .../user/snapshot/DeleteSnapshotCmd.java | 2 +- .../user/snapshot/RevertSnapshotCmd.java | 2 +- .../api/command/user/vm/BaseDeployVMCmd.java | 3 +- .../api/command/user/vm/DestroyVMCmd.java | 6 +- .../api/command/user/vm/RebootVMCmd.java | 5 +- .../api/command/user/vm/StartVMCmd.java | 5 +- .../api/command/user/vm/StopVMCmd.java | 7 +- .../user/vmsnapshot/CreateVMSnapshotCmd.java | 2 +- .../user/vmsnapshot/DeleteVMSnapshotCmd.java | 2 +- .../vmsnapshot/RevertToVMSnapshotCmd.java | 2 +- .../command/user/volume/AttachVolumeCmd.java | 2 +- .../command/user/volume/CreateVolumeCmd.java | 2 +- .../command/user/volume/DestroyVolumeCmd.java | 3 +- .../command/user/volume/DetachVolumeCmd.java | 2 +- .../command/user/volume/MigrateVolumeCmd.java | 2 +- .../cloudstack/jobs/AsyncJobService.java | 2 +- .../com/cloud/agent/manager/AgentAttache.java | 3 - .../cloud/agent/manager/AgentManagerImpl.java | 16 +- .../db/views/cloud.async_job_view.sql | 2 +- .../framework/jobs/AsyncJobManager.java | 2 + .../framework/jobs/dao/AsyncJobDaoImpl.java | 2 +- .../jobs/impl/AsyncJobManagerImpl.java | 107 +++++---- .../vmware/resource/VmwareResource.java | 204 +++-------------- .../vmware/mo/VirtualMachineMO.java | 206 +++++++++++++----- .../hypervisor/vmware/util/VmwareClient.java | 57 +++-- .../hypervisor/vmware/util/VmwareHelper.java | 5 + 37 files changed, 327 insertions(+), 412 deletions(-) delete mode 100644 api/src/main/java/org/apache/cloudstack/api/Cancellable.java delete mode 100644 api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java index 64ba8d390415..969a8da507c8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java @@ -83,8 +83,4 @@ public Long getSyncObjId() { public Object getJob() { return job; } - - public boolean isCancellable() { - return false; - } } diff --git a/api/src/main/java/org/apache/cloudstack/api/Cancellable.java b/api/src/main/java/org/apache/cloudstack/api/Cancellable.java deleted file mode 100644 index d12b4fa9184b..000000000000 --- a/api/src/main/java/org/apache/cloudstack/api/Cancellable.java +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.TYPE; - -@Retention(RetentionPolicy.RUNTIME) -@Target({TYPE}) -public @interface Cancellable { - boolean value() default true; -} diff --git a/api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java b/api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java deleted file mode 100644 index 3dc390151ad7..000000000000 --- a/api/src/main/java/org/apache/cloudstack/api/CancellableCmd.java +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api; - -public interface CancellableCmd { -} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java index b09a112ce6ba..b1ecffb0ea6c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java @@ -29,8 +29,9 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance. Once destroyed, only the administrator can recover it.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, - responseHasSensitiveInfo = true) -public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd, CancellableCmd { + responseHasSensitiveInfo = true, + cancellable = true) +public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd { @Parameter( name = ApiConstants.FORCED, type = CommandType.BOOLEAN, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java index 467b92d415d2..0cbc55b81d9e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java @@ -48,7 +48,8 @@ description = "Attempts Migration of an Instance to a different host or Root volume of the Instance to a different storage pool", responseObject = UserVmResponse.class, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, - responseHasSensitiveInfo = true) + responseHasSensitiveInfo = true, + cancellable = true) public class MigrateVMCmd extends BaseAsyncCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java index ad298513ac0b..4d2a412738a3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java @@ -49,8 +49,9 @@ description = "Attempts Migration of an Instance with its volumes to a different host", responseObject = UserVmResponse.class, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, - responseHasSensitiveInfo = true) -public class MigrateVirtualMachineWithVolumeCmd extends BaseAsyncCmd { + responseHasSensitiveInfo = true, + cancellable = true) +public class MigrateVirtualMachineWithVolumeCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java index fcbf19842cec..c55aaaf0501b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; @@ -26,5 +25,5 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "rebootVirtualMachine", description = "Reboots an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class RebootVMCmdByAdmin extends RebootVMCmd implements AdminCmd, CancellableCmd {} + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) +public class RebootVMCmdByAdmin extends RebootVMCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java index 087abd468948..cecff0bd1c15 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.StartVMCmd; @@ -26,5 +25,5 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "startVirtualMachine", responseObject = UserVmResponse.class, description = "Starts an Instance.", responseView = ResponseView.Full, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StartVMCmdByAdmin extends StartVMCmd implements AdminCmd, CancellableCmd {} + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) +public class StartVMCmdByAdmin extends StartVMCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java index 685c232bcfec..98ea91fb62ed 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command.admin.vm; import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.vm.StopVMCmd; @@ -26,5 +25,5 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "stopVirtualMachine", responseObject = UserVmResponse.class, description = "Stops an Instance.", responseView = ResponseView.Full, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StopVMCmdByAdmin extends StopVMCmd implements AdminCmd, CancellableCmd {} + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) +public class StopVMCmdByAdmin extends StopVMCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java index 6fbdce4dbaa8..cc49ece98517 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.api.response.AsyncJobResponse; import com.cloud.user.Account; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.jobs.AsyncJobService; import javax.inject.Inject; @@ -89,7 +90,7 @@ public Long getInstanceId() { @Override public void execute() { - String status = asyncJobService.cancelAsyncJob(id, "cancel request by user using cancelAsyncJob api"); + String status = asyncJobService.cancelAsyncJob(id, "Cancel requested by {}" + CallContext.current().getCallingUser().toString()); if (StringUtils.isBlank(status)) { AsyncJobResponse response = _responseGenerator.cancelJobResponse(this); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index d03df501847a..8af8a7a2b8dd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -50,7 +50,7 @@ import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = "createSnapshot", description = "Creates an instant Snapshot of a volume.", responseObject = SnapshotResponse.class, entityType = {Snapshot.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class CreateSnapshotCmd extends BaseAsyncCreateCmd { // /////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java index b4eaceb61ba6..b99e186ae034 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java @@ -36,7 +36,7 @@ import com.cloud.user.Account; @APICommand(name = "deleteSnapshot", description = "Deletes a Snapshot of a disk volume.", responseObject = SuccessResponse.class, entityType = {Snapshot.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class DeleteSnapshotCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/RevertSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/RevertSnapshotCmd.java index 59881dfdabe2..17a84f080490 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/RevertSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/RevertSnapshotCmd.java @@ -36,7 +36,7 @@ import com.cloud.user.Account; @APICommand(name = "revertSnapshot", description = "This is supposed to revert a volume Snapshot. This command is only supported with KVM so far", responseObject = SnapshotResponse.class, entityType = {Snapshot.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class RevertSnapshotCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index e331078ae778..844cc0392a5d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -35,7 +35,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.api.BaseAsyncCreateCustomIdCmd; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.DiskOfferingResponse; @@ -69,7 +68,7 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.VmDiskInfo; -public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd, CancellableCmd { +public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { private static final String s_name = "deployvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index debe9f309d83..39a83dbc5463 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -26,7 +26,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -44,8 +43,9 @@ @APICommand(name = "destroyVirtualMachine", description = "Destroys an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, - responseHasSensitiveInfo = true) -public class DestroyVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { + responseHasSensitiveInfo = true, + cancellable = true) +public class DestroyVMCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "destroyvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java index c681fb8c2174..a89ee2da47d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RebootVMCmd.java @@ -25,7 +25,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -41,8 +40,8 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "rebootVirtualMachine", description = "Reboots an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class RebootVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) +public class RebootVMCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "rebootvirtualmachineresponse"; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java index d3de77a3ffd2..41e3410c4e3f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StartVMCmd.java @@ -27,7 +27,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -49,8 +48,8 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "startVirtualMachine", responseObject = UserVmResponse.class, description = "Starts an Instance.", responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StartVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) +public class StartVMCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "startvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java index eb3478b6b063..160c240afb07 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java @@ -24,8 +24,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; -import org.apache.cloudstack.api.Cancellable; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -39,10 +37,9 @@ import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; -@Cancellable @APICommand(name = "stopVirtualMachine", responseObject = UserVmResponse.class, description = "Stops an Instance.", responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class StopVMCmd extends BaseAsyncCmd implements UserCmd, CancellableCmd { + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) +public class StopVMCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "stopvirtualmachineresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java index 35723a3bbb74..33ce3fee7789 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java @@ -38,7 +38,7 @@ @APICommand(name = "createVMSnapshot", description = "Creates Snapshot for an Instance. Running KVM UEFI disk-only snapshots briefly suspend the Instance while copying NVRAM state.", responseObject = VMSnapshotResponse.class, since = "4.2.0", entityType = {VMSnapshot.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class CreateVMSnapshotCmd extends BaseAsyncCreateCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/DeleteVMSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/DeleteVMSnapshotCmd.java index 3373ac534cc8..a1f05919e5bb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/DeleteVMSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/DeleteVMSnapshotCmd.java @@ -36,7 +36,7 @@ import com.cloud.vm.snapshot.VMSnapshot; @APICommand(name = "deleteVMSnapshot", description = "Deletes an Instance Snapshot.", responseObject = SuccessResponse.class, since = "4.2.0", entityType = {VMSnapshot.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class DeleteVMSnapshotCmd extends BaseAsyncCmd { @ACL(accessType = AccessType.OperateEntry) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/RevertToVMSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/RevertToVMSnapshotCmd.java index d44cefca5027..3cff47d43ed7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/RevertToVMSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/RevertToVMSnapshotCmd.java @@ -42,7 +42,7 @@ import com.cloud.vm.snapshot.VMSnapshot; @APICommand(name = "revertToVMSnapshot", description = "Revert Instance from a vmsnapshot.", responseObject = UserVmResponse.class, since = "4.2.0", responseView = ResponseView.Restricted, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) public class RevertToVMSnapshotCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "reverttovmsnapshotresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AttachVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AttachVolumeCmd.java index 8624043afc51..d89c92319509 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AttachVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/AttachVolumeCmd.java @@ -38,7 +38,7 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "attachVolume", description = "Attaches a disk volume to an Instance.", responseObject = VolumeResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class AttachVolumeCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "attachvolumeresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 538e263ae9de..ca83e2e880eb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -47,7 +47,7 @@ @APICommand(name = "createVolume", responseObject = VolumeResponse.class, description = "Creates a disk volume from a disk offering. This disk volume must still be attached to an Instance to make use of it.", responseView = ResponseView.Restricted, entityType = { Volume.class, VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserCmd { private static final String s_name = "createvolumeresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DestroyVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DestroyVolumeCmd.java index e9e388436642..73a393d73b21 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DestroyVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DestroyVolumeCmd.java @@ -39,7 +39,8 @@ since = "4.14.0", authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, requestHasSensitiveInfo = false, - responseHasSensitiveInfo = true) + responseHasSensitiveInfo = true, + cancellable = true) public class DestroyVolumeCmd extends BaseAsyncCmd { private static final String s_name = "destroyvolumeresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java index f6e811da6055..16534943bfe7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/DetachVolumeCmd.java @@ -39,7 +39,7 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "detachVolume", description = "Detaches a disk volume from an Instance.", responseObject = VolumeResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class DetachVolumeCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "detachvolumeresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java index 9927978ad55e..4bf446ee38ff 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/MigrateVolumeCmd.java @@ -34,7 +34,7 @@ import com.cloud.user.Account; @APICommand(name = "migrateVolume", description = "Migrate volume", responseObject = VolumeResponse.class, since = "3.0.0", responseView = ResponseView.Restricted, entityType = { - Volume.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + Volume.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class MigrateVolumeCmd extends BaseAsyncCmd implements UserCmd { private static final String s_name = "migratevolumeresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java b/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java index df35dfe73145..1bf7a5f859de 100644 --- a/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java +++ b/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java @@ -20,7 +20,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; public interface AsyncJobService extends PluggableService { - ConfigKey CancelJobInterval = new ConfigKey("Advanced", Integer.class, "ping.interval", "30", + ConfigKey CancelledJobInterval = new ConfigKey<>("Advanced", Integer.class, "job.cancelled.interval", "5", "Interval in seconds to check the jobs cancelled", false); String cancelAsyncJob(long jobId, String reason); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 3da343e6235a..163008c98c75 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -497,9 +497,6 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE sendNext(seq); } _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); -// if (e instanceof OperationCancelledException) { -// throw e; -// } throw new OperationTimedoutException(req.getCommands(), _id, seq, wait, false); } finally { unregisterListener(seq); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 34ddce6ebb3a..6cd3695cd9d3 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -146,7 +146,7 @@ import com.cloud.utils.nio.Task; import com.cloud.utils.time.InaccurateClock; -import static org.apache.cloudstack.jobs.AsyncJobService.CancelJobInterval; +import static org.apache.cloudstack.jobs.AsyncJobService.CancelledJobInterval; /** * Implementation of the Agent Manager. This class controls the connection to the agents. @@ -218,7 +218,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected ScheduledExecutorService _directAgentExecutor; protected ScheduledExecutorService _cronJobExecutor; protected ScheduledExecutorService _monitorExecutor; - protected ScheduledExecutorService _cancelJobExecutor; + protected ScheduledExecutorService _cancelledJobsCheckExecutor; private int _directAgentThreadCap; @@ -399,7 +399,7 @@ public void onManagementServerCancelPreparingForMaintenance() { public void onManagementServerMaintenance() { logger.debug("Management server maintenance enabled"); _monitorExecutor.shutdownNow(); - _cancelJobExecutor.shutdownNow(); + _cancelledJobsCheckExecutor.shutdownNow(); newAgentConnectionsMonitor.shutdownNow(); if (_connection != null) { _connection.stop(); @@ -451,8 +451,8 @@ private void initAndScheduleMonitorExecutor() { } private void initAndScheduleCancelJobExecutor() { - _cancelJobExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("CancelJob")); - _cancelJobExecutor.scheduleWithFixedDelay(new CancelJobTask(), CancelJobInterval.value(), CancelJobInterval.value(), TimeUnit.SECONDS); + _cancelledJobsCheckExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("CancelledJobsCheck")); + _cancelledJobsCheckExecutor.scheduleWithFixedDelay(new CancelledJobsCheckTask(), CancelledJobInterval.value(), CancelledJobInterval.value(), TimeUnit.SECONDS); } private void initAndScheduleAgentConnectionsMonitor() { @@ -924,7 +924,7 @@ public boolean start() { ManagementServerHostVO msHost = _mshostDao.findByMsid(_nodeId); if (msHost != null && (ManagementServerHost.State.Maintenance.equals(msHost.getState()) || ManagementServerHost.State.PreparingForMaintenance.equals(msHost.getState()))) { _monitorExecutor.shutdownNow(); - _cancelJobExecutor.shutdownNow(); + _cancelledJobsCheckExecutor.shutdownNow(); newAgentConnectionsMonitor.shutdownNow(); return true; } @@ -1104,7 +1104,7 @@ public boolean stop() { _connectExecutor.shutdownNow(); _monitorExecutor.shutdownNow(); - _cancelJobExecutor.shutdownNow(); + _cancelledJobsCheckExecutor.shutdownNow(); newAgentConnectionsMonitor.shutdownNow(); return true; } @@ -2100,7 +2100,7 @@ protected List findAgentsBehindOnPing() { } } - protected class CancelJobTask extends ManagedContextRunnable { + protected class CancelledJobsCheckTask extends ManagedContextRunnable { @Override protected void runInContext() { final List jobs = _asyncJobDao.getCancelledJobs(); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql index 6369cb21fbfa..80c45b55f69e 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql @@ -33,6 +33,7 @@ select user.uuid user_uuid, async_job.id, async_job.uuid, + async_job.related, async_job.job_cmd, async_job.job_status, async_job.job_process_status, @@ -42,7 +43,6 @@ select async_job.removed, async_job.instance_type, async_job.instance_id, - async_job.related, async_job.job_executing_msid, CASE WHEN async_job.instance_type = 'Volume' THEN volumes.uuid diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java index 9f50a545efc7..b187ff48146f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java @@ -40,6 +40,8 @@ public interface AsyncJobManager extends Manager { void completeAsyncJob(long jobId, JobInfo.Status jobStatus, int resultCode, String result); + void completeAsyncJob(long jobId, JobInfo.Status jobStatus, int resultCode, String result, boolean remove); + void updateAsyncJobStatus(long jobId, int processStatus, String resultObject); void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index e680a5f7520c..3082ae819d5f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -325,7 +325,7 @@ public List getCancelledJobs() { @Override public boolean isJobCancelled(Long jobId) { - AsyncJobVO job = findById(jobId); + AsyncJobVO job = findByIdIncludingRemoved(jobId); return job != null && job.getStatus() == JobInfo.Status.CANCELLED; } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 7c83ab9f61b7..e2dfbe7ee000 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -20,7 +20,6 @@ import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -44,9 +43,6 @@ import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseAsyncCmd; -import org.apache.cloudstack.api.Cancellable; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.command.user.job.CancelAsyncJobCmd; import org.apache.cloudstack.command.ReconcileCommandService; import org.apache.cloudstack.context.CallContext; @@ -203,7 +199,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout, HidePassword}; + return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout, HidePassword, CancelledJobInterval}; } @Override @@ -341,6 +337,12 @@ public Long doInTransaction(TransactionStatus status) { @Override @DB public void completeAsyncJob(final long jobId, final Status jobStatus, final int resultCode, final String resultObject) { + completeAsyncJob(jobId, jobStatus, resultCode, resultObject, true); + } + + @Override + @DB + public void completeAsyncJob(final long jobId, final Status jobStatus, final int resultCode, final String resultObject, boolean remove) { String resultObj = null; if (logger.isDebugEnabled()) { resultObj = convertHumanReadableJson(obfuscatePassword(resultObject, HidePassword.value())); @@ -384,7 +386,6 @@ public List doInTransaction(final TransactionStatus status) { if (logger.isDebugEnabled()) { logger.debug("Update db status for job-" + jobId); } - job.setCompleteMsid(getMsid()); job.setStatus(jobStatus); job.setResultCode(resultCode); @@ -396,8 +397,12 @@ public List doInTransaction(final TransactionStatus status) { final Date currentGMTTime = DateUtil.currentGMTTime(); job.setLastUpdated(currentGMTTime); - job.setRemoved(currentGMTTime); job.setExecutingMsid(null); + + if (remove) { + job.setCompleteMsid(getMsid()); + job.setRemoved(currentGMTTime); + } _jobDao.update(jobId, job); if (logger.isDebugEnabled()) { @@ -918,10 +923,36 @@ protected void reallyRun() { List l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); if (l != null && l.size() > 0) { for (SyncQueueItemVO item : l) { - if (logger.isDebugEnabled()) { - logger.debug("Execute sync-queue item: " + item.toString()); + boolean isPurged = false; + if (item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { + AsyncJobVO job = _jobDao.findById(item.getContentId()); + if (job != null && StringUtils.isNotBlank(job.getRelated())) { + AsyncJobVO parentJob = _jobDao.findById(Long.valueOf(job.getRelated())); + //If the parent job is done, do not execute the child. complete it and purge it from queue + if (parentJob != null && parentJob.getStatus().done() && !isPseudoJob(parentJob)) { + logger.debug("Purging sync-queue item: {}", item); + completeAsyncJob(item.getContentId(), parentJob.getStatus(), 0, "Job is not " + + "scheduled for execution as the parent job is done. Parent Job " + + "state:" + " " + parentJob.getStatus()); + _jobMonitor.unregisterByJobId(item.getContentId()); + _queueMgr.purgeItem(item.getId()); + + if (parentJob.getStatus() == Status.CANCELLED) { + parentJob.setCompleteMsid(getMsid()); + final Date currentGMTTime = DateUtil.currentGMTTime(); + parentJob.setLastUpdated(currentGMTTime); + parentJob.setRemoved(currentGMTTime); + _jobDao.update(parentJob.getId(), parentJob); + } + + isPurged = true; + } + } + } + if (!isPurged) { + logger.debug("Execute sync-queue item: {}", item); + executeQueueItem(item, false); } - executeQueueItem(item, false); } } @@ -929,8 +960,17 @@ protected void reallyRun() { for (Long jobId : standaloneWakeupJobs) { // TODO, we assume that all jobs in this category is API job only AsyncJobVO job = _jobDao.findById(jobId); - if (job != null && (job.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) - scheduleExecution(job, false); + if (job != null && (job.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) { + if (job.getStatus() != Status.CANCELLED) { + scheduleExecution(job, false); + } else { + job.setCompleteMsid(getMsid()); + final Date currentGMTTime = DateUtil.currentGMTTime(); + job.setLastUpdated(currentGMTTime); + job.setRemoved(currentGMTTime); + _jobDao.update(job.getId(), job); + } + } } } catch (Throwable e) { logger.error("Unexpected exception when trying to execute queue item, ", e); @@ -1421,52 +1461,27 @@ private static void shutdownAndAwaitTermination(ExecutorService pool) { @Override public String cancelAsyncJob(long jobId, String reason) { - final AsyncJobVO job = _jobDao.findById(jobId); + final AsyncJobVO job = _jobDao.findByIdIncludingRemoved(jobId); String errMessage; if (job == null) { - errMessage = "Cannot cancel, job-" + jobId + " no longer exists."; + errMessage = "Cannot cancel, job no longer exists."; logger.debug(errMessage); - // purge queue items (if any) to avoid blocking _queueMgr.purgeAsyncJobQueueItemId(jobId); return errMessage; } try { Class cmdClass = Class.forName(job.getCmd()); - if (BaseAsyncCmd.class.isAssignableFrom(cmdClass)) { - logger.info(cmdClass.getName() + " is a subclass of BaseAsyncCmd"); - try { - BaseAsyncCmd cmd = (BaseAsyncCmd) cmdClass.getDeclaredConstructor().newInstance(); - boolean cmdCancellable = cmd.isCancellable(); - if (cmdCancellable) { - logger.info("Cmd: " + cmdClass.getName() + " is cancellable"); - } - } catch (NoSuchMethodException e) { - logger.warn("No declared constructor for Cmd: " + cmdClass.getName() + ", not able to check if it is cancellable or not"); - } - } - - boolean cancellable = cmdClass.isAnnotationPresent(Cancellable.class); - if (cancellable) { - logger.info(cmdClass.getName() + " is cancellable"); - } - APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); - if (apiCommand == null || !apiCommand.cancellable() || !CancellableCmd.class.isAssignableFrom(cmdClass)) { - errMessage = "Cannot cancel, job-" + jobId + " as it is not cancellable."; + if (apiCommand == null || !apiCommand.cancellable()) { + errMessage = "Cannot cancel, job " + job.getUuid() + " is not cancellable."; logger.debug(errMessage); return errMessage; } } catch (ClassNotFoundException e) { - errMessage = "Command " + job.getCmd() + " of jobid-" + jobId + " not found."; + errMessage = "Command " + job.getCmd() + " of jobid " + job.getUuid() + " not found."; logger.error(errMessage, e); return errMessage; - } catch (InvocationTargetException e) { - throw new RuntimeException(e); - } catch (InstantiationException e) { - throw new RuntimeException(e); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); } if (job.getStatus() != Status.IN_PROGRESS) { @@ -1476,13 +1491,11 @@ public String cancelAsyncJob(long jobId, String reason) { return errMessage; } - logger.debug("Cancelling job-{} which is in IN_PROGRESS state.", jobId); + logger.debug("Cancelling job-{} which is in progress.", jobId); try { - completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason); + completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason, false); _jobMonitor.unregisterByJobId(jobId); - - // purge the item and resume queue processing _queueMgr.purgeAsyncJobQueueItemId(jobId); return ""; } catch (Throwable t) { @@ -1495,7 +1508,7 @@ public String cancelAsyncJob(long jobId, String reason) { @Override public List> getCommands() { - final List> cmdList = new ArrayList>(); + final List> cmdList = new ArrayList<>(); cmdList.add(CancelAsyncJobCmd.class); return cmdList; } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 1872c1afb1b1..0687e6cc6cde 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -43,7 +43,6 @@ import java.util.Set; import java.util.TimeZone; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import com.cloud.agent.api.CleanupVMCommand; @@ -63,7 +62,10 @@ import com.vmware.vim25.VirtualTPM; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.PrepareForBackupRestorationCommand; +import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.CreateObjectCommand; +import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; @@ -435,10 +437,6 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes protected VirtualRoutingResource _vrResource; - private final ConcurrentHashMap _activeVmTasks = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> _vmNameToTaskSequences = new ConcurrentHashMap<>(); - private static final ThreadLocal currentTaskContext = new ThreadLocal<>(); - protected final static HashMap s_powerStatesTable = new HashMap<>(); static { @@ -456,148 +454,6 @@ public VmwareResource() { _gson = GsonHelper.getGsonLogger(); } - public static class ActiveVmTaskInfo { - final long cmdSequence; - final String vmName; - final String commandType; - final ManagedObjectReference taskMor; - final VmwareContext context; - final long startTime; - - ActiveVmTaskInfo(long cmdSequence, String vmName, String commandType, - ManagedObjectReference taskMor, VmwareContext context) { - this.cmdSequence = cmdSequence; - this.vmName = vmName; - this.commandType = commandType; - this.taskMor = taskMor; - this.context = context; - this.startTime = System.currentTimeMillis(); - } - } - - void registerActiveVmTask(long cmdSequence, String vmName, String commandType, - ManagedObjectReference taskMor, VmwareContext context) { - ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, taskMor, context); - _activeVmTasks.put(cmdSequence, taskInfo); - - _vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); - - logger.debug("Registered active VM task: sequence={}, vm={}, command={}, task={}", - cmdSequence, vmName, commandType, taskMor.getValue()); - } - - private void unregisterActiveVmTask(long cmdSequence) { - ActiveVmTaskInfo taskInfo = _activeVmTasks.remove(cmdSequence); - if (taskInfo != null) { - Set sequences = _vmNameToTaskSequences.get(taskInfo.vmName); - if (sequences != null) { - sequences.remove(cmdSequence); - if (sequences.isEmpty()) { - _vmNameToTaskSequences.remove(taskInfo.vmName); - } - } - logger.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={}ms", - cmdSequence, taskInfo.vmName, taskInfo.commandType, - System.currentTimeMillis() - taskInfo.startTime); - } - } - - public int cancelActiveVmTasks(String vmName) { - Set sequences = _vmNameToTaskSequences.get(vmName); - if (sequences == null || sequences.isEmpty()) { - return 0; - } - - int cancelled = 0; - for (Long seq : sequences) { - if (cancelActiveVmTask(seq)) { - cancelled++; - } - } - return cancelled; - } - - public boolean cancelActiveVmTask(long cmdSequence) { - ActiveVmTaskInfo taskInfo = _activeVmTasks.get(cmdSequence); - if (taskInfo == null) { - return false; - } - - try { - // Cancel the vCenter task - TaskMO taskMo = new TaskMO(taskInfo.context, taskInfo.taskMor); - taskMo.cancelTask(); - logger.info("Cancelled vCenter task: sequence={}, vm={}, command={}, task={}", - cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.taskMor.getValue()); - - // Unregister the task - unregisterActiveVmTask(cmdSequence); - return true; - } catch (Exception e) { - logger.warn("Failed to cancel vCenter task: sequence={}, vm={}, error={}", - cmdSequence, taskInfo.vmName, e.getMessage(), e); - return false; - } - } - - public List getActiveVmTasks(String vmName) { - Set sequences = _vmNameToTaskSequences.get(vmName); - if (sequences == null || sequences.isEmpty()) { - return Collections.emptyList(); - } - - List tasks = new ArrayList<>(); - for (Long seq : sequences) { - ActiveVmTaskInfo taskInfo = _activeVmTasks.get(seq); - if (taskInfo != null) { - tasks.add(taskInfo); - } - } - return tasks; - } - - public List getAllActiveVmTasks() { - return new ArrayList<>(_activeVmTasks.values()); - } - - public static class TaskRegistrationContext { - private final VmwareResource resource; - private final long cmdSequence; - private final String vmName; - private final String commandType; - - public TaskRegistrationContext(VmwareResource resource, long cmdSequence, String vmName, String commandType) { - this.resource = resource; - this.cmdSequence = cmdSequence; - this.vmName = vmName; - this.commandType = commandType; - } - - public void registerTask(ManagedObjectReference taskMor, VmwareContext context) { - if (resource != null) { - resource.registerActiveVmTask(cmdSequence, vmName, commandType, taskMor, context); - } - } - - public void unregisterTask() { - if (resource != null) { - resource.unregisterActiveVmTask(cmdSequence); - } - } - } - - private void setTaskRegistrationContext(long cmdSequence, String vmName, String commandType) { - currentTaskContext.set(new TaskRegistrationContext(this, cmdSequence, vmName, commandType)); - } - - private void clearTaskRegistrationContext() { - currentTaskContext.remove(); - } - - public static TaskRegistrationContext getCurrentTaskContext() { - return currentTaskContext.get(); - } - private String getCommandLogTitle(Command cmd) { StringBuffer sb = new StringBuffer(); if (_hostName != null) { @@ -656,34 +512,33 @@ public Answer executeRequest(Command cmd) { } else if (clz == CheckHealthCommand.class) { answer = execute((CheckHealthCommand) cmd); } else if (clz == StopCommand.class) { - String vmName = null; - String commandType = cmd.getClass().getSimpleName(); - vmName = ((StopCommand) cmd).getVmName(); - setTaskRegistrationContext(cmdSequence, vmName, commandType); - VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); + String vmName = ((StopCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); answer = execute((StopCommand) cmd); } else if (clz == RebootRouterCommand.class) { answer = execute((RebootRouterCommand) cmd); } else if (clz == RebootCommand.class) { + String vmName = ((RebootCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); answer = execute((RebootCommand) cmd); } else if (clz == CheckVirtualMachineCommand.class) { answer = execute((CheckVirtualMachineCommand) cmd); } else if (clz == PrepareForMigrationCommand.class) { answer = execute((PrepareForMigrationCommand) cmd); } else if (clz == MigrateCommand.class) { - String vmName = null; - String commandType = cmd.getClass().getSimpleName(); - vmName = ((MigrateCommand) cmd).getVmName(); - setTaskRegistrationContext(cmdSequence, vmName, commandType); - VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); + String vmName = ((MigrateCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); answer = execute((MigrateCommand) cmd); } else if (clz == MigrateVmToPoolCommand.class) { answer = execute((MigrateVmToPoolCommand) cmd); } else if (clz == MigrateWithStorageCommand.class) { answer = execute((MigrateWithStorageCommand) cmd); } else if (clz == MigrateVolumeCommand.class) { + setTaskContext(cmdSequence, cmd, null); answer = execute((MigrateVolumeCommand) cmd); } else if (clz == DestroyCommand.class) { + String vmName = ((DestroyCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); answer = execute((DestroyCommand) cmd); } else if (clz == CreateStoragePoolCommand.class) { return execute((CreateStoragePoolCommand) cmd); @@ -732,11 +587,9 @@ public Answer executeRequest(Command cmd) { } else if (clz == NetworkUsageCommand.class) { answer = execute((NetworkUsageCommand) cmd); } else if (clz == StartCommand.class) { - String vmName = null; - String commandType = cmd.getClass().getSimpleName(); - vmName = ((StartCommand) cmd).getVirtualMachine().getName(); - setTaskRegistrationContext(cmdSequence, vmName, commandType); - VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); + String vmName = ((StartCommand) cmd).getVirtualMachine().getName(); + setTaskContext(cmdSequence, cmd, vmName); + Thread.sleep(3 * 1000); answer = execute((StartCommand) cmd); } else if (clz == CheckSshCommand.class) { answer = execute((CheckSshCommand) cmd); @@ -749,10 +602,16 @@ public Answer executeRequest(Command cmd) { } else if (clz == UnPlugNicCommand.class) { answer = execute((UnPlugNicCommand) cmd); } else if (cmd instanceof CreateVMSnapshotCommand) { + String vmName = ((CreateVMSnapshotCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); return execute((CreateVMSnapshotCommand) cmd); } else if (cmd instanceof DeleteVMSnapshotCommand) { + String vmName = ((DeleteVMSnapshotCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); return execute((DeleteVMSnapshotCommand) cmd); } else if (cmd instanceof RevertToVMSnapshotCommand) { + String vmName = ((RevertToVMSnapshotCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); return execute((RevertToVMSnapshotCommand) cmd); } else if (clz == ResizeVolumeCommand.class) { return execute((ResizeVolumeCommand) cmd); @@ -761,6 +620,15 @@ public Answer executeRequest(Command cmd) { } else if (clz == CleanupVMCommand.class) { return execute((CleanupVMCommand) cmd); } else if (cmd instanceof StorageSubSystemCommand) { + if (cmd instanceof CreateObjectCommand) { + setTaskContext(cmdSequence, cmd, null); + } if (cmd instanceof AttachCommand) { + String vmName = ((AttachCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); + } else if (cmd instanceof DettachCommand) { + String vmName = ((DettachCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); + } checkStorageProcessorAndHandlerNfsVersionAttribute((StorageSubSystemCommand) cmd); return storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd); } else if (clz == ScaleVmCommand.class) { @@ -824,7 +692,6 @@ public Answer executeRequest(Command cmd) { Thread.currentThread().interrupt(); // Restore interrupt status return new Answer(cmd, false, "Command execution was cancelled: " + e.getMessage()); } finally { - clearTaskRegistrationContext(); VmwareHelper.clearTaskContext(); recycleServiceContext(); ThreadContext.pop(); @@ -836,6 +703,11 @@ public Answer executeRequest(Command cmd) { return answer; } + private void setTaskContext(long cmdSequence, Command cmd, String vmName) { + String commandType = cmd.getClass().getSimpleName(); + VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); + } + private ExecutionResult getSystemVmVersionAndChecksum(String controlIp) { ExecutionResult result; try { @@ -6241,12 +6113,6 @@ public CopyVolumeAnswer execute(CopyVolumeCommand cmd) { @Override public void disconnected() { - List tasks = getAllActiveVmTasks(); - for (ActiveVmTaskInfo task : tasks) { - cancelActiveVmTask(task.cmdSequence); - } - _activeVmTasks.clear(); - _vmNameToTaskSequences.clear(); } @Override diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 4fbbf9c1d889..8a464db14522 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -226,12 +226,6 @@ public boolean powerOn() throws Exception { return true; ManagedObjectReference morTask = _context.getService().powerOnVMTask(_mor, null); - -// VmwareResource.TaskRegistrationContext taskCtx = VmwareResource.getCurrentTaskContext(); -// if (taskCtx != null) { -// taskCtx.registerTask(morTask, _context); -// } - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { taskCtx.registerTask(morTask, _context); @@ -314,9 +308,6 @@ public void run() { } throw e; } finally { -// if (taskCtx != null) { -// taskCtx.unregisterTask(); -// } if (taskCtx != null) { taskCtx.unregisterTask(); } @@ -374,6 +365,10 @@ public boolean safePowerOff(int shutdownWaitMs) throws Exception { private boolean powerOffNoCheck() throws Exception { ManagedObjectReference morTask = _context.getService().powerOffVMTask(_mor); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); + } try { boolean result = _context.getVimClient().waitForTask(morTask); @@ -412,6 +407,10 @@ private boolean powerOffNoCheck() throws Exception { logger.warn("Failed to cancel powerOff task: " + ex.getMessage()); } throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } } } @@ -448,15 +447,34 @@ public VirtualMachinePowerState getPowerState() throws Exception { public boolean reset() throws Exception { ManagedObjectReference morTask = _context.getService().resetVMTask(_mor); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); + } - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware resetVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + try { + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware resetVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("resetVM interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel resetVM task: " + ex.getMessage()); + } + throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } } - return false; } public void shutdown() throws Exception { @@ -482,6 +500,10 @@ public boolean isTemplate() throws Exception { public boolean migrate(ManagedObjectReference morRp, ManagedObjectReference morTargetHost) throws Exception { ManagedObjectReference morTask = _context.getService().migrateVMTask(_mor, morRp, morTargetHost, VirtualMachineMovePriority.DEFAULT_PRIORITY, null); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); + } try { boolean result = _context.getVimClient().waitForTask(morTask); @@ -495,13 +517,17 @@ public boolean migrate(ManagedObjectReference morRp, ManagedObjectReference morT return false; } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.warn("migrate interrupted for VM: " + getVmName() + ", cancelling task"); + logger.warn("migrateVM interrupted for VM: " + getVmName() + ", cancelling task"); try { _context.getVimClient().cancelTask(morTask); } catch (Exception ex) { - logger.warn("Failed to cancel migrate task: " + ex.getMessage()); + logger.warn("Failed to cancel migrateVM task: " + ex.getMessage()); } throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } } } @@ -579,44 +605,63 @@ public ManagedObjectReference createSnapshotGetReference(String snapshotName, St long apiTimeout = _context.getVimClient().getVcenterSessionTimeout(); ManagedObjectReference morTask = _context.getService().createSnapshotTask(_mor, snapshotName, snapshotDescription, dumpMemory, quiesce); boolean result = _context.getVimClient().waitForTask(morTask); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); + } - if (result) { - _context.waitForTaskProgressDone(morTask); + try { + if (result) { + _context.waitForTaskProgressDone(morTask); - ManagedObjectReference morSnapshot = null; + ManagedObjectReference morSnapshot = null; - // We still need to wait until the object appear in vCenter - long startTick = System.currentTimeMillis(); + // We still need to wait until the object appear in vCenter + long startTick = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTick < apiTimeout) { - morSnapshot = getSnapshotMor(snapshotName); + while (System.currentTimeMillis() - startTick < apiTimeout) { + morSnapshot = getSnapshotMor(snapshotName); - if (morSnapshot != null) { - break; + if (morSnapshot != null) { + break; + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.debug("[ignored] interrupted while waiting for snapshot to be done."); + } } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.debug("[ignored] interrupted while waiting for snapshot to be done."); + if (morSnapshot == null) { + logger.error("We've been waiting for over " + apiTimeout + " milli seconds for snapshot MOR to be appearing in vCenter after CreateSnapshot task is done, " + + "but it is still not there?!"); + + return null; } - } - if (morSnapshot == null) { - logger.error("We've been waiting for over " + apiTimeout + " milli seconds for snapshot MOR to be appearing in vCenter after CreateSnapshot task is done, " + - "but it is still not there?!"); + logger.debug("Waited for " + (System.currentTimeMillis() - startTick) + " seconds for snapshot object [" + snapshotName + "] to appear in vCenter."); - return null; + return morSnapshot; + } else { + logger.error("VMware createSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } - logger.debug("Waited for " + (System.currentTimeMillis() - startTick) + " seconds for snapshot object [" + snapshotName + "] to appear in vCenter."); - - return morSnapshot; - } else { - logger.error("VMware createSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + return null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("powerOn interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel powerOn task: " + ex.getMessage()); + } + throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } } - - return null; } public boolean removeSnapshot(String snapshotName, boolean removeChildren) throws Exception { @@ -627,15 +672,35 @@ public boolean removeSnapshot(String snapshotName, boolean removeChildren) throw } ManagedObjectReference morTask = _context.getService().removeSnapshotTask(morSnapshot, removeChildren, true); - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware removeSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); } - return false; + try { + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware removeSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("removeSnapshot interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel removeSnapshot task: " + ex.getMessage()); + } + throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } + } } public boolean revertToSnapshot(String snapshotName) throws Exception { @@ -645,15 +710,35 @@ public boolean revertToSnapshot(String snapshotName) throws Exception { return false; } ManagedObjectReference morTask = _context.getService().revertToSnapshotTask(morSnapshot, _mor, null); - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware revert to snapshot failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); } - return false; + try { + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware revert to snapshot failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("revertToSnapshot interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel revertToSnapshot task: " + ex.getMessage()); + } + throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } + } } /** @@ -1536,6 +1621,11 @@ public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference m reConfigSpec.getDeviceChange().add(deviceConfigSpec); ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask(morTask, _context); + } + boolean result = _context.getVimClient().waitForTask(morTask); if (!result) { @@ -1547,7 +1637,7 @@ public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference m _context.waitForTaskProgressDone(morTask); } - if(logger.isTraceEnabled()) + if (logger.isTraceEnabled()) logger.trace("vCenter API trace - attachDisk() done(successfully)"); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index eb47e8f14c3a..a57c27181f44 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -416,33 +416,22 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa boolean retVal = false; + boolean cancelled = false; if (Thread.currentThread().isInterrupted()) { Thread.currentThread().interrupt(); // Restore interrupt status try { - cancelTask(task); + Pair status = cancelTask(task); + cancelled = status.first(); } catch (Exception e) { LOGGER.warn("Failed to cancel task on interruption: " + task.getValue(), e); } - throw new InterruptedException("Task wait cancelled for task: " + task.getValue()); -// boolean cancellable = false; -// try { -// cancellable = isTaskCancelable(task); -// } catch (Exception e) { -// LOGGER.warn("Failed to check if task is cancelable: " + task.getValue(), e); -// } -// -// if (cancellable) { -// try { -// cancelTask(task); -// LOGGER.info("Cancelled cancelable task on interruption: " + task.getValue()); -// } catch (Exception e) { -// LOGGER.warn("Failed to cancel task on interruption: " + task.getValue(), e); -// } -// throw new InterruptedException("Task wait cancelled for task: " + task.getValue()); -// } else { -// LOGGER.warn("Task " + task.getValue() + " is not cancelable, continuing execution despite interruption"); -// Thread.interrupted(); -// } + + if (cancelled) { + throw new InterruptedException("Task wait cancelled for task: " + task.getValue()); + } else { + // Clear interrupt flag to continue execution + Thread.interrupted(); + } } try { @@ -542,20 +531,30 @@ private synchronized Object[] waitForValues(ManagedObjectReference objmor, Strin List propchgary = null; while (!reached) { if (!isCancelTask && Thread.currentThread().isInterrupted()) { - try { - vimPort.destroyPropertyFilter(filterSpecRef); - } catch (Exception e) { - LOGGER.warn("Failed to destroy property filter on interruption: " + e.getMessage()); - } - Thread.currentThread().interrupt(); + boolean cancelled = false; if (objmor.getType().equals("Task")) { try { - cancelTask(objmor); + Pair status = cancelTask(objmor); + cancelled = status.first(); } catch (Exception e) { LOGGER.warn("Failed to cancel task on interruption: " + objmor.getValue(), e); } + } else { + Thread.currentThread().interrupt(); + throw new InterruptedException("Task wait cancelled for object: " + objmor.getValue()); + } + + if (cancelled) { + try { + vimPort.destroyPropertyFilter(filterSpecRef); + } catch (Exception e) { + LOGGER.warn("Failed to destroy property filter on interruption: " + e.getMessage()); + } + Thread.currentThread().interrupt(); + throw new InterruptedException("Task wait cancelled for object: " + objmor.getValue()); + } else { + Thread.interrupted(); } - throw new InterruptedException("Task wait cancelled for object: " + objmor.getValue()); } updateset = vimPort.waitForUpdatesEx(propertyCollector, version, new WaitOptions()); if (updateset == null || updateset.getFilterSet() == null) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 419769522049..380d716abd35 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -163,6 +163,11 @@ public static class ActiveVmTaskInfo { private static void registerActiveVmTask(long cmdSequence, String vmName, String commandType, ManagedObjectReference taskMor, VmwareContext context) { + if (StringUtils.isBlank(vmName)) { + LOGGER.debug("No active VM for sequence={}, command={}, task={}", cmdSequence, commandType, taskMor); + return; + } + ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, taskMor, context); activeVmTasks.put(cmdSequence, taskInfo); From 49fe187fb7f5ffe9913252d7e0397d02083f557f Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 23 Jan 2026 16:11:43 +0530 Subject: [PATCH 07/29] simulator fix --- .../src/main/java/com/cloud/ha/SimulatorInvestigator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java index 95a6a1291cf1..e0c004b574e0 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java @@ -20,6 +20,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.ha.HAManager; import com.cloud.agent.AgentManager; @@ -103,6 +104,9 @@ public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM { } catch (OperationTimedoutException e) { logger.debug("Operation timed out for " + vm.toString() + ": " + e.getMessage()); throw new UnknownVM(); + } catch (OperationCancelledException e) { + logger.debug("Operation cancelled for " + vm.toString() + ": " + e.getMessage()); + throw new UnknownVM(); } } } From ce33f39d9204d0a86c40f50c1cea57d816a9b79a Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Tue, 3 Feb 2026 14:25:33 +0530 Subject: [PATCH 08/29] Agent and VMware resource changes --- .../com/cloud/resource/ServerResource.java | 16 ++++ .../com/cloud/agent/manager/AgentAttache.java | 8 +- .../cloud/agent/manager/AgentManagerImpl.java | 12 ++- .../agent/manager/DirectAgentAttache.java | 83 +++++++++++++++++++ .../agent/manager/DirectAgentAttacheTest.java | 52 ++++++++++-- .../vmware/resource/VmwareResource.java | 13 ++- .../vmware/mo/VirtualMachineMO.java | 10 ++- .../hypervisor/vmware/util/VmwareClient.java | 34 ++++---- .../hypervisor/vmware/util/VmwareContext.java | 17 ++-- .../hypervisor/vmware/util/VmwareHelper.java | 48 +++++++++-- 10 files changed, 246 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 23d200942a27..6787e6d40b4f 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -92,4 +92,20 @@ default boolean isAppendAgentNameToLogs() { } default void processPingAnswer(PingAnswer answer) {}; + + /** + * Indicates whether a currently executing request sequence can be cancelled or not. + * Resources that cannot determine cancellability should return true. + */ + default boolean isRequestSequenceCancellable(long sequence) { + return true; + } + + /** + * Attempts to cancel a currently executing request sequence at the resource layer. + * Returning false indicates cancellation was not possible and execution should continue. + */ + default boolean cancelRequestSequence(long sequence) { + return true; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 163008c98c75..63253e8bcb7a 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -234,6 +234,10 @@ protected synchronized void cancel(final long seq) { } } + protected boolean isExecutionCancellable(final long seq) { + return true; + } + protected synchronized int findRequest(final Request req) { return Collections.binarySearch(_requests, req, s_reqComparator); } @@ -421,7 +425,7 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE try { for (int i = 0; i < 2; i++) { Answer[] answers = null; - if (_agentMgr._asyncJobDao.isJobCancelled(jobId)) { + if (_agentMgr._asyncJobDao.isJobCancelled(jobId) && isExecutionCancellable(seq)) { throw new OperationCancelledException(req.getCommands(), _id, seq, wait, false); } Command[] cmds = req.getCommands(); @@ -435,7 +439,7 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE } catch (final InterruptedException e) { logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted while waiting for job commands processing"); Thread.currentThread().interrupt(); - if (_agentMgr._asyncJobDao.isJobCancelled(jobId)) { + if (_agentMgr._asyncJobDao.isJobCancelled(jobId) && isExecutionCancellable(seq)) { throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, "Cancelled during waiting for job commands processing"); } } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 6cd3695cd9d3..cdbb9bc02e07 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -63,6 +63,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; +import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.MapUtils; @@ -2109,7 +2110,16 @@ protected void runInContext() { if (sequence != null) { try { final AgentAttache agent = getAttache(sequence.first()); - agent.cancel(sequence.second()); + if (agent.isExecutionCancellable(sequence.second())) { + agent.cancel(sequence.second()); + if (job.getStatus() != JobInfo.Status.CANCELLED) { + job.setStatus(JobInfo.Status.CANCELLED); + _asyncJobDao.update(job.getId(), job); + } + } else { + logger.info("Job {} cancellation requested but sequence {} on host {} is not cancellable, allowing execution to continue.", + job.getId(), sequence.second(), sequence.first()); + } } catch (AgentUnavailableException e) { logger.debug("Agent {} not found", sequence.first()); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index 5f28994c889a..39ab1266fa97 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -17,6 +17,7 @@ package com.cloud.agent.manager; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -52,6 +53,7 @@ public class DirectAgentAttache extends AgentAttache { ServerResource _resource; List> _futures = new ArrayList>(); private final Map> _taskFutures = new ConcurrentHashMap<>(); + private final Map _taskRequests = new ConcurrentHashMap<>(); long _seq = 0; LinkedList tasks = new LinkedList(); AtomicInteger _outstandingTaskCount; @@ -73,9 +75,17 @@ public void disconnect(Status state) { } synchronized (this) { + for (Task task : tasks) { + task._req.cancel(); + } tasks.clear(); } + for (Request request : _taskRequests.values()) { + request.cancel(); + } + _taskRequests.clear(); + for (Future future : _taskFutures.values()) { boolean cancelled = future.cancel(true); logger.debug("Running task {} for [id: {}, uuid: {}, name: {}]", cancelled ? "cancelled" : "not cancelled", _id, _uuid, _name); @@ -92,6 +102,66 @@ public void disconnect(Status state) { cleanup(state); } + @Override + protected synchronized boolean isExecutionCancellable(final long seq) { + if (_taskFutures.containsKey(seq)) { + final ServerResource resource = _resource; + return resource == null || resource.isRequestSequenceCancellable(seq); + } + + if (_taskRequests.containsKey(seq)) { + return true; + } + + for (Task task : tasks) { + if (task._req.getSequence() == seq) { + return true; + } + } + return true; + } + + @Override + protected synchronized void cancel(final long seq) { + Request request = _taskRequests.get(seq); + if (request != null) { + request.cancel(); + } + + final Iterator iterator = tasks.iterator(); + while (iterator.hasNext()) { + final Task task = iterator.next(); + if (task._req.getSequence() == seq) { + task._req.cancel(); + iterator.remove(); + _taskRequests.remove(seq); + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelled queued task."); + super.cancel(seq); + return; + } + } + + final Future future = _taskFutures.get(seq); + if (future != null) { + final ServerResource resource = _resource; + if (resource != null && !resource.isRequestSequenceCancellable(seq)) { + logger.info(LOG_SEQ_FORMATTED_STRING, seq, "Cancellation requested but command is not cancellable, continuing execution."); + return; + } + + if (resource != null) { + resource.cancelRequestSequence(seq); + } + + final boolean cancelled = future.cancel(true); + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Running task " + (cancelled ? "cancelled" : "not cancelled")); + _taskFutures.remove(seq); + } + + _taskRequests.remove(seq); + super.cancel(seq); + } + @Override public boolean equals(Object obj) { if (!(obj instanceof DirectAgentAttache)) { @@ -155,6 +225,7 @@ protected void finalize() throws Throwable { private synchronized void queueTask(Task task) { tasks.add(task); + _taskRequests.put(task._req.getSequence(), task._req); } private synchronized void scheduleFromQueue() { @@ -352,6 +423,17 @@ protected void runInContext() { logger.warn("Resource returned null answer!"); answer = new Answer(cmds[i], false, "Resource returned null answer"); } + + // If cancellation was requested while the current command was running, + // stop scheduling further commands for cancellable sequences. + if ((Thread.currentThread().isInterrupted() || _req.isCancelled()) + && resource.isRequestSequenceCancellable(seq)) { + answers.add(answer); + for (int j = i + 1; j < cmds.length; j++) { + answers.add(new Answer(cmds[j], false, "Command cancelled")); + } + break; + } } else { answer = new Answer(cmds[i], false, "Agent is disconnected"); } @@ -401,6 +483,7 @@ protected void runInContext() { logger.error(LOG_SEQ_FORMATTED_STRING, seq, "Throwable caught in runInContext, this will cause the management to become unpredictable", t); } finally { _taskFutures.remove(seq); + _taskRequests.remove(seq); _outstandingTaskCount.decrementAndGet(); scheduleFromQueue(); } diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java index 4ba276460e3a..0049deebc8cb 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java @@ -16,19 +16,21 @@ // under the License. package com.cloud.agent.manager; +import java.util.UUID; +//import java.util.concurrent.Future; +// +//import com.cloud.agent.api.Command; +//import com.cloud.agent.transport.Request; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; import org.mockito.junit.MockitoJUnitRunner; import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ServerResource; -import java.util.UUID; - @RunWith(MockitoJUnitRunner.class) public class DirectAgentAttacheTest { @Mock @@ -37,23 +39,61 @@ public class DirectAgentAttacheTest { @Mock private ServerResource _resource; +// @Mock +// private ExecutorService directAgentPool; + long _id = 0L; String _uuid = UUID.randomUUID().toString(); @Before public void setup() { +// Mockito.when(_agentMgr.getDirectAgentPool()).thenReturn(directAgentPool); + Mockito.doReturn(2).when(_agentMgr).getDirectAgentThreadCap(); directAgentAttache = new DirectAgentAttache(_agentMgr, _id, _uuid, "myDirectAgentAttache", Hypervisor.HypervisorType.KVM, _resource, false); - - MockitoAnnotations.initMocks(directAgentAttache); } private DirectAgentAttache directAgentAttache; @Test public void testPingTask() throws Exception { DirectAgentAttache.PingTask pt = directAgentAttache.new PingTask(); - Mockito.doReturn(2).when(_agentMgr).getDirectAgentThreadCap(); pt.runInContext(); Mockito.verify(_resource, Mockito.times(1)).getCurrentStatus(_id); } + +// @Test +// public void testCancelRunningTaskCancelsFutureAndRequest() throws Exception { +// final Command command = Mockito.mock(Command.class); +// final Request request = new Request(_id, -1, command, false); +// final long seq = 101L; +// request.setSequence(seq); +// +// @SuppressWarnings("unchecked") +// final Future runningFuture = Mockito.mock(Future.class); +// Mockito.when(directAgentPool.submit(Mockito.any(Runnable.class))).thenReturn(runningFuture); +// +// directAgentAttache.send(request); +// directAgentAttache.cancel(seq); +// +// Mockito.verify(runningFuture, Mockito.times(1)).cancel(true); +// org.junit.Assert.assertTrue(request.isCancelled()); +// } +// +// @Test +// public void testCancelQueuedTaskMarksRequestCancelledAndRemovesItFromQueue() throws Exception { +// final Command command = Mockito.mock(Command.class); +// final Request request = new Request(_id, -1, command, false); +// final long seq = 202L; +// request.setSequence(seq); +// +// Mockito.doReturn(0).when(_agentMgr).getDirectAgentThreadCap(); +// +// directAgentAttache.send(request); +// org.junit.Assert.assertEquals(1, directAgentAttache.tasks.size()); +// +// directAgentAttache.cancel(seq); +// +// org.junit.Assert.assertTrue(request.isCancelled()); +// org.junit.Assert.assertEquals(0, directAgentAttache.tasks.size()); +// } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 0687e6cc6cde..58672d01d80e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -514,6 +514,7 @@ public Answer executeRequest(Command cmd) { } else if (clz == StopCommand.class) { String vmName = ((StopCommand) cmd).getVmName(); setTaskContext(cmdSequence, cmd, vmName); + Thread.sleep(3 * 60 * 1000); // 3 mins answer = execute((StopCommand) cmd); } else if (clz == RebootRouterCommand.class) { answer = execute((RebootRouterCommand) cmd); @@ -589,7 +590,7 @@ public Answer executeRequest(Command cmd) { } else if (clz == StartCommand.class) { String vmName = ((StartCommand) cmd).getVirtualMachine().getName(); setTaskContext(cmdSequence, cmd, vmName); - Thread.sleep(3 * 1000); + Thread.sleep(3 * 60 * 1000); // 3 mins answer = execute((StartCommand) cmd); } else if (clz == CheckSshCommand.class) { answer = execute((CheckSshCommand) cmd); @@ -708,6 +709,16 @@ private void setTaskContext(long cmdSequence, Command cmd, String vmName) { VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); } + @Override + public boolean isRequestSequenceCancellable(long sequence) { + return VmwareHelper.isActiveVmTaskCancellable(sequence); + } + + @Override + public boolean cancelRequestSequence(long sequence) { + return VmwareHelper.cancelActiveVmTask(sequence); + } + private ExecutionResult getSystemVmVersionAndChecksum(String controlIp) { ExecutionResult result; try { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 8a464db14522..1e276003cd23 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -3997,9 +3997,13 @@ public void cancelPendingTasks() throws Exception { vmTasks++; if (!(info.getState().equals(TaskInfoState.SUCCESS) || info.getState().equals(TaskInfoState.ERROR))) { String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown"; - logger.debug(taskName + " task pending for the Instance: " + vmName + ", cancelling it"); - vmPendingTasks++; - _context.getVimClient().cancelTask(task); + if (_context.getVimClient().isTaskCancellable(task)) { + logger.debug(taskName + " task pending for the Instance: " + vmName + ", cancelling it"); + vmPendingTasks++; + _context.getVimClient().cancelTask(task); + } else { + logger.debug(taskName + " task pending for the Instance: " + vmName + ", but not cancellable"); + } } } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index a57c27181f44..b30d7f505fb4 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -825,7 +825,7 @@ public int getVcenterSessionTimeout() { return vCenterSessionTimeout; } - public boolean isTaskCancelable(ManagedObjectReference task) throws Exception { + public boolean isTaskCancellable(ManagedObjectReference task) throws Exception { TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); if (info == null) { LOGGER.warn("Unable to get the task info for task: " + task.getValue()); @@ -849,8 +849,8 @@ public boolean isTaskCancelable(ManagedObjectReference task) throws Exception { public Pair cancelTask(ManagedObjectReference task) throws Exception { TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); if (info == null) { - LOGGER.warn("Unable to get the task info, so couldn't cancel the task"); - return new Pair<>(false, "Unable to get the task info, so couldn't cancel the task"); + LOGGER.warn("Unable to get the VM task info, so couldn't cancel the task"); + return new Pair<>(false, "Unable to get the VM task info, so couldn't cancel the task"); } String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown"; @@ -859,22 +859,22 @@ public Pair cancelTask(ManagedObjectReference task) throws Exce String entityName = StringUtils.isNotBlank(info.getEntityName()) ? info.getEntityName() : ""; if (info.getState().equals(TaskInfoState.SUCCESS)) { - LOGGER.debug(taskName + " task successfully completed for the entity " + entityName + ", can't cancel it"); - return new Pair<>(false, "Task successfully completed for the entity " + entityName); + LOGGER.debug("VM task " + " successfully completed for the entity " + entityName + ", can't cancel it"); + return new Pair<>(false, "VM task successfully completed for the entity " + entityName); } if (info.getState().equals(TaskInfoState.ERROR)) { - LOGGER.debug(taskName + " task execution failed for the entity " + entityName + ", can't cancel it"); - return new Pair<>(false, "Task execution failed for the entity " + entityName); + LOGGER.debug("VM task " + taskName + " execution failed for the entity " + entityName + ", can't cancel it"); + return new Pair<>(false, "VM task execution failed for the entity " + entityName); } - LOGGER.debug(taskName + " task pending for the entity " + entityName + ", trying to cancel"); + LOGGER.debug("VM task " + taskName + " pending for the entity " + entityName + ", trying to cancel"); if (!info.isCancelable()) { - LOGGER.warn(taskName + " task will continue to run on vCenter because it can't be cancelled"); - return new Pair<>(false, "Task will continue to run because it can't be cancelled" + entityName); + LOGGER.warn("VM task " + taskName + " will continue to run on vCenter because it can't be cancelled"); + return new Pair<>(false, "VM task will continue to run because it can't be cancelled" + entityName); } - LOGGER.debug("Cancelling task " + taskName + " of the entity " + entityName); + LOGGER.debug("Cancelling VM task " + taskName + " of the entity " + entityName); getService().cancelTask(task); // Since task cancellation is asynchronous, wait for the task to be cancelled @@ -883,20 +883,20 @@ public Pair cancelTask(ManagedObjectReference task) throws Exce if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { - LOGGER.warn("Failed to cancel" + taskName + " task of the entity " + entityName + ", the task successfully completed"); - return new Pair<>(false, "Task successfully completed for the entity " + entityName); + LOGGER.warn("Failed to cancel VM task " + taskName + " of the entity " + entityName + ", the task successfully completed"); + return new Pair<>(false, "VM task successfully completed for the entity " + entityName); } if (result[1] instanceof LocalizedMethodFault) { MethodFault fault = ((LocalizedMethodFault)result[1]).getFault(); if (fault instanceof RequestCanceled) { - LOGGER.debug(taskName + " task of the entity " + entityName + " was successfully cancelled"); - return new Pair<>(true, "Successfully cancelled"); + LOGGER.debug("VM task " + taskName + " of the entity " + entityName + " was successfully cancelled"); + return new Pair<>(true, "VM task successfully cancelled"); } } else { - LOGGER.warn("Couldn't cancel " + taskName + " task of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage()); + LOGGER.warn("Couldn't cancel VM task " + taskName + " of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage()); } } - return new Pair<>(false, "Unable to cancel " + taskName + " of the entity " + entityName); + return new Pair<>(false, "Unable to cancel VM task " + taskName + " of the entity " + entityName); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index c2b42894c459..6bef56f25bc9 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -345,10 +345,7 @@ public String getDatastoreNameFromPath(String inventoryPath) throws Exception { public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Exception { while (true) { - if (Thread.currentThread().isInterrupted()) { - Thread.currentThread().interrupt(); - throw new InterruptedException("Task progress wait cancelled for task: " + morTask.getValue()); - } + checkThreadInterrupted(morTask); TaskInfo tinfo = (TaskInfo)_vimClient.getDynamicProperty(morTask, "info"); Integer progress = tinfo.getProgress(); @@ -358,14 +355,18 @@ public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Excep if (progress.intValue() >= 100) break; - if (Thread.currentThread().isInterrupted()) { - Thread.currentThread().interrupt(); - throw new InterruptedException("Task progress wait cancelled for task: " + morTask.getValue()); - } + checkThreadInterrupted(morTask); Thread.sleep(1000); } } + private void checkThreadInterrupted(ManagedObjectReference morTask) throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + Thread.currentThread().interrupt(); + throw new InterruptedException("Task progress wait cancelled for task: " + morTask.getValue()); + } + } + public void getFile(String urlString, String localFileFullName) throws Exception { HttpURLConnection conn = getHTTPConnection(urlString); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 380d716abd35..74054609e606 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -48,7 +48,6 @@ import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.hypervisor.vmware.mo.DistributedVirtualSwitchMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; -import com.cloud.hypervisor.vmware.mo.TaskMO; import com.cloud.serializer.GsonHelper; import com.cloud.storage.Volume; import com.cloud.utils.exception.CloudRuntimeException; @@ -163,8 +162,9 @@ public static class ActiveVmTaskInfo { private static void registerActiveVmTask(long cmdSequence, String vmName, String commandType, ManagedObjectReference taskMor, VmwareContext context) { + if (StringUtils.isBlank(vmName)) { - LOGGER.debug("No active VM for sequence={}, command={}, task={}", cmdSequence, commandType, taskMor); + LOGGER.debug("No active VM to registered VM task for sequence={}, command={}, task={}", cmdSequence, commandType, taskMor); return; } @@ -187,7 +187,7 @@ private static void unregisterActiveVmTask(long cmdSequence) { vmNameToTaskSequences.remove(taskInfo.vmName); } } - LOGGER.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={}ms", + LOGGER.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={} ms", cmdSequence, taskInfo.vmName, taskInfo.commandType, System.currentTimeMillis() - taskInfo.startTime); } @@ -208,24 +208,49 @@ public static int cancelActiveVmTasks(String vmName) { return cancelled; } - public static boolean cancelActiveVmTask(long cmdSequence) { + public static boolean isActiveVmTaskCancellable(long cmdSequence) { ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); if (taskInfo == null) { + return true; + } + + try { + return taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor); + } catch (Exception e) { + LOGGER.warn("Failed to verify vCenter task cancellability for sequence={}, vm={}, error={}", + cmdSequence, taskInfo.vmName, e.getMessage(), e); return false; } + } + + public static boolean cancelActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); + if (taskInfo == null) { + return true; + } try { - // Cancel the vCenter task - TaskMO taskMo = new TaskMO(taskInfo.context, taskInfo.taskMor); - taskMo.cancelTask(); - LOGGER.info("Cancelled vCenter task: sequence={}, vm={}, command={}, task={}", + if (!taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor)) { + LOGGER.info("Active vCenter VM task is not cancellable: sequence={}, vm={}, command={}", + cmdSequence, taskInfo.vmName, taskInfo.commandType); + return false; + } + + Pair cancellationResult = taskInfo.context.getVimClient().cancelTask(taskInfo.taskMor); + if (!cancellationResult.first()) { + LOGGER.info("Unable to cancel vCenter VM task: sequence={}, vm={}, reason={}", + cmdSequence, taskInfo.vmName, cancellationResult.second()); + return false; + } + + LOGGER.info("Cancelled vCenter VM task: sequence={}, vm={}, command={}, task={}", cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.taskMor.getValue()); // Unregister the task unregisterActiveVmTask(cmdSequence); return true; } catch (Exception e) { - LOGGER.warn("Failed to cancel vCenter task: sequence={}, vm={}, error={}", + LOGGER.warn("Failed to cancel vCenter VM task: sequence={}, vm={}, error={}", cmdSequence, taskInfo.vmName, e.getMessage(), e); return false; } @@ -272,10 +297,15 @@ public void unregisterTask() { } public static void setTaskContext(long cmdSequence, String vmName, String commandType) { + LOGGER.debug("Setting task context: sequence={}, vm={}, command={}", cmdSequence, vmName, commandType); currentTaskContext.set(new TaskContext(cmdSequence, vmName, commandType)); } public static void clearTaskContext() { + TaskContext taskContext = currentTaskContext.get(); + if (taskContext != null) { + LOGGER.debug("Clearing task context: sequence={}, vm={}, command={}", taskContext.cmdSequence, taskContext.vmName, taskContext.commandType); + } currentTaskContext.remove(); } From 8db8313168af0c6ea2611c45cb07c1fceebc990a Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 6 Feb 2026 12:37:54 +0530 Subject: [PATCH 09/29] changes and some code improvements --- .../api/command/admin/vm/MigrateVMCmd.java | 1 - .../MigrateVirtualMachineWithVolumeCmd.java | 1 - .../RevertToVMSnapshotCmdByAdmin.java | 2 +- .../admin/volume/AttachVolumeCmdByAdmin.java | 2 +- .../admin/volume/CreateVolumeCmdByAdmin.java | 2 +- .../admin/volume/DestroyVolumeCmdByAdmin.java | 3 +- .../admin/volume/DetachVolumeCmdByAdmin.java | 2 +- .../admin/volume/MigrateVolumeCmdByAdmin.java | 2 +- .../user/vmsnapshot/CreateVMSnapshotCmd.java | 1 - .../com/cloud/vm/VmWorkJobHandlerProxy.java | 44 +++++++------ .../cloud/vm/VirtualMachineManagerImpl.java | 63 ++++++++----------- .../com/cloud/vm/VmWorkJobDispatcher.java | 2 +- .../jobs/AsyncJobExecutionContext.java | 45 ++++++------- .../framework/jobs/AsyncJobManager.java | 9 +-- .../jobs/dao/AsyncJobJoinMapDaoImpl.java | 3 +- .../framework/jobs/dao/SyncQueueItemDao.java | 15 ++--- .../jobs/dao/SyncQueueItemDaoImpl.java | 4 +- .../jobs/impl/AsyncJobManagerImpl.java | 39 ++++++------ .../framework/jobs/impl/SyncQueueManager.java | 16 ++--- .../jobs/impl/SyncQueueManagerImpl.java | 10 +-- .../framework/jobs/impl/VmWorkJobVO.java | 6 +- .../vmware/manager/VmwareManagerImpl.java | 4 +- .../vmware/resource/VmwareResource.java | 2 - .../java/com/cloud/api/ApiDispatcher.java | 3 +- .../main/java/com/cloud/api/ApiServer.java | 25 ++++---- .../api/dispatch/CommandCreationWorker.java | 5 +- 26 files changed, 145 insertions(+), 166 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java index 0cbc55b81d9e..45ac0a27bd36 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java @@ -52,7 +52,6 @@ cancellable = true) public class MigrateVMCmd extends BaseAsyncCmd { - ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java index 4d2a412738a3..85c2f5c20274 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java @@ -53,7 +53,6 @@ cancellable = true) public class MigrateVirtualMachineWithVolumeCmd extends BaseAsyncCmd { - ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vmsnapshot/RevertToVMSnapshotCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vmsnapshot/RevertToVMSnapshotCmdByAdmin.java index a57362d62056..0c25c6c693e0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vmsnapshot/RevertToVMSnapshotCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vmsnapshot/RevertToVMSnapshotCmdByAdmin.java @@ -23,5 +23,5 @@ import org.apache.cloudstack.api.response.UserVmResponse; @APICommand(name = "revertToVMSnapshot", description = "Revert Instance from a vmsnapshot.", responseObject = UserVmResponse.class, since = "4.2.0", responseView = ResponseView.Full, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true) public class RevertToVMSnapshotCmdByAdmin extends RevertToVMSnapshotCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/AttachVolumeCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/AttachVolumeCmdByAdmin.java index 20418d589405..5a6ee0f19d16 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/AttachVolumeCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/AttachVolumeCmdByAdmin.java @@ -25,5 +25,5 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "attachVolume", description = "Attaches a disk volume to an Instance.", responseObject = VolumeResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class AttachVolumeCmdByAdmin extends AttachVolumeCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/CreateVolumeCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/CreateVolumeCmdByAdmin.java index e156475b29ae..40b950bfbb33 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/CreateVolumeCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/CreateVolumeCmdByAdmin.java @@ -27,5 +27,5 @@ @APICommand(name = "createVolume", responseObject = VolumeResponse.class, description = "Creates a disk volume from a disk offering. This disk volume must still be attached to an Instance to make use of it.", responseView = ResponseView.Full, entityType = { Volume.class, VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class CreateVolumeCmdByAdmin extends CreateVolumeCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DestroyVolumeCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DestroyVolumeCmdByAdmin.java index de90fee102de..5273d562dc49 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DestroyVolumeCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DestroyVolumeCmdByAdmin.java @@ -33,7 +33,8 @@ since = "4.14.0", authorized = {RoleType.Admin}, requestHasSensitiveInfo = false, - responseHasSensitiveInfo = true) + responseHasSensitiveInfo = true, + cancellable = true) public class DestroyVolumeCmdByAdmin extends DestroyVolumeCmd implements AdminCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DetachVolumeCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DetachVolumeCmdByAdmin.java index 05f9fe6a31a6..c39e487fc04f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DetachVolumeCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DetachVolumeCmdByAdmin.java @@ -25,5 +25,5 @@ import com.cloud.vm.VirtualMachine; @APICommand(name = "detachVolume", description = "Detaches a disk volume from an Instance.", responseObject = VolumeResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class DetachVolumeCmdByAdmin extends DetachVolumeCmd implements AdminCmd {} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/MigrateVolumeCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/MigrateVolumeCmdByAdmin.java index 9dd544cae581..f30f3f788fbc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/MigrateVolumeCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/MigrateVolumeCmdByAdmin.java @@ -25,6 +25,6 @@ import com.cloud.storage.Volume; @APICommand(name = "migrateVolume", description = "Migrate volume", responseObject = VolumeResponse.class, since = "3.0.0", responseView = ResponseView.Full, entityType = { - Volume.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + Volume.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class MigrateVolumeCmdByAdmin extends MigrateVolumeCmd implements AdminCmd { } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java index 33ce3fee7789..fd7a93e02aa1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java @@ -41,7 +41,6 @@ requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true) public class CreateVMSnapshotCmd extends BaseAsyncCreateCmd { - @ACL(accessType = AccessType.OperateEntry) @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, required = true, entityType = UserVmResponse.class, description = "The ID of the Instance") private Long vmId; diff --git a/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java b/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java index ef1c71e6b014..92edfdcb1cd4 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java @@ -93,36 +93,34 @@ private Method getHandlerMethod(Class paramType) { public Pair handleVmWorkJob(VmWork work) throws Exception { Method method = getHandlerMethod(work.getClass()); - if (method != null) { + if (method == null) { + logger.error("Unable to find handler for VM work job: {} {}", work.getClass().getName(), work); + RuntimeException ex = new RuntimeException("Unable to find handler for VM work job: " + work.getClass().getName()); + return new Pair<>(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); + } - try { - if (logger.isDebugEnabled()) - logger.debug("Execute VM work job: " + work.getClass().getName() + work); + try { + if (logger.isDebugEnabled()) + logger.debug("Execute VM work job: {}{}", work.getClass().getName(), work); - Object obj = method.invoke(_target, work); + Object obj = method.invoke(_target, work); - if (logger.isDebugEnabled()) - logger.debug("Done executing VM work job: " + work.getClass().getName() + work); + if (logger.isDebugEnabled()) + logger.debug("Done executing VM work job: {}{}", work.getClass().getName(), work); - assert (obj instanceof Pair); - return (Pair)obj; - } catch (InvocationTargetException e) { - logger.error("Invocation exception, caused by: " + e.getCause()); + assert (obj instanceof Pair); + return (Pair)obj; + } catch (InvocationTargetException e) { + logger.error("Invocation exception, caused by: {}", String.valueOf(e.getCause())); - // legacy CloudStack code relies on checked exception for error handling - // we need to re-throw the real exception here - if (e.getCause() != null && e.getCause() instanceof Exception) { - logger.info("Rethrow exception " + e.getCause()); - throw (Exception)e.getCause(); - } - - throw e; + // legacy CloudStack code relies on checked exception for error handling + // we need to re-throw the real exception here + if (e.getCause() != null && e.getCause() instanceof Exception) { + logger.info("Rethrow exception {}", String.valueOf(e.getCause())); + throw (Exception)e.getCause(); } - } else { - logger.error("Unable to find handler for VM work job: {} {}", work.getClass().getName(), work); - RuntimeException ex = new RuntimeException("Unable to find handler for VM work job: " + work.getClass().getName()); - return new Pair<>(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); + throw e; } } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index a7525c0a7145..c75f564c04dd 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -663,7 +663,7 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t } VirtualMachineGuru getVmGuru(final VirtualMachine vm) { - if(vm != null) { + if (vm != null) { return _vmGurus.get(vm.getType()); } return null; @@ -1008,7 +1008,6 @@ protected boolean checkWorkItems(final VMInstanceVO vm, final State state) throw } logger.debug("Waited some more to make sure there's no activity on " + vm); } - } @DB @@ -1298,7 +1297,6 @@ protected void updateExternalVmFromPrepareAnswer(VirtualMachineTO vmTO, VirtualM } updateExternalVmDataFromPrepareAnswer(vmTO, updatedTO); updateExternalVmNicsFromPrepareAnswer(vmTO, updatedTO); - return; } protected void processPrepareExternalProvisioning(boolean firstStart, Host host, @@ -2043,7 +2041,7 @@ private void syncDiskChainChange(final StartAnswer answer) { _volsDao.update(volume.getId(), volume); } - if(vol.getPath() != null) { + if (vol.getPath() != null) { volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID()); } else { volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID()); @@ -2061,7 +2059,6 @@ public void stop(final String vmUuid) throws ResourceUnavailableException { } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException(String.format("Unable to stop vm [%s] because of a concurrent operation", vmUuid), e); } - } @Override @@ -2257,7 +2254,7 @@ protected Pair sendStop(final VirtualMachineGuru guru, final Vi final StopCommand stop = stpCmd; try { Answer answer = null; - if(vm.getHostId() != null) { + if (vm.getHostId() != null) { answer = _agentMgr.send(vm.getHostId(), stop); } if (answer != null && answer instanceof StopAnswer) { @@ -2294,7 +2291,6 @@ protected Pair sendStop(final VirtualMachineGuru guru, final Vi logger.error(errorMsg); return new Pair<>(false, errorMsg); } - } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { String errorMsg = String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()); logger.warn(errorMsg, e); @@ -2343,7 +2339,6 @@ protected Pair cleanup(final VirtualMachineGuru guru, final Vir } } } - } else if (state == State.Stopping) { if (vm.getHostId() != null) { Pair result = sendStop(guru, profile, cleanUpEvenIfUnableToStop, false); @@ -2413,7 +2408,6 @@ public void advanceStop(final String vmUuid, final boolean cleanUpEvenIfUnableTo _workJobDao.expunge(placeHolder.getId()); } } - } else { final Outcome outcome = stopVmThroughJobQueue(vmUuid, cleanUpEvenIfUnableToStop); @@ -2774,13 +2768,12 @@ public void doInTransactionWithoutResult(final TransactionStatus status) throws * @param expunge indicates if vm should be expunged */ private void deleteVMSnapshots(VMInstanceVO vm, boolean expunge) { - if (! vm.getHypervisorType().equals(HypervisorType.VMware)) { + if (!vm.getHypervisorType().equals(HypervisorType.VMware)) { if (!_vmSnapshotMgr.deleteAllVMSnapshots(vm.getId(), null)) { logger.debug("Unable to delete all Snapshots for {}", vm); throw new CloudRuntimeException("Unable to delete Instance Snapshots for " + vm); } - } - else { + } else { if (expunge) { _vmSnapshotMgr.deleteVMSnapshotsFromDB(vm.getId(), false); } @@ -3018,7 +3011,7 @@ private void checkDestinationForTags(StoragePool destPool, VMInstanceVO vm) { for(Volume vol : vols) { DiskOfferingVO diskOffering = _diskOfferingDao.findById(vol.getDiskOfferingId()); List volumeTags = StringUtils.csvTagsToList(diskOffering.getTags()); - if(! matches(volumeTags, storageTags)) { + if (!matches(volumeTags, storageTags)) { String msg = String.format("destination pool '%s' with tags '%s', does not support the volume diskoffering for volume '%s' (tags: '%s') ", destPool.getName(), StringUtils.listToCsvTags(storageTags), @@ -3566,7 +3559,7 @@ protected void createStoragePoolMappingsForVolumes(VirtualMachineProfile profile executeManagedStorageChecksWhenTargetStoragePoolNotProvided(targetHost, currentPool, volume); if (ScopeType.HOST.equals(currentPool.getScope()) || isStorageCrossClusterMigration(plan.getClusterId(), currentPool)) { createVolumeToStoragePoolMappingIfPossible(profile, plan, volumeToPoolObjectMap, volume, currentPool); - } else if (shouldMapVolume(profile, currentPool)){ + } else if (shouldMapVolume(profile, currentPool)) { volumeToPoolObjectMap.put(volume, currentPool); } } @@ -3603,7 +3596,7 @@ protected void executeManagedStorageChecksWhenTargetStoragePoolNotProvided(Host } /** - * Return true if the VM migration is a cross cluster migration. To execute that, we check if the volume current storage pool cluster is different from the target cluster. + * Return true if the VM migration is a cross-cluster migration. To execute that, we check if the volume current storage pool cluster is different from the target cluster. */ protected boolean isStorageCrossClusterMigration(Long clusterId, StoragePoolVO currentPool) { return clusterId != null && ScopeType.CLUSTER.equals(currentPool.getScope()) && !currentPool.getClusterId().equals(clusterId); @@ -4136,7 +4129,7 @@ public void advanceReboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ConcurrentOperationException, + private void orchestrateReboot(final String vmUuid, final Map params) throws ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { @@ -4203,7 +4196,7 @@ private void orchestrateReboot(final String vmUuid, final Map vmMetadatum) { boolean found = false; for(Pair, Pair> vmDetail : vmDetails ) { Pair vmNameTypePair = vmDetail.first(); - if(vmNameTypePair.first().equals(name)) { + if (vmNameTypePair.first().equals(name)) { found = true; - if(vmNameTypePair.second() == VirtualMachine.Type.User) { + if (vmNameTypePair.second() == VirtualMachine.Type.User) { Pair detailPair = vmDetail.second(); String platformDetail = detailPair.second(); @@ -4369,9 +4362,9 @@ public void syncVMMetaData(final Map vmMetadatum) { } } - if(!found) { + if (!found) { VMInstanceVO vm = _vmDao.findVMByInstanceName(name); - if(vm != null && vm.getType() == VirtualMachine.Type.User) { + if (vm != null && vm.getType() == VirtualMachine.Type.User) { updateVmMetaData(vm.getId(), platform); } } @@ -4381,12 +4374,12 @@ public void syncVMMetaData(final Map vmMetadatum) { private void updateVmMetaData(Long vmId, String platform) { UserVmVO userVm = _userVmDao.findById(vmId); _userVmDao.loadDetails(userVm); - if ( userVm.details.containsKey(VmDetailConstants.TIME_OFFSET)) { + if (userVm.details.containsKey(VmDetailConstants.TIME_OFFSET)) { userVm.details.remove(VmDetailConstants.TIME_OFFSET); } userVm.setDetail(VmDetailConstants.PLATFORM, platform); String pvdriver = "xenserver56"; - if ( platform.contains("device_id")) { + if (platform.contains("device_id")) { pvdriver = "xenserver61"; } if (!userVm.details.containsKey(VmDetailConstants.HYPERVISOR_TOOLS_VERSION) || !userVm.details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION).equals(pvdriver)) { @@ -4403,7 +4396,7 @@ public boolean isRecurring() { @Override public boolean processAnswers(final long agentId, final long seq, final Answer[] answers) { for (final Answer answer : answers) { - if ( answer instanceof ClusterVMMetaDataSyncAnswer) { + if (answer instanceof ClusterVMMetaDataSyncAnswer) { final ClusterVMMetaDataSyncAnswer cvms = (ClusterVMMetaDataSyncAnswer)answer; if (!cvms.isExecuted()) { syncVMMetaData(cvms.getVMMetaDatum()); @@ -4593,7 +4586,7 @@ protected void checkIfNewOfferingStorageScopeMatchesStoragePool(VirtualMachine v public boolean isRootVolumeOnLocalStorage(long vmId) { ScopeType poolScope = ScopeType.ZONE; List volumes = _volsDao.findByInstanceAndType(vmId, Type.ROOT); - if(CollectionUtils.isNotEmpty(volumes)) { + if (CollectionUtils.isNotEmpty(volumes)) { VolumeVO rootDisk = volumes.get(0); Long poolId = rootDisk.getPoolId(); if (poolId != null) { @@ -4703,7 +4696,7 @@ private NicProfile orchestrateAddVmToNetwork(final VirtualMachine vm, final Netw logger.debug("Nic is plugged successfully for vm {} in network {}. VM is a part of network now.", vm, network); final long isDefault = nic.isDefaultNic() ? 1 : 0; - if(VirtualMachine.Type.User.equals(vmVO.getType())) { + if (VirtualMachine.Type.User.equals(vmVO.getType())) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(), Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay()); } @@ -5485,9 +5478,6 @@ private ApiCommandResourceType getApiCommandResourceTypeForVm(VirtualMachine vm) } private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { - Host host = _hostDao.findById(vm.getHostId()); - Host poweredHost = _hostDao.findById(vm.getPowerHostId()); - switch (vm.getState()) { case Starting: logger.info("VM {} is at {} and we received a power-on report while there is no pending jobs on it.", vm.getInstanceName(), vm.getState()); @@ -5510,6 +5500,7 @@ private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { if (vm.getHostId() != null && !vm.getHostId().equals(vm.getPowerHostId())) { logger.info("Detected out of band VM migration from host {} to host {}", () -> _hostDao.findById(vm.getHostId()), () -> _hostDao.findById(vm.getPowerHostId())); } + stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOnReport, vm.getPowerHostId()); } catch (final NoTransitionException e) { logger.warn("Unexpected VM state transition exception, race-condition?", e); @@ -5568,7 +5559,7 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { case Migrating: logger.info("VM {} is at {} and we received a {} report while there is no pending jobs on it" , vm, vm.getState(), vm.getPowerState()); - if((HighAvailabilityManager.ForceHA.value() || vm.isHaEnabled()) && vm.getState() == State.Running + if ((HighAvailabilityManager.ForceHA.value() || vm.isHaEnabled()) && vm.getState() == State.Running && HaVmRestartHostUp.value() && vm.getHypervisorType() != HypervisorType.VMware && vm.getHypervisorType() != HypervisorType.Hyperv) { @@ -5854,7 +5845,7 @@ public Outcome migrateVmThroughJobQueue(final String vmUuid, fin Map volumeStorageMap = dest.getStorageForDisks(); if (volumeStorageMap != null) { for (Volume vol : volumeStorageMap.keySet()) { - checkConcurrentJobsPerDatastoreThreshhold(volumeStorageMap.get(vol)); + checkConcurrentJobsPerDatastoreThreshold(volumeStorageMap.get(vol)); } } @@ -5949,7 +5940,7 @@ public Outcome migrateVmForScaleThroughJobQueue( return new VmJobVirtualMachineOutcome(workJob, vmId); } - private void checkConcurrentJobsPerDatastoreThreshhold(final StoragePool destPool) { + private void checkConcurrentJobsPerDatastoreThreshold(final StoragePool destPool) { final Long threshold = VolumeApiService.ConcurrentMigrationsThresholdPerDatastore.value(); if (threshold != null && threshold > 0) { long count = _jobMgr.countPendingJobs("\"storageid\":\"" + destPool.getUuid() + "\"", MigrateVMCmd.class.getName(), MigrateVolumeCmd.class.getName(), MigrateVolumeCmdByAdmin.class.getName()); @@ -5964,7 +5955,7 @@ public Outcome migrateVmStorageThroughJobQueue(final String vmUu Set uniquePoolIds = new HashSet<>(poolIds); for (Long poolId : uniquePoolIds) { StoragePoolVO pool = _storagePoolDao.findById(poolId); - checkConcurrentJobsPerDatastoreThreshhold(pool); + checkConcurrentJobsPerDatastoreThreshold(pool); } String commandName = VmWorkStorageMigration.class.getName(); @@ -6138,7 +6129,7 @@ private Pair orchestrateStart(final VmWorkStart work) th try { orchestrateStart(vm.getUuid(), work.getParams(), work.getPlan(), _dpMgr.getDeploymentPlannerByName(work.getDeploymentPlanner())); - } catch (CloudRuntimeException e){ + } catch (CloudRuntimeException e) { logger.error("Unable to orchestrate start {} due to [{}].", vm, e.getMessage()); CloudRuntimeException ex = new CloudRuntimeException(String.format("Unable to orchestrate the start of VM instance %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(vm, "instanceName", "uuid"))); @@ -6280,7 +6271,7 @@ private VmWorkJobVO createPlaceHolderWork(final long instanceId, String secondar workJob.setStep(VmWorkJobVO.Step.Starting); workJob.setVmType(VirtualMachine.Type.Instance); workJob.setVmInstanceId(instanceId); - if(org.apache.commons.lang3.StringUtils.isNotBlank(secondaryObjectIdentifier)) { + if (org.apache.commons.lang3.StringUtils.isNotBlank(secondaryObjectIdentifier)) { workJob.setSecondaryObjectIdentifier(secondaryObjectIdentifier); } workJob.setInitMsid(ManagementServerNode.getManagementServerId()); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java index d6c55d7038bb..2d802dc0b455 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java @@ -69,7 +69,7 @@ public void runJob(AsyncJob job) { } work = VmWorkSerializer.deserialize(workClz, job.getCmdInfo()); - if(work == null) { + if (work == null) { logger.error("Unable to deserialize VM work {}, job info: {}, job origin: {}", job.getCmd(), job.getCmdInfo(), job.getRelated()); _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, "Unable to deserialize VM work"); return; diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java index 2d39e18edea3..cf61b4fd1ec9 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java @@ -46,7 +46,7 @@ public static void init(AsyncJobManager jobMgr, AsyncJobJoinMapDao joinMapDao) { s_joinMapDao = joinMapDao; } - private static ManagedThreadLocal s_currentExectionContext = new ManagedThreadLocal(); + private static ManagedThreadLocal s_currentExecutionContext = new ManagedThreadLocal(); public AsyncJobExecutionContext() { } @@ -73,10 +73,7 @@ public void setJob(AsyncJob job) { public boolean isJobDispatchedBy(String jobDispatcherName) { assert (jobDispatcherName != null); - if (_job != null && _job.getDispatcher() != null && _job.getDispatcher().equals(jobDispatcherName)) - return true; - - return false; + return _job != null && _job.getDispatcher() != null && _job.getDispatcher().equals(jobDispatcherName); } public void completeAsyncJob(JobInfo.Status jobStatus, int resultCode, String resultObject) { @@ -118,7 +115,7 @@ public void joinJob(long joinJobId, String wakeupHandler, String wakeupDispatche // // check failure exception before we disjoin the worker job, work job usually fails with exception - // this will help propogate exception between jobs + // this will help propagate exception between jobs // TODO : it is ugly and this will become unnecessary after we switch to full-async mode // public void disjoinJob(long joinedJobId) throws InsufficientCapacityException, @@ -133,24 +130,21 @@ public void disjoinJob(long joinedJobId) throws InsufficientCapacityException, Object exception = JobSerializerHelper.fromObjectSerializedString(record.getJoinResult()); if (exception != null && exception instanceof Exception) { if (exception instanceof InsufficientCapacityException) { - LOGGER.error("Job " + joinedJobId + " failed with InsufficientCapacityException"); + LOGGER.error("Job {} failed with InsufficientCapacityException", joinedJobId); throw (InsufficientCapacityException)exception; - } - else if (exception instanceof ConcurrentOperationException) { - LOGGER.error("Job " + joinedJobId + " failed with ConcurrentOperationException"); + } else if (exception instanceof ConcurrentOperationException) { + LOGGER.error("Job {} failed with ConcurrentOperationException", joinedJobId); throw (ConcurrentOperationException)exception; - } - else if (exception instanceof ResourceUnavailableException) { - LOGGER.error("Job " + joinedJobId + " failed with ResourceUnavailableException"); + } else if (exception instanceof ResourceUnavailableException) { + LOGGER.error("Job {} failed with ResourceUnavailableException", joinedJobId); throw (ResourceUnavailableException)exception; - } - else { - LOGGER.error("Job " + joinedJobId + " failed with exception"); + } else { + LOGGER.error("Job {} failed with exception", joinedJobId); throw new RuntimeException((Exception)exception); } } } else { - LOGGER.error("Job " + joinedJobId + " failed without providing an error object"); + LOGGER.error("Job {} failed without providing an error object", joinedJobId); throw new RuntimeException("Job " + joinedJobId + " failed without providing an error object"); } } @@ -168,28 +162,29 @@ public void completeJobAndJoin(JobInfo.Status joinStatus, String joinResult) { } public static AsyncJobExecutionContext getCurrentExecutionContext() { - AsyncJobExecutionContext context = s_currentExectionContext.get(); + AsyncJobExecutionContext context = s_currentExecutionContext.get(); if (context == null) { // TODO, this has security implications, operations carried from API layer should always // set its context, otherwise, the fall-back here will use system security context // - LOGGER.warn("Job is executed without a context, setup psudo job for the executing thread"); - if (CallContext.current() != null) + LOGGER.warn("Job is executed without a context, setup pseudo job for the executing thread"); + if (CallContext.current() != null) { context = registerPseudoExecutionContext(CallContext.current().getCallingAccountId(), CallContext.current().getCallingUserId()); - else + } else { context = registerPseudoExecutionContext(Account.ACCOUNT_ID_SYSTEM, User.UID_SYSTEM); + } } return context; } // return currentExecutionContext without create it public static AsyncJobExecutionContext getCurrent() { - return s_currentExectionContext.get(); + return s_currentExecutionContext.get(); } public static AsyncJobExecutionContext registerPseudoExecutionContext(long accountId, long userId) { - AsyncJobExecutionContext context = s_currentExectionContext.get(); + AsyncJobExecutionContext context = s_currentExecutionContext.get(); if (context == null) { context = new AsyncJobExecutionContext(); context.setJob(s_jobMgr.getPseudoJob(accountId, userId)); @@ -200,14 +195,14 @@ public static AsyncJobExecutionContext registerPseudoExecutionContext(long accou } public static AsyncJobExecutionContext unregister() { - AsyncJobExecutionContext context = s_currentExectionContext.get(); + AsyncJobExecutionContext context = s_currentExecutionContext.get(); setCurrentExecutionContext(null); return context; } // This is intended to be package level access for AsyncJobManagerImpl only. public static void setCurrentExecutionContext(AsyncJobExecutionContext currentContext) { - s_currentExectionContext.set(currentContext); + s_currentExecutionContext.set(currentContext); } public static String getOriginJobId() { diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java index b187ff48146f..872dc4124101 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java @@ -27,8 +27,8 @@ public interface AsyncJobManager extends Manager { - public static final String API_JOB_POOL_THREAD_PREFIX = "API-Job-Executor"; - public static final String WORK_JOB_POOL_THREAD_PREFIX = "Work-Job-Executor"; + String API_JOB_POOL_THREAD_PREFIX = "API-Job-Executor"; + String WORK_JOB_POOL_THREAD_PREFIX = "Work-Job-Executor"; AsyncJobVO getAsyncJob(long jobId); @@ -45,6 +45,7 @@ public interface AsyncJobManager extends Manager { void updateAsyncJobStatus(long jobId, int processStatus, String resultObject); void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId); + void logJobJournal(long jobId, AsyncJob.JournalType journalType, String journalText, String journalObjJson); @@ -117,14 +118,14 @@ void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakupDispa * This method will be deprecated after all code has been migrated to fully-asynchronous mode * that uses async-feature of joinJob/disjoinJob * - * @param wakupTopicsOnMessageBus topic on message bus to wakeup the wait + * @param wakeupTopicsOnMessageBus topic on message bus to wake up the wait * @param checkIntervalInMilliSeconds time to break out wait for checking predicate condition * @param timeoutInMilliseconds time out to break out the whole wait process * @param predicate * @return true, predicate condition is satisfied * false, wait is timed out */ - boolean waitAndCheck(AsyncJob job, String[] wakupTopicsOnMessageBus, long checkIntervalInMilliSeconds, long timeoutInMilliseconds, Predicate predicate); + boolean waitAndCheck(AsyncJob job, String[] wakeupTopicsOnMessageBus, long checkIntervalInMilliSeconds, long timeoutInMilliseconds, Predicate predicate); AsyncJob queryJob(long jobId, boolean updatePollTime); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java index 09a88939492e..ab2b54b83d3f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java @@ -144,7 +144,7 @@ public void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinR @Override public List findJobsToWake(long joinedJobId) { - // TODO: We should fix this. We shouldn't be crossing daos in a dao code. + // TODO: We should fix this. We shouldn't be crossing daos in a dao code. List standaloneList = new ArrayList(); TransactionLegacy txn = TransactionLegacy.currentTxn(); String sql = "SELECT job_id FROM async_job_join_map WHERE join_job_id = ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)"; @@ -189,5 +189,4 @@ public List findJobsToWakeBetween(Date cutDate) { throw new CloudRuntimeException("Unable to handle SQL exception", e); } } - } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java index c87d88457c75..909a7e54c49a 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java @@ -23,16 +23,17 @@ import com.cloud.utils.db.GenericDao; public interface SyncQueueItemDao extends GenericDao { - public SyncQueueItemVO getNextQueueItem(long queueId); - public int getActiveQueueItemCount(long queueId); + SyncQueueItemVO getNextQueueItem(long queueId); - public List getNextQueueItems(int maxItems); + int getActiveQueueItemCount(long queueId); - public List getActiveQueueItems(Long msid, boolean exclusive); + List getNextQueueItems(int maxItems); - public List getQueuedItems(long queueId); + List getActiveQueueItems(Long msid, boolean exclusive); - public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + List getQueuedItems(long queueId); - public Long getQueueItemIdByContentIdAndType(long contentId, String contentType); + List getBlockedQueueItems(long thresholdMs, boolean exclusive); + + Long getQueueItemIdByContentIdAndType(long contentId, String contentType); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java index 50e10e75e439..03c0b6707ed3 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java @@ -162,14 +162,14 @@ public List getBlockedQueueItems(long thresholdMs, boolean excl sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessNumber(), SearchCriteria.Op.NNULL); sbItem.and("lastProcessTime", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.NNULL); sbItem.and("lastProcessTime2", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.LT); - sbItem.done(); SearchCriteria sc = sbItem.create(); sc.setParameters("lastProcessTime2", new Date(cutTime.getTime() - thresholdMs)); - if(exclusive) + if (exclusive) { return lockRows(sc, null, true); + } return listBy(sc, null); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index e2dfbe7ee000..788e1cd24e1d 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -126,8 +126,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private static final ConfigKey VmJobLockTimeout = new ConfigKey("Advanced", Integer.class, "vm.job.lock.timeout", "1800", "Time in seconds to wait in acquiring lock to submit a vm worker job", false); - private static final ConfigKey HidePassword = new ConfigKey("Advanced", Boolean.class, "log.hide.password", "true", "If set to true, the password is hidden", true, ConfigKey.Scope.Global); - + private static final ConfigKey HidePassword = new ConfigKey("Advanced", Boolean.class, "log.hide.password", "true", + "If set to true, the password is hidden", true, ConfigKey.Scope.Global); private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; // 3 seconds @@ -308,8 +308,8 @@ public long submitAsyncJob(final AsyncJob job, final String syncObjType, final l publishOnEventBus(job, "submit"); - if (!_vmInstanceDao.lockInLockTable(String.valueOf(syncObjId), VmJobLockTimeout.value())){ - throw new CloudRuntimeException("Failed to acquire lock in submitting async job: " + job.getCmd() + " with timeout value = " + VmJobLockTimeout.value()); + if (!_vmInstanceDao.lockInLockTable(String.valueOf(syncObjId), VmJobLockTimeout.value())) { + throw new CloudRuntimeException("Failed to acquire lock in submitting async job: " + job.getCmd() + " within vm job lock timeout value = " + VmJobLockTimeout.value()); } try { @@ -637,8 +637,9 @@ private Runnable getExecutorRunnable(final AsyncJob job) { @Override public void run() { // register place-holder context to avoid installing system account call context - if (CallContext.current() == null) + if (CallContext.current() == null) { CallContext.registerPlaceHolderContext(); + } String related = job.getRelated(); String logContext = job.getShortUuid(); @@ -646,7 +647,7 @@ public void run() { ThreadContext.push("job-" + related + "/" + "job-" + job.getId()); AsyncJob relatedJob = _jobDao.findByIdIncludingRemoved(Long.parseLong(related)); if (relatedJob != null) { - logContext = relatedJob.getShortUuid(); + logContext = relatedJob.getShortUuid() + "/" + job.getShortUuid(); } } else { ThreadContext.push("job-" + job.getId()); @@ -683,14 +684,14 @@ protected void runInContext() { if (related != null && !related.isEmpty()) { AsyncJob relatedJob = _jobDao.findByIdIncludingRemoved(Long.parseLong(related)); if (relatedJob != null) { - logContext = relatedJob.getShortUuid(); + logContext = relatedJob.getShortUuid() + "/" + job.getShortUuid(); } } ThreadContext.put("logcontextid", logContext); // execute the job if (logger.isDebugEnabled()) { - logger.debug("Executing " + StringUtils.cleanString(job.toString())); + logger.debug("Executing {}", StringUtils.cleanString(job.toString())); } if ((getAndResetPendingSignals(job) & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) { @@ -708,19 +709,18 @@ protected void runInContext() { jobDispatcher.runJob(job); } else { logger.error("Unable to find job dispatcher, job will be cancelled"); - completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null); + completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "Unable to find job dispatcher: " + job.getDispatcher()); } } if (logger.isDebugEnabled()) { - logger.debug("Done executing " + job.getCmd() + " for job-" + job.getId()); + logger.debug("Done executing {} for {}job-{}", job.getCmd(), related != null && !related.isEmpty() ? "job-" + related + "/" : "", job.getId()); } - } catch (Throwable e) { logger.error("Unexpected exception", e); completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null); } finally { - // guard final clause as well + // guard the final clause as well try { if (job.getSyncSource() != null) { // here check queue item one more time to double make sure that queue item is removed in case of any uncaught exception @@ -760,7 +760,7 @@ private int getAndResetPendingSignals(AsyncJob job) { return signals; } - private void executeQueueItem(SyncQueueItemVO item, boolean fromPreviousSession) { + private void executeQueueItem(SyncQueueItemVO item) { AsyncJobVO job = _jobDao.findById(item.getContentId()); if (job != null) { if (logger.isDebugEnabled()) { @@ -883,7 +883,7 @@ private void checkQueue(long queueId) { logger.debug("Executing sync queue item: " + item.toString()); } - executeQueueItem(item, false); + executeQueueItem(item); } else { break; } @@ -951,7 +951,7 @@ protected void reallyRun() { } if (!isPurged) { logger.debug("Execute sync-queue item: {}", item); - executeQueueItem(item, false); + executeQueueItem(item); } } } @@ -1005,11 +1005,12 @@ public void reallyRun() { // forcefully cancel blocking queue items if they've been staying there for too long List blockItems = _queueMgr.getBlockedQueueItems(JobCancelThresholdMinutes.value() * 60000, false); if (blockItems != null && blockItems.size() > 0) { + logger.debug("Found {} blocking queue items for over {} minutes, will cancel them and purge from queue", blockItems.size(), JobCancelThresholdMinutes.value()); for (SyncQueueItemVO item : blockItems) { try { if (item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { - logger.info("Remove Job-" + item.getContentId() + " from Queue-" + item.getId() + " since it has been blocked for too long"); - completeAsyncJob(item.getContentId(), JobInfo.Status.FAILED, 0, "Job is cancelled as it has been blocking others for too long"); + logger.info("Remove Job-{} from Queue-{} since it has been blocked for too long", item.getContentId(), item.getId()); + completeAsyncJob(item.getContentId(), Status.CANCELLED, 0, "Job is cancelled as it has been blocking others for too long"); _jobMonitor.unregisterByJobId(item.getContentId()); } @@ -1147,10 +1148,10 @@ public boolean configure(String name, Map params) throws Configu int apiPoolSize = cloudMaxActive / 2; int workPoolSize = (cloudMaxActive * 2) / 3; - logger.info("Start AsyncJobManager API executor thread pool in size " + apiPoolSize); + logger.info("Start AsyncJobManager API executor thread pool in size {}", apiPoolSize); _apiJobExecutor = Executors.newFixedThreadPool(apiPoolSize, new NamedThreadFactory(AsyncJobManager.API_JOB_POOL_THREAD_PREFIX)); - logger.info("Start AsyncJobManager Work executor thread pool in size " + workPoolSize); + logger.info("Start AsyncJobManager Work executor thread pool in size {}", workPoolSize); _workerJobExecutor = Executors.newFixedThreadPool(workPoolSize, new NamedThreadFactory(AsyncJobManager.WORK_JOB_POOL_THREAD_PREFIX)); } catch (final Exception e) { throw new ConfigurationException("Unable to load db.properties to configure AsyncJobManagerImpl"); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java index 32d84647a2d4..d611807a33b2 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java @@ -21,21 +21,21 @@ import com.cloud.utils.component.Manager; public interface SyncQueueManager extends Manager { - public SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit); + SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit); - public SyncQueueItemVO dequeueFromOne(long queueId, Long msid); + SyncQueueItemVO dequeueFromOne(long queueId, Long msid); - public List dequeueFromAny(Long msid, int maxItems); + List dequeueFromAny(Long msid, int maxItems); - public void purgeItem(long queueItemId); + void purgeItem(long queueItemId); - public void returnItem(long queueItemId); + void returnItem(long queueItemId); - public List getActiveQueueItems(Long msid, boolean exclusive); + List getActiveQueueItems(Long msid, boolean exclusive); - public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + List getBlockedQueueItems(long thresholdMs, boolean exclusive); void purgeAsyncJobQueueItemId(long asyncJobId); - public void cleanupActiveQueueItems(Long msid, boolean exclusive); + void cleanupActiveQueueItems(Long msid, boolean exclusive); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java index 3397daa58191..3ffef5385075 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java @@ -50,18 +50,19 @@ public SyncQueueVO queue(final String syncObjType, final long syncObjId, final S public SyncQueueVO doInTransaction(TransactionStatus status) { _syncQueueDao.ensureQueue(syncObjType, syncObjId); SyncQueueVO queueVO = _syncQueueDao.find(syncObjType, syncObjId); - if (queueVO == null) + if (queueVO == null) { throw new CloudRuntimeException("Unable to queue item into DB, DB is full?"); + } queueVO.setQueueSizeLimit(queueSizeLimit); _syncQueueDao.update(queueVO.getId(), queueVO); - Date dt = DateUtil.currentGMTTime(); + Date date = DateUtil.currentGMTTime(); SyncQueueItemVO item = new SyncQueueItemVO(); item.setQueueId(queueVO.getId()); item.setContentType(itemType); item.setContentId(itemId); - item.setCreated(dt); + item.setCreated(date); _syncQueueItemDao.persist(item); return queueVO; @@ -181,9 +182,8 @@ public void purgeItem(final long queueItemId) { @Override public void doInTransactionWithoutResult(TransactionStatus status) { SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId); - if(itemVO != null) { + if (itemVO != null) { SyncQueueVO queueVO = _syncQueueDao.findById(itemVO.getQueueId()); - _syncQueueItemDao.expunge(itemVO.getId()); // if item is active, reset queue information diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java index 050fe4e5215c..f4206ff72027 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java @@ -113,11 +113,11 @@ public void setSecondaryObjectIdentifier(String secondaryObjectIdentifier) { public String toString() { StringBuffer sb = new StringBuffer(); sb.append("VmWorkJobVO : {"). - append(", step: ").append(getStep()). + append("vmInstanceId: ").append(getVmInstanceId()). append(", vmType: ").append(getVmType()). - append(", vmInstanceId: ").append(getVmInstanceId()). + append(", step: ").append(getStep()). append(", secondaryObjectIdentifier: ").append(getSecondaryObjectIdentifier()). - append(super.toString()). + append(", ").append(super.toString()). append("}"); return sb.toString(); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index bdc55d792e55..70c38b41eb84 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -690,8 +690,8 @@ public boolean needRecycle(String workerTag) { Instant start = Instant.ofEpochMilli(startTick); Instant end = start.plusSeconds(2 * (AsyncJobManagerImpl.JobExpireMinutes.value() + AsyncJobManagerImpl.JobCancelThresholdMinutes.value()) * SECONDS_PER_MINUTE); Instant now = Instant.now(); - if(s_vmwareCleanOldWorderVMs.value() && now.isAfter(end)) { - if(logger.isInfoEnabled()) { + if (s_vmwareCleanOldWorderVMs.value() && now.isAfter(end)) { + if (logger.isInfoEnabled()) { logger.info("Worker VM expired, seconds elapsed: " + Duration.between(start,now).getSeconds()); } return true; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 58672d01d80e..53f43eaeb348 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -514,7 +514,6 @@ public Answer executeRequest(Command cmd) { } else if (clz == StopCommand.class) { String vmName = ((StopCommand) cmd).getVmName(); setTaskContext(cmdSequence, cmd, vmName); - Thread.sleep(3 * 60 * 1000); // 3 mins answer = execute((StopCommand) cmd); } else if (clz == RebootRouterCommand.class) { answer = execute((RebootRouterCommand) cmd); @@ -590,7 +589,6 @@ public Answer executeRequest(Command cmd) { } else if (clz == StartCommand.class) { String vmName = ((StartCommand) cmd).getVirtualMachine().getName(); setTaskContext(cmdSequence, cmd, vmName); - Thread.sleep(3 * 60 * 1000); // 3 mins answer = execute((StartCommand) cmd); } else if (clz == CheckSshCommand.class) { answer = execute((CheckSshCommand) cmd); diff --git a/server/src/main/java/com/cloud/api/ApiDispatcher.java b/server/src/main/java/com/cloud/api/ApiDispatcher.java index 90cbb6afc8e5..f1a647041cc0 100644 --- a/server/src/main/java/com/cloud/api/ApiDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiDispatcher.java @@ -121,7 +121,7 @@ public void dispatch(final BaseCmd cmd, final Map params, final final CallContext ctx = CallContext.current(); ctx.setEventDisplayEnabled(cmd.isDisplay()); - if(params.get(ApiConstants.PROJECT_ID) != null) { + if (params.get(ApiConstants.PROJECT_ID) != null) { Project project = _entityMgr.findByUuidIncludingRemoved(Project.class, params.get(ApiConstants.PROJECT_ID)); ctx.setProject(project); } @@ -135,7 +135,6 @@ public void dispatch(final BaseCmd cmd, final Map params, final // TODO This if shouldn't be here. Use polymorphism and move it to validateSpecificParameters if (cmd instanceof BaseAsyncCmd) { - final BaseAsyncCmd asyncCmd = (BaseAsyncCmd)cmd; final String startEventId = params.get(ApiConstants.CTX_START_EVENT_ID); ctx.setStartEventId(Long.parseLong(startEventId)); diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index 61c8ae68c5b1..f85c281f998b 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -387,7 +387,7 @@ public void handleAsyncJobPublishEvent(String subject, String senderAddress, Obj String jobEvent = eventInfo.second(); if (logger.isTraceEnabled()) - logger.trace("Handle asyjob publish event {}", jobEvent); + logger.trace("Handle async job publish event {}", jobEvent); if (eventDistributor == null) { setEventDistributor(ComponentContext.getComponent(EventDistributor.class)); } @@ -418,7 +418,7 @@ public void handleAsyncJobPublishEvent(String subject, String senderAddress, Obj logger.debug("Unable to locate cmdEventType marker in job info. publish as unknown event"); } String contextDetails = cmdInfo.get("ctxDetails"); - if(contextDetails != null) { + if (contextDetails != null) { Type objectMapType = new TypeToken>() {}.getType(); Map ctxDetails = ApiGsonHelper.getBuilder().create().fromJson(contextDetails, objectMapType); cmdInfoObj.put("ctxDetails", ctxDetails); @@ -443,7 +443,7 @@ public void handleAsyncJobPublishEvent(String subject, String senderAddress, Obj eventDescription.put("jobResult", ApiSerializerHelper.fromSerializedStringToMap(StringUtils.obfuscatePasswordInJsonLikeString(job.getResult()))); eventDescription.put("cmdInfo", cmdInfoObj); eventDescription.put("status", "" + job.getStatus()); - // If the event.accountinfo boolean value is set, get the human readable value for the username / domainname + // If the event.accountinfo boolean value is set, get the human-readable value for the username / domainname if (UseEventAccountInfo.value()) { DomainVO domain = domainDao.findById(jobOwner.getDomainId()); eventDescription.put("username", userJobOwner.getUsername()); @@ -494,12 +494,12 @@ public boolean start() { } for (final Class cmdClass : cmdClasses) { - final APICommand at = cmdClass.getAnnotation(APICommand.class); - if (at == null) { + final APICommand apiCmdAnnotation = cmdClass.getAnnotation(APICommand.class); + if (apiCmdAnnotation == null) { throw new CloudRuntimeException(String.format("%s is claimed as a API command, but it doesn't have @APICommand annotation", cmdClass.getName())); } - String apiName = at.name(); + String apiName = apiCmdAnnotation.name(); List> apiCmdList = s_apiNameCmdClassMap.get(apiName); if (apiCmdList == null) { apiCmdList = new ArrayList<>(); @@ -540,18 +540,18 @@ public void handle(final HttpRequest request, final HttpResponse response, final // Use Multimap as the parameter map should be in the form (name=String, value=String[]) // So parameter values are stored in a list for the same name key - // APITODO: Use Guava's (import com.google.common.collect.Multimap;) + // API TODO: Use Guava's (import com.google.common.collect.Multimap;) // (Immutable)Multimap paramMultiMap = HashMultimap.create(); // Map> parameterMap = paramMultiMap.asMap(); final Map parameterMap = new HashMap(); String responseType = HttpUtils.RESPONSE_TYPE_XML; - if(paramList != null) { + if (paramList != null) { for (final NameValuePair param : paramList) { if (param.getName().equalsIgnoreCase("response")) { responseType = param.getValue(); continue; } - if(parameterMap.putIfAbsent(param.getName(), new String[]{param.getValue()}) != null) { + if (parameterMap.putIfAbsent(param.getName(), new String[]{param.getValue()}) != null) { String message = String.format("Query parameter '%s' has multiple values [%s, %s]. Only the last value will be respected." + "It is advised to pass only a single parameter", param.getName(), param.getValue(), parameterMap.get(param.getName())); logger.warn(StringUtils.cleanString(message)); @@ -755,7 +755,7 @@ public String handleRequest(final Map params, final String responseType, final S logger.info(ex.getDescription()); throw ex; } catch (final Exception ex) { - logger.error("unhandled exception executing api command: " + ((command == null) ? "null" : command), ex); + logger.error("unhandled exception executing api command: {}", (command == null) ? "null" : command, ex); String errorMsg = ex.getMessage(); if (!accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId())) { // hide internal details to non-admin user for security reason @@ -815,7 +815,6 @@ private String queueCommand(final BaseCmd cmdObj, final Map para // in the api logs. log.append(response); return response; - } else { dispatcher.dispatch(cmdObj, params, false); @@ -1184,7 +1183,7 @@ private boolean commandAvailable(final InetAddress remoteAddress, final String c " for user."; throw new ServerApiException(ApiErrorCode.UNAUTHORIZED , errorMessage); } catch (final OriginDeniedException ex) { - // in this case we can remove the session with extreme prejudice + // in this case, we can remove the session with extreme prejudice final String errorMessage = String.format("The user '%s' is not allowed to execute commands from ip address '%s'.", user, remoteAddress.getHostName()); logger.debug(errorMessage); return false; @@ -1490,7 +1489,7 @@ private void checkCommandAvailable(final User user, final String commandName, fi @Override public Class getCmdClass(String cmdName) { List> cmdList = s_apiNameCmdClassMap.get(cmdName); - if (cmdList == null || cmdList.size() == 0) + if (cmdList == null || cmdList.isEmpty()) return null; else if (cmdList.size() == 1) return cmdList.get(0); diff --git a/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java b/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java index 25846f908332..2b58338be001 100644 --- a/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java @@ -33,7 +33,7 @@ */ public class CommandCreationWorker implements DispatchWorker { - private static final String ATTEMP_TO_CREATE_NON_CREATION_CMD = + private static final String ATTEMPT_TO_CREATE_NON_CREATION_CMD = "Trying to invoke creation on a Command that is not " + BaseAsyncCreateCmd.class.getName(); @@ -51,8 +51,7 @@ public void handle(final DispatchTask task) { } } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - ATTEMP_TO_CREATE_NON_CREATION_CMD); + ATTEMPT_TO_CREATE_NON_CREATION_CMD); } } - } From b85fc76a46cf5e8903b9ecfce9496ba8c44b3306 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 27 Feb 2026 18:43:39 +0530 Subject: [PATCH 10/29] Async job, Agent and XenServer resource changes, and some code improvements --- .../org/apache/cloudstack/api/BaseCmd.java | 25 +- .../command/user/job/CancelAsyncJobCmd.java | 2 +- .../cloudstack/context/CallContext.java | 8 +- .../cloudstack/jobs/AsyncJobService.java | 2 +- .../cloud/agent/manager/AgentManagerImpl.java | 33 +-- .../agent/manager/DirectAgentAttache.java | 45 +--- .../framework/jobs/dao/AsyncJobDao.java | 2 + .../framework/jobs/dao/AsyncJobDaoImpl.java | 12 + .../jobs/impl/AsyncJobManagerImpl.java | 45 ++-- .../framework/jobs/impl/AsyncJobMonitor.java | 37 ++- .../vmware/resource/VmwareResource.java | 4 +- .../xenserver/resource/CitrixHelper.java | 218 ++++++++++++++++++ .../resource/CitrixResourceBase.java | 97 +++++++- .../ConfigurationManagerImpl.java | 2 + .../com/cloud/event/ActionEventUtils.java | 4 +- .../vmware/mo/VirtualMachineMO.java | 77 +++++-- .../hypervisor/vmware/util/VmwareHelper.java | 24 +- 17 files changed, 502 insertions(+), 135 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index 483fa83630ad..c87c76cc1ef0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -415,25 +415,26 @@ public void validateSpecificParameters(final Map params){ * display flag is used to control the display of the resource only to the end user. It doesn't affect Root Admin. * @return display flag */ - public boolean isDisplay(){ + public boolean isDisplay() { CallContext context = CallContext.current(); Map contextMap = context.getContextParameters(); boolean isDisplay = true; // Iterate over all the first class entities in context and check their display property. - for(Map.Entry entry : contextMap.entrySet()){ - try{ + for (Map.Entry entry : contextMap.entrySet()) { + try { Object key = entry.getKey(); Class clz = Class.forName((String)key); - if(Displayable.class.isAssignableFrom(clz)){ + if (Displayable.class.isAssignableFrom(clz)) { final Object objVO = getEntityVO(clz, entry.getValue()); isDisplay = ((Displayable) objVO).isDisplay(); } - // If the flag is false break immediately - if(!isDisplay) + // If the flag is false, break immediately + if (!isDisplay) { break; - } catch (Exception e){ + } + } catch (Exception e) { logger.trace("Caught exception while checking first class entities for display property, continuing on", e); } } @@ -443,19 +444,19 @@ public boolean isDisplay(){ } - private Object getEntityVO(Class entityType, Object entityId){ + private Object getEntityVO(Class entityType, Object entityId) { // entityId can be internal db id or UUID so accordingly call findbyId or findByUUID - if (entityId instanceof Long){ + if (entityId instanceof Long) { // Its internal db id - use findById return _entityMgr.findById(entityType, (Long)entityId); - } else if(entityId instanceof String){ - try{ + } else if (entityId instanceof String) { + try { // In case its an async job the internal db id would be a string because of json deserialization Long internalId = Long.valueOf((String) entityId); return _entityMgr.findById(entityType, internalId); - } catch (NumberFormatException e){ + } catch (NumberFormatException e) { // It is uuid - use findByUuid` return _entityMgr.findByUuid(entityType, (String)entityId); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java index cc49ece98517..5cdcd26d90d0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java @@ -90,7 +90,7 @@ public Long getInstanceId() { @Override public void execute() { - String status = asyncJobService.cancelAsyncJob(id, "Cancel requested by {}" + CallContext.current().getCallingUser().toString()); + String status = asyncJobService.cancelAsyncJob(id, "Cancel requested by " + CallContext.current().getCallingUser().toString()); if (StringUtils.isBlank(status)) { AsyncJobResponse response = _responseGenerator.cancelJobResponse(this); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/context/CallContext.java b/api/src/main/java/org/apache/cloudstack/context/CallContext.java index fcfb5b6b1e00..b4e4cf3de3a0 100644 --- a/api/src/main/java/org/apache/cloudstack/context/CallContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/CallContext.java @@ -401,9 +401,11 @@ public Map getContextParameters() { return context; } - public void putContextParameters(Map details){ - if (details == null) return; - for(Map.Entryentry : details.entrySet()){ + public void putContextParameters(Map details) { + if (details == null) { + return; + } + for (Map.Entryentry : details.entrySet()) { putContextParameter(entry.getKey(), entry.getValue()); } } diff --git a/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java b/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java index 1bf7a5f859de..fc50e4364d23 100644 --- a/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java +++ b/api/src/main/java/org/apache/cloudstack/jobs/AsyncJobService.java @@ -20,7 +20,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; public interface AsyncJobService extends PluggableService { - ConfigKey CancelledJobInterval = new ConfigKey<>("Advanced", Integer.class, "job.cancelled.interval", "5", + ConfigKey CancelledJobInterval = new ConfigKey<>("Advanced", Integer.class, "job.cancelled.interval", "1", "Interval in seconds to check the jobs cancelled", false); String cancelAsyncJob(long jobId, String reason); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index cdbb9bc02e07..b7c40c1e78c7 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -43,6 +43,7 @@ import javax.naming.ConfigurationException; import com.cloud.exception.OperationCancelledException; +import com.cloud.utils.DateUtil; import com.cloud.utils.StringUtils; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; @@ -63,7 +64,6 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; -import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.MapUtils; @@ -165,7 +165,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected List _loadingAgents = new ArrayList<>(); protected Map _commandTimeouts = new HashMap<>(); private int _monitorId = 0; - protected Map> _sequenceMap = new HashMap<>(); + protected Map> _jobToHostIdAndReqSequenceMap = new HashMap<>(); @Inject protected CAManager caService; @@ -693,7 +693,7 @@ public Answer[] send(final Long hostId, final Commands commands, int timeout) th final Request req = new Request(hostId, agent.getName(), _nodeId, cmds, commands.stopOnError(), true); req.setSequence(agent.getNextSequence()); if (jobId != null) { - _sequenceMap.put(jobId, new Pair<>(hostId, req.getSequence())); + _jobToHostIdAndReqSequenceMap.put(jobId, new Pair<>(hostId, req.getSequence())); } reconcileCommandService.persistReconcileCommands(hostId, req.getSequence(), cmds); @@ -707,7 +707,7 @@ public Answer[] send(final Long hostId, final Commands commands, int timeout) th return answers; } finally { if (jobId != null) { - _sequenceMap.remove(jobId); + _jobToHostIdAndReqSequenceMap.remove(jobId); } } } @@ -2104,26 +2104,29 @@ protected List findAgentsBehindOnPing() { protected class CancelledJobsCheckTask extends ManagedContextRunnable { @Override protected void runInContext() { +// logger.info("Started cancelled jobs check task."); final List jobs = _asyncJobDao.getCancelledJobs(); for (final AsyncJobVO job : jobs) { - Pair sequence = _sequenceMap.get(job.getId()); - if (sequence != null) { + logger.info("Job-{} with instance type: {}, id: {} cancelled.", job.getId(), job.getInstanceType(), job.getInstanceId()); + Pair hostAndSequence = _jobToHostIdAndReqSequenceMap.get(job.getId()); + if (hostAndSequence != null) { try { - final AgentAttache agent = getAttache(sequence.first()); - if (agent.isExecutionCancellable(sequence.second())) { - agent.cancel(sequence.second()); - if (job.getStatus() != JobInfo.Status.CANCELLED) { - job.setStatus(JobInfo.Status.CANCELLED); - _asyncJobDao.update(job.getId(), job); - } + final AgentAttache agent = getAttache(hostAndSequence.first()); + if (agent.isExecutionCancellable(hostAndSequence.second())) { + logger.debug("Job-{} cancellation requested, sending cancel command to agent {} for sequence {}", job.getId(), hostAndSequence.first(), hostAndSequence.second()); + agent.cancel(hostAndSequence.second()); } else { logger.info("Job {} cancellation requested but sequence {} on host {} is not cancellable, allowing execution to continue.", - job.getId(), sequence.second(), sequence.first()); + job.getId(), hostAndSequence.second(), hostAndSequence.first()); } } catch (AgentUnavailableException e) { - logger.debug("Agent {} not found", sequence.first()); + logger.debug("Agent {} not found", hostAndSequence.first()); } + } else { + logger.debug("Job-{} host and sequence not found", job.getId()); } + job.setRemoved(DateUtil.currentGMTTime()); + _asyncJobDao.update(job.getId(), job); } } } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index 39ab1266fa97..0f5487bb7cd0 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -51,11 +51,11 @@ public class DirectAgentAttache extends AgentAttache { protected final ConfigKey _HostPingRetryTimer = new ConfigKey("Advanced", Integer.class, "host.ping.retry.timer", "5", "Interval to wait before retrying a host ping while waiting for check results", true); ServerResource _resource; - List> _futures = new ArrayList>(); + List> _futures = new ArrayList<>(); private final Map> _taskFutures = new ConcurrentHashMap<>(); private final Map _taskRequests = new ConcurrentHashMap<>(); long _seq = 0; - LinkedList tasks = new LinkedList(); + LinkedList tasks = new LinkedList<>(); AtomicInteger _outstandingTaskCount; AtomicInteger _outstandingCronTaskCount; @@ -87,7 +87,7 @@ public void disconnect(Status state) { _taskRequests.clear(); for (Future future : _taskFutures.values()) { - boolean cancelled = future.cancel(true); + boolean cancelled = future.cancel(false); logger.debug("Running task {} for [id: {}, uuid: {}, name: {}]", cancelled ? "cancelled" : "not cancelled", _id, _uuid, _name); } _taskFutures.clear(); @@ -126,6 +126,8 @@ protected synchronized void cancel(final long seq) { Request request = _taskRequests.get(seq); if (request != null) { request.cancel(); + _taskRequests.remove(seq); + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Removed task request."); } final Iterator iterator = tasks.iterator(); @@ -134,7 +136,6 @@ protected synchronized void cancel(final long seq) { if (task._req.getSequence() == seq) { task._req.cancel(); iterator.remove(); - _taskRequests.remove(seq); logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelled queued task."); super.cancel(seq); return; @@ -150,6 +151,7 @@ protected synchronized void cancel(final long seq) { } if (resource != null) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancel request sequence."); resource.cancelRequestSequence(seq); } @@ -158,7 +160,6 @@ protected synchronized void cancel(final long seq) { _taskFutures.remove(seq); } - _taskRequests.remove(seq); super.cancel(seq); } @@ -386,25 +387,15 @@ protected void runInContext() { Command[] cmds = _req.getCommands(); if (Thread.currentThread().isInterrupted() || _req.isCancelled()) { throw new InterruptedException("Task execution cancelled"); -// Answer[] answers = new Answer[cmds.length]; -// for (int i = 0; i < cmds.length; i++) { -// answers[i] = new Answer(cmds[i], false, "Task cancelled"); -// } -// Response resp = new Response(_req, answers); -// processAnswers(seq, resp); - -// handleCancellation(seq, false, "Request cancelled before execution"); -// return; } ServerResource resource = _resource; boolean stopOnError = _req.stopOnError(); logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Executing request"); - ArrayList answers = new ArrayList(cmds.length); + ArrayList answers = new ArrayList<>(cmds.length); for (int i = 0; i < cmds.length; i++) { if (Thread.currentThread().isInterrupted() || _req.isCancelled()) { -// throw new InterruptedException("Task execution cancelled"); for (int j = i; j < cmds.length; j++) { answers.add(new Answer(cmds[j], false, "Command cancelled")); } @@ -437,24 +428,6 @@ protected void runInContext() { } else { answer = new Answer(cmds[i], false, "Agent is disconnected"); } -// } catch (InterruptedException e) { -// logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Command interrupted: " + cmds[i].getClass().getSimpleName()); -// Thread.currentThread().interrupt(); - -// boolean cancelled = resource.cancelActiveVmTask(cmds[i]); //cmdSequence -// if (cancelled) { -// logger.info("Successfully cancelled task: " + cmds[i]); -// } -// int cancelled = resource.cancelActiveVmTasks(vmName); -// logger.info("Cancelled " + cancelled + " tasks for VM: " + vmName); - -// answer = new Answer(cmds[i], false, "Command execution cancelled: " + e.getMessage()); -// answers.add(answer); -// -// for (int j = i + 1; j < cmds.length; j++) { -// answers.add(new Answer(cmds[j], false, "Command cancelled - previous command was interrupted")); -// } -// break; } catch (Throwable t) { // Catch Throwable as all exceptions will otherwise be eaten by the executor framework logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Throwable caught while executing command", t); @@ -477,7 +450,7 @@ protected void runInContext() { logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Task interrupted"); Thread.currentThread().interrupt(); // Send cancellation response - handleCancellation(seq, true, "Task interrupted: " + e.getMessage()); + handleCancellation(seq, "Task interrupted: " + e.getMessage()); } catch (Throwable t) { // This is pretty serious as processAnswers might not be called and the calling process is stuck waiting for the full timeout logger.error(LOG_SEQ_FORMATTED_STRING, seq, "Throwable caught in runInContext, this will cause the management to become unpredictable", t); @@ -489,7 +462,7 @@ protected void runInContext() { } } - private void handleCancellation(long seq, boolean isActive, String reason) { + private void handleCancellation(long seq, String reason) { try { Command[] cmds = _req.getCommands(); ArrayList answers = new ArrayList<>(cmds.length); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index f4f74a56eb7f..9a3134840ba1 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -71,6 +71,8 @@ public interface AsyncJobDao extends GenericDao { List listPendingJobIdsForAccount(long accountId); + AsyncJobVO getRelatedJob(String jobId); + List getCancelledJobs(); boolean isJobCancelled(Long jobId); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 3082ae819d5f..39be874687f1 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -50,6 +50,7 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements private final SearchBuilder byIdResourceIdResourceTypeSearch; private final GenericSearchBuilder asyncJobTypeSearch; private final GenericSearchBuilder pendingNonPseudoAsyncJobsSearch; + private final SearchBuilder relatedAsyncJobSearch; private final SearchBuilder cancelledAsyncJobSearch; public AsyncJobDaoImpl() { @@ -118,6 +119,10 @@ public AsyncJobDaoImpl() { pendingNonPseudoAsyncJobsSearch.and("jobStatusEQ", pendingNonPseudoAsyncJobsSearch.entity().getStatus(), SearchCriteria.Op.EQ); pendingNonPseudoAsyncJobsSearch.and("executingMsidIN", pendingNonPseudoAsyncJobsSearch.entity().getExecutingMsid(), SearchCriteria.Op.IN); + relatedAsyncJobSearch = createSearchBuilder(); + relatedAsyncJobSearch.and("related", relatedAsyncJobSearch.entity().getRelated(), SearchCriteria.Op.EQ); + relatedAsyncJobSearch.done(); + cancelledAsyncJobSearch = createSearchBuilder(); cancelledAsyncJobSearch.and("status", cancelledAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ); cancelledAsyncJobSearch.and("completeMsId", cancelledAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NULL); @@ -316,6 +321,13 @@ public List listPendingJobIdsForAccount(long accountId) { return customSearch(sc, null); } + @Override + public AsyncJobVO getRelatedJob(String jobId) { + SearchCriteria sc = relatedAsyncJobSearch.create(); + sc.setParameters("related", jobId); + return findOneIncludingRemovedBy(sc); + } + @Override public List getCancelledJobs() { SearchCriteria sc = cancelledAsyncJobSearch.create(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 788e1cd24e1d..80a8ef2514ec 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -920,9 +920,9 @@ protected void reallyRun() { return; } - List l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); - if (l != null && l.size() > 0) { - for (SyncQueueItemVO item : l) { + List items = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); + if (items != null && items.size() > 0) { + for (SyncQueueItemVO item : items) { boolean isPurged = false; if (item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { AsyncJobVO job = _jobDao.findById(item.getContentId()); @@ -1011,7 +1011,6 @@ public void reallyRun() { if (item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { logger.info("Remove Job-{} from Queue-{} since it has been blocked for too long", item.getContentId(), item.getId()); completeAsyncJob(item.getContentId(), Status.CANCELLED, 0, "Job is cancelled as it has been blocking others for too long"); - _jobMonitor.unregisterByJobId(item.getContentId()); } @@ -1031,7 +1030,6 @@ public void reallyRun() { for (AsyncJobVO job : unfinishedJobs) { try { logger.info("Expunging unfinished job-" + job.getId()); - _jobMonitor.unregisterByJobId(job.getId()); expungeAsyncJob(job); } catch (Throwable e) { @@ -1044,7 +1042,6 @@ public void reallyRun() { for (AsyncJobVO job : completedJobs) { try { logger.info("Expunging completed job-" + job.getId()); - expungeAsyncJob(job); } catch (Throwable e) { logger.error("Unexpected exception when trying to expunge job-" + job.getId(), e); @@ -1471,21 +1468,23 @@ public String cancelAsyncJob(long jobId, String reason) { return errMessage; } - try { - Class cmdClass = Class.forName(job.getCmd()); - APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); - if (apiCommand == null || !apiCommand.cancellable()) { - errMessage = "Cannot cancel, job " + job.getUuid() + " is not cancellable."; - logger.debug(errMessage); + if (isActiveJob(jobId)) { + try { + Class cmdClass = Class.forName(job.getCmd()); + APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); + if (apiCommand == null || !apiCommand.cancellable()) { + errMessage = "Cannot cancel, job " + job.getUuid() + " is not cancellable."; + logger.debug(errMessage); + return errMessage; + } + } catch (ClassNotFoundException e) { + errMessage = "Command " + job.getCmd() + " of jobid " + job.getUuid() + " not found."; + logger.error(errMessage, e); return errMessage; } - } catch (ClassNotFoundException e) { - errMessage = "Command " + job.getCmd() + " of jobid " + job.getUuid() + " not found."; - logger.error(errMessage, e); - return errMessage; } - if (job.getStatus() != Status.IN_PROGRESS) { + if (job.getStatus() != null && job.getStatus().done()) { errMessage = "Cannot cancel, job-" + jobId + " is not running. Current job status is " + job.getStatus() + "."; logger.debug(errMessage); _queueMgr.purgeAsyncJobQueueItemId(jobId); @@ -1497,7 +1496,6 @@ public String cancelAsyncJob(long jobId, String reason) { try { completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason, false); _jobMonitor.unregisterByJobId(jobId); - _queueMgr.purgeAsyncJobQueueItemId(jobId); return ""; } catch (Throwable t) { errMessage = "Unexpected exception when cancelling async job with id: " + jobId; @@ -1507,6 +1505,17 @@ public String cancelAsyncJob(long jobId, String reason) { return errMessage; } + private boolean isActiveJob(long jobId) { + // check if the job (and it's related job) is active or not. + // if not, it means the job is not running, and we can safely purge it from the queue. + final AsyncJobVO relatedJob = _jobDao.getRelatedJob(String.valueOf(jobId)); + if (relatedJob != null) { + return _jobMonitor.isActiveJob(relatedJob.getId()); + } + + return _jobMonitor.isActiveJob(jobId); + } + @Override public List> getCommands() { final List> cmdList = new ArrayList<>(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java index b2216cb75025..12877478bed4 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java @@ -17,8 +17,10 @@ package org.apache.cloudstack.framework.jobs.impl; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Set; import java.util.Timer; import java.util.concurrent.atomic.AtomicInteger; @@ -39,7 +41,8 @@ public class AsyncJobMonitor extends ManagerBase { @Inject private MessageBus _messageBus; - private final Map _activeTasks = new HashMap(); + private final Map _runNumberToActiveTasksMap = new HashMap<>(); + private final Set _activeJobs = new HashSet<>(); private final Timer _timer = new Timer(); private final AtomicInteger _activePoolThreads = new AtomicInteger(); @@ -72,7 +75,7 @@ public void setInactivityWarningThresholdMs(long thresholdMs) { public void onJobHeartbeatNotify(String subject, String senderAddress, Object args) { if (args != null && args instanceof Long) { synchronized (this) { - ActiveTaskRecord record = _activeTasks.get(args); + ActiveTaskRecord record = _runNumberToActiveTasksMap.get(args); if (record != null) { record.updateJobHeartbeatTick(); } @@ -82,7 +85,7 @@ public void onJobHeartbeatNotify(String subject, String senderAddress, Object ar private void heartbeat() { synchronized (this) { - for (Map.Entry entry : _activeTasks.entrySet()) { + for (Map.Entry entry : _runNumberToActiveTasksMap.entrySet()) { if (entry.getValue().millisSinceLastJobHeartbeat() > _inactivityWarningThresholdMs) { logger.warn("Task (job-" + entry.getValue().getJobId() + ") has been pending for " + entry.getValue().millisSinceLastJobHeartbeat() / 1000 + " seconds"); @@ -109,23 +112,24 @@ protected void runInContext() { public void registerActiveTask(long runNumber, long jobId) { synchronized (this) { logger.info("Add job-" + jobId + " into job monitoring"); - - assert (_activeTasks.get(runNumber) == null); + assert (_runNumberToActiveTasksMap.get(runNumber) == null); long threadId = Thread.currentThread().getId(); boolean fromPoolThread = Thread.currentThread().getName().contains(AsyncJobManager.API_JOB_POOL_THREAD_PREFIX); ActiveTaskRecord record = new ActiveTaskRecord(jobId, threadId, fromPoolThread); - _activeTasks.put(runNumber, record); - if (fromPoolThread) + _runNumberToActiveTasksMap.put(runNumber, record); + _activeJobs.add(jobId); + if (fromPoolThread) { _activePoolThreads.incrementAndGet(); - else + } else { _activeInplaceThreads.incrementAndGet(); + } } } public void unregisterActiveTask(long runNumber) { synchronized (this) { - ActiveTaskRecord record = _activeTasks.get(runNumber); + ActiveTaskRecord record = _runNumberToActiveTasksMap.get(runNumber); assert (record != null); if (record != null) { logger.info("Remove job-" + record.getJobId() + " from job monitoring"); @@ -135,18 +139,19 @@ public void unregisterActiveTask(long runNumber) { else _activeInplaceThreads.decrementAndGet(); - _activeTasks.remove(runNumber); + _runNumberToActiveTasksMap.remove(runNumber); + _activeJobs.remove(record.getJobId()); } } } public void unregisterByJobId(long jobId) { synchronized (this) { - Iterator> it = _activeTasks.entrySet().iterator(); + Iterator> it = _runNumberToActiveTasksMap.entrySet().iterator(); while (it.hasNext()) { Map.Entry entry = it.next(); if (entry.getValue().getJobId() == jobId) { - logger.info("Remove Job-" + entry.getValue().getJobId() + " from job monitoring due to job cancelling"); + logger.info("Remove Job-{} from job monitoring due to job cancelling", entry.getValue().getJobId()); if (entry.getValue().isPoolThread()) _activePoolThreads.decrementAndGet(); @@ -154,11 +159,19 @@ public void unregisterByJobId(long jobId) { _activeInplaceThreads.decrementAndGet(); it.remove(); + _activeJobs.add(jobId); + break; } } } } + public boolean isActiveJob(long jobId) { + synchronized (this) { + return _activeJobs.contains(jobId); + } + } + public int getActivePoolThreads() { return _activePoolThreads.get(); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 53f43eaeb348..62cfd15ea992 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -687,7 +687,7 @@ public Answer executeRequest(Command cmd) { } } } catch (InterruptedException e) { - logger.warn("Command execution interrupted: " + cmd.getClass().getSimpleName()); + logger.warn("Command execution interrupted: {}", cmd.getClass().getSimpleName()); Thread.currentThread().interrupt(); // Restore interrupt status return new Answer(cmd, false, "Command execution was cancelled: " + e.getMessage()); } finally { @@ -709,11 +709,13 @@ private void setTaskContext(long cmdSequence, Command cmd, String vmName) { @Override public boolean isRequestSequenceCancellable(long sequence) { + logger.info("Check if request sequence: {} is cancellable or not", sequence); return VmwareHelper.isActiveVmTaskCancellable(sequence); } @Override public boolean cancelRequestSequence(long sequence) { + logger.info("Cancel request sequence: {}", sequence); return VmwareHelper.cancelActiveVmTask(sequence); } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java index a9ee1dd76082..3ec483d3f0a6 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java @@ -17,13 +17,231 @@ package com.cloud.hypervisor.xenserver.resource; import com.cloud.storage.Storage; +import com.xensource.xenapi.Connection; import com.xensource.xenapi.Host; +import com.xensource.xenapi.Task; +import com.xensource.xenapi.Types; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; /** * Reduce bloat inside CitrixResourceBase * */ public class CitrixHelper { + protected static Logger LOGGER = LogManager.getLogger(CitrixHelper.class); + + /** + * Tracks active VM tasks (startVM, shutdownVM, migrateVM) by command sequence. + * Key: command sequence, Value: ActiveVmTaskInfo + */ + private static final ConcurrentHashMap activeVmTasks = new ConcurrentHashMap<>(); + + /** + * Tracks active VM tasks by VM name for quick lookup. + * Key: VM name, Value: Set of command sequences + */ + private static final ConcurrentHashMap> vmNameToTaskSequences = new ConcurrentHashMap<>(); + + /** + * ThreadLocal to store current command context for task registration. + */ + private static final ThreadLocal currentTaskContext = new ThreadLocal<>(); + + public static class ActiveVmTaskInfo { + final Task task; + final Connection connection; + final long cmdSequence; + final String vmName; + final String commandType; + final long startTime; + + ActiveVmTaskInfo(long cmdSequence, String vmName, String commandType, + Task task, Connection connection) { + this.cmdSequence = cmdSequence; + this.vmName = vmName; + this.commandType = commandType; + this.task = task; + this.connection = connection; + this.startTime = System.currentTimeMillis(); + } + } + + /** + * Registers an active XenServer task for tracking. + */ + private static void registerActiveVmTask(long cmdSequence, String vmName, String commandType, + Task task, Connection connection) { + if (StringUtils.isBlank(vmName)) { + LOGGER.debug("No active VM to registered VM task for sequence={}, command={}, task={}", cmdSequence, commandType, task); + return; + } + + ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, task, connection); + activeVmTasks.put(cmdSequence, taskInfo); + + vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); + + LOGGER.debug("Registered active VM task: sequence={}, vm={}, command={}, task={}", cmdSequence, vmName, commandType, task); + } + + /** + * Unregisters an active XenServer task. + */ + private static void unregisterActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = activeVmTasks.remove(cmdSequence); + if (taskInfo != null) { + Set sequences = vmNameToTaskSequences.get(taskInfo.vmName); + if (sequences != null) { + sequences.remove(cmdSequence); + if (sequences.isEmpty()) { + vmNameToTaskSequences.remove(taskInfo.vmName); + } + } + LOGGER.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={} ms", + cmdSequence, taskInfo.vmName, taskInfo.commandType, + System.currentTimeMillis() - taskInfo.startTime); + } + } + + /** + * Cancels all active XenServer tasks for a given VM name. + */ + public static int cancelActiveVmTasks(String vmName) { + Set sequences = vmNameToTaskSequences.get(vmName); + if (sequences == null || sequences.isEmpty()) { + return 0; + } + + int cancelled = 0; + for (Long seq : sequences) { + if (cancelActiveVmTask(seq)) { + cancelled++; + } + } + return cancelled; + } + + /** + * Checks whether an active XenServer task is Cancellable or not. + */ + public static boolean isActiveVmTaskCancellable(long cmdSequence) { + ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); + if (taskInfo == null) { + return true; + } + + try { + return taskInfo.task.getStatus(taskInfo.connection) == Types.TaskStatusType.PENDING; + } catch (Exception e) { + LOGGER.warn("Failed to verify XenServer task cancellability for sequence={}, vm={}, error={}", + cmdSequence, taskInfo.vmName, e.getMessage(), e); + return false; + } + } + + /** + * Cancels an active XenServer task by command sequence. + */ + public static boolean cancelActiveVmTask(long cmdSequence) { + ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); + if (taskInfo == null) { + return true; + } + + try { + if (taskInfo.task.getStatus(taskInfo.connection) != Types.TaskStatusType.PENDING) { + LOGGER.info("XenServer VM task is not cancellable: sequence={}, vm={}, command={}", + cmdSequence, taskInfo.vmName, taskInfo.commandType); + return false; + } + + // Cancel the XenServer task + taskInfo.task.cancel(taskInfo.connection); + LOGGER.info("Cancelled XenServer VM task: sequence={}, vm={}, command={}, task={}", + cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.task.getUuid(taskInfo.connection)); + + // Unregister the task + unregisterActiveVmTask(cmdSequence); + return true; + } catch (Exception e) { + LOGGER.warn("Failed to cancel XenServer VM task: sequence={}, vm={}, error={}", + cmdSequence, taskInfo.vmName, e.getMessage(), e); + return false; + } + } + + /** + * Gets all active XenServer tasks for a given VM name. + */ + public static List getActiveVmTasks(String vmName) { + Set sequences = vmNameToTaskSequences.get(vmName); + if (sequences == null || sequences.isEmpty()) { + return Collections.emptyList(); + } + + List tasks = new ArrayList<>(); + for (Long seq : sequences) { + ActiveVmTaskInfo taskInfo = activeVmTasks.get(seq); + if (taskInfo != null) { + tasks.add(taskInfo); + } + } + return tasks; + } + + /** + * Gets all active XenServer tasks. + */ + public static List getAllActiveVmTasks() { + return new ArrayList<>(activeVmTasks.values()); + } + + public static class TaskContext { + private final CitrixResourceBase resource; + private final long cmdSequence; + private final String vmName; + private final String commandType; + + public TaskContext(CitrixResourceBase resource, long cmdSequence, String vmName, String commandType) { + this.resource = resource; + this.cmdSequence = cmdSequence; + this.vmName = vmName; + this.commandType = commandType; + } + + public void registerTask(Task task, Connection connection) { + registerActiveVmTask(cmdSequence, vmName, commandType, task, connection); + } + + public void unregisterTask() { + unregisterActiveVmTask(cmdSequence); + } + } + + public static void setTaskContext(CitrixResourceBase resource, long cmdSequence, String vmName, String commandType) { + LOGGER.debug("Setting task context: sequence={}, vm={}, command={}", cmdSequence, vmName, commandType); + currentTaskContext.set(new TaskContext(resource, cmdSequence, vmName, commandType)); + } + + public static void clearTaskContext() { + TaskContext taskContext = currentTaskContext.get(); + if (taskContext != null) { + LOGGER.debug("Clearing task context: sequence={}, vm={}, command={}", taskContext.cmdSequence, taskContext.vmName, taskContext.commandType); + } + currentTaskContext.remove(); + } + + public static TaskContext getCurrentTaskContext() { + return currentTaskContext.get(); + } public static String getProductVersion(final Host.Record record) { String prodVersion = record.softwareVersion.get("product_version"); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index cdb4d7434aef..39f543650592 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -51,6 +51,7 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.agent.api.MigrateCommand; import com.xensource.xenapi.VTPM; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageAnswer; @@ -305,6 +306,8 @@ private static boolean isAlienVm(final VM vm, final Connection conn) throws XenA protected String _configDriveSRName = "ConfigDriveISOs"; public String _attachIsoDeviceNum = "3"; + protected volatile long _cmdSequence = 1; + protected XenServerUtilitiesHelper xenServerUtilitiesHelper = new XenServerUtilitiesHelper(); protected int _wait; @@ -1775,14 +1778,46 @@ public ExecutionResult executeInVR(final String routerIP, final String script, f @Override public Answer executeRequest(final Command cmd) { + if (Thread.currentThread().isInterrupted()) { + String msg = "Command " + cmd.getClass().getSimpleName() + " was cancelled before execution"; + logger.warn(msg); + return new Answer(cmd, false, msg); + } + final CitrixRequestWrapper wrapper = CitrixRequestWrapper.getInstance(); try { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Command execution cancelled"); + } + long cmdSequence = _cmdSequence++; + Class clz = cmd.getClass(); + if (clz == StartCommand.class) { + String vmName = ((StartCommand) cmd).getVirtualMachine().getName(); + setTaskContext(cmdSequence, cmd, vmName); + } else if (clz == StopCommand.class) { + String vmName = ((StopCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); + } else if (clz == RebootCommand.class) { + String vmName = ((RebootCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); + } else if (clz == MigrateCommand.class) { + String vmName = ((MigrateCommand) cmd).getVmName(); + setTaskContext(cmdSequence, cmd, vmName); + } + return wrapper.execute(cmd, this); } catch (final Exception e) { return Answer.createUnsupportedCommandAnswer(cmd); + } finally { + CitrixHelper.clearTaskContext(); } } + private void setTaskContext(long cmdSequence, Command cmd, String vmName) { + String commandType = cmd.getClass().getSimpleName(); + CitrixHelper.setTaskContext(this, cmdSequence, vmName, commandType); + } + protected void fillHostInfo(final Connection conn, final StartupRoutingCommand cmd) { final StringBuilder caps = new StringBuilder(); try { @@ -3948,10 +3983,14 @@ protected String logX(final XenAPIObject obj, final String msg) { public void migrateVM(final Connection conn, final Host destHost, final VM vm, final String vmName) throws Exception { Task task = null; + CitrixHelper.TaskContext taskContext = CitrixHelper.getCurrentTaskContext(); try { final Map other = new HashMap<>(); other.put("live", "true"); task = vm.poolMigrateAsync(conn, destHost, other); + if (taskContext != null) { + taskContext.registerTask(task, conn); + } try { // poll every 1 seconds final long timeout = _migratewait * 1000L; @@ -3963,6 +4002,10 @@ public void migrateVM(final Connection conn, final Host destHost, final VM vm, f return; } throw new CloudRuntimeException("migrate VM catch HandleInvalid and VM is not running on dest host"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("migrate VM interrupted for VM: " + vmName); + throw e; } } catch (final XenAPIException e) { final String msg = "Unable to migrate VM(" + vmName + ") from host(" + _host.getUuid() + ")"; @@ -3976,6 +4019,10 @@ public void migrateVM(final Connection conn, final Host destHost, final VM vm, f logger.debug("unable to destroy task(" + task + ") on host(" + _host.getUuid() + ") due to " + e1); } } + + if (taskContext != null) { + taskContext.unregisterTask(); + } } } @@ -5043,8 +5090,9 @@ public synchronized Network setupvSwitchNetwork(final Connection conn) { return null; } - public void shutdownVM(final Connection conn, final VM vm, final String vmName, final boolean forcedStop) throws XmlRpcException { + public void shutdownVM(final Connection conn, final VM vm, final String vmName, final boolean forcedStop) throws Exception { Task task = null; + CitrixHelper.TaskContext taskContext = CitrixHelper.getCurrentTaskContext(); try { if (forcedStop) { task = vm.hardShutdownAsync(conn); @@ -5052,6 +5100,10 @@ public void shutdownVM(final Connection conn, final VM vm, final String vmName, task = vm.cleanShutdownAsync(conn); } + if (taskContext != null) { + taskContext.registerTask(task, conn); + } + try { // poll every 1 seconds , timeout after 10 minutes waitForTask(conn, task, 1000, 10 * 60 * 1000); @@ -5062,6 +5114,10 @@ public void shutdownVM(final Connection conn, final VM vm, final String vmName, return; } throw new CloudRuntimeException("Shutdown VM catch HandleInvalid and VM is not in HALTED state"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Shutdown VM interrupted for VM: " + vmName); + throw e; } } catch (final XenAPIException e) { logger.debug("Unable to shutdown VM(" + vmName + ") with force=" + forcedStop + " on host(" + _host.getUuid() + ") due to " + e); @@ -5097,6 +5153,10 @@ public void shutdownVM(final Connection conn, final VM vm, final String vmName, logger.debug("unable to destroy task(" + task + ") on host(" + _host.getUuid() + ") due to " + e1); } } + + if (taskContext != null) { + taskContext.unregisterTask(); + } } } @@ -5107,8 +5167,12 @@ public boolean start() { public void startVM(final Connection conn, final Host host, final VM vm, final String vmName) throws Exception { Task task = null; + CitrixHelper.TaskContext taskContext = CitrixHelper.getCurrentTaskContext(); try { task = vm.startOnAsync(conn, host, false, true); + if (taskContext != null) { + taskContext.registerTask(task, conn); + } try { // poll every 1 seconds , timeout after 10 minutes waitForTask(conn, task, 1000, 10 * 60 * 1000); @@ -5127,6 +5191,10 @@ public void startVM(final Connection conn, final Host host, final VM vm, final S return; } throw new CloudRuntimeException("Start VM " + vmName + " catch BadAsyncResult and VM is not in RUNNING state"); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Start VM interrupted for VM: " + vmName); + throw e; } } catch (final XenAPIException e) { final String msg = "Unable to start VM(" + vmName + ") on host(" + _host.getUuid() + ") due to " + e; @@ -5140,6 +5208,9 @@ public void startVM(final Connection conn, final Host host, final VM vm, final S logger.debug("unable to destroy task(" + task + ") on host(" + _host.getUuid() + ") due to " + e1); } } + if (taskContext != null) { + taskContext.unregisterTask(); + } } } @@ -5278,18 +5349,38 @@ public String upgradeSnapshot(final Connection conn, final String templatePath, } } - public void waitForTask(final Connection c, final Task task, final long pollInterval, final long timeout) throws XenAPIException, XmlRpcException, TimeoutException { + public void waitForTask(final Connection c, final Task task, final long pollInterval, final long timeout) throws XenAPIException, XmlRpcException, TimeoutException, InterruptedException { final long beginTime = System.currentTimeMillis(); if (logger.isTraceEnabled()) { logger.trace("Task " + task.getNameLabel(c) + " (" + task.getUuid(c) + ") sent to " + c.getSessionReference() + " is pending completion with a " + timeout + "ms timeout"); } while (task.getStatus(c) == Types.TaskStatusType.PENDING) { + // Check for thread interruption before each iteration + if (Thread.currentThread().isInterrupted()) { + Thread.currentThread().interrupt(); // Restore interrupt status + try { + task.cancel(c); + logger.info("Cancelled XenServer task: " + task.getUuid(c) + " due to interruption"); + } catch (Exception e) { + logger.warn("Failed to cancel task: " + task.getUuid(c), e); + } + throw new InterruptedException("Task wait cancelled for task: " + task.getUuid(c)); + } + try { if (logger.isTraceEnabled()) { logger.trace("Task " + task.getNameLabel(c) + " (" + task.getUuid(c) + ") is pending, sleeping for " + pollInterval + "ms"); } Thread.sleep(pollInterval); - } catch (final InterruptedException ignored) { + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // Restore interrupt status + try { + task.cancel(c); + logger.info("Cancelled XenServer task: " + task.getUuid(c) + " due to interruption"); + } catch (Exception ex) { + logger.warn("Failed to cancel task: " + task.getUuid(c), ex); + } + throw new InterruptedException("Task wait interrupted for task: " + task.getUuid(c)); } if (System.currentTimeMillis() - beginTime > timeout) { final String msg = "Async " + timeout / 1000 + " seconds timeout for task " + task; diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index c68dc390df2a..01b76ada39b4 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -122,6 +122,7 @@ import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.jobs.AsyncJobService; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.region.PortableIp; @@ -630,6 +631,7 @@ protected void populateConfigValuesForValidationSet() { configValuesForValidation.add(VMLeaseManager.InstanceLeaseExpiryEventSchedulerInterval.key()); configValuesForValidation.add(VMLeaseManager.InstanceLeaseExpiryEventDaysBefore.key()); configValuesForValidation.add(AutoScaleManager.AutoScaleErroredInstanceThreshold.key()); + configValuesForValidation.add(AsyncJobService.CancelledJobInterval.key()); } protected void weightBasedParametersForValidation() { diff --git a/server/src/main/java/com/cloud/event/ActionEventUtils.java b/server/src/main/java/com/cloud/event/ActionEventUtils.java index 0ebb266fd7c2..c96bac866127 100644 --- a/server/src/main/java/com/cloud/event/ActionEventUtils.java +++ b/server/src/main/java/com/cloud/event/ActionEventUtils.java @@ -275,10 +275,10 @@ private static Ternary getResourceDetailsUsingEntityClassA String entityUuid = null; Long entityId = null; Object param = context.getContextParameter(entityClass); - if(param != null){ + if (param != null) { try { entityUuid = getEntityUuid(entityClass, param); - } catch (Exception e){ + } catch (Exception e) { LOGGER.debug("Caught exception while finding entityUUID, moving on"); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 1e276003cd23..2cf34dbbb706 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -228,7 +228,7 @@ public boolean powerOn() throws Exception { ManagedObjectReference morTask = _context.getService().powerOnVMTask(_mor, null); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("powerOn", morTask, _context); } // Monitor VM questions @@ -367,7 +367,7 @@ private boolean powerOffNoCheck() throws Exception { ManagedObjectReference morTask = _context.getService().powerOffVMTask(_mor); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("powerOff", morTask, _context); } try { @@ -449,7 +449,7 @@ public boolean reset() throws Exception { ManagedObjectReference morTask = _context.getService().resetVMTask(_mor); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("reset", morTask, _context); } try { @@ -502,7 +502,7 @@ public boolean migrate(ManagedObjectReference morRp, ManagedObjectReference morT ManagedObjectReference morTask = _context.getService().migrateVMTask(_mor, morRp, morTargetHost, VirtualMachineMovePriority.DEFAULT_PRIORITY, null); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("migrateVM", morTask, _context); } try { @@ -607,7 +607,7 @@ public ManagedObjectReference createSnapshotGetReference(String snapshotName, St boolean result = _context.getVimClient().waitForTask(morTask); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("createSnapshot", morTask, _context); } try { @@ -650,11 +650,11 @@ public ManagedObjectReference createSnapshotGetReference(String snapshotName, St return null; } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.warn("powerOn interrupted for VM: " + getVmName() + ", cancelling task"); + logger.warn("createSnapshot interrupted for VM: " + getVmName() + ", cancelling task"); try { _context.getVimClient().cancelTask(morTask); } catch (Exception ex) { - logger.warn("Failed to cancel powerOn task: " + ex.getMessage()); + logger.warn("Failed to cancel createSnapshot task: " + ex.getMessage()); } throw e; } finally { @@ -674,7 +674,7 @@ public boolean removeSnapshot(String snapshotName, boolean removeChildren) throw ManagedObjectReference morTask = _context.getService().removeSnapshotTask(morSnapshot, removeChildren, true); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("removeSnapshot", morTask, _context); } try { @@ -712,7 +712,7 @@ public boolean revertToSnapshot(String snapshotName) throws Exception { ManagedObjectReference morTask = _context.getService().revertToSnapshotTask(morSnapshot, _mor, null); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("revertToSnapshot", morTask, _context); } try { @@ -1623,18 +1623,33 @@ public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference m ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); if (taskCtx != null) { - taskCtx.registerTask(morTask, _context); + taskCtx.registerTask("reconfigVM/attachDisk", morTask, _context); } - boolean result = _context.getVimClient().waitForTask(morTask); + try { + boolean result = _context.getVimClient().waitForTask(morTask); - if (!result) { - if (logger.isTraceEnabled()) - logger.trace("vCenter API trace - attachDisk() done(failed)"); - throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } + if (!result) { + if (logger.isTraceEnabled()) + logger.trace("vCenter API trace - attachDisk() done(failed)"); + throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } - _context.waitForTaskProgressDone(morTask); + _context.waitForTaskProgressDone(morTask); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("reconfigVM/attachDisk interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel reconfigVM/attachDisk task: " + ex.getMessage()); + } + throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } + } } if (logger.isTraceEnabled()) @@ -1690,12 +1705,32 @@ public List> detachDisk(String vmdkDatastor reConfigSpec.getDeviceChange().add(deviceConfigSpec); ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - boolean result = _context.getVimClient().waitForTask(morTask); + VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); + if (taskCtx != null) { + taskCtx.registerTask("reconfigVM/detachDisk", morTask, _context); + } - if (!result) { - throw new CloudRuntimeException(String.format("Failed to detach disk from instance [%s] due to [%s].", getVmName(), TaskMO.getTaskFailureInfo(_context, morTask))); + try { + boolean result = _context.getVimClient().waitForTask(morTask); + + if (!result) { + throw new CloudRuntimeException(String.format("Failed to detach disk from instance [%s] due to [%s].", getVmName(), TaskMO.getTaskFailureInfo(_context, morTask))); + } + _context.waitForTaskProgressDone(morTask); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("reconfigVM/detachDisk interrupted for VM: " + getVmName() + ", cancelling task"); + try { + _context.getVimClient().cancelTask(morTask); + } catch (Exception ex) { + logger.warn("Failed to cancel reconfigVM/detachDisk task: " + ex.getMessage()); + } + throw e; + } finally { + if (taskCtx != null) { + taskCtx.unregisterTask(); + } } - _context.waitForTaskProgressDone(morTask); // VMware does not update snapshot references to the detached disk, we have to work around it SnapshotDescriptor snapshotDescriptor = null; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 74054609e606..169eae68fa3a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -160,9 +160,8 @@ public static class ActiveVmTaskInfo { } } - private static void registerActiveVmTask(long cmdSequence, String vmName, String commandType, + private static void registerActiveVmTask(String taskName, long cmdSequence, String vmName, String commandType, ManagedObjectReference taskMor, VmwareContext context) { - if (StringUtils.isBlank(vmName)) { LOGGER.debug("No active VM to registered VM task for sequence={}, command={}, task={}", cmdSequence, commandType, taskMor); return; @@ -170,11 +169,10 @@ private static void registerActiveVmTask(long cmdSequence, String vmName, String ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, taskMor, context); activeVmTasks.put(cmdSequence, taskInfo); - vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); - LOGGER.debug("Registered active VM task: sequence={}, vm={}, command={}, task={}", - cmdSequence, vmName, commandType, taskMor.getValue()); + LOGGER.debug("Registered active VM task: {} - sequence={}, vm={}, command={}, task={}", + taskName, cmdSequence, vmName, commandType, taskMor.getValue()); } private static void unregisterActiveVmTask(long cmdSequence) { @@ -209,13 +207,17 @@ public static int cancelActiveVmTasks(String vmName) { } public static boolean isActiveVmTaskCancellable(long cmdSequence) { + LOGGER.info("Check if active vm task for sequence: {} is cancellable or not", cmdSequence); ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); if (taskInfo == null) { + LOGGER.debug("No active vm task registered for sequence: {}", cmdSequence); return true; } try { - return taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor); + boolean cancellable = taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor); + LOGGER.debug("Active vm task registered for sequence: {} is ", cmdSequence, cancellable ? "cancellable" : "not cancellable"); + return cancellable; } catch (Exception e) { LOGGER.warn("Failed to verify vCenter task cancellability for sequence={}, vm={}, error={}", cmdSequence, taskInfo.vmName, e.getMessage(), e); @@ -224,21 +226,23 @@ public static boolean isActiveVmTaskCancellable(long cmdSequence) { } public static boolean cancelActiveVmTask(long cmdSequence) { + LOGGER.info("Cancel active vm tasks for sequence: {}", cmdSequence); ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); if (taskInfo == null) { + LOGGER.debug("No active vm tasks for sequence: {}", cmdSequence); return true; } try { if (!taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor)) { - LOGGER.info("Active vCenter VM task is not cancellable: sequence={}, vm={}, command={}", + LOGGER.debug("Active vCenter VM task is not cancellable: sequence={}, vm={}, command={}", cmdSequence, taskInfo.vmName, taskInfo.commandType); return false; } Pair cancellationResult = taskInfo.context.getVimClient().cancelTask(taskInfo.taskMor); if (!cancellationResult.first()) { - LOGGER.info("Unable to cancel vCenter VM task: sequence={}, vm={}, reason={}", + LOGGER.debug("Unable to cancel vCenter VM task: sequence={}, vm={}, reason={}", cmdSequence, taskInfo.vmName, cancellationResult.second()); return false; } @@ -287,8 +291,8 @@ public TaskContext(long cmdSequence, String vmName, String commandType) { this.commandType = commandType; } - public void registerTask(ManagedObjectReference taskMor, VmwareContext context) { - registerActiveVmTask(cmdSequence, vmName, commandType, taskMor, context); + public void registerTask(String taskName, ManagedObjectReference taskMor, VmwareContext context) { + registerActiveVmTask(taskName, cmdSequence, vmName, commandType, taskMor, context); } public void unregisterTask() { From 676278c406049e839932ac0679bb4cca19b5058d Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 03:00:31 +0530 Subject: [PATCH 11/29] changes --- .../command/admin/vm/DestroyVMCmdByAdmin.java | 1 - .../command/user/job/ListAsyncJobsCmd.java | 23 -------- .../cloud/vm/VirtualMachineManagerImpl.java | 4 +- .../orchestration/StorageOrchestrator.java | 3 +- .../StorageSystemDataMotionStrategy.java | 2 +- .../vmsnapshot/DefaultVMSnapshotStrategy.java | 2 +- .../jobs/impl/AsyncJobManagerImpl.java | 38 ++++++++++--- .../backup/KbossBackupProvider.java | 9 +-- .../backup/KbossBackupProviderTest.java | 29 +++++----- .../backup/NASBackupProviderTest.java | 14 ++--- .../motion/LinstorDataMotionStrategy.java | 6 +- .../vmsnapshot/OntapVMSnapshotStrategy.java | 4 ++ .../com/cloud/api/query/QueryManagerImpl.java | 20 +++---- .../cloud/storage/clvm/ClvmPoolManager.java | 7 ++- .../backup/KVMBackupExportServiceImpl.java | 19 ++++--- .../vm/UnmanagedVMsManagerImpl.java | 2 +- .../cloud/storage/ClvmPoolManagerTest.java | 57 ++++++++++--------- .../vm/UnmanagedVMsManagerImplTest.java | 2 +- 18 files changed, 125 insertions(+), 117 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java index b1ecffb0ea6c..7ab285b18ed4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java @@ -18,7 +18,6 @@ import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.CancellableCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index e2348d659b49..bb4ab6c1f97b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -23,7 +23,6 @@ import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; -import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; @@ -62,12 +61,6 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the async jobs started or completed within that period up to now.", since = "4.23") private Integer duration; - @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource.", since = "4.23") - private String resourceType; - - @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING, description = "the id of the resource", since = "4.23") - private String resourceId; - ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -116,22 +109,6 @@ public Integer getDuration() { return duration; } - public ApiCommandResourceType getResourceType() { - if (resourceType != null) { - ApiCommandResourceType jobInstanceType = com.cloud.utils.EnumUtils.getEnumIgnoreCase(ApiCommandResourceType.class, resourceType, ApiCommandResourceType.None); - if (jobInstanceType == ApiCommandResourceType.None) { - throw new InvalidParameterValueException("Not a valid resource type"); - } - return jobInstanceType; - } - - return null; - } - - public String getResourceId() { - return resourceId; - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index c75f564c04dd..74f516a1192e 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -6464,7 +6464,7 @@ private boolean orchestrateUpdateVmNic(final VirtualMachine vm, final Nic nic, f logger.warn("Unable to update VM %s NIC [{}].", vm.getName(), nic.getUuid()); return false; } - } catch (final OperationTimedoutException e) { + } catch (final OperationTimedoutException | OperationCancelledException e) { throw new AgentUnavailableException(String.format("Unable to update NIC %s for VM %s.", nic.getUuid(), vm.getUuid()), vm.getHostId(), e); } } @@ -6582,7 +6582,7 @@ private void executePreMigrationCommand(VMInstanceVO vm, VirtualMachineTO to, lo throw new CloudRuntimeException(msg); } logger.info("Successfully prepared source host {} for migration of VM {}", srcHostUuid, vmInstanceName); - } catch (final AgentUnavailableException | OperationTimedoutException e) { + } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Failed to send PreMigrationCommand to source host {}: {}", srcHostUuid, e.getMessage(), e); throw new CloudRuntimeException("Failed to prepare source host for migration: " + e.getMessage(), e); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 403e83bccf99..a909a94c4d0b 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -45,6 +45,7 @@ import com.cloud.agent.api.MigrateBetweenSecondaryStoragesCommandAnswer; import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -424,7 +425,7 @@ private void migrateKvmIncrementalBackupChain(DataObject chosenFileForMigration, throw new CloudRuntimeException("Unable to migrate KVM incremental backups to another secondary storage"); } - } catch (final OperationTimedoutException | AgentUnavailableException e) { + } catch (final OperationTimedoutException | AgentUnavailableException | OperationCancelledException e) { throw new CloudRuntimeException("Error while migrating KVM incremental backup chain. Check the logs for more information.", e); } finally { if (answer != null) { diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 86f628891128..dcd274e79b25 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -2458,7 +2458,7 @@ private void sendClvmLockCommand(long hostId, StoragePoolVO pool, VolumeInfo vol String details = answer != null ? answer.getDetails() : "null answer"; logger.warn("CLVM lock command [{}] failed for LV [{}] on host [{}]: {}", operation, lvPath, hostId, details); } - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.warn("Exception sending CLVM lock command [{}] for LV [{}] on host [{}]: {}", operation, lvPath, hostId, e.getMessage()); } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index 5f0e05637e64..20d04219b86c 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -269,7 +269,7 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) { processAnswer(vmSnapshotVO, userVm, answer, hostId); throw new CloudRuntimeException("Delete Instance Snapshot " + vmSnapshot.getName() + " of Instance " + userVm.getInstanceName() + " failed due to " + errMsg); } - } catch (OperationTimedoutException | AgentUnavailableException e) { + } catch (OperationTimedoutException | AgentUnavailableException | OperationCancelledException e) { throw new CloudRuntimeException("Delete Instance Snapshot " + vmSnapshot.getName() + " of Instance " + userVm.getInstanceName() + " failed due to " + e.getMessage()); } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 80a8ef2514ec..0f5dbec1b0fe 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -20,6 +20,7 @@ import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -67,6 +68,7 @@ import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageDetector; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.jobs.AsyncJobService; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.jobs.JobInfo.Status; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -186,6 +188,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private volatile long _executionRunNumber = 1; private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat")); + private final ExecutorService _eventBusPublisher = Executors.newSingleThreadExecutor(new NamedThreadFactory("AsyncJobMgr-EventBus")); private ExecutorService _apiJobExecutor; private ExecutorService _workerJobExecutor; @@ -979,6 +982,11 @@ protected void reallyRun() { }; } + private boolean isPseudoJob(AsyncJob job) { + return AsyncJobVO.JOB_DISPATCHER_PSEUDO.equals(job.getDispatcher()) && AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE + .equals(job.getInstanceType()); + } + @DB private Runnable getGCTask() { return new ManagedContextRunnable() { @@ -1421,17 +1429,15 @@ public boolean start() { @Override public boolean stop() { cancelPendingJobs(); - shutdownAndAwaitTermination(_workerJobExecutor); - shutdownAndAwaitTermination(_apiJobExecutor); _heartbeatScheduler.shutdown(); _eventBusPublisher.shutdown(); - _apiJobExecutor.shutdown(); - _workerJobExecutor.shutdown(); + shutdownAndAwaitTermination(_apiJobExecutor); + shutdownAndAwaitTermination(_workerJobExecutor); return true; } private void cancelPendingJobs() { - List jobs = _syncQueueItemDao.getActiveQueueItems(getMsid(), false); + List jobs = _queueItemDao.getActiveQueueItems(getMsid(), false); for (SyncQueueItemVO job : jobs) { AsyncJobVO childJob = _jobDao.findById(job.getContentId()); if (childJob != null && StringUtils.isNotBlank(childJob.getRelated())) { @@ -1537,8 +1543,26 @@ protected AsyncJobManagerImpl() { } private void publishOnEventBus(AsyncJob job, String jobEvent) { - _messageBus.publish(null, AsyncJob.Topics.JOB_EVENT_PUBLISH, PublishScope.LOCAL, - new Pair(job, jobEvent)); + try { + _eventBusPublisher.submit(new ManagedContextRunnable() { + @Override + protected void runInContext() { + publishJobEvent(job, jobEvent); + } + }); + } catch (RejectedExecutionException e) { + logger.warn("Failed to publish async job event, event bus publisher is shut down", e); + } + } + + private void publishJobEvent(AsyncJob job, String jobEvent) { + try { + _messageBus.publish(null, AsyncJob.Topics.JOB_EVENT_PUBLISH, PublishScope.LOCAL, + new Pair<>(job, jobEvent)); + } catch (Throwable t) { + logger.warn("Failed to publish async job event on message bus. jobId={}, jobEvent={}", + job != null ? job.getId() : null, jobEvent, t); + } } @Override diff --git a/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java b/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java index 5edac430a673..a79023695fea 100644 --- a/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java +++ b/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java @@ -44,6 +44,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -552,7 +553,7 @@ public Boolean orchestrateDeleteBackup(Backup backup, boolean forced) { Answer[] deleteAnswers; try { deleteAnswers = sendBackupCommands(endPoint.getId(), deleteCommands); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException(e); } @@ -652,7 +653,7 @@ public Boolean orchestrateRestoreVMFromBackup(Backup backup, VirtualMachine vm, try { answers = sendBackupCommands(host.getId(), commands); - } catch (OperationTimedoutException | AgentUnavailableException e) { + } catch (OperationTimedoutException | AgentUnavailableException | OperationCancelledException e) { throw new CloudRuntimeException(e); } @@ -978,7 +979,7 @@ public void prepareVmForSnapshotRevert(VMSnapshot vmSnapshot, VirtualMachine vir Answer[] answers; try { answers = sendBackupCommands(hostId, commands); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException(e); } @@ -2628,7 +2629,7 @@ protected Answer sendBackupCommand(long hostId, Command cmd) { return agentManager.easySend(hostId, cmd); } - protected Answer[] sendBackupCommands(Long hostId, Commands cmds) throws OperationTimedoutException, AgentUnavailableException { + protected Answer[] sendBackupCommands(Long hostId, Commands cmds) throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { for (Command cmd : cmds) { cmd.setWait(backupTimeout.value()); } diff --git a/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java b/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java index 17ee3c8f6722..2342ad8b8f31 100644 --- a/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java +++ b/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java @@ -45,6 +45,7 @@ import java.util.Map; import java.util.Set; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.backup.dao.BackupDao; @@ -877,7 +878,7 @@ public void orchestrateDeleteBackupTestBackupStateIsNotOk() { } @Test - public void orchestrateDeleteBackupTestDeleteFailedBackup() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteFailedBackup() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).validateBackupStateForRemoval(backupId); @@ -892,7 +893,7 @@ public void orchestrateDeleteBackupTestDeleteFailedBackup() throws OperationTime } @Test - public void orchestrateDeleteBackupTestDeleteBackupWithLiveChildren() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteBackupWithLiveChildren() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).validateBackupStateForRemoval(backupId); @@ -910,7 +911,7 @@ public void orchestrateDeleteBackupTestDeleteBackupWithLiveChildren() throws Ope } @Test - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenFailedToMerge() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenFailedToMerge() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).validateBackupStateForRemoval(backupId); @@ -928,7 +929,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenFailedTo } @Test (expected = CloudRuntimeException.class) - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentNoEndPoint() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentNoEndPoint() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -954,7 +955,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithPare } @Test (expected = CloudRuntimeException.class) - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentTimedoutException() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentTimedoutException() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -981,7 +982,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithPare } @Test - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetNotEmpty() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetNotEmpty() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -1011,7 +1012,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithPare } @Test - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetEmpty() throws OperationTimedoutException, AgentUnavailableException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetEmpty() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -1065,7 +1066,7 @@ public void orchestrateRestoreVMFromBackupTestCurrentBackupNoHostToRestore() thr } @Test (expected = CloudRuntimeException.class) - public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupTimeOut() throws AgentUnavailableException, OperationTimedoutException { + public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupTimeOut() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, false); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1089,7 +1090,7 @@ public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupTimeOut() throw } @Test - public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupNullAnswers() throws AgentUnavailableException, OperationTimedoutException { + public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupNullAnswers() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, false); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1114,7 +1115,7 @@ public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupNullAnswers() t } @Test - public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupAnswerFalse() throws AgentUnavailableException, OperationTimedoutException { + public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupAnswerFalse() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, false); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1140,7 +1141,7 @@ public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupAnswerFalse() t } @Test - public void orchestrateRestoreVMFromBackupTestSameVmQuickRestoreCurrentBackupAnswerTrue() throws AgentUnavailableException, OperationTimedoutException { + public void orchestrateRestoreVMFromBackupTestSameVmQuickRestoreCurrentBackupAnswerTrue() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, true); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1431,7 +1432,7 @@ public void prepareVmForSnapshotRevertTestCurrentBackupBeforeVmSnapshot() { } @Test (expected = CloudRuntimeException.class) - public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() throws OperationTimedoutException, AgentUnavailableException { + public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); @@ -1446,7 +1447,7 @@ public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() } @Test (expected = CloudRuntimeException.class) - public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer() throws OperationTimedoutException, AgentUnavailableException { + public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); @@ -1461,7 +1462,7 @@ public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer } @Test - public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotSuccess() throws OperationTimedoutException, AgentUnavailableException { + public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotSuccess() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index 4eba917aa5ed..a97af9a5d956 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -532,7 +532,7 @@ public void allVolumesOnCheckpointCapableStorageFalseWhenPoolUnresolvable() { * is called with the active_checkpoint_id key. */ @Test - public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableException, OperationTimedoutException { + public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long vmId = 7L; Long hostId = 8L; Long backupOfferingId = 9L; @@ -582,7 +582,7 @@ public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableExcepti */ @Test public void restoreBackedUpVolumeClearsTargetVmActiveCheckpoint() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long targetVmId = 42L; Long backupOfferingId = 9L; String targetVmName = "i-2-42-VM"; @@ -652,7 +652,7 @@ public void restoreBackedUpVolumeClearsTargetVmActiveCheckpoint() */ @Test public void deleteWithLiveChildMarksDeletePendingAndPreservesFile() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -728,7 +728,7 @@ public void deleteWithLiveChildMarksDeletePendingAndPreservesFile() */ @Test public void deletingLeafSweepsUpDeletePendingParent() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -813,7 +813,7 @@ public void deletingLeafSweepsUpDeletePendingParent() @Test public void deletingLastLiveMemberCollectsDeeperOrphanTombstones() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -890,7 +890,7 @@ public void deletingLastLiveMemberCollectsDeeperOrphanTombstones() @Test public void deletingAncestorOfTombstoneWithLiveDescendantTombstonesIt() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -959,7 +959,7 @@ public void deletingAncestorOfTombstoneWithLiveDescendantTombstonesIt() @Test public void sweepContinuesPastFailedTombstoneDelete() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java index 0dd0de70464f..d1f410a65b04 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.motion; +import com.cloud.exception.OperationCancelledException; import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRcList; @@ -453,6 +454,9 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } catch (final OperationTimedoutException e) { errMsg = String.format("Operation timed out due to %s", e.getMessage()); throw new AgentUnavailableException(errMsg, destHost.getId()); + } catch (final OperationCancelledException e) { + errMsg = String.format("Operation cancelled due to %s", e.getMessage()); + throw new AgentUnavailableException(errMsg, destHost.getId()); } VMInstanceVO vm = _vmDao.findById(vmTO.getId()); @@ -485,7 +489,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException(errMsg); } - } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException | CloudRuntimeException ex) { errMsg = String.format( "Copy volume(s) of VM [%s] to storage(s) [%s] and VM to host [%s] failed in LinstorDataMotionStrategy.copyAsync. Error message: [%s].", vmTO, srcHost, destHost, ex.getMessage()); diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java index 7fa80a0b3fae..91936a212c2e 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java @@ -27,6 +27,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.VMSnapshotOptions; @@ -473,6 +474,9 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { } catch (OperationTimedoutException e) { logger.error("takeVMSnapshot: ONTAP VM Snapshot [{}] timed out: {}", vmSnapshot.getName(), e.getMessage()); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " timed out: " + e.getMessage()); + } catch (OperationCancelledException e) { + logger.error("takeVMSnapshot: ONTAP VM Snapshot [{}] cancelled: {}", vmSnapshot.getName(), e.getMessage()); + throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " cancelled: " + e.getMessage()); } catch (AgentUnavailableException e) { logger.error("takeVMSnapshot: ONTAP VM Snapshot [{}] failed, agent unavailable: {}", vmSnapshot.getName(), e.getMessage()); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.getMessage()); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 0a4cfee03727..27c0537e4f12 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -39,7 +39,6 @@ import javax.inject.Inject; -import com.cloud.api.ApiDBUtils; import com.cloud.network.PublicIpQuarantine; import com.cloud.network.dao.PublicIpQuarantineDao; import com.cloud.network.vo.PublicIpQuarantineVO; @@ -3305,21 +3304,16 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync sc.addAnd("created", SearchCriteria.Op.SC, scc); } - ApiCommandResourceType resourceType = cmd.getResourceType(); - if (resourceType != null) { - sc.addAnd("instanceType", SearchCriteria.Op.EQ, resourceType); - - if (cmd.getResourceId() != null) { - Long resourceId = ApiDBUtils.findResourceId(resourceType, cmd.getResourceId()); - if (resourceId == null) { - throw new InvalidParameterValueException("Invalid resource id for the resource type " + resourceType); - } + if (cmd.getResourceType() != null) { + ApiCommandResourceType resourceType = getResourceType(cmd.getResourceType()); + sc.addAnd("instanceType", SearchCriteria.Op.EQ, resourceType.toString()); - sc.addAnd("instanceUuid", SearchCriteria.Op.EQ, cmd.getResourceId()); -// sc.addAnd("instanceId", SearchCriteria.Op.EQ, resourceId); + final String resourceId = getResourceUuid(cmd.getResourceId()); + if (resourceId != null) { + sc.addAnd("instanceUuid", SearchCriteria.Op.EQ, resourceId); } } else if (cmd.getResourceId() != null) { - throw new InvalidParameterValueException("Resource type must be specified for the resource id"); + throw new InvalidParameterValueException(String.format("%s parameter must be used with %s parameter", ApiConstants.RESOURCE_ID, ApiConstants.RESOURCE_TYPE)); } if (cmd.getManagementServerId() != null) { diff --git a/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java b/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java index febef471e683..b21284ef69a1 100644 --- a/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java +++ b/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java @@ -28,6 +28,7 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -288,7 +289,7 @@ private Boolean querySingleHostLockState(Long hostId, String lvPath, String volu logger.debug("Host {} reports volume {} active={} (attr={})", hostId, volumeUuid, queryAnswer.isActive(), queryAnswer.getLvAttributes()); return queryAnswer.isActive(); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.debug("Could not query host {} for lock state: {}", hostId, e.getMessage()); return null; } @@ -363,7 +364,7 @@ private Long tryActivateExclusivelyOnHost(Long volumeId, String volumeUuid, Stri logger.warn("Recovery activation of volume {} on host {} failed: {}", volumeUuid, targetHostId, activateAnswer != null ? activateAnswer.getDetails() : "null answer"); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.warn("Recovery activation of volume {} on host {} threw exception: {}", volumeUuid, targetHostId, e.getMessage()); } @@ -465,7 +466,7 @@ public boolean transferClvmVolumeLock(String volumeUuid, Long volumeId, String v return true; - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Exception during CLVM lock transfer for volume {}: {}", volumeUuid, e.getMessage(), e); return false; } diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index ea90fc6b4b53..8f46a19b4eb4 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -33,6 +33,7 @@ import javax.inject.Inject; +import com.cloud.exception.OperationCancelledException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; @@ -329,7 +330,7 @@ public Backup startBackup(StartBackupCmd cmd) { StartBackupAnswer answer; try { answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { removeFailedBackup(backup); logger.error("Failed to communicate with agent on {} for {} start", host, backup, e); throw new CloudRuntimeException("Failed to communicate with agent", e); @@ -407,7 +408,7 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { StopBackupAnswer answer; try { answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { removeFailedBackup(backup); throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -493,7 +494,7 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu imageTransfer = imageTransferDao.persist(imageTransfer); return imageTransfer; - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } } @@ -538,7 +539,7 @@ private void startNBDServer(String transferId, String direction, Long hostId, St ); try { nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(hostId, nbdServerCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!nbdServerAnswer.getResult()) { @@ -627,7 +628,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer CreateImageTransferAnswer transferAnswer; try { transferAnswer = (CreateImageTransferAnswer) agentManager.send(imageTransfer.getHostId(), transferCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -721,7 +722,7 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { Answer answer; try { answer = agentManager.send(backup.getHostId(), finalizeCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -745,7 +746,7 @@ private boolean stopNBDServer(ImageTransferVO imageTransfer) { Answer answer; try { answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Failed to stop NBD server on image transfer finalization", e); return false; } @@ -763,7 +764,7 @@ private String finalizeUploadImageTransferWithResult(ImageTransferVO imageTransf Answer answer; try { answer = agentManager.send(imageTransfer.getHostId(), finalizeCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { return "Failed to communicate with agent"; } @@ -881,7 +882,7 @@ private void sendDeleteCheckpointCommand(VMInstanceVO vm, String checkpointId) { Answer answer; try { answer = agentManager.send(hostId, deleteCmd); - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Failed to communicate with agent to delete checkpoint for VM {}", vm.getId(), e); throw new CloudRuntimeException("Failed to communicate with agent", e); } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 0542ff9301c8..4a1c199cc8c8 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -2261,7 +2261,7 @@ private void cleanupConvertedDisks(String sourceVM, HostVO convertHost, DataStor logger.warn("Failed to cleanup the converted disks for the VM {} through " + "the conversion host {}: {}", sourceVM, convertHost.getName(), cleanupAnswer.getDetails()); } - } catch (AgentUnavailableException | OperationTimedoutException e) { + } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { logger.error("Error cleaning up converted disks for VM {} through the conversion host {}", sourceVM, convertHost.getName(), e); } diff --git a/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java b/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java index 1678b3760919..cb62193b072d 100644 --- a/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java +++ b/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java @@ -21,6 +21,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -162,7 +163,7 @@ public void testClearClvmLockHostDetail_NoDetail() { } @Test - public void testTransferClvmVolumeLock_Success() throws AgentUnavailableException, OperationTimedoutException { + public void testTransferClvmVolumeLock_Success() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn("/" + VG_NAME); @@ -192,7 +193,7 @@ public void testTransferClvmVolumeLock_NullPool() { } @Test - public void testTransferClvmVolumeLock_SameHost() throws AgentUnavailableException, OperationTimedoutException { + public void testTransferClvmVolumeLock_SameHost() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn("/" + VG_NAME); @@ -207,7 +208,7 @@ public void testTransferClvmVolumeLock_SameHost() throws AgentUnavailableExcepti } @Test - public void testTransferClvmVolumeLock_ActivationFails() throws AgentUnavailableException, OperationTimedoutException { + public void testTransferClvmVolumeLock_ActivationFails() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn(VG_NAME); @@ -221,7 +222,7 @@ public void testTransferClvmVolumeLock_ActivationFails() throws AgentUnavailable } @Test - public void testTransferClvmVolumeLock_AgentUnavailable() throws AgentUnavailableException, OperationTimedoutException { + public void testTransferClvmVolumeLock_AgentUnavailable() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn(VG_NAME); @@ -261,7 +262,7 @@ public void testQueryCurrentLockHolder_NoHostsInCluster() { } @Test - public void testQueryCurrentLockHolder_ZoneScopedPool() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_ZoneScopedPool() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(null); when(pool.getDataCenterId()).thenReturn(1L); @@ -284,7 +285,7 @@ public void testQueryCurrentLockHolder_ZoneScopedPool() throws AgentUnavailableE } @Test - public void testQueryCurrentLockHolder_SuccessfulQuery() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_SuccessfulQuery() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -306,7 +307,7 @@ public void testQueryCurrentLockHolder_SuccessfulQuery() throws AgentUnavailable } @Test - public void testQueryCurrentLockHolder_VolumeNotLocked() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_VolumeNotLocked() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -331,7 +332,7 @@ public void testQueryCurrentLockHolder_VolumeNotLocked() throws AgentUnavailable } @Test - public void testQueryCurrentLockHolder_EmptyHostname() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_EmptyHostname() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -356,7 +357,7 @@ public void testQueryCurrentLockHolder_EmptyHostname() throws AgentUnavailableEx } @Test - public void testQueryCurrentLockHolder_HostnameNotResolved() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_HostnameNotResolved() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -377,7 +378,7 @@ public void testQueryCurrentLockHolder_HostnameNotResolved() throws AgentUnavail } @Test - public void testQueryCurrentLockHolder_QueryFails() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_QueryFails() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -397,7 +398,7 @@ public void testQueryCurrentLockHolder_QueryFails() throws AgentUnavailableExcep } @Test - public void testQueryCurrentLockHolder_NullAnswer() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_NullAnswer() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -416,7 +417,7 @@ public void testQueryCurrentLockHolder_NullAnswer() throws AgentUnavailableExcep } @Test - public void testQueryCurrentLockHolder_AgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_AgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -436,7 +437,7 @@ public void testQueryCurrentLockHolder_AgentUnavailableException() throws AgentU } @Test - public void testQueryCurrentLockHolder_OperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_OperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -456,7 +457,7 @@ public void testQueryCurrentLockHolder_OperationTimedoutException() throws Agent } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_MatchingValue() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_UpdateDatabase_MatchingValue() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); Mockito.lenient().when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -484,7 +485,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_MatchingValue() throws Age } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_DifferentValue() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_UpdateDatabase_DifferentValue() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -517,7 +518,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_DifferentValue() throws Ag } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_NoExistingDetail() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_UpdateDatabase_NoExistingDetail() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -540,7 +541,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_NoExistingDetail() throws } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_RemoveDetailWhenUnlocked() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_UpdateDatabase_RemoveDetailWhenUnlocked() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -579,7 +580,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_RemoveDetailWhenUnlocked() } @Test - public void testQueryCurrentLockHolder_SkipsNonKVMHosts() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_SkipsNonKVMHosts() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -602,7 +603,7 @@ public void testQueryCurrentLockHolder_SkipsNonKVMHosts() throws AgentUnavailabl } @Test - public void testQueryCurrentLockHolder_SkipsDownHosts() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_SkipsDownHosts() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -625,7 +626,7 @@ public void testQueryCurrentLockHolder_SkipsDownHosts() throws AgentUnavailableE } @Test - public void testQueryCurrentLockHolder_PathWithLeadingSlash() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_PathWithLeadingSlash() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -649,7 +650,7 @@ public void testQueryCurrentLockHolder_PathWithLeadingSlash() throws AgentUnavai * No fan-out should occur. */ @Test - public void testQueryCurrentLockHolder_FastPath_HitOnDbHost() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_FastPath_HitOnDbHost() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn(VG_NAME); @@ -677,7 +678,7 @@ public void testQueryCurrentLockHolder_FastPath_HitOnDbHost() throws AgentUnavai * HOST_ID_1 should NOT be queried again during fan-out. */ @Test - public void testQueryCurrentLockHolder_FastPath_MissDbHost_FanOutFindsOther() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_FastPath_MissDbHost_FanOutFindsOther() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); when(pool.getPath()).thenReturn(VG_NAME); @@ -710,7 +711,7 @@ public void testQueryCurrentLockHolder_FastPath_MissDbHost_FanOutFindsOther() th * Fast path skip: DB host is DOWN. Fan-out proceeds to all UP hosts. */ @Test - public void testQueryCurrentLockHolder_FastPath_DbHostDown_FanOut() throws AgentUnavailableException, OperationTimedoutException { + public void testQueryCurrentLockHolder_FastPath_DbHostDown_FanOut() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); when(pool.getPath()).thenReturn(VG_NAME); @@ -741,7 +742,7 @@ public void testQueryCurrentLockHolder_FastPath_DbHostDown_FanOut() throws Agent */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnDbHost() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -785,7 +786,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnDbHost() */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnClusterHostWhenNoDbRecord() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -825,7 +826,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnClusterHost */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_SkipsDownDbHost_ActivatesOnClusterHost() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -868,7 +869,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_SkipsDownDbHost_Activa */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_ActivationThrows_ReturnsNull() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -905,7 +906,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_ActivationThrows_Retur */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_NoEligibleHost_ReturnsNull() - throws AgentUnavailableException, OperationTimedoutException { + throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index 82fece462295..b14a05add794 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -1034,7 +1034,7 @@ public void testImportVmFromVmwareToKvmExistingVcenterConvertFailure() throws Op } @Test(expected = ServerApiException.class) - public void testImportVmFromVmwareToKvmForceMsMutuallyExclusiveWithUseVddk() throws OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmForceMsMutuallyExclusiveWithUseVddk() throws OperationCancelledException, OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.FORCE_MS_AND_USE_VDDK, false, false); } From 3c9b7b70937116eb547f0eb63a1fb8ce07f69310 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 10:06:28 +0530 Subject: [PATCH 12/29] api, job framework changes and some code improvements --- .../exception/AgentUnavailableException.java | 11 - .../exception/OperationTimedoutException.java | 9 +- .../api/ApiCommandResourceType.java | 4 +- .../apache/cloudstack/api/ApiConstants.java | 2 +- .../apache/cloudstack/api/BaseAsyncCmd.java | 1 + .../org/apache/cloudstack/api/BaseCmd.java | 25 +- .../api/command/user/event/ListEventsCmd.java | 8 + .../command/user/job/CancelAsyncJobCmd.java | 2 +- .../command/user/job/ListAsyncJobsCmd.java | 45 ++- .../cloudstack/context/CallContext.java | 8 +- .../jobs/JobCancellationHandler.java | 43 +++ .../com/cloud/vm/VirtualMachineManager.java | 5 +- .../com/cloud/vm/VmWorkJobHandlerProxy.java | 44 ++- .../com/cloud/vm/VmWorkJobDispatcher.java | 2 +- .../orchestration/StorageOrchestrator.java | 3 +- .../main/java/com/cloud/event/EventVO.java | 11 + .../META-INF/db/schema-42300to2400.sql | 4 + .../META-INF/db/views/cloud.event_view.sql | 8 +- .../vmsnapshot/DefaultVMSnapshotStrategy.java | 9 +- .../vmsnapshot/StorageVMSnapshotStrategy.java | 8 +- .../storage/RemoteHostEndPoint.java | 6 +- .../storage/helper/HypervisorHelperImpl.java | 9 +- .../jobs/AsyncJobExecutionContext.java | 45 ++- .../framework/jobs/AsyncJobManager.java | 18 +- .../framework/jobs/dao/AsyncJobDao.java | 4 +- .../framework/jobs/dao/AsyncJobDaoImpl.java | 10 +- .../jobs/dao/AsyncJobJoinMapDaoImpl.java | 3 +- .../framework/jobs/dao/SyncQueueItemDao.java | 15 +- .../jobs/dao/SyncQueueItemDaoImpl.java | 19 +- .../jobs/impl/AsyncJobManagerImpl.java | 358 +++++++++++------- .../framework/jobs/impl/AsyncJobMonitor.java | 3 +- .../framework/jobs/impl/SyncQueueManager.java | 16 +- .../jobs/impl/SyncQueueManagerImpl.java | 10 +- .../framework/jobs/impl/VmWorkJobVO.java | 6 +- .../vmware/manager/VmwareManagerImpl.java | 7 +- .../lb/ElasticLoadBalancerManagerImpl.java | 4 - .../lb/InternalLoadBalancerVMManagerImpl.java | 4 - .../InternalLBVMManagerTest.java | 6 +- .../vmsnapshot/OntapVMSnapshotStrategy.java | 4 - .../com/cloud/api/ApiAsyncJobDispatcher.java | 45 +-- .../java/com/cloud/api/ApiDispatcher.java | 3 +- .../main/java/com/cloud/api/ApiServer.java | 25 +- .../api/dispatch/CommandCreationWorker.java | 5 +- .../com/cloud/api/query/QueryManagerImpl.java | 32 +- .../com/cloud/api/query/vo/EventJoinVO.java | 14 + .../com/cloud/event/ActionEventUtils.java | 26 +- .../com/cloud/event/dao/EventJoinDaoImpl.java | 1 + .../discoverer/LibvirtServerDiscoverer.java | 5 +- .../network/router/NetworkHelperImpl.java | 4 - ...VpcVirtualNetworkApplianceManagerImpl.java | 3 +- .../security/SecurityGroupManagerImpl.java | 9 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 9 - .../apache/cloudstack/ca/CAManagerImpl.java | 7 +- 53 files changed, 580 insertions(+), 407 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java diff --git a/api/src/main/java/com/cloud/exception/AgentUnavailableException.java b/api/src/main/java/com/cloud/exception/AgentUnavailableException.java index 2adba254e153..89193243e1d1 100644 --- a/api/src/main/java/com/cloud/exception/AgentUnavailableException.java +++ b/api/src/main/java/com/cloud/exception/AgentUnavailableException.java @@ -27,8 +27,6 @@ public class AgentUnavailableException extends ResourceUnavailableException { private static final long serialVersionUID = SerialVersionUID.AgentUnavailableException; - private boolean isCancelled; - public AgentUnavailableException(String msg, long agentId) { this(msg, agentId, null); } @@ -40,13 +38,4 @@ public AgentUnavailableException(long agentId) { public AgentUnavailableException(String msg, long agentId, Throwable cause) { super("Host " + agentId + ": " + msg, Host.class, agentId, cause); } - - public AgentUnavailableException(String msg, long agentId, boolean isCancelled) { - this(msg, agentId, null); - this.isCancelled = isCancelled; - } - - public boolean isCancelled() { - return isCancelled; - } } diff --git a/api/src/main/java/com/cloud/exception/OperationTimedoutException.java b/api/src/main/java/com/cloud/exception/OperationTimedoutException.java index ec16cea045a2..66b607100d97 100644 --- a/api/src/main/java/com/cloud/exception/OperationTimedoutException.java +++ b/api/src/main/java/com/cloud/exception/OperationTimedoutException.java @@ -19,8 +19,6 @@ import com.cloud.agent.api.Command; import com.cloud.utils.SerialVersionUID; -import java.util.Arrays; - /** * wait timeout. */ @@ -40,10 +38,9 @@ public class OperationTimedoutException extends CloudException { // transient Command[] _cmds; boolean _isActive; - boolean _isCancelled; public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) { - super("Commands: " + Arrays.toString(cmds) + " to Host " + agentId + " with seqId " + seqId + " timed out after " + time + " secs"); + super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs"); _agentId = agentId; _seqId = seqId; _time = time; @@ -70,8 +67,4 @@ public Command[] getCommands() { public boolean isActive() { return _isActive; } - - public boolean isCancelled() { - return _isCancelled; - } } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 165a662c2ad1..d13b30cf0f3d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -92,9 +92,7 @@ public enum ApiCommandResourceType { ExtensionCustomAction(org.apache.cloudstack.extension.ExtensionCustomAction.class), KmsKey(org.apache.cloudstack.kms.KMSKey.class), HsmProfile(org.apache.cloudstack.kms.HSMProfile.class), - Job(org.apache.cloudstack.jobs.JobInfo.class); // org.apache.cloudstack.framework.jobs.AsyncJob.class - // Job(null); - + Job(org.apache.cloudstack.jobs.JobInfo.class); private final Class clazz; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 28ff3ebe9881..f74c46161180 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -339,6 +339,7 @@ public class ApiConstants { public static final String IP_ADDRESS_ID = "ipaddressid"; public static final String IS_2FA_ENABLED = "is2faenabled"; public static final String IS_2FA_VERIFIED = "is2faverified"; + public static final String IS_2FA_MANDATED = "is2famandated"; public static final String IS_ACTIVE = "isactive"; public static final String IS_ASYNC = "isasync"; @@ -674,7 +675,6 @@ public class ApiConstants { public static final String USAGE_ID = "usageid"; public static final String USAGE_NAME = "usagename"; public static final String USAGE_TYPE = "usagetype"; - public static final String USAGE_SERVER = "usageserver"; public static final String INCLUDE_TAGS = "includetags"; public static final String VLAN = "vlan"; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java index 969a8da507c8..6859b0a7f406 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseAsyncCmd.java @@ -83,4 +83,5 @@ public Long getSyncObjId() { public Object getJob() { return job; } + } diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index c87c76cc1ef0..483fa83630ad 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -415,26 +415,25 @@ public void validateSpecificParameters(final Map params){ * display flag is used to control the display of the resource only to the end user. It doesn't affect Root Admin. * @return display flag */ - public boolean isDisplay() { + public boolean isDisplay(){ CallContext context = CallContext.current(); Map contextMap = context.getContextParameters(); boolean isDisplay = true; // Iterate over all the first class entities in context and check their display property. - for (Map.Entry entry : contextMap.entrySet()) { - try { + for(Map.Entry entry : contextMap.entrySet()){ + try{ Object key = entry.getKey(); Class clz = Class.forName((String)key); - if (Displayable.class.isAssignableFrom(clz)) { + if(Displayable.class.isAssignableFrom(clz)){ final Object objVO = getEntityVO(clz, entry.getValue()); isDisplay = ((Displayable) objVO).isDisplay(); } - // If the flag is false, break immediately - if (!isDisplay) { + // If the flag is false break immediately + if(!isDisplay) break; - } - } catch (Exception e) { + } catch (Exception e){ logger.trace("Caught exception while checking first class entities for display property, continuing on", e); } } @@ -444,19 +443,19 @@ public boolean isDisplay() { } - private Object getEntityVO(Class entityType, Object entityId) { + private Object getEntityVO(Class entityType, Object entityId){ // entityId can be internal db id or UUID so accordingly call findbyId or findByUUID - if (entityId instanceof Long) { + if (entityId instanceof Long){ // Its internal db id - use findById return _entityMgr.findById(entityType, (Long)entityId); - } else if (entityId instanceof String) { - try { + } else if(entityId instanceof String){ + try{ // In case its an async job the internal db id would be a string because of json deserialization Long internalId = Long.valueOf((String) entityId); return _entityMgr.findById(entityType, internalId); - } catch (NumberFormatException e) { + } catch (NumberFormatException e){ // It is uuid - use findByUuid` return _entityMgr.findByUuid(entityType, (String)entityId); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java index b86d1f8b956a..2587044d7199 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -64,6 +65,9 @@ public class ListEventsCmd extends BaseListProjectAndAccountResourcesCmd { @Parameter(name = ApiConstants.START_ID, type = CommandType.UUID, entityType = EventResponse.class, description = "The parent/start ID of the event, when provided this will list all the events with the start/parent ID including the parent event") private Long startId; + @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, description = "the ID of the async job that raised the events", since = "24.0") + private Long jobId; + @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING, description = "The ID of the resource associated with the event", since="4.17.0") private String resourceId; @@ -112,6 +116,10 @@ public Long getStartId() { return startId; } + public Long getJobId() { + return jobId; + } + public String getResourceId() { return resourceId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java index 5cdcd26d90d0..bdcfbe4305c1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/CancelAsyncJobCmd.java @@ -36,7 +36,7 @@ import javax.inject.Inject; @APICommand(name = CancelAsyncJobCmd.APINAME, description = "Cancels the asynchronous job.", responseObject = AsyncJobResponse.class, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}, since = "4.23") + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}, since = "24.0") public class CancelAsyncJobCmd extends BaseCmd { public static final String APINAME = "cancelAsyncJob"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index bb4ab6c1f97b..e345616142ed 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -16,10 +16,14 @@ // under the License. package org.apache.cloudstack.api.command.user.job; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Locale; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.StringUtils; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; @@ -54,11 +58,11 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") private String resourceType; - @Parameter(name = ApiConstants.JOB_STATUS, type = CommandType.LIST, collectionType = CommandType.LONG, description = "Comma-seperated list of job statuses (0 - Pending, 1 - Success, 2 - Failed, 3 - Cancelled) to list the async jobs. " + - "Only pending jobs are listed by default.", since = "4.23") - private List jobStatuses; + @Parameter(name = ApiConstants.JOB_STATUS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "Comma-separated list of job statuses to list the async jobs by. " + + "Accepts names (IN_PROGRESS, SUCCEEDED, FAILED, CANCELLED) or ordinals (0, 1, 2, 3). Only pending jobs are listed by default.", since = "24.0") + private List jobStatuses; - @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the async jobs started or completed within that period up to now.", since = "4.23") + @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the async jobs started or completed within that period up to now.", since = "24.0") private Integer duration; ///////////////////////////////////////////////////// @@ -94,15 +98,34 @@ public List getJobStatuses() { throw new InvalidParameterValueException("Empty job status"); } - for (Long status : jobStatuses) { - try { - JobInfo.Status.fromValue(Math.toIntExact(status)); - } catch (IllegalArgumentException e) { - throw new InvalidParameterValueException("Invalid job status: " + status); - } + List statuses = new ArrayList<>(jobStatuses.size()); + for (String status : jobStatuses) { + statuses.add((long)parseJobStatus(status).value()); + } + return statuses; + } + + private JobInfo.Status parseJobStatus(String status) { + if (StringUtils.isBlank(status)) { + throw new InvalidParameterValueException("Empty job status"); } - return jobStatuses; + String value = status.trim(); + try { + return JobInfo.Status.fromValue(Integer.parseInt(value)); + } catch (NumberFormatException e) { + // not an ordinal, fall through and try the name + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(String.format("Invalid job status: %s. Valid values are %s or their ordinals", + status, Arrays.toString(JobInfo.Status.values()))); + } + + try { + return JobInfo.Status.valueOf(value.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(String.format("Invalid job status: %s. Valid values are %s or their ordinals", + status, Arrays.toString(JobInfo.Status.values()))); + } } public Integer getDuration() { diff --git a/api/src/main/java/org/apache/cloudstack/context/CallContext.java b/api/src/main/java/org/apache/cloudstack/context/CallContext.java index b4e4cf3de3a0..fcfb5b6b1e00 100644 --- a/api/src/main/java/org/apache/cloudstack/context/CallContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/CallContext.java @@ -401,11 +401,9 @@ public Map getContextParameters() { return context; } - public void putContextParameters(Map details) { - if (details == null) { - return; - } - for (Map.Entryentry : details.entrySet()) { + public void putContextParameters(Map details){ + if (details == null) return; + for(Map.Entryentry : details.entrySet()){ putContextParameter(entry.getKey(), entry.getValue()); } } diff --git a/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java b/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java new file mode 100644 index 000000000000..c51da8035d29 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.jobs; + +/** + * Lets the job layer ask whoever is running a job's backend work whether that work can be stopped, + * and then stop it. + * + * This is an interface rather than a direct call to the agent manager because the job framework is + * built before the agent layer and the agent layer already depends on it; calling the other way + * round would close the cycle. + */ +public interface JobCancellationHandler { + + /** + * Whether everything this job currently has in flight can be stopped. + * + * Asked before the job is marked cancelled. A job whose backend work cannot be stopped is + * refused outright, rather than recorded as cancelled while the operation runs to completion -- + * that divergence between what CloudStack believes and what the hypervisor did is the whole + * reason this check exists. + */ + boolean isJobExecutionCancellable(long jobId); + + /** + * Stops whatever this job has in flight. Returns true only if all of it was actually stopped. + */ + boolean cancelJobExecution(long jobId, String reason); +} diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 4df7345bc7f1..c7238a513693 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.vm; -import com.cloud.exception.OperationCancelledException; import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import java.net.URI; @@ -167,11 +166,11 @@ void advanceStart(String vmUuid, Map params ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; void orchestrateStart(String vmUuid, Map params, DeploymentPlan planToDeploy, DeploymentPlanner planner) throws InsufficientCapacityException, - ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; + ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; void advanceStop(String vmUuid, boolean cleanupEvenIfUnableToStop) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException; - void advanceExpunge(String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException, OperationCancelledException; + void advanceExpunge(String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException; void destroy(String vmUuid, boolean expunge) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException; diff --git a/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java b/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java index 92edfdcb1cd4..ef1c71e6b014 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VmWorkJobHandlerProxy.java @@ -93,34 +93,36 @@ private Method getHandlerMethod(Class paramType) { public Pair handleVmWorkJob(VmWork work) throws Exception { Method method = getHandlerMethod(work.getClass()); - if (method == null) { - logger.error("Unable to find handler for VM work job: {} {}", work.getClass().getName(), work); - RuntimeException ex = new RuntimeException("Unable to find handler for VM work job: " + work.getClass().getName()); - return new Pair<>(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); - } + if (method != null) { - try { - if (logger.isDebugEnabled()) - logger.debug("Execute VM work job: {}{}", work.getClass().getName(), work); + try { + if (logger.isDebugEnabled()) + logger.debug("Execute VM work job: " + work.getClass().getName() + work); - Object obj = method.invoke(_target, work); + Object obj = method.invoke(_target, work); - if (logger.isDebugEnabled()) - logger.debug("Done executing VM work job: {}{}", work.getClass().getName(), work); + if (logger.isDebugEnabled()) + logger.debug("Done executing VM work job: " + work.getClass().getName() + work); - assert (obj instanceof Pair); - return (Pair)obj; - } catch (InvocationTargetException e) { - logger.error("Invocation exception, caused by: {}", String.valueOf(e.getCause())); + assert (obj instanceof Pair); + return (Pair)obj; + } catch (InvocationTargetException e) { + logger.error("Invocation exception, caused by: " + e.getCause()); - // legacy CloudStack code relies on checked exception for error handling - // we need to re-throw the real exception here - if (e.getCause() != null && e.getCause() instanceof Exception) { - logger.info("Rethrow exception {}", String.valueOf(e.getCause())); - throw (Exception)e.getCause(); + // legacy CloudStack code relies on checked exception for error handling + // we need to re-throw the real exception here + if (e.getCause() != null && e.getCause() instanceof Exception) { + logger.info("Rethrow exception " + e.getCause()); + throw (Exception)e.getCause(); + } + + throw e; } + } else { + logger.error("Unable to find handler for VM work job: {} {}", work.getClass().getName(), work); - throw e; + RuntimeException ex = new RuntimeException("Unable to find handler for VM work job: " + work.getClass().getName()); + return new Pair<>(JobInfo.Status.FAILED, JobSerializerHelper.toObjectSerializedString(ex)); } } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java index 2d802dc0b455..d6c55d7038bb 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkJobDispatcher.java @@ -69,7 +69,7 @@ public void runJob(AsyncJob job) { } work = VmWorkSerializer.deserialize(workClz, job.getCmdInfo()); - if (work == null) { + if(work == null) { logger.error("Unable to deserialize VM work {}, job info: {}, job origin: {}", job.getCmd(), job.getCmdInfo(), job.getRelated()); _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, "Unable to deserialize VM work"); return; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index a909a94c4d0b..403e83bccf99 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -45,7 +45,6 @@ import com.cloud.agent.api.MigrateBetweenSecondaryStoragesCommandAnswer; import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -425,7 +424,7 @@ private void migrateKvmIncrementalBackupChain(DataObject chosenFileForMigration, throw new CloudRuntimeException("Unable to migrate KVM incremental backups to another secondary storage"); } - } catch (final OperationTimedoutException | AgentUnavailableException | OperationCancelledException e) { + } catch (final OperationTimedoutException | AgentUnavailableException e) { throw new CloudRuntimeException("Error while migrating KVM incremental backup chain. Check the logs for more information.", e); } finally { if (answer != null) { diff --git a/engine/schema/src/main/java/com/cloud/event/EventVO.java b/engine/schema/src/main/java/com/cloud/event/EventVO.java index 24c3e8cd0641..47c5cf36d383 100644 --- a/engine/schema/src/main/java/com/cloud/event/EventVO.java +++ b/engine/schema/src/main/java/com/cloud/event/EventVO.java @@ -73,6 +73,9 @@ public class EventVO implements Event { @Column(name = "start_id") private long startId; + @Column(name = "async_job_id") + private Long asyncJobId; + @Column(name = "parameters", length = 1024) private String parameters; @@ -209,6 +212,14 @@ public void setStartId(long startId) { this.startId = startId; } + public Long getAsyncJobId() { + return asyncJobId; + } + + public void setAsyncJobId(Long asyncJobId) { + this.asyncJobId = asyncJobId; + } + @Override public String getParameters() { return parameters; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 7c11013a17d2..085122899248 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -18,3 +18,7 @@ --; -- Schema upgrade from 4.23.0.0 to 24.0.0 --; + +-- Link every action event to the API job that raised it, so a job can be traced to its events +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.event', 'async_job_id', 'BIGINT UNSIGNED DEFAULT NULL COMMENT ''The async job that raised this event'' '); +CALL `cloud`.`IDEMPOTENT_ADD_KEY`('i_event__async_job_id', 'cloud.event', '(`async_job_id`)'); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.event_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.event_view.sql index 0a15ae4c0c91..1066f567e95e 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.event_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.event_view.sql @@ -34,6 +34,8 @@ select `event`.`parameters` AS `parameters`, `event`.`start_id` AS `start_id`, `eve`.`uuid` AS `start_uuid`, + `event`.`async_job_id` AS `async_job_id`, + `async_job`.`uuid` AS `async_job_uuid`, `event`.`user_id` AS `user_id`, `event`.`archived` AS `archived`, `event`.`display` AS `display`, @@ -50,7 +52,7 @@ select `projects`.`uuid` AS `project_uuid`, `projects`.`name` AS `project_name` from - (((((`event` + ((((((`event` join `account` on ((`event`.`account_id` = `account`.`id`))) join `domain` on @@ -60,4 +62,6 @@ join `user` on left join `projects` on ((`projects`.`project_account_id` = `event`.`account_id`))) left join `event` `eve` on - ((`event`.`start_id` = `eve`.`id`))); + ((`event`.`start_id` = `eve`.`id`))) +left join `async_job` on + ((`event`.`async_job_id` = `async_job`.`id`))); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index 20d04219b86c..665c3a4659ca 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -25,7 +25,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Snapshot; import com.cloud.storage.Storage; @@ -209,10 +208,6 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { } catch (OperationTimedoutException e) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); - } catch (OperationCancelledException e) { - logger.debug("Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); - throw new CloudRuntimeException("Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); - } catch (AgentUnavailableException e) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed", e); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); @@ -269,7 +264,7 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) { processAnswer(vmSnapshotVO, userVm, answer, hostId); throw new CloudRuntimeException("Delete Instance Snapshot " + vmSnapshot.getName() + " of Instance " + userVm.getInstanceName() + " failed due to " + errMsg); } - } catch (OperationTimedoutException | AgentUnavailableException | OperationCancelledException e) { + } catch (OperationTimedoutException | AgentUnavailableException e) { throw new CloudRuntimeException("Delete Instance Snapshot " + vmSnapshot.getName() + " of Instance " + userVm.getInstanceName() + " failed due to " + e.getMessage()); } } @@ -454,7 +449,7 @@ public boolean revertVMSnapshot(VMSnapshot vmSnapshot) { logger.error(errMsg); throw new CloudRuntimeException(String.format("Unable to revert VM %s to snapshot %s.", userVm.getInstanceName(), vmSnapshotVO.getName())); } - } catch (OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { logger.debug("Failed to revert Instance Snapshot", e); throw new CloudRuntimeException(e.getMessage()); } catch (AgentUnavailableException e) { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java index 76e88cb4a02d..fd306414ad49 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java @@ -26,7 +26,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; @@ -211,11 +210,6 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); throw new CloudRuntimeException( "Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.toString()); - } catch (OperationCancelledException e) { - logger.debug("Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); - throw new CloudRuntimeException( - "Creating Instance snapshot: " + vmSnapshot.getName() + " cancelled: " + e.toString()); - } catch (AgentUnavailableException e) { logger.debug("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed", e); throw new CloudRuntimeException( @@ -228,7 +222,7 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { .convert(elapsedTime(startFreeze), TimeUnit.NANOSECONDS))); try { thawAnswer = (FreezeThawVMAnswer) agentMgr.send(hostId, thawCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.debug("Could not unfreeze the VM due to " + e); } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java index 2b5c293c832c..bd4bce29b0a0 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/RemoteHostEndPoint.java @@ -23,7 +23,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -132,7 +131,10 @@ public Answer sendMessage(Command cmd) { setId(newHostId); } return agentMgr.send(newHostId, cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException e) { + errMsg = e.toString(); + logger.debug("Failed to send command, due to Agent:" + getId() + ", " + e.toString()); + } catch (OperationTimedoutException e) { errMsg = e.toString(); logger.debug("Failed to send command, due to Agent:" + getId() + ", " + e.toString()); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java index 2b17deb2453c..10af5d55d619 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java @@ -23,7 +23,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -140,7 +139,9 @@ public VMSnapshotTO quiesceVm(VirtualMachine virtualMachine) { String errMsg = (answer != null) ? answer.getDetails() : null; throw new CloudRuntimeException("Failed to quiesce vm, due to " + errMsg); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Failed to quiesce vm", e); + } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Failed to quiesce vm", e); } return vmSnapshotTO; @@ -161,7 +162,9 @@ public boolean unquiesceVM(VirtualMachine virtualMachine, VMSnapshotTO vmSnapsho String errMsg = (answer != null) ? answer.getDetails() : null; throw new CloudRuntimeException("Failed to unquiesce vm, due to " + errMsg); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Failed to unquiesce vm", e); + } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Failed to unquiesce vm", e); } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java index cf61b4fd1ec9..2d39e18edea3 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java @@ -46,7 +46,7 @@ public static void init(AsyncJobManager jobMgr, AsyncJobJoinMapDao joinMapDao) { s_joinMapDao = joinMapDao; } - private static ManagedThreadLocal s_currentExecutionContext = new ManagedThreadLocal(); + private static ManagedThreadLocal s_currentExectionContext = new ManagedThreadLocal(); public AsyncJobExecutionContext() { } @@ -73,7 +73,10 @@ public void setJob(AsyncJob job) { public boolean isJobDispatchedBy(String jobDispatcherName) { assert (jobDispatcherName != null); - return _job != null && _job.getDispatcher() != null && _job.getDispatcher().equals(jobDispatcherName); + if (_job != null && _job.getDispatcher() != null && _job.getDispatcher().equals(jobDispatcherName)) + return true; + + return false; } public void completeAsyncJob(JobInfo.Status jobStatus, int resultCode, String resultObject) { @@ -115,7 +118,7 @@ public void joinJob(long joinJobId, String wakeupHandler, String wakeupDispatche // // check failure exception before we disjoin the worker job, work job usually fails with exception - // this will help propagate exception between jobs + // this will help propogate exception between jobs // TODO : it is ugly and this will become unnecessary after we switch to full-async mode // public void disjoinJob(long joinedJobId) throws InsufficientCapacityException, @@ -130,21 +133,24 @@ public void disjoinJob(long joinedJobId) throws InsufficientCapacityException, Object exception = JobSerializerHelper.fromObjectSerializedString(record.getJoinResult()); if (exception != null && exception instanceof Exception) { if (exception instanceof InsufficientCapacityException) { - LOGGER.error("Job {} failed with InsufficientCapacityException", joinedJobId); + LOGGER.error("Job " + joinedJobId + " failed with InsufficientCapacityException"); throw (InsufficientCapacityException)exception; - } else if (exception instanceof ConcurrentOperationException) { - LOGGER.error("Job {} failed with ConcurrentOperationException", joinedJobId); + } + else if (exception instanceof ConcurrentOperationException) { + LOGGER.error("Job " + joinedJobId + " failed with ConcurrentOperationException"); throw (ConcurrentOperationException)exception; - } else if (exception instanceof ResourceUnavailableException) { - LOGGER.error("Job {} failed with ResourceUnavailableException", joinedJobId); + } + else if (exception instanceof ResourceUnavailableException) { + LOGGER.error("Job " + joinedJobId + " failed with ResourceUnavailableException"); throw (ResourceUnavailableException)exception; - } else { - LOGGER.error("Job {} failed with exception", joinedJobId); + } + else { + LOGGER.error("Job " + joinedJobId + " failed with exception"); throw new RuntimeException((Exception)exception); } } } else { - LOGGER.error("Job {} failed without providing an error object", joinedJobId); + LOGGER.error("Job " + joinedJobId + " failed without providing an error object"); throw new RuntimeException("Job " + joinedJobId + " failed without providing an error object"); } } @@ -162,29 +168,28 @@ public void completeJobAndJoin(JobInfo.Status joinStatus, String joinResult) { } public static AsyncJobExecutionContext getCurrentExecutionContext() { - AsyncJobExecutionContext context = s_currentExecutionContext.get(); + AsyncJobExecutionContext context = s_currentExectionContext.get(); if (context == null) { // TODO, this has security implications, operations carried from API layer should always // set its context, otherwise, the fall-back here will use system security context // - LOGGER.warn("Job is executed without a context, setup pseudo job for the executing thread"); - if (CallContext.current() != null) { + LOGGER.warn("Job is executed without a context, setup psudo job for the executing thread"); + if (CallContext.current() != null) context = registerPseudoExecutionContext(CallContext.current().getCallingAccountId(), CallContext.current().getCallingUserId()); - } else { + else context = registerPseudoExecutionContext(Account.ACCOUNT_ID_SYSTEM, User.UID_SYSTEM); - } } return context; } // return currentExecutionContext without create it public static AsyncJobExecutionContext getCurrent() { - return s_currentExecutionContext.get(); + return s_currentExectionContext.get(); } public static AsyncJobExecutionContext registerPseudoExecutionContext(long accountId, long userId) { - AsyncJobExecutionContext context = s_currentExecutionContext.get(); + AsyncJobExecutionContext context = s_currentExectionContext.get(); if (context == null) { context = new AsyncJobExecutionContext(); context.setJob(s_jobMgr.getPseudoJob(accountId, userId)); @@ -195,14 +200,14 @@ public static AsyncJobExecutionContext registerPseudoExecutionContext(long accou } public static AsyncJobExecutionContext unregister() { - AsyncJobExecutionContext context = s_currentExecutionContext.get(); + AsyncJobExecutionContext context = s_currentExectionContext.get(); setCurrentExecutionContext(null); return context; } // This is intended to be package level access for AsyncJobManagerImpl only. public static void setCurrentExecutionContext(AsyncJobExecutionContext currentContext) { - s_currentExecutionContext.set(currentContext); + s_currentExectionContext.set(currentContext); } public static String getOriginJobId() { diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java index 872dc4124101..16a0b7ea58a5 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/AsyncJobManager.java @@ -27,8 +27,8 @@ public interface AsyncJobManager extends Manager { - String API_JOB_POOL_THREAD_PREFIX = "API-Job-Executor"; - String WORK_JOB_POOL_THREAD_PREFIX = "Work-Job-Executor"; + public static final String API_JOB_POOL_THREAD_PREFIX = "API-Job-Executor"; + public static final String WORK_JOB_POOL_THREAD_PREFIX = "Work-Job-Executor"; AsyncJobVO getAsyncJob(long jobId); @@ -40,12 +40,20 @@ public interface AsyncJobManager extends Manager { void completeAsyncJob(long jobId, JobInfo.Status jobStatus, int resultCode, String result); + /** + * As above, but with remove=false the row is left un-finalised: executing_msid is kept so the + * management server running the job can find it, and complete_msid stays null until that + * server has stopped the job's in-flight work and called finalizeCancelledJob. + */ void completeAsyncJob(long jobId, JobInfo.Status jobStatus, int resultCode, String result, boolean remove); + List listCancelledJobsExecutingOn(long msid); + + void finalizeCancelledJob(long jobId); + void updateAsyncJobStatus(long jobId, int processStatus, String resultObject); void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId); - void logJobJournal(long jobId, AsyncJob.JournalType journalType, String journalText, String journalObjJson); @@ -118,14 +126,14 @@ void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakupDispa * This method will be deprecated after all code has been migrated to fully-asynchronous mode * that uses async-feature of joinJob/disjoinJob * - * @param wakeupTopicsOnMessageBus topic on message bus to wake up the wait + * @param wakupTopicsOnMessageBus topic on message bus to wakeup the wait * @param checkIntervalInMilliSeconds time to break out wait for checking predicate condition * @param timeoutInMilliseconds time out to break out the whole wait process * @param predicate * @return true, predicate condition is satisfied * false, wait is timed out */ - boolean waitAndCheck(AsyncJob job, String[] wakeupTopicsOnMessageBus, long checkIntervalInMilliSeconds, long timeoutInMilliseconds, Predicate predicate); + boolean waitAndCheck(AsyncJob job, String[] wakupTopicsOnMessageBus, long checkIntervalInMilliSeconds, long timeoutInMilliseconds, Predicate predicate); AsyncJob queryJob(long jobId, boolean updatePollTime); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 9a3134840ba1..fe311fb11f5d 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -73,7 +73,5 @@ public interface AsyncJobDao extends GenericDao { AsyncJobVO getRelatedJob(String jobId); - List getCancelledJobs(); - - boolean isJobCancelled(Long jobId); + List getCancelledJobs(long executingMsid); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 39be874687f1..7c91df7b290b 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -126,6 +126,7 @@ public AsyncJobDaoImpl() { cancelledAsyncJobSearch = createSearchBuilder(); cancelledAsyncJobSearch.and("status", cancelledAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ); cancelledAsyncJobSearch.and("completeMsId", cancelledAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NULL); + cancelledAsyncJobSearch.and("executingMsid", cancelledAsyncJobSearch.entity().getExecutingMsid(), SearchCriteria.Op.EQ); cancelledAsyncJobSearch.done(); } @@ -329,15 +330,10 @@ public AsyncJobVO getRelatedJob(String jobId) { } @Override - public List getCancelledJobs() { + public List getCancelledJobs(long executingMsid) { SearchCriteria sc = cancelledAsyncJobSearch.create(); sc.setParameters("status", JobInfo.Status.CANCELLED); + sc.setParameters("executingMsid", executingMsid); return listBy(sc); } - - @Override - public boolean isJobCancelled(Long jobId) { - AsyncJobVO job = findByIdIncludingRemoved(jobId); - return job != null && job.getStatus() == JobInfo.Status.CANCELLED; - } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java index ab2b54b83d3f..09a88939492e 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java @@ -144,7 +144,7 @@ public void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinR @Override public List findJobsToWake(long joinedJobId) { - // TODO: We should fix this. We shouldn't be crossing daos in a dao code. + // TODO: We should fix this. We shouldn't be crossing daos in a dao code. List standaloneList = new ArrayList(); TransactionLegacy txn = TransactionLegacy.currentTxn(); String sql = "SELECT job_id FROM async_job_join_map WHERE join_job_id = ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)"; @@ -189,4 +189,5 @@ public List findJobsToWakeBetween(Date cutDate) { throw new CloudRuntimeException("Unable to handle SQL exception", e); } } + } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java index 909a7e54c49a..7b6eed75a008 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java @@ -23,17 +23,14 @@ import com.cloud.utils.db.GenericDao; public interface SyncQueueItemDao extends GenericDao { - SyncQueueItemVO getNextQueueItem(long queueId); + public SyncQueueItemVO getNextQueueItem(long queueId); + public int getActiveQueueItemCount(long queueId); - int getActiveQueueItemCount(long queueId); + public List getNextQueueItems(int maxItems); - List getNextQueueItems(int maxItems); + public List getActiveQueueItems(Long msid, boolean exclusive); - List getActiveQueueItems(Long msid, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, boolean exclusive); - List getQueuedItems(long queueId); - - List getBlockedQueueItems(long thresholdMs, boolean exclusive); - - Long getQueueItemIdByContentIdAndType(long contentId, String contentType); + public Long getQueueItemIdByContentIdAndType(long contentId, String contentType); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java index 03c0b6707ed3..756cbb7efb0f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java @@ -43,7 +43,6 @@ public class SyncQueueItemDaoImpl extends GenericDaoBase implements SyncQueueItemDao { final GenericSearchBuilder queueIdSearch; final GenericSearchBuilder queueActiveItemSearch; - final SearchBuilder queuedItemsSearch; public SyncQueueItemDaoImpl() { super(); @@ -58,10 +57,6 @@ public SyncQueueItemDaoImpl() { queueActiveItemSearch.and("processNumber", queueActiveItemSearch.entity().getLastProcessNumber(), Op.NNULL); queueActiveItemSearch.select(null, Func.COUNT, queueActiveItemSearch.entity().getId()); queueActiveItemSearch.done(); - - queuedItemsSearch = createSearchBuilder(); - queuedItemsSearch.and("queueId", queuedItemsSearch.entity().getQueueId(), SearchCriteria.Op.EQ); - queuedItemsSearch.done(); } @Override @@ -143,16 +138,6 @@ public List getActiveQueueItems(Long msid, boolean exclusive) { return listBy(sc, filter); } - @Override - public List getQueuedItems(long queueId) { - SearchCriteria sc = queuedItemsSearch.create(); - sc.setParameters("queueId", queueId); - - Filter filter = new Filter(SyncQueueItemVO.class, "created", true, null, null); - List queueItems = search(sc, filter); - return queueItems; - } - @Override public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { Date cutTime = DateUtil.currentGMTTime(); @@ -162,14 +147,14 @@ public List getBlockedQueueItems(long thresholdMs, boolean excl sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessNumber(), SearchCriteria.Op.NNULL); sbItem.and("lastProcessTime", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.NNULL); sbItem.and("lastProcessTime2", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.LT); + sbItem.done(); SearchCriteria sc = sbItem.create(); sc.setParameters("lastProcessTime2", new Date(cutTime.getTime() - thresholdMs)); - if (exclusive) { + if(exclusive) return lockRows(sc, null, true); - } return listBy(sc, null); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 0f5dbec1b0fe..e4e7b5d4e48c 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -36,11 +36,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.storage.SnapshotVO; -import com.cloud.vm.snapshot.VMSnapshot; -import com.cloud.vm.snapshot.VMSnapshotService; -import com.cloud.vm.snapshot.VMSnapshotVO; -import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; @@ -69,20 +64,23 @@ import org.apache.cloudstack.framework.messagebus.MessageDetector; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.jobs.AsyncJobService; +import org.apache.cloudstack.jobs.JobCancellationHandler; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.jobs.JobInfo.Status; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.logging.log4j.ThreadContext; import com.cloud.cluster.ClusterManagerListener; import com.cloud.network.Network; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; @@ -116,10 +114,13 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; - -import org.apache.logging.log4j.ThreadContext; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotService; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable, AsyncJobService { + // Advanced public static final ConfigKey JobExpireMinutes = new ConfigKey("Advanced", Long.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", true, ConfigKey.Scope.Global); @@ -128,8 +129,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private static final ConfigKey VmJobLockTimeout = new ConfigKey("Advanced", Integer.class, "vm.job.lock.timeout", "1800", "Time in seconds to wait in acquiring lock to submit a vm worker job", false); - private static final ConfigKey HidePassword = new ConfigKey("Advanced", Boolean.class, "log.hide.password", "true", - "If set to true, the password is hidden", true, ConfigKey.Scope.Global); + private static final ConfigKey HidePassword = new ConfigKey("Advanced", Boolean.class, "log.hide.password", "true", "If set to true, the password is hidden", true, ConfigKey.Scope.Global); + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; // 3 seconds @@ -184,6 +185,10 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private NetworkOrchestrationService networkOrchestrationService; @Inject private ReconcileCommandService reconcileCommandService; + // A list so that the job framework still starts in contexts that have no agent layer; + // Spring leaves it empty rather than failing to wire. + @Inject + private List jobCancellationHandlers; private volatile long _executionRunNumber = 1; @@ -311,8 +316,8 @@ public long submitAsyncJob(final AsyncJob job, final String syncObjType, final l publishOnEventBus(job, "submit"); - if (!_vmInstanceDao.lockInLockTable(String.valueOf(syncObjId), VmJobLockTimeout.value())) { - throw new CloudRuntimeException("Failed to acquire lock in submitting async job: " + job.getCmd() + " within vm job lock timeout value = " + VmJobLockTimeout.value()); + if (!_vmInstanceDao.lockInLockTable(String.valueOf(syncObjId), VmJobLockTimeout.value())){ + throw new CloudRuntimeException("Failed to acquire lock in submitting async job: " + job.getCmd() + " with timeout value = " + VmJobLockTimeout.value()); } try { @@ -345,13 +350,14 @@ public void completeAsyncJob(final long jobId, final Status jobStatus, final int @Override @DB - public void completeAsyncJob(final long jobId, final Status jobStatus, final int resultCode, final String resultObject, boolean remove) { + public void completeAsyncJob(final long jobId, final Status jobStatus, final int resultCode, final String resultObject, final boolean remove) { String resultObj = null; if (logger.isDebugEnabled()) { resultObj = convertHumanReadableJson(obfuscatePassword(resultObject, HidePassword.value())); logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus + ", resultCode: " + resultCode + ", result: " + resultObj); } + final AsyncJobVO job = _jobDao.findById(jobId); if (job == null) { if (logger.isDebugEnabled()) { @@ -373,7 +379,7 @@ public void completeAsyncJob(final long jobId, final Status jobStatus, final int } if (resultObject != null) { - job.setResult(resultObject); + job.updateResultWithEncryptionIfNeeded(resultObject); } if (logger.isDebugEnabled()) { @@ -393,18 +399,17 @@ public List doInTransaction(final TransactionStatus status) { job.setResultCode(resultCode); if (resultObject != null) { - job.setResult(resultObject); + job.updateResultWithEncryptionIfNeeded(resultObject); } else { - job.setResult(null); + job.updateResultWithEncryptionIfNeeded(null); } final Date currentGMTTime = DateUtil.currentGMTTime(); job.setLastUpdated(currentGMTTime); - job.setExecutingMsid(null); - if (remove) { job.setCompleteMsid(getMsid()); job.setRemoved(currentGMTTime); + job.setExecutingMsid(null); } _jobDao.update(jobId, job); @@ -468,7 +473,7 @@ public void updateAsyncJobStatus(final long jobId, final int processStatus, fina public void doInTransactionWithoutResult(TransactionStatus status) { job.setProcessStatus(processStatus); if (resultObject != null) { - job.setResult(resultObject); + job.updateResultWithEncryptionIfNeeded(resultObject); } job.setLastUpdated(DateUtil.currentGMTTime()); _jobDao.update(jobId, job); @@ -566,23 +571,11 @@ public AsyncJob queryJob(final long jobId, final boolean updatePollTime) { return job; } - public String obfuscatePassword(String result, boolean hidePassword) { - if (hidePassword) { - String pattern = "\"password\":"; - if (result != null) { - if (result.contains(pattern)) { - String[] resp = result.split(pattern); - String psswd = resp[1].toString().split(",")[0]; - if (psswd.endsWith("}")) { - psswd = psswd.substring(0, psswd.length() - 1); - result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "}," + resp[1].split(",", 2)[1]; - } else { - result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1]; - } - } - } + public String obfuscatePassword(String result, boolean hidePassword) { + if (!hidePassword) { + return result; } - return result; + return StringUtils.obfuscatePasswordInJsonLikeString(result); } private void scheduleExecution(final AsyncJobVO job) { @@ -640,9 +633,8 @@ private Runnable getExecutorRunnable(final AsyncJob job) { @Override public void run() { // register place-holder context to avoid installing system account call context - if (CallContext.current() == null) { + if (CallContext.current() == null) CallContext.registerPlaceHolderContext(); - } String related = job.getRelated(); String logContext = job.getShortUuid(); @@ -650,7 +642,7 @@ public void run() { ThreadContext.push("job-" + related + "/" + "job-" + job.getId()); AsyncJob relatedJob = _jobDao.findByIdIncludingRemoved(Long.parseLong(related)); if (relatedJob != null) { - logContext = relatedJob.getShortUuid() + "/" + job.getShortUuid(); + logContext = relatedJob.getShortUuid(); } } else { ThreadContext.push("job-" + job.getId()); @@ -687,14 +679,14 @@ protected void runInContext() { if (related != null && !related.isEmpty()) { AsyncJob relatedJob = _jobDao.findByIdIncludingRemoved(Long.parseLong(related)); if (relatedJob != null) { - logContext = relatedJob.getShortUuid() + "/" + job.getShortUuid(); + logContext = relatedJob.getShortUuid(); } } ThreadContext.put("logcontextid", logContext); // execute the job if (logger.isDebugEnabled()) { - logger.debug("Executing {}", StringUtils.cleanString(job.toString())); + logger.debug("Executing " + StringUtils.cleanString(job.toString())); } if ((getAndResetPendingSignals(job) & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) { @@ -712,18 +704,19 @@ protected void runInContext() { jobDispatcher.runJob(job); } else { logger.error("Unable to find job dispatcher, job will be cancelled"); - completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "Unable to find job dispatcher: " + job.getDispatcher()); + completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null); } } if (logger.isDebugEnabled()) { - logger.debug("Done executing {} for {}job-{}", job.getCmd(), related != null && !related.isEmpty() ? "job-" + related + "/" : "", job.getId()); + logger.debug("Done executing " + job.getCmd() + " for job-" + job.getId()); } + } catch (Throwable e) { logger.error("Unexpected exception", e); completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null); } finally { - // guard the final clause as well + // guard final clause as well try { if (job.getSyncSource() != null) { // here check queue item one more time to double make sure that queue item is removed in case of any uncaught exception @@ -763,7 +756,7 @@ private int getAndResetPendingSignals(AsyncJob job) { return signals; } - private void executeQueueItem(SyncQueueItemVO item) { + private void executeQueueItem(SyncQueueItemVO item, boolean fromPreviousSession) { AsyncJobVO job = _jobDao.findById(item.getContentId()); if (job != null) { if (logger.isDebugEnabled()) { @@ -886,7 +879,7 @@ private void checkQueue(long queueId) { logger.debug("Executing sync queue item: " + item.toString()); } - executeQueueItem(item); + executeQueueItem(item, false); } else { break; } @@ -923,39 +916,16 @@ protected void reallyRun() { return; } - List items = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); - if (items != null && items.size() > 0) { - for (SyncQueueItemVO item : items) { - boolean isPurged = false; - if (item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { - AsyncJobVO job = _jobDao.findById(item.getContentId()); - if (job != null && StringUtils.isNotBlank(job.getRelated())) { - AsyncJobVO parentJob = _jobDao.findById(Long.valueOf(job.getRelated())); - //If the parent job is done, do not execute the child. complete it and purge it from queue - if (parentJob != null && parentJob.getStatus().done() && !isPseudoJob(parentJob)) { - logger.debug("Purging sync-queue item: {}", item); - completeAsyncJob(item.getContentId(), parentJob.getStatus(), 0, "Job is not " - + "scheduled for execution as the parent job is done. Parent Job " + - "state:" + " " + parentJob.getStatus()); - _jobMonitor.unregisterByJobId(item.getContentId()); - _queueMgr.purgeItem(item.getId()); - - if (parentJob.getStatus() == Status.CANCELLED) { - parentJob.setCompleteMsid(getMsid()); - final Date currentGMTTime = DateUtil.currentGMTTime(); - parentJob.setLastUpdated(currentGMTTime); - parentJob.setRemoved(currentGMTTime); - _jobDao.update(parentJob.getId(), parentJob); - } - - isPurged = true; - } - } + List l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); + if (l != null && l.size() > 0) { + for (SyncQueueItemVO item : l) { + if (isChildOfFinishedJob(item)) { + continue; } - if (!isPurged) { - logger.debug("Execute sync-queue item: {}", item); - executeQueueItem(item); + if (logger.isDebugEnabled()) { + logger.debug("Execute sync-queue item: " + item.toString()); } + executeQueueItem(item, false); } } @@ -964,14 +934,10 @@ protected void reallyRun() { // TODO, we assume that all jobs in this category is API job only AsyncJobVO job = _jobDao.findById(jobId); if (job != null && (job.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) { - if (job.getStatus() != Status.CANCELLED) { - scheduleExecution(job, false); + if (job.getStatus() == Status.CANCELLED) { + finalizeCancelledJob(job.getId()); } else { - job.setCompleteMsid(getMsid()); - final Date currentGMTTime = DateUtil.currentGMTTime(); - job.setLastUpdated(currentGMTTime); - job.setRemoved(currentGMTTime); - _jobDao.update(job.getId(), job); + scheduleExecution(job, false); } } } @@ -982,11 +948,6 @@ protected void reallyRun() { }; } - private boolean isPseudoJob(AsyncJob job) { - return AsyncJobVO.JOB_DISPATCHER_PSEUDO.equals(job.getDispatcher()) && AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE - .equals(job.getInstanceType()); - } - @DB private Runnable getGCTask() { return new ManagedContextRunnable() { @@ -1013,12 +974,12 @@ public void reallyRun() { // forcefully cancel blocking queue items if they've been staying there for too long List blockItems = _queueMgr.getBlockedQueueItems(JobCancelThresholdMinutes.value() * 60000, false); if (blockItems != null && blockItems.size() > 0) { - logger.debug("Found {} blocking queue items for over {} minutes, will cancel them and purge from queue", blockItems.size(), JobCancelThresholdMinutes.value()); for (SyncQueueItemVO item : blockItems) { try { if (item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { - logger.info("Remove Job-{} from Queue-{} since it has been blocked for too long", item.getContentId(), item.getId()); - completeAsyncJob(item.getContentId(), Status.CANCELLED, 0, "Job is cancelled as it has been blocking others for too long"); + logger.info("Remove Job-" + item.getContentId() + " from Queue-" + item.getId() + " since it has been blocked for too long"); + completeAsyncJob(item.getContentId(), JobInfo.Status.CANCELLED, 0, "Job is cancelled as it has been blocking others for too long"); + _jobMonitor.unregisterByJobId(item.getContentId()); } @@ -1038,6 +999,7 @@ public void reallyRun() { for (AsyncJobVO job : unfinishedJobs) { try { logger.info("Expunging unfinished job-" + job.getId()); + _jobMonitor.unregisterByJobId(job.getId()); expungeAsyncJob(job); } catch (Throwable e) { @@ -1050,6 +1012,7 @@ public void reallyRun() { for (AsyncJobVO job : completedJobs) { try { logger.info("Expunging completed job-" + job.getId()); + expungeAsyncJob(job); } catch (Throwable e) { logger.error("Unexpected exception when trying to expunge job-" + job.getId(), e); @@ -1057,6 +1020,9 @@ public void reallyRun() { } logger.trace("End cleanup expired async-jobs"); + + cleanupNetworksStuckInImplementing(); + } catch (Throwable e) { logger.error("Unexpected exception when trying to execute queue item, ", e); } @@ -1064,6 +1030,15 @@ public void reallyRun() { }; } + private void cleanupNetworksStuckInImplementing() { + // Cleanup orphaned networks stuck in Implementing state without async jobs + try { + cleanupOrphanedNetworks(); + } catch (Throwable e) { + logger.error("Unexpected exception when trying to cleanup orphaned networks", e); + } + } + @DB protected void expungeAsyncJob(final AsyncJobVO job) { Transaction.execute(new TransactionCallbackNoReturn() { @@ -1153,10 +1128,10 @@ public boolean configure(String name, Map params) throws Configu int apiPoolSize = cloudMaxActive / 2; int workPoolSize = (cloudMaxActive * 2) / 3; - logger.info("Start AsyncJobManager API executor thread pool in size {}", apiPoolSize); + logger.info("Start AsyncJobManager API executor thread pool in size " + apiPoolSize); _apiJobExecutor = Executors.newFixedThreadPool(apiPoolSize, new NamedThreadFactory(AsyncJobManager.API_JOB_POOL_THREAD_PREFIX)); - logger.info("Start AsyncJobManager Work executor thread pool in size {}", workPoolSize); + logger.info("Start AsyncJobManager Work executor thread pool in size " + workPoolSize); _workerJobExecutor = Executors.newFixedThreadPool(workPoolSize, new NamedThreadFactory(AsyncJobManager.WORK_JOB_POOL_THREAD_PREFIX)); } catch (final Exception e) { throw new ConfigurationException("Unable to load db.properties to configure AsyncJobManagerImpl"); @@ -1209,7 +1184,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { cleanupResources(job); job.setStatus(JobInfo.Status.FAILED); job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode()); - job.setResult("job cancelled because of management server restart or shutdown"); + job.updateResultWithEncryptionIfNeeded("job cancelled because of management server restart or shutdown"); job.setCompleteMsid(msid); final Date currentGMTTime = DateUtil.currentGMTTime(); job.setLastUpdated(currentGMTTime); @@ -1401,6 +1376,74 @@ private void cleanupFailedSnapshotsCreatedWithDefaultStrategy(final long msid) { } } + /** + * Cleanup networks that are stuck in Implementing state without associated async jobs. + * This only processes networks that have been stuck for longer than the job expiration threshold. + */ + private void cleanupOrphanedNetworks() { + try { + SearchCriteria sc = networkDao.createSearchCriteria(); + sc.addAnd("state", SearchCriteria.Op.EQ, Network.State.Implementing); + sc.addAnd("removed", SearchCriteria.Op.NULL); + List implementingNetworks = networkDao.search(sc, null); + + if (implementingNetworks == null || implementingNetworks.isEmpty()) { + return; + } + + logger.debug("Found {} networks in Implementing state, checking for orphaned networks", implementingNetworks.size()); + + final long expireMinutes = JobExpireMinutes.value(); + final Date cutoffTime = new Date(System.currentTimeMillis() - (expireMinutes * 60 * 1000)); + + for (NetworkVO network : implementingNetworks) { + if (network.getCreated().after(cutoffTime)) { + logger.trace("Network {} in Implementing state is only {} minutes old (threshold: {} minutes), skipping cleanup", + network.getId(), + (System.currentTimeMillis() - network.getCreated().getTime()) / 60000, + expireMinutes); + continue; + } + + List jobs = _jobDao.findInstancePendingAsyncJobs("Network", network.getAccountId()); + boolean hasActiveJob = false; + for (AsyncJobVO job : jobs) { + if (job.getInstanceId() != null && job.getInstanceId().equals(network.getId())) { + hasActiveJob = true; + break; + } + } + + if (hasActiveJob) { + logger.debug("Network {} in Implementing state has active async job, skipping cleanup", network.getId()); + continue; + } + + logger.warn("Found orphaned network {} in Implementing state without async job. " + + "Network created: {}, age: {} minutes, expiration threshold: {} minutes. Transitioning to Shutdown state.", + network.getId(), network.getCreated(), + (System.currentTimeMillis() - network.getCreated().getTime()) / 60000, + expireMinutes); + updateNetworkState(network); + + } + } catch (Exception e) { + logger.error("Error while cleaning up orphaned networks", e); + } + } + + private void updateNetworkState(NetworkVO network) { + try { + networkOrchestrationService.stateTransitTo(network, Network.Event.OperationFailed); + logger.info("Successfully transitioned orphaned network {} to Shutdown state using state machine", network.getId()); + } catch (final NoTransitionException e) { + logger.debug("State transition failed for orphaned network {}, forcing state update", network.getId()); + network.setState(Network.State.Shutdown); + networkDao.update(network.getId(), network); + logger.info("Successfully forced orphaned network {} to Shutdown state", network.getId()); + } + } + @Override public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @@ -1428,43 +1471,48 @@ public boolean start() { @Override public boolean stop() { - cancelPendingJobs(); _heartbeatScheduler.shutdown(); _eventBusPublisher.shutdown(); - shutdownAndAwaitTermination(_apiJobExecutor); - shutdownAndAwaitTermination(_workerJobExecutor); + _apiJobExecutor.shutdown(); + _workerJobExecutor.shutdown(); return true; } - private void cancelPendingJobs() { - List jobs = _queueItemDao.getActiveQueueItems(getMsid(), false); - for (SyncQueueItemVO job : jobs) { - AsyncJobVO childJob = _jobDao.findById(job.getContentId()); - if (childJob != null && StringUtils.isNotBlank(childJob.getRelated())) { - long parentJobId = Long.parseLong(childJob.getRelated()); - try { - logger.debug("Cancel left-over async. job-{}", parentJobId); - cancelAsyncJob(parentJobId, "Management Server shutdown"); - } catch (Exception e) { - logger.error("Exception while cancelling job-{}", parentJobId, e); - } - } - _queueMgr.purgeItem(job.getId()); + /** + * A queued child (VM work) job whose parent has already finished must not run: the parent was + * cancelled or failed while this child waited its turn. Completes the child with the parent's + * status and drops it from the queue. + */ + private boolean isChildOfFinishedJob(final SyncQueueItemVO item) { + if (!SyncQueueItem.AsyncJobContentType.equalsIgnoreCase(item.getContentType())) { + return false; + } + final AsyncJobVO job = _jobDao.findById(item.getContentId()); + if (job == null || StringUtils.isBlank(job.getRelated())) { + return false; + } + final AsyncJobVO parentJob = _jobDao.findById(Long.parseLong(job.getRelated())); + if (parentJob == null || !parentJob.getStatus().done() || isPseudoJob(parentJob)) { + return false; } - } - private static void shutdownAndAwaitTermination(ExecutorService pool) { - pool.shutdown(); - try { - if (!pool.awaitTermination(60, TimeUnit.SECONDS)) { - pool.shutdownNow(); - } - } catch (InterruptedException ignore) { + logger.debug("Not executing sync-queue item {}: parent job-{} is already {}", item, parentJob.getId(), parentJob.getStatus()); + completeAsyncJob(item.getContentId(), parentJob.getStatus(), 0, + "Job is not scheduled for execution as the parent job is done. Parent job state: " + parentJob.getStatus()); + _jobMonitor.unregisterByJobId(item.getContentId()); + _queueMgr.purgeItem(item.getId()); + if (parentJob.getStatus() == Status.CANCELLED) { + finalizeCancelledJob(parentJob.getId()); } + return true; + } + + private boolean isPseudoJob(final AsyncJob job) { + return AsyncJobVO.JOB_DISPATCHER_PSEUDO.equals(job.getDispatcher()) && AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE.equals(job.getInstanceType()); } @Override - public String cancelAsyncJob(long jobId, String reason) { + public String cancelAsyncJob(final long jobId, final String reason) { final AsyncJobVO job = _jobDao.findByIdIncludingRemoved(jobId); String errMessage; if (job == null) { @@ -1474,16 +1522,18 @@ public String cancelAsyncJob(long jobId, String reason) { return errMessage; } - if (isActiveJob(jobId)) { + // A job that is still queued can always be dropped. One that is executing -- here or on + // another management server -- only if its command opted in. + if (job.getExecutingMsid() != null || isActiveJob(jobId)) { try { - Class cmdClass = Class.forName(job.getCmd()); - APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); + final Class cmdClass = Class.forName(job.getCmd()); + final APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); if (apiCommand == null || !apiCommand.cancellable()) { errMessage = "Cannot cancel, job " + job.getUuid() + " is not cancellable."; logger.debug(errMessage); return errMessage; } - } catch (ClassNotFoundException e) { + } catch (final ClassNotFoundException e) { errMessage = "Command " + job.getCmd() + " of jobid " + job.getUuid() + " not found."; logger.error(errMessage, e); return errMessage; @@ -1497,28 +1547,70 @@ public String cancelAsyncJob(long jobId, String reason) { return errMessage; } - logger.debug("Cancelling job-{} which is in progress.", jobId); + // Ask before changing anything. If the backend work cannot be stopped, refusing is the + // honest answer: recording the job as cancelled while the hypervisor finishes the operation + // is the divergence this whole feature exists to prevent. + for (final JobCancellationHandler handler : getJobCancellationHandlers()) { + if (!handler.isJobExecutionCancellable(jobId)) { + errMessage = "Cannot cancel job-" + jobId + ", the operation it is running cannot be stopped at this point."; + logger.info(errMessage); + return errMessage; + } + } + logger.debug("Cancelling job-{} which is in progress.", jobId); try { - completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason, false); - _jobMonitor.unregisterByJobId(jobId); + for (final JobCancellationHandler handler : getJobCancellationHandlers()) { + if (!handler.cancelJobExecution(jobId, reason)) { + errMessage = "Cannot cancel job-" + jobId + ", the operation it is running could not be stopped."; + logger.info(errMessage); + return errMessage; + } + } + + // If no other management server is executing the job there is nobody left to + // acknowledge the cancellation, so finish the row here. Otherwise keep executing_msid + // so that server's poller can find the job, stop its in-flight commands and finish it. + final Long executingMsid = job.getExecutingMsid(); + final boolean finalizeNow = executingMsid == null || executingMsid == getMsid(); + completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason, finalizeNow); return ""; - } catch (Throwable t) { + } catch (final Throwable t) { errMessage = "Unexpected exception when cancelling async job with id: " + jobId; logger.error(errMessage, t); } - return errMessage; } - private boolean isActiveJob(long jobId) { - // check if the job (and it's related job) is active or not. - // if not, it means the job is not running, and we can safely purge it from the queue. + @Override + public List listCancelledJobsExecutingOn(final long msid) { + return _jobDao.getCancelledJobs(msid); + } + + @Override + public void finalizeCancelledJob(final long jobId) { + final AsyncJobVO job = _jobDao.findById(jobId); + if (job == null || job.getStatus() != JobInfo.Status.CANCELLED) { + return; + } + final Date now = DateUtil.currentGMTTime(); + job.setCompleteMsid(getMsid()); + job.setLastUpdated(now); + job.setRemoved(now); + job.setExecutingMsid(null); + _jobDao.update(jobId, job); + } + + private List getJobCancellationHandlers() { + return jobCancellationHandlers != null ? jobCancellationHandlers : Collections.emptyList(); + } + + private boolean isActiveJob(final long jobId) { + // a VM work child runs on behalf of its API parent, so check whichever of the two is active final AsyncJobVO relatedJob = _jobDao.getRelatedJob(String.valueOf(jobId)); if (relatedJob != null) { return _jobMonitor.isActiveJob(relatedJob.getId()); } - return _jobMonitor.isActiveJob(jobId); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java index 12877478bed4..d69c5978456f 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java @@ -159,10 +159,9 @@ public void unregisterByJobId(long jobId) { _activeInplaceThreads.decrementAndGet(); it.remove(); - _activeJobs.add(jobId); - break; } } + _activeJobs.remove(jobId); } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java index d611807a33b2..32d84647a2d4 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java @@ -21,21 +21,21 @@ import com.cloud.utils.component.Manager; public interface SyncQueueManager extends Manager { - SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit); + public SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit); - SyncQueueItemVO dequeueFromOne(long queueId, Long msid); + public SyncQueueItemVO dequeueFromOne(long queueId, Long msid); - List dequeueFromAny(Long msid, int maxItems); + public List dequeueFromAny(Long msid, int maxItems); - void purgeItem(long queueItemId); + public void purgeItem(long queueItemId); - void returnItem(long queueItemId); + public void returnItem(long queueItemId); - List getActiveQueueItems(Long msid, boolean exclusive); + public List getActiveQueueItems(Long msid, boolean exclusive); - List getBlockedQueueItems(long thresholdMs, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, boolean exclusive); void purgeAsyncJobQueueItemId(long asyncJobId); - void cleanupActiveQueueItems(Long msid, boolean exclusive); + public void cleanupActiveQueueItems(Long msid, boolean exclusive); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java index 3ffef5385075..3397daa58191 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java @@ -50,19 +50,18 @@ public SyncQueueVO queue(final String syncObjType, final long syncObjId, final S public SyncQueueVO doInTransaction(TransactionStatus status) { _syncQueueDao.ensureQueue(syncObjType, syncObjId); SyncQueueVO queueVO = _syncQueueDao.find(syncObjType, syncObjId); - if (queueVO == null) { + if (queueVO == null) throw new CloudRuntimeException("Unable to queue item into DB, DB is full?"); - } queueVO.setQueueSizeLimit(queueSizeLimit); _syncQueueDao.update(queueVO.getId(), queueVO); - Date date = DateUtil.currentGMTTime(); + Date dt = DateUtil.currentGMTTime(); SyncQueueItemVO item = new SyncQueueItemVO(); item.setQueueId(queueVO.getId()); item.setContentType(itemType); item.setContentId(itemId); - item.setCreated(date); + item.setCreated(dt); _syncQueueItemDao.persist(item); return queueVO; @@ -182,8 +181,9 @@ public void purgeItem(final long queueItemId) { @Override public void doInTransactionWithoutResult(TransactionStatus status) { SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId); - if (itemVO != null) { + if(itemVO != null) { SyncQueueVO queueVO = _syncQueueDao.findById(itemVO.getQueueId()); + _syncQueueItemDao.expunge(itemVO.getId()); // if item is active, reset queue information diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java index f4206ff72027..050fe4e5215c 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/VmWorkJobVO.java @@ -113,11 +113,11 @@ public void setSecondaryObjectIdentifier(String secondaryObjectIdentifier) { public String toString() { StringBuffer sb = new StringBuffer(); sb.append("VmWorkJobVO : {"). - append("vmInstanceId: ").append(getVmInstanceId()). - append(", vmType: ").append(getVmType()). append(", step: ").append(getStep()). + append(", vmType: ").append(getVmType()). + append(", vmInstanceId: ").append(getVmInstanceId()). append(", secondaryObjectIdentifier: ").append(getSecondaryObjectIdentifier()). - append(", ").append(super.toString()). + append(super.toString()). append("}"); return sb.toString(); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 70c38b41eb84..8052317817ad 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -43,7 +43,6 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; -import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; import com.cloud.hypervisor.vmware.util.VmwareClient; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; @@ -690,8 +689,8 @@ public boolean needRecycle(String workerTag) { Instant start = Instant.ofEpochMilli(startTick); Instant end = start.plusSeconds(2 * (AsyncJobManagerImpl.JobExpireMinutes.value() + AsyncJobManagerImpl.JobCancelThresholdMinutes.value()) * SECONDS_PER_MINUTE); Instant now = Instant.now(); - if (s_vmwareCleanOldWorderVMs.value() && now.isAfter(end)) { - if (logger.isInfoEnabled()) { + if(s_vmwareCleanOldWorderVMs.value() && now.isAfter(end)) { + if(logger.isInfoEnabled()) { logger.info("Worker VM expired, seconds elapsed: " + Duration.between(start,now).getSeconds()); } return true; @@ -1583,7 +1582,7 @@ public List listVsphereStoragePolicyCompatibleStoragePools(ListVsph if (result) { compatiblePools.add(pool); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Could not verify if storage policy " + storagePolicy.getName() + " is compatible with storage pool " + pool.getName()); } } diff --git a/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index e2cdea6dad7f..9dc083015c3b 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -30,7 +30,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -171,9 +170,6 @@ private boolean sendCommandsToRouter(final DomainRouterVO elbVm, Commands cmds) } catch (OperationTimedoutException e) { logger.warn("ELB: Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual elbVm ", elbVm.getHostId(), e); - } catch (OperationCancelledException e) { - logger.warn("ELB: Cancelled", e); - throw new AgentUnavailableException("Unable to send commands to virtual elbVm ", elbVm.getHostId(), e); } if (answers == null) { diff --git a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index 29df6ea5df6e..cda139b5a62b 100644 --- a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -36,7 +36,6 @@ import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -975,9 +974,6 @@ protected boolean sendCommandsToInternalLbVm(final VirtualRouter internalLbVm, f } catch (final OperationTimedoutException e) { logger.warn("Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", internalLbVm.getHostId(), e); - } catch (final OperationCancelledException e) { - logger.warn("Cancelled", e); - throw new AgentUnavailableException("Unable to send commands to virtual router ", internalLbVm.getHostId(), e); } if (answers == null) { diff --git a/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java b/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java index 9b80008004e6..1550649f847a 100644 --- a/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java +++ b/plugins/network-elements/internal-loadbalancer/src/test/java/org/apache/cloudstack/internallbvmmgr/InternalLBVMManagerTest.java @@ -26,7 +26,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; import org.apache.cloudstack.network.lb.InternalLoadBalancerVMManager; import org.junit.Before; @@ -156,7 +155,10 @@ public void setUp() { try { Mockito.when(_agentMgr.send(nullable(Long.class), nullable(Commands.class))).thenReturn(answers); - } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (final AgentUnavailableException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (final OperationTimedoutException e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java index 91936a212c2e..7fa80a0b3fae 100644 --- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java +++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/vmsnapshot/OntapVMSnapshotStrategy.java @@ -27,7 +27,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.VMSnapshotOptions; @@ -474,9 +473,6 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { } catch (OperationTimedoutException e) { logger.error("takeVMSnapshot: ONTAP VM Snapshot [{}] timed out: {}", vmSnapshot.getName(), e.getMessage()); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " timed out: " + e.getMessage()); - } catch (OperationCancelledException e) { - logger.error("takeVMSnapshot: ONTAP VM Snapshot [{}] cancelled: {}", vmSnapshot.getName(), e.getMessage()); - throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " cancelled: " + e.getMessage()); } catch (AgentUnavailableException e) { logger.error("takeVMSnapshot: ONTAP VM Snapshot [{}] failed, agent unavailable: {}", vmSnapshot.getName(), e.getMessage()); throw new CloudRuntimeException("Creating Instance Snapshot: " + vmSnapshot.getName() + " failed: " + e.getMessage()); diff --git a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java index 1407ff3a4b37..3f6e82f54ce1 100644 --- a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java @@ -118,30 +118,31 @@ public void runJob(final AsyncJob job) { CallContext.unregister(); } } catch (Throwable e) { - //Get the latest job status from DB to check if it has been cancelled during execution + // A job cancelled mid-execution has already reached a terminal state; re-reading it here + // stops the failure that cancellation caused from overwriting the CANCELLED result. AsyncJobVO jobFromDb = _asyncJobMgr.getAsyncJob(job.getId()); - if (!jobFromDb.getStatus().done()) { - String errorMsg = null; - int errorCode = ApiErrorCode.INTERNAL_ERROR.getHttpCode(); - if (!(e instanceof ServerApiException)) { - logger.error("Unexpected exception while executing {}", job.getCmd(), e); - errorMsg = e.getMessage(); - } else { - ServerApiException sApiEx = (ServerApiException) e; - errorMsg = sApiEx.getDescription(); - errorCode = sApiEx.getErrorCode().getHttpCode(); - } - - ExceptionResponse response = new ExceptionResponse(); - response.setErrorCode(errorCode); - response.setErrorText(errorMsg); - response.setResponseName((cmdObj == null) ? "unknowncommandresponse" : cmdObj.getCommandName()); - - // FIXME: setting resultCode to ApiErrorCode.INTERNAL_ERROR is not right, usually executors have their exception handling - // and we need to preserve that as much as possible here - - _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), ApiSerializerHelper.toSerializedString(response)); + if (jobFromDb != null && jobFromDb.getStatus().done()) { + logger.debug("Not recording failure for job-{}, it is already in {}", job.getId(), jobFromDb.getStatus()); + return; } + + String errorMsg = null; + int errorCode = ApiErrorCode.INTERNAL_ERROR.getHttpCode(); + if (!(e instanceof ServerApiException)) { + logger.error("Unexpected exception while executing {}", job.getCmd(), e); + errorMsg = e.getMessage(); + } else { + ServerApiException sApiEx = (ServerApiException)e; + errorMsg = sApiEx.getDescription(); + errorCode = sApiEx.getErrorCode().getHttpCode(); + } + + ExceptionResponse response = new ExceptionResponse(); + response.setErrorCode(errorCode); + response.setErrorText(errorMsg); + response.setResponseName((cmdObj == null) ? "unknowncommandresponse" : cmdObj.getCommandName()); + + _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, errorCode, ApiSerializerHelper.toSerializedString(response)); } } } diff --git a/server/src/main/java/com/cloud/api/ApiDispatcher.java b/server/src/main/java/com/cloud/api/ApiDispatcher.java index f1a647041cc0..90cbb6afc8e5 100644 --- a/server/src/main/java/com/cloud/api/ApiDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiDispatcher.java @@ -121,7 +121,7 @@ public void dispatch(final BaseCmd cmd, final Map params, final final CallContext ctx = CallContext.current(); ctx.setEventDisplayEnabled(cmd.isDisplay()); - if (params.get(ApiConstants.PROJECT_ID) != null) { + if(params.get(ApiConstants.PROJECT_ID) != null) { Project project = _entityMgr.findByUuidIncludingRemoved(Project.class, params.get(ApiConstants.PROJECT_ID)); ctx.setProject(project); } @@ -135,6 +135,7 @@ public void dispatch(final BaseCmd cmd, final Map params, final // TODO This if shouldn't be here. Use polymorphism and move it to validateSpecificParameters if (cmd instanceof BaseAsyncCmd) { + final BaseAsyncCmd asyncCmd = (BaseAsyncCmd)cmd; final String startEventId = params.get(ApiConstants.CTX_START_EVENT_ID); ctx.setStartEventId(Long.parseLong(startEventId)); diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index f85c281f998b..61c8ae68c5b1 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -387,7 +387,7 @@ public void handleAsyncJobPublishEvent(String subject, String senderAddress, Obj String jobEvent = eventInfo.second(); if (logger.isTraceEnabled()) - logger.trace("Handle async job publish event {}", jobEvent); + logger.trace("Handle asyjob publish event {}", jobEvent); if (eventDistributor == null) { setEventDistributor(ComponentContext.getComponent(EventDistributor.class)); } @@ -418,7 +418,7 @@ public void handleAsyncJobPublishEvent(String subject, String senderAddress, Obj logger.debug("Unable to locate cmdEventType marker in job info. publish as unknown event"); } String contextDetails = cmdInfo.get("ctxDetails"); - if (contextDetails != null) { + if(contextDetails != null) { Type objectMapType = new TypeToken>() {}.getType(); Map ctxDetails = ApiGsonHelper.getBuilder().create().fromJson(contextDetails, objectMapType); cmdInfoObj.put("ctxDetails", ctxDetails); @@ -443,7 +443,7 @@ public void handleAsyncJobPublishEvent(String subject, String senderAddress, Obj eventDescription.put("jobResult", ApiSerializerHelper.fromSerializedStringToMap(StringUtils.obfuscatePasswordInJsonLikeString(job.getResult()))); eventDescription.put("cmdInfo", cmdInfoObj); eventDescription.put("status", "" + job.getStatus()); - // If the event.accountinfo boolean value is set, get the human-readable value for the username / domainname + // If the event.accountinfo boolean value is set, get the human readable value for the username / domainname if (UseEventAccountInfo.value()) { DomainVO domain = domainDao.findById(jobOwner.getDomainId()); eventDescription.put("username", userJobOwner.getUsername()); @@ -494,12 +494,12 @@ public boolean start() { } for (final Class cmdClass : cmdClasses) { - final APICommand apiCmdAnnotation = cmdClass.getAnnotation(APICommand.class); - if (apiCmdAnnotation == null) { + final APICommand at = cmdClass.getAnnotation(APICommand.class); + if (at == null) { throw new CloudRuntimeException(String.format("%s is claimed as a API command, but it doesn't have @APICommand annotation", cmdClass.getName())); } - String apiName = apiCmdAnnotation.name(); + String apiName = at.name(); List> apiCmdList = s_apiNameCmdClassMap.get(apiName); if (apiCmdList == null) { apiCmdList = new ArrayList<>(); @@ -540,18 +540,18 @@ public void handle(final HttpRequest request, final HttpResponse response, final // Use Multimap as the parameter map should be in the form (name=String, value=String[]) // So parameter values are stored in a list for the same name key - // API TODO: Use Guava's (import com.google.common.collect.Multimap;) + // APITODO: Use Guava's (import com.google.common.collect.Multimap;) // (Immutable)Multimap paramMultiMap = HashMultimap.create(); // Map> parameterMap = paramMultiMap.asMap(); final Map parameterMap = new HashMap(); String responseType = HttpUtils.RESPONSE_TYPE_XML; - if (paramList != null) { + if(paramList != null) { for (final NameValuePair param : paramList) { if (param.getName().equalsIgnoreCase("response")) { responseType = param.getValue(); continue; } - if (parameterMap.putIfAbsent(param.getName(), new String[]{param.getValue()}) != null) { + if(parameterMap.putIfAbsent(param.getName(), new String[]{param.getValue()}) != null) { String message = String.format("Query parameter '%s' has multiple values [%s, %s]. Only the last value will be respected." + "It is advised to pass only a single parameter", param.getName(), param.getValue(), parameterMap.get(param.getName())); logger.warn(StringUtils.cleanString(message)); @@ -755,7 +755,7 @@ public String handleRequest(final Map params, final String responseType, final S logger.info(ex.getDescription()); throw ex; } catch (final Exception ex) { - logger.error("unhandled exception executing api command: {}", (command == null) ? "null" : command, ex); + logger.error("unhandled exception executing api command: " + ((command == null) ? "null" : command), ex); String errorMsg = ex.getMessage(); if (!accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId())) { // hide internal details to non-admin user for security reason @@ -815,6 +815,7 @@ private String queueCommand(final BaseCmd cmdObj, final Map para // in the api logs. log.append(response); return response; + } else { dispatcher.dispatch(cmdObj, params, false); @@ -1183,7 +1184,7 @@ private boolean commandAvailable(final InetAddress remoteAddress, final String c " for user."; throw new ServerApiException(ApiErrorCode.UNAUTHORIZED , errorMessage); } catch (final OriginDeniedException ex) { - // in this case, we can remove the session with extreme prejudice + // in this case we can remove the session with extreme prejudice final String errorMessage = String.format("The user '%s' is not allowed to execute commands from ip address '%s'.", user, remoteAddress.getHostName()); logger.debug(errorMessage); return false; @@ -1489,7 +1490,7 @@ private void checkCommandAvailable(final User user, final String commandName, fi @Override public Class getCmdClass(String cmdName) { List> cmdList = s_apiNameCmdClassMap.get(cmdName); - if (cmdList == null || cmdList.isEmpty()) + if (cmdList == null || cmdList.size() == 0) return null; else if (cmdList.size() == 1) return cmdList.get(0); diff --git a/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java b/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java index 2b58338be001..25846f908332 100644 --- a/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/CommandCreationWorker.java @@ -33,7 +33,7 @@ */ public class CommandCreationWorker implements DispatchWorker { - private static final String ATTEMPT_TO_CREATE_NON_CREATION_CMD = + private static final String ATTEMP_TO_CREATE_NON_CREATION_CMD = "Trying to invoke creation on a Command that is not " + BaseAsyncCreateCmd.class.getName(); @@ -51,7 +51,8 @@ public void handle(final DispatchTask task) { } } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - ATTEMPT_TO_CREATE_NON_CREATION_CMD); + ATTEMP_TO_CREATE_NON_CREATION_CMD); } } + } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 27c0537e4f12..41a406ba044b 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -20,9 +20,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,6 +31,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -926,6 +924,7 @@ private Pair, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) { Integer entryTime = cmd.getEntryTime(); Integer duration = cmd.getDuration(); Long startId = cmd.getStartId(); + final Long jobId = cmd.getJobId(); final String resourceUuid = getResourceUuid(cmd.getResourceId()); final ApiCommandResourceType resourceType = getResourceType(cmd.getResourceType()); final String stateStr = cmd.getState(); @@ -983,6 +982,7 @@ private Pair, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) { eventSearchBuilder.and("archived", eventSearchBuilder.entity().getArchived(), SearchCriteria.Op.EQ); eventSearchBuilder.and("resourceId", eventSearchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); eventSearchBuilder.and("resourceType", eventSearchBuilder.entity().getResourceType(), SearchCriteria.Op.EQ); + eventSearchBuilder.and("asyncJobId", eventSearchBuilder.entity().getAsyncJobId(), SearchCriteria.Op.EQ); if (keyword != null) { eventSearchBuilder.and().op("keywordType", eventSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE); @@ -1011,6 +1011,10 @@ private Pair, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) { } } + if (jobId != null) { + sc.setParameters("asyncJobId", jobId); + } + if (keyword != null) { sc.setParameters("keywordType", "%" + keyword + "%"); sc.setParameters("keywordDescription", "%" + keyword + "%"); @@ -3221,12 +3225,15 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync Boolean isRecursive = domainIdRecursiveListProject.second(); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); - boolean pendingJobsOnly = cmd.getJobStatuses() == null; + // A job is soft-deleted the moment it completes, so anything but the default pending-only + // listing has to reach into removed rows or it can never return a finished job. + final boolean filterByStatus = cmd.getJobStatuses() != null; + final boolean includeRemoved = filterByStatus || cmd.getEndDate() != null || cmd.getDuration() != null; Filter searchFilter = new Filter(AsyncJobJoinVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); SearchBuilder sb = _jobJoinDao.createSearchBuilder(); - if (!pendingJobsOnly) { + if (filterByStatus) { sb.and("statuses", sb.entity().getStatus(), SearchCriteria.Op.IN); } sb.and("instanceTypeNEQ", sb.entity().getInstanceType(), SearchCriteria.Op.NEQ); @@ -3260,7 +3267,7 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync Object endDate = cmd.getEndDate(); SearchCriteria sc = sb.create(); - if (!pendingJobsOnly) { + if (filterByStatus) { sc.setParameters("statuses", cmd.getJobStatuses().toArray()); } sc.setParameters("instanceTypeNEQ", AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE); @@ -3292,11 +3299,9 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync } if (cmd.getDuration() != null) { - ZoneId systemZoneId = ZoneId.systemDefault(); - LocalDateTime nowLocal = LocalDateTime.now(); - ZonedDateTime nowZoned = nowLocal.atZone(systemZoneId); - ZonedDateTime lastDateTimeZoned = nowZoned.minusHours(cmd.getDuration()); - Date lastDate = Date.from(lastDateTimeZoned.toInstant()); + // Rows are written with DateUtil.currentGMTTime(); anchoring the window in the system + // zone would shift it by the offset on any management server that is not on UTC. + Date lastDate = new Date(DateUtil.currentGMTTime().getTime() - TimeUnit.HOURS.toMillis(cmd.getDuration())); SearchCriteria scc = _jobJoinDao.createSearchCriteria(); scc.addOr("created", SearchCriteria.Op.GTEQ, lastDate); @@ -3318,10 +3323,13 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync if (cmd.getManagementServerId() != null) { ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); + if (msHost == null) { + throw new InvalidParameterValueException("Unable to find a management server with the specified id"); + } sc.setParameters("executingMsid", msHost.getMsid()); } - return _jobJoinDao.searchAndCount(sc, searchFilter); + return _jobJoinDao.searchAndCount(sc, searchFilter, includeRemoved); } @Override diff --git a/server/src/main/java/com/cloud/api/query/vo/EventJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/EventJoinVO.java index bf731e0dbd6a..450616a61d22 100644 --- a/server/src/main/java/com/cloud/api/query/vo/EventJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/EventJoinVO.java @@ -69,6 +69,12 @@ public class EventJoinVO extends BaseViewVO implements ControlledViewEntity { @Column(name = "start_uuid") private String startUuid; + @Column(name = "async_job_id") + private Long asyncJobId; + + @Column(name = "async_job_uuid") + private String asyncJobUuid; + @Column(name = "parameters", length = 1024) private String parameters; @@ -229,6 +235,14 @@ public String getStartUuid() { return startUuid; } + public Long getAsyncJobId() { + return asyncJobId; + } + + public String getAsyncJobUuid() { + return asyncJobUuid; + } + public String getParameters() { return parameters; } diff --git a/server/src/main/java/com/cloud/event/ActionEventUtils.java b/server/src/main/java/com/cloud/event/ActionEventUtils.java index c96bac866127..7fa2800f36b5 100644 --- a/server/src/main/java/com/cloud/event/ActionEventUtils.java +++ b/server/src/main/java/com/cloud/event/ActionEventUtils.java @@ -32,6 +32,9 @@ import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.events.EventDistributor; import org.apache.commons.lang3.ObjectUtils; @@ -183,6 +186,7 @@ private static Event persistActionEvent(Long userId, Long accountId, Long domain event.setState(state); event.setDescription(description); event.setDisplay(eventDisplayEnabled); + event.setAsyncJobId(currentJobId()); if (domainId != null) { event.setDomainId(domainId); @@ -205,6 +209,24 @@ private static Event persistActionEvent(Long userId, Long accountId, Long domain return event; } + /** + * The API job on whose behalf this thread is working, or null outside a job. VM work jobs run on + * behalf of an API job (their "related" id); that parent is what the user can see and list, so it + * is the one recorded. + */ + private static Long currentJobId() { + final AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrent(); + if (context == null || context.getJob() == null) { + return null; + } + final AsyncJob job = context.getJob(); + if (AsyncJobVO.JOB_DISPATCHER_PSEUDO.equals(job.getDispatcher())) { + return null; + } + final String related = job.getRelated(); + return StringUtils.isNotEmpty(related) ? Long.valueOf(related) : job.getId(); + } + private static void publishOnEventBus(Event eventRecord, long userId, long accountId, Long domainId, String eventCategory, String eventType, Event.State state, String description, String resourceUuid, String resourceType) { @@ -275,10 +297,10 @@ private static Ternary getResourceDetailsUsingEntityClassA String entityUuid = null; Long entityId = null; Object param = context.getContextParameter(entityClass); - if (param != null) { + if(param != null){ try { entityUuid = getEntityUuid(entityClass, param); - } catch (Exception e) { + } catch (Exception e){ LOGGER.debug("Caught exception while finding entityUUID, moving on"); } } diff --git a/server/src/main/java/com/cloud/event/dao/EventJoinDaoImpl.java b/server/src/main/java/com/cloud/event/dao/EventJoinDaoImpl.java index b4316051e439..00ec0b9faf15 100644 --- a/server/src/main/java/com/cloud/event/dao/EventJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/event/dao/EventJoinDaoImpl.java @@ -106,6 +106,7 @@ public EventResponse newEventResponse(EventJoinVO event) { responseEvent.setId(event.getUuid()); responseEvent.setLevel(event.getLevel()); responseEvent.setParentId(event.getStartUuid()); + responseEvent.setJobId(event.getAsyncJobUuid()); responseEvent.setState(event.getState()); responseEvent.setUsername(event.getUserName()); if (event.getArchived()) { diff --git a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index 4b57d5a38b8f..b9fa3f0ebae7 100644 --- a/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -29,7 +29,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.direct.download.DirectDownloadManager; @@ -487,7 +486,9 @@ public DeleteHostAnswer deleteHost(HostVO host, boolean isForced, boolean isForc try { ShutdownCommand cmd = new ShutdownCommand(ShutdownCommand.DeleteHost, null, !ADD_HOST_ON_SERVICE_RESTART_KVM.value()); agentMgr.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException e) { + logger.warn("Sending ShutdownCommand failed: ", e); + } catch (OperationTimedoutException e) { logger.warn("Sending ShutdownCommand failed: ", e); } diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index 406c5f507b1f..56534a8ee7df 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -28,7 +28,6 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.utils.validation.ChecksumUtil; @@ -209,9 +208,6 @@ public boolean sendCommandsToRouter(final VirtualRouter router, final Commands c } catch (final OperationTimedoutException e) { logger.warn("Timed Out", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); - } catch (final OperationCancelledException e) { - logger.warn("Operation Cancelled", e); - throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } if (answers == null || answers.length != cmds.size()) { diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index ee9cc02e4b5c..49c2243d1ecf 100644 --- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -28,7 +28,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import com.cloud.network.dao.NetworkDao; import com.cloud.network.vpc.dao.VpcDao; import org.apache.cloudstack.agent.routing.ManageServiceCommand; @@ -931,7 +930,7 @@ public boolean startRemoteAccessVpn(final RemoteAccessVpn vpn, final VirtualRout try { _agentMgr.send(router.getHostId(), cmds); - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (final OperationTimedoutException e) { logger.debug("Failed to start remote access VPN: ", e); throw new AgentUnavailableException("Unable to send commands to virtual router ", router.getHostId(), e); } diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index e621dd3b3a13..585b65aa4d94 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -37,7 +37,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import com.cloud.host.dao.HostDao; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.command.user.securitygroup.AuthorizeSecurityGroupEgressCmd; @@ -573,7 +572,9 @@ protected void handleVmMigrated(VMInstanceVO vm) { cmds = new Commands(nrc); try { _agentMgr.send(vm.getHostId(), cmds); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException e) { + logger.debug(e.toString()); + } catch (OperationTimedoutException e) { logger.debug(e.toString()); } @@ -1479,7 +1480,9 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool cmds = new Commands(cmd); try { _agentMgr.send(vm.getHostId(), cmds); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException e) { + logger.debug(e.toString()); + } catch (OperationTimedoutException e) { logger.debug(e.toString()); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a0c5774e8757..9bbbdc503508 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -67,7 +67,6 @@ import javax.xml.parsers.ParserConfigurationException; import com.cloud.agent.api.StartCommand; -import com.cloud.exception.OperationCancelledException; import com.cloud.network.NetworkService; import com.cloud.vm.snapshot.dao.VMSnapshotDetailsDao; import org.apache.cloudstack.acl.ControlledEntity; @@ -856,8 +855,6 @@ protected void runInContext() { } } catch (OperationTimedoutException e) { logger.warn("Timed Out", e); - } catch (OperationCancelledException e) { - logger.warn("Operation Cancelled", e); } catch (AgentUnavailableException e) { logger.warn("Agent Unavailable ", e); } finally { @@ -2743,9 +2740,6 @@ public boolean expunge(UserVmVO vm) { } catch (OperationTimedoutException e) { logger.warn("Operation time out on expunging " + vm, e); return false; - } catch (OperationCancelledException e) { - logger.warn("Operation cancelled on expunging " + vm, e); - return false; } catch (ConcurrentOperationException e) { logger.warn("Concurrent operations on expunging " + vm, e); return false; @@ -5726,9 +5720,6 @@ public boolean setupVmForPvlan(boolean add, Long hostId, NicProfile nic) { } catch (OperationTimedoutException e) { logger.warn("Timed Out", e); return false; - } catch (OperationCancelledException e) { - logger.warn("Operation Cancelled", e); - return false; } catch (AgentUnavailableException e) { logger.warn("Agent Unavailable ", e); return false; diff --git a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java index 5d094f2f7785..73ff79301fb7 100644 --- a/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java @@ -57,7 +57,6 @@ import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.lang3.math.NumberUtils; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.ca.IssueCertificateCmd; @@ -224,7 +223,7 @@ public boolean provisionCertificate(final Host host, final Boolean reconnect, fi } final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); - } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (final AgentUnavailableException | OperationTimedoutException e) { logger.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent {}, due to: ", host, e); throw new CloudRuntimeException(String.format("Failed to generate keystore and get CSR from the host/agent %s", host)); } @@ -366,7 +365,7 @@ private boolean provisionSystemVmViaSsh(Host host, Boolean reconnect, String caP } @Override - public String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException { final SetupKeyStoreCommand cmd = new SetupKeyStoreCommand(CertValidityPeriod.value()); if (sshAccessDetails != null && !sshAccessDetails.isEmpty()) { cmd.setAccessDetail(sshAccessDetails); @@ -378,7 +377,7 @@ public String generateKeyStoreAndCsr(final Host host, final Map @Override public boolean deployCertificate(final Host host, final Certificate certificate, final Boolean reconnect, final Map sshAccessDetails) - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { final SetupCertificateCommand cmd = new SetupCertificateCommand(certificate); if (sshAccessDetails != null && !sshAccessDetails.isEmpty()) { cmd.setAccessDetail(sshAccessDetails); From d3f7b7de3eae2d80f73ae9cb2c43f393db33d610 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 10:09:20 +0530 Subject: [PATCH 13/29] agent changes --- .../main/java/com/cloud/agent/api/Answer.java | 19 +++ .../OperationCancelledException.java | 11 +- .../java/com/cloud/vm/VirtualMachine.java | 1 - .../com/cloud/resource/ServerResource.java | 11 +- .../java/com/cloud/agent/AgentManager.java | 18 +- .../com/cloud/agent/manager/AgentAttache.java | 34 ++-- .../cloud/agent/manager/AgentManagerImpl.java | 155 +++++++++++++----- .../agent/manager/DirectAgentAttache.java | 99 ++++++----- .../cloud/vm/VirtualMachineManagerImpl.java | 92 +++++------ .../agent/manager/DirectAgentAttacheTest.java | 123 +++++++++----- 10 files changed, 350 insertions(+), 213 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/Answer.java b/api/src/main/java/com/cloud/agent/api/Answer.java index 1166c075a78e..e6df7f64b4cc 100644 --- a/api/src/main/java/com/cloud/agent/api/Answer.java +++ b/api/src/main/java/com/cloud/agent/api/Answer.java @@ -21,6 +21,9 @@ public class Answer extends Command { protected boolean result; protected String details; + // Set when the command was stopped because its async job was cancelled, so callers can tell a + // cancellation from an ordinary failure without matching on the detail text. + protected boolean cancelled; protected Answer() { this(null); @@ -47,6 +50,20 @@ public String getDetails() { return details; } + public boolean isCancelled() { + return cancelled; + } + + public void setCancelled(final boolean cancelled) { + this.cancelled = cancelled; + } + + public static Answer createCancelledAnswer(final Command command, final String details) { + Answer answer = new Answer(command, false, details); + answer.setCancelled(true); + return answer; + } + @Override public boolean executeInSequence() { return false; @@ -69,6 +86,7 @@ public boolean equals(Object o) { Answer answer = (Answer) o; if (result != answer.result) return false; + if (cancelled != answer.cancelled) return false; if (details != null ? !details.equals(answer.details) : answer.details != null) return false; return true; @@ -78,6 +96,7 @@ public boolean equals(Object o) { public int hashCode() { int result1 = super.hashCode(); result1 = 31 * result1 + (result ? 1 : 0); + result1 = 31 * result1 + (cancelled ? 1 : 0); result1 = 31 * result1 + (details != null ? details.hashCode() : 0); return result1; } diff --git a/api/src/main/java/com/cloud/exception/OperationCancelledException.java b/api/src/main/java/com/cloud/exception/OperationCancelledException.java index 8e6393b28fad..25b81d6177f1 100644 --- a/api/src/main/java/com/cloud/exception/OperationCancelledException.java +++ b/api/src/main/java/com/cloud/exception/OperationCancelledException.java @@ -18,10 +18,19 @@ import com.cloud.agent.api.Command; import com.cloud.utils.SerialVersionUID; +import com.cloud.utils.exception.CloudRuntimeException; import java.util.Arrays; -public class OperationCancelledException extends CloudException { +/** + * Thrown when an operation was stopped because its async job was cancelled. + * + * Unchecked on purpose: a cancellation is not an alternative outcome that every caller between the + * agent layer and the job layer should have to name and handle. Almost all of them would only + * re-label it as a timeout or an unavailable agent, which is exactly what hides the cancellation + * from the job result. Callers that genuinely need to distinguish it still can. + */ +public class OperationCancelledException extends CloudRuntimeException { private static final long serialVersionUID = SerialVersionUID.OperationCancelledException; long _agentId; long _seqId; diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index 46adb9032195..4b437b4bd8e9 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -106,7 +106,6 @@ public static StateMachine2 getStat s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.AgentReportStopped, State.Stopped, null)); s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.AgentReportShutdowned, State.Stopped, null)); s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.StopRequested, State.Stopping, null)); -// s_fsm.addTransition(new Transition(State.Starting, VirtualMachine.Event.OperationCancelled, State.Stopping, null)); s_fsm.addTransition(new Transition(State.Destroyed, VirtualMachine.Event.RecoveryRequested, State.Stopped, Arrays.asList(new Impact[]{Impact.USAGE}))); s_fsm.addTransition(new Transition(State.Destroyed, VirtualMachine.Event.ExpungeOperation, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Running, VirtualMachine.Event.MigrationRequested, State.Migrating, null)); diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 6787e6d40b4f..92d768a4f292 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -94,18 +94,19 @@ default boolean isAppendAgentNameToLogs() { default void processPingAnswer(PingAnswer answer) {}; /** - * Indicates whether a currently executing request sequence can be cancelled or not. - * Resources that cannot determine cancellability should return true. + * Indicates whether a currently executing request sequence can be cancelled at the resource layer. + * The default is false: a resource that has not implemented cancellation must not claim it, + * or the job is marked cancelled while the backend operation runs to completion. */ default boolean isRequestSequenceCancellable(long sequence) { - return true; + return false; } /** * Attempts to cancel a currently executing request sequence at the resource layer. - * Returning false indicates cancellation was not possible and execution should continue. + * Returns true only when the backend operation was actually stopped. */ default boolean cancelRequestSequence(long sequence) { - return true; + return false; } } diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index 6bf97758dfc8..e94d40a1bf6e 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -18,7 +18,6 @@ import java.util.Map; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Answer; @@ -34,10 +33,12 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ServerResource; +import org.apache.cloudstack.jobs.JobCancellationHandler; + /** * AgentManager manages hosts. It directly coordinates between the DAOs and the connections it manages. */ -public interface AgentManager { +public interface AgentManager extends JobCancellationHandler { ConfigKey Wait = new ConfigKey("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return", true); ConfigKey EnableKVMAutoEnableDisable = new ConfigKey<>(Boolean.class, @@ -86,7 +87,7 @@ enum TapAgentsAction { * @return an Answer */ - Answer send(Long hostId, Command cmd) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; + Answer send(Long hostId, Command cmd) throws AgentUnavailableException, OperationTimedoutException; /** * Synchronous sending a list of commands to the agent. @@ -101,9 +102,9 @@ enum TapAgentsAction { * should the agent stop execution on the first error. * @return an array of Answer */ - Answer[] send(Long hostId, Commands cmds) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; + Answer[] send(Long hostId, Commands cmds) throws AgentUnavailableException, OperationTimedoutException; - Answer[] send(Long hostId, Commands cmds, int timeout) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; + Answer[] send(Long hostId, Commands cmds, int timeout) throws AgentUnavailableException, OperationTimedoutException; /** * Asynchronous sending of a command to the agent. @@ -181,4 +182,11 @@ enum TapAgentsAction { int getHostSshPort(HostVO host); Long getAsyncJobId(); + + /** + * True when the given job has been cancelled. Answered from an in-memory view refreshed by the + * cancelled-jobs poller, so it is cheap enough to consult on the command path. + */ + boolean isJobCancelled(Long jobId); + } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 63253e8bcb7a..9a99771dd636 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -39,8 +39,6 @@ import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.agent.lb.SetupMSListCommand; import org.apache.cloudstack.command.ReconcileAnswer; -import org.apache.cloudstack.framework.jobs.AsyncJob; -import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.logging.log4j.Logger; @@ -416,16 +414,16 @@ public void send(final Request req, final Listener listener) throws AgentUnavail } } - public Answer[] send(final Request req, final int wait) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public Answer[] send(final Request req, final int wait) throws AgentUnavailableException, OperationTimedoutException { SynchronousListener sl = new SynchronousListener(null); - Long jobId = _agentMgr.getAsyncJobId(); + final Long jobId = _agentMgr.getAsyncJobId(); long seq = req.getSequence(); send(req, sl); try { for (int i = 0; i < 2; i++) { Answer[] answers = null; - if (_agentMgr._asyncJobDao.isJobCancelled(jobId) && isExecutionCancellable(seq)) { + if (isJobCancelled(jobId) && isExecutionCancellable(seq)) { throw new OperationCancelledException(req.getCommands(), _id, seq, wait, false); } Command[] cmds = req.getCommands(); @@ -437,18 +435,16 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE try { answers = sl.waitFor(wait); } catch (final InterruptedException e) { - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted while waiting for job commands processing"); + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted while waiting for the answer"); Thread.currentThread().interrupt(); - if (_agentMgr._asyncJobDao.isJobCancelled(jobId) && isExecutionCancellable(seq)) { - throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, "Cancelled during waiting for job commands processing"); + if (isJobCancelled(jobId) && isExecutionCancellable(seq)) { + throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, "Cancelled while waiting for the answer"); } } } if (answers != null) { for (Answer answer : answers) { - if (answer != null && !answer.getResult() && - answer.getDetails() != null && - answer.getDetails().contains("cancelled")) { + if (answer != null && answer.isCancelled()) { throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, answer.getDetails()); } } @@ -486,7 +482,7 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE } throw e; } catch (OperationCancelledException e) { - logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Operation cancelled: " + req.toString()); + logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Cancelled: " + req.toString()); cancel(seq); final Long current = _currentSequence; if (req.executeInSequence() && (current != null && current == seq)) { @@ -507,18 +503,8 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE } } - private Long getAsyncJobId() { - Long jobId = null; - final AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrent(); - if (context != null && context.getJob() != null) { - AsyncJob job = context.getJob(); - if (job.getRelated() != null && !job.getRelated().isEmpty()) { - jobId = Long.parseLong(job.getRelated()); - } else { - jobId = job.getId(); - } - } - return jobId; + private boolean isJobCancelled(final Long jobId) { + return jobId != null && _agentMgr.isJobCancelled(jobId); } private Answer[] waitForAnswerOfReconcileCommand(SynchronousListener sl, final long seq, final Command command, final int wait) { diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index b7c40c1e78c7..129251ab3789 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -43,7 +43,6 @@ import javax.naming.ConfigurationException; import com.cloud.exception.OperationCancelledException; -import com.cloud.utils.DateUtil; import com.cloud.utils.StringUtils; import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; @@ -57,7 +56,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; -import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.maintenance.ManagementServerMaintenanceListener; import org.apache.cloudstack.maintenance.ManagementServerMaintenanceManager; @@ -66,6 +65,7 @@ import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; @@ -165,7 +165,14 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected List _loadingAgents = new ArrayList<>(); protected Map _commandTimeouts = new HashMap<>(); private int _monitorId = 0; - protected Map> _jobToHostIdAndReqSequenceMap = new HashMap<>(); + // A job can have several commands in flight, on more than one host, so this maps to a set. + // Written from every worker thread in send() and read by the cancelled-jobs poller. + protected final Map>> _jobToHostIdAndReqSequenceMap = new ConcurrentHashMap<>(); + // Jobs known to be cancelled, keyed by id with the time first seen. Deliberately sticky rather + // than a mirror of the database: a worker thread keeps consulting this after the job row has + // been finalised, and the row then no longer shows up as cancelled-and-unacknowledged. + protected final Map _cancelledJobs = new ConcurrentHashMap<>(); + private static final long CANCELLED_JOB_MEMORY_MS = TimeUnit.HOURS.toMillis(1); @Inject protected CAManager caService; @@ -190,7 +197,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl @Inject protected ClusterDao _clusterDao = null; @Inject - public AsyncJobDao _asyncJobDao = null; + protected AsyncJobManager asyncJobManager = null; @Inject protected HighAvailabilityManager _haMgr = null; @@ -432,7 +439,7 @@ public void onManagementServerCancelMaintenance() { if (_monitorExecutor.isShutdown()) { initAndScheduleMonitorExecutor(); } - if (_connectExecutor.isShutdown()) { + if (_cancelledJobsCheckExecutor.isShutdown()) { initAndScheduleCancelJobExecutor(); } if (newAgentConnectionsMonitor.isShutdown()) { @@ -528,7 +535,7 @@ public Answer sendTo(final Long dcId, final HypervisorType type, final Command c } @Override - public Answer send(final Long hostId, final Command cmd) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public Answer send(final Long hostId, final Command cmd) throws AgentUnavailableException, OperationTimedoutException { final Commands cmds = new Commands(Command.OnError.Stop); cmds.addCommand(cmd); send(hostId, cmds, cmd.getWait()); @@ -650,15 +657,15 @@ private Map getCommandTimeoutsMap(String commandWaits) { } @Override - public Answer[] send(final Long hostId, final Commands commands, int timeout) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public Answer[] send(final Long hostId, final Commands commands, int timeout) throws AgentUnavailableException, OperationTimedoutException { assert hostId != null : "Who's not checking the agent id before sending? ... (finger wagging)"; if (hostId == null) { throw new AgentUnavailableException(-1); } - Long jobId = getAsyncJobId(); - if (jobId != null && _asyncJobDao.isJobCancelled(jobId)) { - logger.debug("job-{} for host: {}, with commands: {} is cancelled", jobId, hostId, commands.toString()); + final Long jobId = getAsyncJobId(); + if (jobId != null && isJobCancelled(jobId)) { + logger.debug("job-{} for host: {}, with commands: {} is cancelled", jobId, hostId, commands); throw new OperationCancelledException(commands.toCommands(), hostId, 0, 0, false); } @@ -689,13 +696,14 @@ public Answer[] send(final Long hostId, final Commands commands, int timeout) th throw new AgentUnavailableException("agent not logged into this management server", hostId); } - try { - final Request req = new Request(hostId, agent.getName(), _nodeId, cmds, commands.stopOnError(), true); - req.setSequence(agent.getNextSequence()); - if (jobId != null) { - _jobToHostIdAndReqSequenceMap.put(jobId, new Pair<>(hostId, req.getSequence())); - } + final Request req = new Request(hostId, agent.getName(), _nodeId, cmds, commands.stopOnError(), true); + req.setSequence(agent.getNextSequence()); + final Pair inFlight = new Pair<>(hostId, req.getSequence()); + if (jobId != null) { + _jobToHostIdAndReqSequenceMap.computeIfAbsent(jobId, k -> ConcurrentHashMap.newKeySet()).add(inFlight); + } + try { reconcileCommandService.persistReconcileCommands(hostId, req.getSequence(), cmds); final Answer[] answers = agent.send(req, wait); @@ -707,7 +715,10 @@ public Answer[] send(final Long hostId, final Commands commands, int timeout) th return answers; } finally { if (jobId != null) { - _jobToHostIdAndReqSequenceMap.remove(jobId); + _jobToHostIdAndReqSequenceMap.computeIfPresent(jobId, (k, v) -> { + v.remove(inFlight); + return v.isEmpty() ? null : v; + }); } } } @@ -1334,7 +1345,7 @@ public Answer easySend(final Long hostId, final Command cmd) { } @Override - public Answer[] send(final Long hostId, final Commands cmds) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public Answer[] send(final Long hostId, final Commands cmds) throws AgentUnavailableException, OperationTimedoutException { int wait = 0; if (cmds.size() > 1) { logger.debug("Checking the wait time in seconds to be used for the following commands : {}. If there are multiple commands sent at once," + @@ -2101,36 +2112,104 @@ protected List findAgentsBehindOnPing() { } } + /** + * Refreshes the local view of cancelled jobs and stops whatever they still have in flight. + * + * Scoped to jobs executing on this management server: another server holds the agent connection + * for its own jobs, and acting on them here would race it for work we cannot reach. + */ protected class CancelledJobsCheckTask extends ManagedContextRunnable { @Override protected void runInContext() { -// logger.info("Started cancelled jobs check task."); - final List jobs = _asyncJobDao.getCancelledJobs(); - for (final AsyncJobVO job : jobs) { - logger.info("Job-{} with instance type: {}, id: {} cancelled.", job.getId(), job.getInstanceType(), job.getInstanceId()); - Pair hostAndSequence = _jobToHostIdAndReqSequenceMap.get(job.getId()); - if (hostAndSequence != null) { - try { - final AgentAttache agent = getAttache(hostAndSequence.first()); - if (agent.isExecutionCancellable(hostAndSequence.second())) { - logger.debug("Job-{} cancellation requested, sending cancel command to agent {} for sequence {}", job.getId(), hostAndSequence.first(), hostAndSequence.second()); - agent.cancel(hostAndSequence.second()); - } else { - logger.info("Job {} cancellation requested but sequence {} on host {} is not cancellable, allowing execution to continue.", - job.getId(), hostAndSequence.second(), hostAndSequence.first()); + try { + final long now = System.currentTimeMillis(); + _cancelledJobs.values().removeIf(firstSeen -> now - firstSeen > CANCELLED_JOB_MEMORY_MS); + + for (final AsyncJobVO job : asyncJobManager.listCancelledJobsExecutingOn(_nodeId)) { + _cancelledJobs.putIfAbsent(job.getId(), now); + if (_jobToHostIdAndReqSequenceMap.containsKey(job.getId())) { + logger.info("Job-{} on {} {} was cancelled, stopping its in-flight commands", + job.getId(), job.getInstanceType(), job.getInstanceId()); + if (!cancelJobExecution(job.getId(), "Job was cancelled")) { + logger.warn("Not every in-flight command of cancelled job-{} could be stopped; the rest will run to completion", job.getId()); } - } catch (AgentUnavailableException e) { - logger.debug("Agent {} not found", hostAndSequence.first()); } - } else { - logger.debug("Job-{} host and sequence not found", job.getId()); + asyncJobManager.finalizeCancelledJob(job.getId()); } - job.setRemoved(DateUtil.currentGMTTime()); - _asyncJobDao.update(job.getId(), job); + } catch (final Throwable e) { + logger.error("Unexpected exception in the cancelled jobs check task", e); } } } + @Override + public boolean isJobCancelled(final Long jobId) { + return jobId != null && _cancelledJobs.containsKey(jobId); + } + + @Override + public boolean isJobExecutionCancellable(final long jobId) { + final Set> inFlight = _jobToHostIdAndReqSequenceMap.get(jobId); + if (CollectionUtils.isEmpty(inFlight)) { + // Nothing is running on an agent for this job, so cancelling it strands no backend work. + return true; + } + + for (final Pair request : inFlight) { + final AgentAttache attache = findAttache(request.first()); + if (attache == null) { + logger.debug("Agent {} for job-{} is no longer attached, treating sequence {} as not cancellable", + request.first(), jobId, request.second()); + return false; + } + if (!attache.isExecutionCancellable(request.second())) { + logger.info("Job-{} cannot be cancelled: sequence {} on host {} is not cancellable", + jobId, request.second(), request.first()); + return false; + } + } + return true; + } + + @Override + public boolean cancelJobExecution(final long jobId, final String reason) { + // Mark it before touching the agents, so a worker thread that catches the interruption and + // carries on to its next command is refused there. + _cancelledJobs.putIfAbsent(jobId, System.currentTimeMillis()); + final Set> inFlight = _jobToHostIdAndReqSequenceMap.get(jobId); + if (CollectionUtils.isEmpty(inFlight)) { + return true; + } + + boolean allCancelled = true; + for (final Pair request : inFlight) { + final AgentAttache attache = findAttache(request.first()); + if (attache == null) { + logger.debug("Agent {} for job-{} is no longer attached, cannot cancel sequence {}", + request.first(), jobId, request.second()); + allCancelled = false; + continue; + } + if (!attache.isExecutionCancellable(request.second())) { + logger.info("Not cancelling sequence {} on host {} for job-{}: it is not cancellable, letting it run to completion", + request.second(), request.first(), jobId); + allCancelled = false; + continue; + } + logger.debug("Cancelling sequence {} on host {} for job-{} ({})", request.second(), request.first(), jobId, reason); + attache.cancel(request.second()); + } + return allCancelled; + } + + private AgentAttache findAttache(final Long hostId) { + try { + return getAttache(hostId); + } catch (final AgentUnavailableException e) { + return null; + } + } + protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable { @Override protected void runInContext() { diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index 0f5487bb7cd0..f915b715c511 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -102,65 +102,82 @@ public void disconnect(Status state) { cleanup(state); } - @Override - protected synchronized boolean isExecutionCancellable(final long seq) { - if (_taskFutures.containsKey(seq)) { - final ServerResource resource = _resource; - return resource == null || resource.isRequestSequenceCancellable(seq); + private synchronized boolean isQueued(final long seq) { + for (Task task : tasks) { + if (task._req.getSequence() == seq) { + return true; + } } + return false; + } - if (_taskRequests.containsKey(seq)) { + @Override + protected boolean isExecutionCancellable(final long seq) { + // Still queued: it has not reached the resource, so dropping it strands nothing. + if (isQueued(seq)) { return true; } - for (Task task : tasks) { - if (task._req.getSequence() == seq) { - return true; - } + if (!_taskFutures.containsKey(seq)) { + // Neither queued nor running here. We have nothing to stop and no way to know what the + // backend is doing, so do not claim it can be cancelled. + return false; } - return true; + + final ServerResource resource = _resource; + // Deliberately not holding the attache lock: asking the resource means a call out to the + // hypervisor, and the send path needs the lock to keep queueing work. + return resource != null && resource.isRequestSequenceCancellable(seq); } @Override - protected synchronized void cancel(final long seq) { - Request request = _taskRequests.get(seq); + protected void cancel(final long seq) { + if (removeQueuedTask(seq)) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelled queued task."); + super.cancel(seq); + return; + } + + final Future future = _taskFutures.get(seq); + if (future == null) { + super.cancel(seq); + return; + } + + final ServerResource resource = _resource; + if (resource == null || !resource.isRequestSequenceCancellable(seq)) { + logger.info(LOG_SEQ_FORMATTED_STRING, seq, "Cancellation requested but the command is not cancellable, letting it run to completion."); + return; + } + + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelling request sequence at the resource."); + if (!resource.cancelRequestSequence(seq)) { + logger.info(LOG_SEQ_FORMATTED_STRING, seq, "Resource could not cancel the command, letting it run to completion."); + return; + } + + final Request request = _taskRequests.get(seq); if (request != null) { request.cancel(); - _taskRequests.remove(seq); - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Removed task request."); } + final boolean cancelled = future.cancel(true); + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Running task " + (cancelled ? "cancelled" : "not cancelled")); + super.cancel(seq); + } + + private synchronized boolean removeQueuedTask(final long seq) { final Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { final Task task = iterator.next(); if (task._req.getSequence() == seq) { task._req.cancel(); iterator.remove(); - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelled queued task."); - super.cancel(seq); - return; - } - } - - final Future future = _taskFutures.get(seq); - if (future != null) { - final ServerResource resource = _resource; - if (resource != null && !resource.isRequestSequenceCancellable(seq)) { - logger.info(LOG_SEQ_FORMATTED_STRING, seq, "Cancellation requested but command is not cancellable, continuing execution."); - return; - } - - if (resource != null) { - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancel request sequence."); - resource.cancelRequestSequence(seq); + _taskRequests.remove(seq); + return true; } - - final boolean cancelled = future.cancel(true); - logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Running task " + (cancelled ? "cancelled" : "not cancelled")); - _taskFutures.remove(seq); } - - super.cancel(seq); + return false; } @Override @@ -397,7 +414,7 @@ protected void runInContext() { for (int i = 0; i < cmds.length; i++) { if (Thread.currentThread().isInterrupted() || _req.isCancelled()) { for (int j = i; j < cmds.length; j++) { - answers.add(new Answer(cmds[j], false, "Command cancelled")); + answers.add(Answer.createCancelledAnswer(cmds[j], "Command cancelled")); } break; } @@ -421,7 +438,7 @@ protected void runInContext() { && resource.isRequestSequenceCancellable(seq)) { answers.add(answer); for (int j = i + 1; j < cmds.length; j++) { - answers.add(new Answer(cmds[j], false, "Command cancelled")); + answers.add(Answer.createCancelledAnswer(cmds[j], "Command cancelled")); } break; } @@ -467,7 +484,7 @@ private void handleCancellation(long seq, String reason) { Command[] cmds = _req.getCommands(); ArrayList answers = new ArrayList<>(cmds.length); for (Command cmd : cmds) { - answers.add(new Answer(cmd, false, reason != null ? reason : "Task cancelled")); + answers.add(Answer.createCancelledAnswer(cmd, reason != null ? reason : "Task cancelled")); } Response resp = new Response(_req, answers.toArray(new Answer[answers.size()])); processAnswers(seq, resp); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 74f516a1192e..f54ddd879e12 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -51,7 +51,6 @@ import javax.persistence.EntityExistsException; import com.cloud.agent.api.PostMigrationCommand; -import com.cloud.exception.OperationCancelledException; import com.cloud.storage.clvm.ClvmPoolManager; import com.cloud.hypervisor.KVMGuru; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -211,6 +210,7 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; @@ -675,15 +675,13 @@ public void expunge(final String vmUuid) throws ResourceUnavailableException { advanceExpunge(vmUuid); } catch (final OperationTimedoutException e) { throw new CloudRuntimeException("Operation timed out", e); - } catch (final OperationCancelledException e) { - throw new CloudRuntimeException("Operation cancelled", e); } catch (final ConcurrentOperationException e) { throw new CloudRuntimeException("Concurrent operation ", e); } } @Override - public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException, OperationCancelledException { + public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); advanceExpunge(vm); } @@ -701,7 +699,7 @@ private boolean isVmDestroyed(VMInstanceVO vm) { return false; } - protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException, OperationCancelledException { + protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException { if (isVmDestroyed(vm)) { return; } @@ -787,7 +785,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti } private void handleUnsuccessfulExpungeOperation(List finalizeExpungeCommands, List nicExpungeCommands, - VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException { if ((CollectionUtils.isNotEmpty(finalizeExpungeCommands) || CollectionUtils.isNotEmpty(nicExpungeCommands)) && hostId != null) { final Commands cmds = new Commands(Command.OnError.Stop); addAllExpungeCommandsFromList(finalizeExpungeCommands, cmds, vm); @@ -1141,7 +1139,7 @@ public void advanceStart(final String vmUuid, final Map sshAccessDetails, final VirtualMachine vm) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + private void setupAgentSecurity(final Host vmHost, final Map sshAccessDetails, final VirtualMachine vm) throws AgentUnavailableException, OperationTimedoutException { final String csr = caManager.generateKeyStoreAndCsr(vmHost, sshAccessDetails); if (org.apache.commons.lang3.StringUtils.isNotEmpty(csr)) { final Map ipAddressDetails = new HashMap<>(sshAccessDetails); @@ -1332,7 +1330,7 @@ protected void processPrepareExternalProvisioning(boolean firstStart, Host host, CloudRuntimeException cre = new CloudRuntimeException("Failed to prepare VM"); try { answer = _agentMgr.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Failed PrepareExternalProvisioningCommand due to : {}", e.getMessage(), e); throw cre; } @@ -1679,7 +1677,14 @@ public void orchestrateStart(final String vmUuid, final Map sendStop(final VirtualMachineGuru guru, final Vi logger.error(errorMsg); return new Pair<>(false, errorMsg); } - } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { String errorMsg = String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()); logger.warn(errorMsg, e); if (!force) { @@ -2489,8 +2494,7 @@ private Pair getVMNetworkDetails(NetworkVO networkVO, boolean i return null; } - private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, - ConcurrentOperationException { + private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException { final State state = vm.getState(); if (state == State.Stopped) { logger.debug("VM is already stopped: {}", vm); @@ -2612,7 +2616,7 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl throw new CloudRuntimeException("Invalid answer received in response to a StopCommand on " + vm.instanceName); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { agentExceptionDetail = e.getMessage(); logger.warn("Unable to stop {} due to [{}].", profile.toString(), e.toString(), e); } finally { @@ -2780,7 +2784,7 @@ private void deleteVMSnapshots(VMInstanceVO vm, boolean expunge) { } } - protected boolean checkVmOnHost(final VirtualMachine vm, final long hostId) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + protected boolean checkVmOnHost(final VirtualMachine vm, final long hostId) throws AgentUnavailableException, OperationTimedoutException { final Answer answer = _agentMgr.send(hostId, new CheckVirtualMachineCommand(vm.getInstanceName())); if (answer == null || !answer.getResult()) { return false; @@ -2877,7 +2881,7 @@ private Answer[] attemptHypervisorMigration(VMInstanceVO vm, Map volumeToPool) throws ResourceUnavailableException, - ConcurrentOperationException { + private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); @@ -3803,8 +3802,6 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo logger.debug("Deleted config drive ISO for vm {} in host {}", vm.getInstanceName(), srcHost); } catch (OperationTimedoutException e) { logger.error("TIme out occurred while executing command AttachOrDettachConfigDrive {}", e.getMessage(), e); - } catch (OperationCancelledException e) { - logger.error("Operation cancelled while executing command AttachOrDettachConfigDrive {}", e.getMessage(), e); } } } @@ -3825,7 +3822,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo String errorDetails = (cleanupResult.second() != null) ? " due to " + cleanupResult.second() : ""; throw new CloudRuntimeException("VM not found on destination host. Unable to complete migration for " + vm + errorDetails); } - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { logger.error("Error while checking the vm {} is on host {}", vm, destHost, e); } migrated = true; @@ -3886,7 +3883,7 @@ protected void recreateCheckpointsKvmOnVmAfterMigration(VMInstanceVO vm, long ho try { logger.debug(String.format("Recreating the volume checkpoints with URLs [%s] of volumes [%s] on %s as part of the migration process.", volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm)); answer = _agentMgr.send(hostId, recreateCheckpointsCommand); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error(String.format("Exception while sending command to host [%s] to recreate checkpoints with URLs [%s] of volumes [%s] on %s due to: [%s].", hostId, volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm, e.getMessage()), e); throw new CloudRuntimeException(e); } finally { @@ -4152,8 +4149,7 @@ public void advanceReboot(final String vmUuid, final Map params) throws ConcurrentOperationException, - ResourceUnavailableException { + private void orchestrateReboot(final String vmUuid, final Map params) throws ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (_vmSnapshotMgr.hasActiveVMSnapshotTasks(vm.getId())) { logger.error("Unable to reboot Instance: {} due to: {} has active Instance Snapshot tasks", vm, vm.getInstanceName()); @@ -4195,7 +4191,7 @@ private void orchestrateReboot(final String vmUuid, final Map customParameters, final ExcludeList excludes) throws InsufficientCapacityException, ConcurrentOperationException, - ResourceUnavailableException { + public void findHostAndMigrate(final String vmUuid, final Long newSvcOfferingId, final Map customParameters, final ExcludeList excludes) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); if (vm == null) { @@ -5037,8 +5031,6 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI } } catch (final OperationTimedoutException e1) { throw new AgentUnavailableException("Operation timed out ", dstHostId); - } catch (final OperationCancelledException e1) { - throw new AgentUnavailableException("Operation cancelled ", dstHostId); } finally { if (pfma == null) { work.setStep(Step.Done); @@ -5076,9 +5068,6 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI _haMgr.scheduleRestart(vm, true); } throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId, e); - } catch (final OperationCancelledException e) { - _haMgr.scheduleRestart(vm, true); - throw new AgentUnavailableException("Operation cancelled on migrating " + vm, dstHostId, e); } try { @@ -5104,7 +5093,7 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI String errorDetails = (cleanupResult.second() != null) ? " due to " + cleanupResult.second() : ""; throw new CloudRuntimeException("Unable to complete migration for " + vm + errorDetails); } - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { logger.debug("Error while checking the {} on {}", vm, dstHost, e); } @@ -5141,8 +5130,7 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI } @Override - public boolean replugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final Host host) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { + public boolean replugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final Host host) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { boolean result = true; final VMInstanceVO router = _vmDao.findById(vm.getId()); @@ -5157,7 +5145,7 @@ public boolean replugNic(final Network network, final NicTO nic, final VirtualMa logger.warn("Unable to replug nic for vm {}", vm.getName()); result = false; } - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, host.getId(), e); } } else { @@ -5170,8 +5158,7 @@ public boolean replugNic(final Network network, final NicTO nic, final VirtualMa return result; } - public boolean plugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { + public boolean plugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { boolean result = true; final VMInstanceVO router = _vmDao.findById(vm.getId()); @@ -5193,7 +5180,7 @@ public boolean plugNic(final Network network, final NicTO nic, final VirtualMach logger.warn("Unable to plug nic for vm {}", vm.getName()); result = false; } - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, dest.getHost().getId(), e); } } else { @@ -5207,8 +5194,7 @@ public boolean plugNic(final Network network, final NicTO nic, final VirtualMach return result; } - public boolean unplugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, - ResourceUnavailableException { + public boolean unplugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException { boolean result = true; final VMInstanceVO router = _vmDao.findById(vm.getId()); @@ -5233,7 +5219,7 @@ public boolean unplugNic(final Network network, final NicTO nic, final VirtualMa logger.warn("Unable to unplug nic from router {}", router); result = false; } - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { throw new AgentUnavailableException("Unable to unplug nic from rotuer " + router + " from network " + network, dest.getHost().getId(), e); } } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) { @@ -5356,8 +5342,6 @@ private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering old } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Operation timed out on reconfiguring " + vm, dstHostId); - } catch (final OperationCancelledException e) { - throw new AgentUnavailableException("Operation cancelled on reconfiguring " + vm, dstHostId); } catch (final AgentUnavailableException e) { throw e; } @@ -6464,7 +6448,7 @@ private boolean orchestrateUpdateVmNic(final VirtualMachine vm, final Nic nic, f logger.warn("Unable to update VM %s NIC [{}].", vm.getName(), nic.getUuid()); return false; } - } catch (final OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { throw new AgentUnavailableException(String.format("Unable to update NIC %s for VM %s.", nic.getUuid(), vm.getUuid()), vm.getHostId(), e); } } @@ -6582,7 +6566,7 @@ private void executePreMigrationCommand(VMInstanceVO vm, VirtualMachineTO to, lo throw new CloudRuntimeException(msg); } logger.info("Successfully prepared source host {} for migration of VM {}", srcHostUuid, vmInstanceName); - } catch (final AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Failed to send PreMigrationCommand to source host {}: {}", srcHostUuid, e.getMessage(), e); throw new CloudRuntimeException("Failed to prepare source host for migration: " + e.getMessage(), e); } diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java index 0049deebc8cb..7b3cc8ce7264 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java @@ -17,10 +17,10 @@ package com.cloud.agent.manager; import java.util.UUID; -//import java.util.concurrent.Future; -// -//import com.cloud.agent.api.Command; -//import com.cloud.agent.transport.Request; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; + +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -28,6 +28,8 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.agent.api.CheckHealthCommand; +import com.cloud.agent.transport.Request; import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ServerResource; @@ -39,61 +41,94 @@ public class DirectAgentAttacheTest { @Mock private ServerResource _resource; -// @Mock -// private ExecutorService directAgentPool; + @Mock + private ScheduledExecutorService directAgentPool; + + @Mock + private Future runningTask; long _id = 0L; String _uuid = UUID.randomUUID().toString(); + private DirectAgentAttache directAgentAttache; + @Before public void setup() { -// Mockito.when(_agentMgr.getDirectAgentPool()).thenReturn(directAgentPool); - Mockito.doReturn(2).when(_agentMgr).getDirectAgentThreadCap(); directAgentAttache = new DirectAgentAttache(_agentMgr, _id, _uuid, "myDirectAgentAttache", Hypervisor.HypervisorType.KVM, _resource, false); } - private DirectAgentAttache directAgentAttache; @Test public void testPingTask() throws Exception { DirectAgentAttache.PingTask pt = directAgentAttache.new PingTask(); + Mockito.doReturn(2).when(_agentMgr).getDirectAgentThreadCap(); pt.runInContext(); Mockito.verify(_resource, Mockito.times(1)).getCurrentStatus(_id); } -// @Test -// public void testCancelRunningTaskCancelsFutureAndRequest() throws Exception { -// final Command command = Mockito.mock(Command.class); -// final Request request = new Request(_id, -1, command, false); -// final long seq = 101L; -// request.setSequence(seq); -// -// @SuppressWarnings("unchecked") -// final Future runningFuture = Mockito.mock(Future.class); -// Mockito.when(directAgentPool.submit(Mockito.any(Runnable.class))).thenReturn(runningFuture); -// -// directAgentAttache.send(request); -// directAgentAttache.cancel(seq); -// -// Mockito.verify(runningFuture, Mockito.times(1)).cancel(true); -// org.junit.Assert.assertTrue(request.isCancelled()); -// } -// -// @Test -// public void testCancelQueuedTaskMarksRequestCancelledAndRemovesItFromQueue() throws Exception { -// final Command command = Mockito.mock(Command.class); -// final Request request = new Request(_id, -1, command, false); -// final long seq = 202L; -// request.setSequence(seq); -// -// Mockito.doReturn(0).when(_agentMgr).getDirectAgentThreadCap(); -// -// directAgentAttache.send(request); -// org.junit.Assert.assertEquals(1, directAgentAttache.tasks.size()); -// -// directAgentAttache.cancel(seq); -// -// org.junit.Assert.assertTrue(request.isCancelled()); -// org.junit.Assert.assertEquals(0, directAgentAttache.tasks.size()); -// } + @Test + public void testCancelQueuedTaskDropsItBeforeItReachesTheResource() throws Exception { + // no free thread, so the task stays queued + Mockito.doReturn(0).when(_agentMgr).getDirectAgentThreadCap(); + final Request request = newRequest(202L); + + directAgentAttache.send(request, null); + Assert.assertEquals(1, directAgentAttache.tasks.size()); + Assert.assertTrue(directAgentAttache.isExecutionCancellable(202L)); + + directAgentAttache.cancel(202L); + + Assert.assertTrue(request.isCancelled()); + Assert.assertEquals(0, directAgentAttache.tasks.size()); + Mockito.verify(_resource, Mockito.never()).cancelRequestSequence(Mockito.anyLong()); + } + + @Test + public void testCancelRunningTaskStopsTheResourceThenTheThread() throws Exception { + final Request request = submitRunning(101L); + Mockito.doReturn(true).when(_resource).isRequestSequenceCancellable(101L); + Mockito.doReturn(true).when(_resource).cancelRequestSequence(101L); + + Assert.assertTrue(directAgentAttache.isExecutionCancellable(101L)); + directAgentAttache.cancel(101L); + + Mockito.verify(_resource).cancelRequestSequence(101L); + Mockito.verify(runningTask).cancel(true); + Assert.assertTrue(request.isCancelled()); + } + + @Test + public void testCancelLeavesNonCancellableRunningTaskAlone() throws Exception { + // a resource that cannot stop the backend work must not have its thread interrupted either, + // or the job is recorded as cancelled while the hypervisor finishes the operation + final Request request = submitRunning(303L); + Mockito.doReturn(false).when(_resource).isRequestSequenceCancellable(303L); + + Assert.assertFalse(directAgentAttache.isExecutionCancellable(303L)); + directAgentAttache.cancel(303L); + + Mockito.verify(_resource, Mockito.never()).cancelRequestSequence(Mockito.anyLong()); + Mockito.verify(runningTask, Mockito.never()).cancel(Mockito.anyBoolean()); + Assert.assertFalse(request.isCancelled()); + } + + @Test + public void testUnknownSequenceIsNotClaimedCancellable() { + Assert.assertFalse(directAgentAttache.isExecutionCancellable(999L)); + } + + private Request newRequest(final long seq) { + final Request request = new Request(_id, -1, new CheckHealthCommand(), false); + request.setSequence(seq); + return request; + } + + private Request submitRunning(final long seq) throws Exception { + Mockito.doReturn(2).when(_agentMgr).getDirectAgentThreadCap(); + Mockito.doReturn(directAgentPool).when(_agentMgr).getDirectAgentPool(); + Mockito.doReturn(runningTask).when(directAgentPool).submit(Mockito.any(Runnable.class)); + final Request request = newRequest(seq); + directAgentAttache.send(request, null); + return request; + } } From 9902fb9d5d012561d4f1be8574b95089d9259000 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 10:10:50 +0530 Subject: [PATCH 14/29] usage server job changes --- .../java/org/apache/cloudstack/api/ApiConstants.java | 1 + .../api/command/admin/usage/ListUsageJobsCmd.java | 2 +- .../java/com/cloud/server/ManagementServerImpl.java | 7 +------ .../main/java/com/cloud/usage/UsageServiceImpl.java | 12 ++++-------- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index f74c46161180..8d82a8034f2d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -675,6 +675,7 @@ public class ApiConstants { public static final String USAGE_ID = "usageid"; public static final String USAGE_NAME = "usagename"; public static final String USAGE_TYPE = "usagetype"; + public static final String USAGE_SERVER = "usageserver"; public static final String INCLUDE_TAGS = "includetags"; public static final String VLAN = "vlan"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java index f72f884863e3..eecbe27ebc9a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java @@ -26,7 +26,7 @@ import org.apache.cloudstack.api.response.UsageJobResponse; @APICommand(name = "listUsageJobs", description = "Lists the usage jobs.", responseObject = UsageJobResponse.class, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.23") + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0") public class ListUsageJobsCmd extends BaseListCmd { ///////////////////////////////////////////////////// diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index d3428311e3fc..cd37d7897d9c 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -45,7 +45,6 @@ import org.apache.cloudstack.acl.ApiKeyPairVO; import com.cloud.api.query.MutualExclusiveIdsManagerBase; -import com.cloud.exception.OperationCancelledException; import com.cloud.network.vpc.VpcVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; @@ -3470,7 +3469,7 @@ public Pair setConsoleAccessForVm(long vmId, String sessionUuid Answer answer; try { answer = _agentMgr.send(hostVO.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { String errorMsg = "Could not send allow session command to CPVM: " + e.getMessage(); logger.error(errorMsg, e); return new Pair<>(false, errorMsg); @@ -5962,10 +5961,6 @@ private Pair patchSystemVm(VMInstanceVO systemVM, boolean force String errMsg = "SystemVM live patch failed"; logger.error(errMsg, e); return new Pair<>(false, String.format("%s due to: %s", errMsg, e.getMessage())); - } catch (OperationCancelledException e) { - String errMsg = "SystemVM live patch cancelled"; - logger.error(errMsg, e); - return new Pair<>(false, String.format("%s due to: %s", errMsg, e.getMessage())); } logger.info(String.format("Successfully patched system VM %s", systemVM.getInstanceName())); List routerTypes = new ArrayList<>(); diff --git a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java index 204974d582b3..3d57d7831790 100644 --- a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java +++ b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java @@ -16,14 +16,12 @@ // under the License. package com.cloud.usage; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -527,11 +525,9 @@ public ListResponse getUsageJobs(ListUsageJobsCmd cmd) { } if (cmd.getDuration() != null) { - ZoneId systemZoneId = ZoneId.systemDefault(); - LocalDateTime nowLocal = LocalDateTime.now(); - ZonedDateTime nowZoned = nowLocal.atZone(systemZoneId); - ZonedDateTime lastDateTimeZoned = nowZoned.minusHours(cmd.getDuration()); - Date lastDate = Date.from(lastDateTimeZoned.toInstant()); + // Usage job timestamps are GMT; anchoring the window in the system zone would shift it + // by the offset on any management server that is not on UTC. + Date lastDate = new Date(DateUtil.currentGMTTime().getTime() - TimeUnit.HOURS.toMillis(cmd.getDuration())); SearchCriteria scc = _usageJobDao.createSearchCriteria(); scc.addOr("startDate", SearchCriteria.Op.GTEQ, lastDate); From a7b0c5210c86a4887333a463e382cae7ed1e2cd3 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 10:46:28 +0530 Subject: [PATCH 15/29] fixes for agent changes Drop the OperationCancelledException catch and throws plumbing that the exception no longer needs now that it is unchecked: every one of those handlers only relabelled a cancellation as a timeout or an unavailable agent. Pass the agent request sequence to the resource thread (RequestExecutionContext) so a resource can track its backend work by the same sequence the agent layer cancels by. --- .../org/apache/cloudstack/ca/CAManager.java | 5 +- .../resource/RequestExecutionContext.java | 45 +++ .../manager/ClusteredAgentManagerImpl.java | 5 +- .../agent/manager/DirectAgentAttache.java | 5 + .../vm/VirtualMachineManagerImplTest.java | 25 +- ...vmNonManagedStorageDataMotionStrategy.java | 3 +- .../StorageSystemDataMotionStrategy.java | 15 +- ...NonManagedStorageSystemDataMotionTest.java | 9 +- .../vmsnapshot/VMSnapshotStrategyTest.java | 7 +- .../image/BaseImageStoreDriverImpl.java | 3 +- .../manager/ExtensionsManagerImpl.java | 5 - .../manager/ExtensionsManagerImplTest.java | 9 +- .../backup/KbossBackupProvider.java | 9 +- .../backup/KbossBackupProviderTest.java | 29 +- .../cloudstack/backup/NASBackupProvider.java | 14 - .../backup/NASBackupProviderTest.java | 21 +- .../BaremetalKickStartServiceImpl.java | 3 +- .../discoverer/HypervServerDiscoverer.java | 3 - .../motion/HypervStorageMotionStrategy.java | 4 - .../com/cloud/ovm/hypervisor/OvmFencer.java | 6 - .../ovm3/resources/Ovm3FenceBuilder.java | 7 - .../java/com/cloud/ha/SimulatorFencer.java | 6 - .../com/cloud/ha/SimulatorInvestigator.java | 4 - .../vmware/resource/VmwareResource.java | 74 +--- .../motion/VmwareStorageMotionStrategy.java | 7 - .../java/com/cloud/ha/XenServerFencer.java | 6 - .../discoverer/XcpServerDiscoverer.java | 3 - .../XenServerStorageMotionStrategy.java | 7 - .../management/ManagementServerMock.java | 3 - .../network/ovs/OvsTunnelManagerImpl.java | 9 +- .../motion/LinstorDataMotionStrategy.java | 6 +- .../manager/ScaleIOSDCManagerImpl.java | 5 +- .../motion/StorPoolDataMotionStrategy.java | 6 +- .../main/java/com/cloud/api/ApiDBUtils.java | 215 ------------ .../cloud/ha/AbstractInvestigatorImpl.java | 6 - .../cloud/ha/CheckOnAgentInvestigator.java | 4 - .../src/main/java/com/cloud/ha/KVMFencer.java | 5 - .../RollingMaintenanceManagerImpl.java | 5 +- .../com/cloud/storage/StorageManagerImpl.java | 5 +- .../cloud/storage/clvm/ClvmPoolManager.java | 7 +- .../backup/KVMBackupExportServiceImpl.java | 19 +- .../ConsoleAccessManagerImpl.java | 3 +- .../download/DirectDownloadManagerImpl.java | 3 +- .../vm/UnmanagedVMsManagerImpl.java | 9 +- .../test/java/com/cloud/ha/KVMFencerTest.java | 7 +- .../network/as/AutoScaleManagerImplTest.java | 3 +- .../network/router/NetworkHelperImplTest.java | 9 +- .../cloud/storage/ClvmPoolManagerTest.java | 57 ++- .../cloud/storage/StorageManagerImplTest.java | 7 +- .../vm/UnmanagedVMsManagerImplTest.java | 21 +- .../vmware/mo/VirtualMachineMO.java | 330 ++++-------------- .../hypervisor/vmware/util/VmwareClient.java | 107 +----- .../hypervisor/vmware/util/VmwareContext.java | 10 - .../hypervisor/vmware/util/VmwareHelper.java | 182 ---------- 54 files changed, 284 insertions(+), 1098 deletions(-) create mode 100644 core/src/main/java/com/cloud/resource/RequestExecutionContext.java diff --git a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java index 607c1c28a521..d2ebdc25f1bd 100644 --- a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java +++ b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; -import com.cloud.exception.OperationCancelledException; import com.trilead.ssh2.Connection; import org.apache.cloudstack.framework.ca.CAProvider; @@ -171,7 +170,7 @@ public interface CAManager extends CAService, Configurable, PluggableService { * @throws AgentUnavailableException * @throws OperationTimedoutException */ - String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; + String generateKeyStoreAndCsr(final Host host, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException; /** * Deploys a Certificate payload to a provided host @@ -183,7 +182,7 @@ public interface CAManager extends CAService, Configurable, PluggableService { * @throws AgentUnavailableException * @throws OperationTimedoutException */ - boolean deployCertificate(final Host host, final Certificate certificate, final Boolean reconnect, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException; + boolean deployCertificate(final Host host, final Certificate certificate, final Boolean reconnect, final Map sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException; /** * Removes the host from an internal active client/certificate map diff --git a/core/src/main/java/com/cloud/resource/RequestExecutionContext.java b/core/src/main/java/com/cloud/resource/RequestExecutionContext.java new file mode 100644 index 000000000000..925a06f0f4c1 --- /dev/null +++ b/core/src/main/java/com/cloud/resource/RequestExecutionContext.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.resource; + +/** + * The agent request sequence the current thread is executing a command for. + * + * Set by the direct-agent attache around each {@link ServerResource#executeRequest} call and read by + * resources that track backend work per request, so that a cancellation aimed at a request sequence can + * find the hypervisor task that request created. Resources used to number commands with their own + * counter, which never matched the sequence the agent layer cancels by. + */ +public final class RequestExecutionContext { + + private static final ThreadLocal REQUEST_SEQUENCE = new ThreadLocal<>(); + + private RequestExecutionContext() { + } + + public static void setRequestSequence(final long sequence) { + REQUEST_SEQUENCE.set(sequence); + } + + public static Long getRequestSequence() { + return REQUEST_SEQUENCE.get(); + } + + public static void clear() { + REQUEST_SEQUENCE.remove(); + } +} diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index f61b3328a758..38a198b73040 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -42,7 +42,6 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; -import com.cloud.exception.OperationCancelledException; import com.cloud.resource.ResourceState; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.config.ConfigDepot; @@ -1237,7 +1236,7 @@ private String handleScheduleHostScanTaskCommand(final ScheduleHostScanTaskComma return _gson.toJson(answers); } - public Answer[] sendToAgent(final Long hostId, final Command[] cmds, final boolean stopOnError) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public Answer[] sendToAgent(final Long hostId, final Command[] cmds, final boolean stopOnError) throws AgentUnavailableException, OperationTimedoutException { final Commands commands = new Commands(stopOnError ? Command.OnError.Stop : Command.OnError.Continue); for (final Command cmd : cmds) { commands.addCommand(cmd); @@ -1356,8 +1355,6 @@ public String dispatch(final ClusterServicePdu pdu) { logger.warn("Agent is unavailable", e); } catch (final OperationTimedoutException e) { logger.warn("Timed Out", e); - } catch (OperationCancelledException e) { - logger.warn("Operation Cancelled", e); } return null; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index f915b715c511..bb1701943dc2 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -41,6 +41,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor; +import com.cloud.resource.RequestExecutionContext; import com.cloud.resource.ServerResource; import org.apache.logging.log4j.ThreadContext; @@ -334,6 +335,7 @@ private void bailout() { @Override protected void runInContext() { long seq = _req.getSequence(); + RequestExecutionContext.setRequestSequence(seq); try { if (_outstandingCronTaskCount.incrementAndGet() >= _agentMgr.getDirectAgentThreadCap()) { logger.warn( @@ -385,6 +387,7 @@ protected void runInContext() { } catch (Exception e) { logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Exception caught ", e); } finally { + RequestExecutionContext.clear(); _outstandingCronTaskCount.decrementAndGet(); } } @@ -400,6 +403,7 @@ public Task(Request req) { @Override protected void runInContext() { long seq = _req.getSequence(); + RequestExecutionContext.setRequestSequence(seq); try { Command[] cmds = _req.getCommands(); if (Thread.currentThread().isInterrupted() || _req.isCancelled()) { @@ -472,6 +476,7 @@ protected void runInContext() { // This is pretty serious as processAnswers might not be called and the calling process is stuck waiting for the full timeout logger.error(LOG_SEQ_FORMATTED_STRING, seq, "Throwable caught in runInContext, this will cause the management to become unpredictable", t); } finally { + RequestExecutionContext.clear(); _taskFutures.remove(seq); _taskRequests.remove(seq); _outstandingTaskCount.decrementAndGet(); diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 215ceff55f06..3b680f0bc74a 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -57,7 +57,6 @@ import com.cloud.api.ApiDBUtils; import com.cloud.event.ActionEventUtils; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.OperationCancelledException; import com.cloud.ha.HighAvailabilityManager; import com.cloud.network.Network; import com.cloud.network.NetworkModel; @@ -1406,7 +1405,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfNotKvm() { } @Test - public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfVolumesDoNotHaveCheckpoints() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfVolumesDoNotHaveCheckpoints() throws OperationTimedoutException, AgentUnavailableException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(new ArrayList()).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1416,7 +1415,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfVolumesDoNotHave } @Test (expected = CloudRuntimeException.class) - public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1429,7 +1428,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCl } @Test (expected = CloudRuntimeException.class) - public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExceptionThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExceptionThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1442,7 +1441,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExcepti } @Test - public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws OperationTimedoutException, AgentUnavailableException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1455,7 +1454,7 @@ public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws } @Test - public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationSucceeds() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationSucceeds() throws OperationTimedoutException, AgentUnavailableException { Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); @@ -1668,7 +1667,7 @@ public void updateExternalVmNicsFromPrepareAnswer_emptyNics_noAction() { } @Test - public void processPrepareExternalProvisioning_nonExternalHypervisor_noAction() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void processPrepareExternalProvisioning_nonExternalHypervisor_noAction() throws OperationTimedoutException, AgentUnavailableException { Host host = mock(Host.class); VirtualMachineProfile vmProfile = mock(VirtualMachineProfile.class); VirtualMachineTemplate template = mock(VirtualMachineTemplate.class); @@ -1679,7 +1678,7 @@ public void processPrepareExternalProvisioning_nonExternalHypervisor_noAction() } @Test - public void processPrepareExternalProvisioning_externalHypervisor_sendsCommand() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void processPrepareExternalProvisioning_externalHypervisor_sendsCommand() throws OperationTimedoutException, AgentUnavailableException { Host host = mock(Host.class); VirtualMachineProfile vmProfile = mock(VirtualMachineProfile.class); VirtualMachineTemplate template = mock(VirtualMachineTemplate.class); @@ -1788,7 +1787,7 @@ public void testPrepVMSpecForUnmanageInstance() { } @Test - public void testPersistDomainForKvmForRunningVmSuccess() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testPersistDomainForKvmForRunningVmSuccess() throws AgentUnavailableException, OperationTimedoutException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); UnmanageInstanceAnswer successAnswer = new UnmanageInstanceAnswer(null, true, "success"); @@ -1801,7 +1800,7 @@ public void testPersistDomainForKvmForRunningVmSuccess() throws AgentUnavailable } @Test - public void testPersistDomainForKvmForStoppedVmSuccess() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testPersistDomainForKvmForStoppedVmSuccess() throws AgentUnavailableException, OperationTimedoutException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Stopped); VirtualMachineTO vmTO = new VirtualMachineTO() {}; vmTO.setName(vmName); @@ -1833,7 +1832,7 @@ public void testPersistDomainForKvmForStoppedVmNoHost() { } @Test - public void testPersistDomainForKvmForRunningVmAgentFailure() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testPersistDomainForKvmForRunningVmAgentFailure() throws AgentUnavailableException, OperationTimedoutException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); UnmanageInstanceAnswer failureAnswer = new UnmanageInstanceAnswer(null, false, "failure"); @@ -1843,7 +1842,7 @@ public void testPersistDomainForKvmForRunningVmAgentFailure() throws AgentUnavai } @Test - public void testPersistDomainForKvmAgentUnavailable() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testPersistDomainForKvmAgentUnavailable() throws AgentUnavailableException, OperationTimedoutException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); doThrow(new AgentUnavailableException("Agent down", hostMockId)).when(agentManagerMock).send(anyLong(), any(UnmanageInstanceCommand.class)); @@ -1860,7 +1859,7 @@ public void testUnmanagePendingHaWork() { } @Test - public void testPersistDomainForKvmOperationTimedOut() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testPersistDomainForKvmOperationTimedOut() throws AgentUnavailableException, OperationTimedoutException { when(vmInstanceMock.getState()).thenReturn(VirtualMachine.State.Running); when(vmInstanceMock.getHostId()).thenReturn(hostMockId); doThrow(new OperationTimedoutException(null, hostMockId, 123L, 60, false)).when(agentManagerMock).send(anyLong(), any(UnmanageInstanceCommand.class)); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index a6198b26870c..867470dac040 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -24,7 +24,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; @@ -278,7 +277,7 @@ protected Answer sendCopyCommand(Host destHost, TemplateObjectTO sourceTemplate, Answer copyCommandAnswer = agentManager.send(destHost.getId(), copyCommand); logInCaseOfTemplateCopyFailure(copyCommandAnswer, sourceTemplate, destDataStore); return copyCommandAnswer; - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException(generateFailToCopyTemplateMessage(sourceTemplate, destDataStore), e); } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index dcd274e79b25..7674f1ce25a1 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -35,7 +35,6 @@ import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.PrepareForMigrationAnswer; -import com.cloud.exception.OperationCancelledException; import com.cloud.resource.ResourceManager; import com.cloud.storage.clvm.ClvmPoolManager; import org.apache.cloudstack.backup.InternalBackupService; @@ -1885,7 +1884,7 @@ private CopyCmdAnswer copyImageToVolume(DataObject srcDataObject, VolumeInfo des copyCmdAnswer = (CopyCmdAnswer)agentManager.send(hostVO.getId(), copyCommand); } - catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException | OperationCancelledException ex) { + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { String msg = "Failed to copy image : "; logger.warn(msg, ex); @@ -2160,8 +2159,6 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Operation timed out", destHost.getId()); - } catch (final OperationCancelledException e) { - throw new AgentUnavailableException("Operation cancelled", destHost.getId()); } for (VolumeInfo vol : samePoolClvmVolumes) { @@ -2209,7 +2206,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach try { migrateAnswer = (MigrateAnswer)agentManager.send(srcHost.getId(), migrateCommand); success = migrateAnswer != null && migrateAnswer.getResult(); - } catch (OperationTimedoutException | OperationCancelledException ex) { + } catch (OperationTimedoutException ex) { if (HypervisorType.KVM.equals(vm.getHypervisorType())) { final Answer answer = agentManager.send(destHost.getId(), new CheckVirtualMachineCommand(vm.getInstanceName())); if (answer != null && answer.getResult() && answer instanceof CheckVirtualMachineAnswer) { @@ -2238,7 +2235,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException(errMsg); } } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException | CloudRuntimeException ex) { + } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { String volumesAndStorages = volumeDataStoreMap.entrySet().stream().map(entry -> formatEntryOfVolumesAndStoragesAsJsonToDisplayOnLog(entry)).collect(Collectors.joining(",")); errMsg = String.format("Copy volume(s) to storage(s) [%s] and VM to host [%s] failed in StorageSystemDataMotionStrategy.copyAsync. Error message: [%s].", volumesAndStorages, formatMigrationElementsAsJsonToDisplayOnLog("vm", vmTO.getId(), srcHost.getId(), destHost.getId()), ex.getMessage()); @@ -2458,7 +2455,7 @@ private void sendClvmLockCommand(long hostId, StoragePoolVO pool, VolumeInfo vol String details = answer != null ? answer.getDetails() : "null answer"; logger.warn("CLVM lock command [{}] failed for LV [{}] on host [{}]: {}", operation, lvPath, hostId, details); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.warn("Exception sending CLVM lock command [{}] for LV [{}] on host [{}]: {}", operation, lvPath, hostId, e.getMessage()); } } @@ -3122,7 +3119,7 @@ private CopyCmdAnswer performResignature(DataObject dataObj, HostVO hostVO, Map< answer = (ResignatureAnswer)agentManager.send(hostVO.getId(), command); } - catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException | OperationCancelledException ex) { + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { keepGrantedAccess = false; String msg = "Failed to resign the DataObject with the following ID: " + dataObj.getId(); @@ -3331,7 +3328,7 @@ private CopyCmdAnswer performCopyOfVdi(VolumeInfo volumeInfo, SnapshotInfo snaps copyCmdAnswer = (CopyCmdAnswer)agentManager.send(hostVO.getId(), copyCommand); } - catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException | OperationCancelledException ex) { + catch (CloudRuntimeException | AgentUnavailableException | OperationTimedoutException ex) { String msg = "Failed to perform VDI copy : "; logger.warn(msg, ex); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 9af60b18f85a..6f0776b27c8c 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Map; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -282,21 +281,21 @@ public void shouldMigrateVolumeTest() { } @Test - public void sendCopyCommandTest() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void sendCopyCommandTest() throws AgentUnavailableException, OperationTimedoutException { configureAndTestSendCommandTest(null); } @Test(expected = CloudRuntimeException.class) - public void sendCopyCommandTestThrowAgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void sendCopyCommandTestThrowAgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException { configureAndTestSendCommandTest(AgentUnavailableException.class); } @Test(expected = CloudRuntimeException.class) - public void sendCopyCommandTestThrowOperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void sendCopyCommandTestThrowOperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException { configureAndTestSendCommandTest(OperationTimedoutException.class); } - private void configureAndTestSendCommandTest(Class exception) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + private void configureAndTestSendCommandTest(Class exception) throws AgentUnavailableException, OperationTimedoutException { Host destHost = new HostVO("guid"); TemplateObjectTO sourceTemplate = new TemplateObjectTO(); sourceTemplate.setName("name"); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java index 6e1b837bc3e5..a20b52fac2dd 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyTest.java @@ -25,7 +25,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import com.cloud.storage.dao.SnapshotDao; import com.cloud.vm.snapshot.dao.VMSnapshotDetailsDao; import org.apache.cloudstack.backup.BackupManager; @@ -108,7 +107,7 @@ public void setUp() { } @Test - public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTimedoutException { Long hostId = 1L; Long vmId = 1L; Long guestOsId = 1L; @@ -152,7 +151,7 @@ public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTi } @Test - public void testRevertSnapshot() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testRevertSnapshot() throws AgentUnavailableException, OperationTimedoutException { Long hostId = 1L; Long vmId = 1L; Long guestOsId = 1L; @@ -200,7 +199,7 @@ public void testRevertSnapshot() throws AgentUnavailableException, OperationTime } @Test - public void testDeleteVMSnapshot() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testDeleteVMSnapshot() throws AgentUnavailableException, OperationTimedoutException { Long hostId = 1L; Long vmId = 1L; Long guestOsId = 1L; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 81f43380099a..26b39e30776f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -29,7 +29,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -448,7 +447,7 @@ private Answer sendToLeastBusyEndpoint(List eps, CopyCommand cmd) { answer = agentMgr.send(endPoint.getId(), cmd); answer.setContextParam("cmd", cmdExecId.toString()); return answer; - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { errMsg = e.toString(); logger.debug("Failed to send command, due to Agent [id: {}, uuid: {}]: {}", endPoint.getId(), endPoint.getUuid(), e.toString()); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index 0d02b8de1508..a209acf6442e 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -48,7 +48,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; @@ -1959,10 +1958,6 @@ private CustomActionResultResponse runVirtualMachineCustomAction(VirtualMachine String msg = "Running custom action timed out, please try again"; logger.error(msg, e); result.put(ApiConstants.DETAILS, msg); - } catch (OperationCancelledException e) { - String msg = "Running custom action cancelled"; - logger.error(msg, e); - result.put(ApiConstants.DETAILS, msg); } response.setResult(result); return response; diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java index c16dbf407d74..fa8c97742697 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java @@ -121,7 +121,6 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; @@ -487,7 +486,7 @@ public void disableExtensionChangesStateToDisabled() { } @Test - public void updateAllExtensionHostsRemovesHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void updateAllExtensionHostsRemovesHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException { Extension extension = mock(Extension.class); when(extension.getId()).thenReturn(1L); Long clusterId = 100L; @@ -498,7 +497,7 @@ public void updateAllExtensionHostsRemovesHostsSuccessfully() throws OperationTi } @Test - public void updateAllExtensionHostsAddsHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void updateAllExtensionHostsAddsHostsSuccessfully() throws OperationTimedoutException, AgentUnavailableException { Extension extension = mock(Extension.class); when(extension.getId()).thenReturn(1L); Long clusterId = 100L; @@ -509,7 +508,7 @@ public void updateAllExtensionHostsAddsHostsSuccessfully() throws OperationTimed } @Test - public void updateAllExtensionHostsHandlesEmptyHostListGracefully() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void updateAllExtensionHostsHandlesEmptyHostListGracefully() throws OperationTimedoutException, AgentUnavailableException { Extension extension = mock(Extension.class); Long clusterId = 100L; when(hostDao.listIdsByClusterId(clusterId)).thenReturn(Collections.emptyList()); @@ -518,7 +517,7 @@ public void updateAllExtensionHostsHandlesEmptyHostListGracefully() throws Opera } @Test - public void updateAllExtensionHostsHandlesNullClusterId() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void updateAllExtensionHostsHandlesNullClusterId() throws OperationTimedoutException, AgentUnavailableException { Extension extension = mock(Extension.class); when(extension.getId()).thenReturn(1L); when(extensionResourceMapDao.listResourceIdsByExtensionIdAndType(eq(1L), any())).thenReturn(Collections.emptyList()); diff --git a/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java b/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java index a79023695fea..5edac430a673 100644 --- a/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java +++ b/plugins/backup/kboss/src/main/java/org/apache/cloudstack/backup/KbossBackupProvider.java @@ -44,7 +44,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -553,7 +552,7 @@ public Boolean orchestrateDeleteBackup(Backup backup, boolean forced) { Answer[] deleteAnswers; try { deleteAnswers = sendBackupCommands(endPoint.getId(), deleteCommands); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException(e); } @@ -653,7 +652,7 @@ public Boolean orchestrateRestoreVMFromBackup(Backup backup, VirtualMachine vm, try { answers = sendBackupCommands(host.getId(), commands); - } catch (OperationTimedoutException | AgentUnavailableException | OperationCancelledException e) { + } catch (OperationTimedoutException | AgentUnavailableException e) { throw new CloudRuntimeException(e); } @@ -979,7 +978,7 @@ public void prepareVmForSnapshotRevert(VMSnapshot vmSnapshot, VirtualMachine vir Answer[] answers; try { answers = sendBackupCommands(hostId, commands); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException(e); } @@ -2629,7 +2628,7 @@ protected Answer sendBackupCommand(long hostId, Command cmd) { return agentManager.easySend(hostId, cmd); } - protected Answer[] sendBackupCommands(Long hostId, Commands cmds) throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + protected Answer[] sendBackupCommands(Long hostId, Commands cmds) throws OperationTimedoutException, AgentUnavailableException { for (Command cmd : cmds) { cmd.setWait(backupTimeout.value()); } diff --git a/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java b/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java index 2342ad8b8f31..17ee3c8f6722 100644 --- a/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java +++ b/plugins/backup/kboss/src/test/java/org/apache/cloudstack/backup/KbossBackupProviderTest.java @@ -45,7 +45,6 @@ import java.util.Map; import java.util.Set; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.backup.dao.BackupDao; @@ -878,7 +877,7 @@ public void orchestrateDeleteBackupTestBackupStateIsNotOk() { } @Test - public void orchestrateDeleteBackupTestDeleteFailedBackup() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteFailedBackup() throws OperationTimedoutException, AgentUnavailableException { doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).validateBackupStateForRemoval(backupId); @@ -893,7 +892,7 @@ public void orchestrateDeleteBackupTestDeleteFailedBackup() throws OperationTime } @Test - public void orchestrateDeleteBackupTestDeleteBackupWithLiveChildren() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteBackupWithLiveChildren() throws OperationTimedoutException, AgentUnavailableException { doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).validateBackupStateForRemoval(backupId); @@ -911,7 +910,7 @@ public void orchestrateDeleteBackupTestDeleteBackupWithLiveChildren() throws Ope } @Test - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenFailedToMerge() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenFailedToMerge() throws OperationTimedoutException, AgentUnavailableException { doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); doNothing().when(kbossBackupProviderSpy).validateVmState(any(), any(), any()); doReturn(true).when(kbossBackupProviderSpy).validateBackupStateForRemoval(backupId); @@ -929,7 +928,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenFailedTo } @Test (expected = CloudRuntimeException.class) - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentNoEndPoint() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentNoEndPoint() throws OperationTimedoutException, AgentUnavailableException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -955,7 +954,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithPare } @Test (expected = CloudRuntimeException.class) - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentTimedoutException() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentTimedoutException() throws OperationTimedoutException, AgentUnavailableException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -982,7 +981,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithPare } @Test - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetNotEmpty() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetNotEmpty() throws OperationTimedoutException, AgentUnavailableException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -1012,7 +1011,7 @@ public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithPare } @Test - public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetEmpty() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void orchestrateDeleteBackupTestDeleteCurrentBackupWithNoChildrenWithParentFailedSetEmpty() throws OperationTimedoutException, AgentUnavailableException { long parentBackupId = 12; doReturn(parentBackupId).when(internalBackupJoinVoMock).getParentId(); doReturn(virtualMachineMock).when(virtualMachineManagerMock).findById(vmId); @@ -1066,7 +1065,7 @@ public void orchestrateRestoreVMFromBackupTestCurrentBackupNoHostToRestore() thr } @Test (expected = CloudRuntimeException.class) - public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupTimeOut() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupTimeOut() throws AgentUnavailableException, OperationTimedoutException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, false); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1090,7 +1089,7 @@ public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupTimeOut() throw } @Test - public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupNullAnswers() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupNullAnswers() throws AgentUnavailableException, OperationTimedoutException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, false); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1115,7 +1114,7 @@ public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupNullAnswers() t } @Test - public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupAnswerFalse() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupAnswerFalse() throws AgentUnavailableException, OperationTimedoutException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, false); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1141,7 +1140,7 @@ public void orchestrateRestoreVMFromBackupTestSameVmCurrentBackupAnswerFalse() t } @Test - public void orchestrateRestoreVMFromBackupTestSameVmQuickRestoreCurrentBackupAnswerTrue() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void orchestrateRestoreVMFromBackupTestSameVmQuickRestoreCurrentBackupAnswerTrue() throws AgentUnavailableException, OperationTimedoutException { doNothing().when(kbossBackupProviderSpy).validateNoVmSnapshots(virtualMachineMock); doNothing().when(kbossBackupProviderSpy).validateQuickRestore(backupVoMock, true); doReturn(new Pair<>(true, backupVoMock)).when(kbossBackupProviderSpy).validateCompressionStateForRestoreAndGetBackup(backupId); @@ -1432,7 +1431,7 @@ public void prepareVmForSnapshotRevertTestCurrentBackupBeforeVmSnapshot() { } @Test (expected = CloudRuntimeException.class) - public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() throws OperationTimedoutException, AgentUnavailableException { doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); @@ -1447,7 +1446,7 @@ public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotTimeout() } @Test (expected = CloudRuntimeException.class) - public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer() throws OperationTimedoutException, AgentUnavailableException { doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); @@ -1462,7 +1461,7 @@ public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotNullAnswer } @Test - public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotSuccess() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void prepareVmForSnapshotRevertTestCurrentBackupAfterVmSnapshotSuccess() throws OperationTimedoutException, AgentUnavailableException { doReturn(internalBackupJoinVoMock).when(internalBackupJoinDaoMock).findCurrent(vmId); doReturn(Date.from(Instant.now())).when(internalBackupJoinVoMock).getDate(); doReturn(Date.from(Instant.EPOCH)).when(vmSnapshotVoMock).getCreated(); diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index d498ecac213e..d08b4775c8bd 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -18,7 +18,6 @@ import com.cloud.agent.AgentManager; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.configuration.Resource; import com.cloud.host.Host; @@ -610,11 +609,6 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce backupVO.setStatus(Backup.Status.Failed); backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); - } catch (OperationCancelledException e) { - logger.error("Operation to initiate backup cancelled for VM {}", vm.getInstanceName()); - backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); - throw new CloudRuntimeException("Operation to initiate backup cancelled"); } if (answer != null && answer.getResult()) { @@ -736,8 +730,6 @@ private Pair restoreVMBackup(VirtualMachine vm, Backup backup) throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); - } catch (OperationCancelledException e) { - throw new CloudRuntimeException("Operation to restore backup cancelled"); } if (answer == null) { throw new CloudRuntimeException(String.format("No answer received from the backend control plane while restoring VM %s from backup %s", vm.getInstanceName(), backup.getUuid())); @@ -864,8 +856,6 @@ public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeI throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backed up volume timed out, please try again"); - } catch (OperationCancelledException e) { - throw new CloudRuntimeException("Operation to restore backed up volume cancelled"); } if (answer.getResult()) { @@ -1002,8 +992,6 @@ private boolean deleteBackupFileAndRow(Backup backup, BackupRepository repo, Hos throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to delete backup timed out, please try again"); - } catch (OperationCancelledException e) { - throw new CloudRuntimeException("Operation to delete backup cancelled"); } if (answer == null || !answer.getResult()) { logger.error("Failed to delete backup file for {} ({}); leaving DB row intact", @@ -1244,8 +1232,6 @@ public void syncBackupStorageStats(Long zoneId) { logger.warn("Unable to contact backend control plane to get backup stats for repository: {}", repository.getName()); } catch (OperationTimedoutException e) { logger.warn("Operation to get backup stats timed out for the repository: " + repository.getName()); - } catch (OperationCancelledException e) { - logger.warn("Operation to get backup stats cancelled for the repository: " + repository.getName()); } } } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index a97af9a5d956..f1d5613ab7fc 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -24,7 +24,6 @@ import java.util.Objects; import java.util.Optional; -import com.cloud.exception.OperationCancelledException; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.junit.Assert; import org.junit.Test; @@ -128,7 +127,7 @@ public class NASBackupProviderTest { private ResourceLimitService resourceLimitMgr; @Test - public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { Long hostId = 1L; BackupVO backup = new BackupVO(); backup.setBackupOfferingId(1L); @@ -154,7 +153,7 @@ public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailab } @Test - public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", "nfs", "address", "sync", 1024L, null); @@ -205,7 +204,7 @@ public void testGetBackupStorageStats() { } @Test - public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException { Long vmId = 1L; Long hostId = 2L; Long backupOfferingId = 3L; @@ -532,7 +531,7 @@ public void allVolumesOnCheckpointCapableStorageFalseWhenPoolUnresolvable() { * is called with the active_checkpoint_id key. */ @Test - public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableException, OperationTimedoutException { Long vmId = 7L; Long hostId = 8L; Long backupOfferingId = 9L; @@ -582,7 +581,7 @@ public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableExcepti */ @Test public void restoreBackedUpVolumeClearsTargetVmActiveCheckpoint() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { Long targetVmId = 42L; Long backupOfferingId = 9L; String targetVmName = "i-2-42-VM"; @@ -652,7 +651,7 @@ public void restoreBackedUpVolumeClearsTargetVmActiveCheckpoint() */ @Test public void deleteWithLiveChildMarksDeletePendingAndPreservesFile() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -728,7 +727,7 @@ public void deleteWithLiveChildMarksDeletePendingAndPreservesFile() */ @Test public void deletingLeafSweepsUpDeletePendingParent() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -813,7 +812,7 @@ public void deletingLeafSweepsUpDeletePendingParent() @Test public void deletingLastLiveMemberCollectsDeeperOrphanTombstones() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -890,7 +889,7 @@ public void deletingLastLiveMemberCollectsDeeperOrphanTombstones() @Test public void deletingAncestorOfTombstoneWithLiveDescendantTombstonesIt() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; @@ -959,7 +958,7 @@ public void deletingAncestorOfTombstoneWithLiveDescendantTombstonesIt() @Test public void sweepContinuesPastFailedTombstoneDelete() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { Long zoneId = 1L; Long vmId = 2L; Long hostId = 3L; diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java index b9e9d3a9f059..6e1f422526d9 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartServiceImpl.java @@ -29,7 +29,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.AddBaremetalKickStartPxeCmd; import org.apache.cloudstack.api.AddBaremetalPxeCmd; import org.apache.cloudstack.api.ListBaremetalPxeServersCmd; @@ -174,7 +173,7 @@ public File getSystemVMKeyFile() { return keyFile; } - private boolean preparePxeInBasicZone(VirtualMachineProfile profile, NicProfile nic, DeployDestination dest, ReservationContext context) throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + private boolean preparePxeInBasicZone(VirtualMachineProfile profile, NicProfile nic, DeployDestination dest, ReservationContext context) throws AgentUnavailableException, OperationTimedoutException { NetworkVO nwVO = _nwDao.findById(nic.getNetworkId()); QueryBuilder sc = QueryBuilder.create(BaremetalPxeVO.class); sc.and(sc.entity().getDeviceType(), Op.EQ, BaremetalPxeType.KICK_START.toString()); diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java index f23958ee1d1c..a5fe1442e56e 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/discoverer/HypervServerDiscoverer.java @@ -49,7 +49,6 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConnectionException; import com.cloud.exception.DiscoveryException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostEnvironment; @@ -170,8 +169,6 @@ public final void processConnect(final Host agent, final StartupCommand cmd, fin logger.warn(String.format("Unable to setup agent %s because it became unavailable.", agent), e); } catch (OperationTimedoutException e) { logger.warn(String.format("Unable to setup agent %s because it timed out", agent), e); - } catch (OperationCancelledException e) { - logger.warn(String.format("Unable to setup agent %s because it is cancelled", agent), e); } throw new ConnectionException(true, "Reinitialize agent after setup."); } diff --git a/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java b/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java index 3565b2032c54..55944a082427 100644 --- a/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java +++ b/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java @@ -24,7 +24,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -141,9 +140,6 @@ private Answer migrateVmWithVolumes(VMInstanceVO vm, VirtualMachineTO to, Host s } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); - } catch (OperationCancelledException e) { - logger.error("Error while migrating vm " + vm + " to host " + destHost, e); - throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } diff --git a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java index 9c95d821454d..60792f0ddb7d 100644 --- a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java +++ b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmFencer.java @@ -27,7 +27,6 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.ha.FenceBuilder; import com.cloud.host.Host; @@ -101,11 +100,6 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { logger.debug("Moving on to the next host because " + h.toString() + " is unavailable", e); } continue; - } catch (OperationCancelledException e) { - if (logger.isDebugEnabled()) { - logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); - } - break; } if (answer != null && answer.getResult()) { diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java index 7a2047e8d534..eb83572fc2b8 100755 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3FenceBuilder.java @@ -28,7 +28,6 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.ha.FenceBuilder; import com.cloud.host.Host; @@ -98,12 +97,6 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { + h.toString() + " is unavailable", e); } continue; - } catch (OperationCancelledException e) { - if (logger.isDebugEnabled()) { - logger.debug("Not moving on to the next host because " - + h.toString() + " is unavailable, and operation cancelled", e); - } - break; } if (answer != null && answer.getResult()) { return true; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java index 1d905bdd4823..b37960e26ab9 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorFencer.java @@ -27,7 +27,6 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -97,11 +96,6 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { logger.debug("Moving on to the next host because " + h.toString() + " is unavailable", e); } continue; - } catch (OperationCancelledException e) { - if (logger.isDebugEnabled()) { - logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); - } - break; } if (answer != null && answer.getResult()) { return true; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java index e0c004b574e0..95a6a1291cf1 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/ha/SimulatorInvestigator.java @@ -20,7 +20,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.ha.HAManager; import com.cloud.agent.AgentManager; @@ -104,9 +103,6 @@ public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM { } catch (OperationTimedoutException e) { logger.debug("Operation timed out for " + vm.toString() + ": " + e.getMessage()); throw new UnknownVM(); - } catch (OperationCancelledException e) { - logger.debug("Operation cancelled for " + vm.toString() + ": " + e.getMessage()); - throw new UnknownVM(); } } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 62cfd15ea992..89b5b111e14f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -62,10 +62,7 @@ import com.vmware.vim25.VirtualTPM; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.PrepareForBackupRestorationCommand; -import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.CreateObjectCommand; -import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; @@ -470,20 +467,10 @@ private String getCommandLogTitle(Command cmd) { @Override public Answer executeRequest(Command cmd) { - if (Thread.currentThread().isInterrupted()) { - String msg = "Command " + cmd.getClass().getSimpleName() + " was cancelled before execution"; - logger.warn(msg); - return new Answer(cmd, false, msg); - } - logCommand(cmd); Answer answer; ThreadContext.push(getCommandLogTitle(cmd)); try { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException("Command execution cancelled"); - } - long cmdSequence = _cmdSequence++; Date startTime = DateUtil.currentGMTTime(); PropertyMapDynamicBean mbean = new PropertyMapDynamicBean(); @@ -512,33 +499,24 @@ public Answer executeRequest(Command cmd) { } else if (clz == CheckHealthCommand.class) { answer = execute((CheckHealthCommand) cmd); } else if (clz == StopCommand.class) { - String vmName = ((StopCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); answer = execute((StopCommand) cmd); } else if (clz == RebootRouterCommand.class) { answer = execute((RebootRouterCommand) cmd); } else if (clz == RebootCommand.class) { - String vmName = ((RebootCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); answer = execute((RebootCommand) cmd); } else if (clz == CheckVirtualMachineCommand.class) { answer = execute((CheckVirtualMachineCommand) cmd); } else if (clz == PrepareForMigrationCommand.class) { answer = execute((PrepareForMigrationCommand) cmd); } else if (clz == MigrateCommand.class) { - String vmName = ((MigrateCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); answer = execute((MigrateCommand) cmd); } else if (clz == MigrateVmToPoolCommand.class) { answer = execute((MigrateVmToPoolCommand) cmd); } else if (clz == MigrateWithStorageCommand.class) { answer = execute((MigrateWithStorageCommand) cmd); } else if (clz == MigrateVolumeCommand.class) { - setTaskContext(cmdSequence, cmd, null); answer = execute((MigrateVolumeCommand) cmd); } else if (clz == DestroyCommand.class) { - String vmName = ((DestroyCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); answer = execute((DestroyCommand) cmd); } else if (clz == CreateStoragePoolCommand.class) { return execute((CreateStoragePoolCommand) cmd); @@ -587,8 +565,6 @@ public Answer executeRequest(Command cmd) { } else if (clz == NetworkUsageCommand.class) { answer = execute((NetworkUsageCommand) cmd); } else if (clz == StartCommand.class) { - String vmName = ((StartCommand) cmd).getVirtualMachine().getName(); - setTaskContext(cmdSequence, cmd, vmName); answer = execute((StartCommand) cmd); } else if (clz == CheckSshCommand.class) { answer = execute((CheckSshCommand) cmd); @@ -601,16 +577,10 @@ public Answer executeRequest(Command cmd) { } else if (clz == UnPlugNicCommand.class) { answer = execute((UnPlugNicCommand) cmd); } else if (cmd instanceof CreateVMSnapshotCommand) { - String vmName = ((CreateVMSnapshotCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); return execute((CreateVMSnapshotCommand) cmd); } else if (cmd instanceof DeleteVMSnapshotCommand) { - String vmName = ((DeleteVMSnapshotCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); return execute((DeleteVMSnapshotCommand) cmd); } else if (cmd instanceof RevertToVMSnapshotCommand) { - String vmName = ((RevertToVMSnapshotCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); return execute((RevertToVMSnapshotCommand) cmd); } else if (clz == ResizeVolumeCommand.class) { return execute((ResizeVolumeCommand) cmd); @@ -619,15 +589,6 @@ public Answer executeRequest(Command cmd) { } else if (clz == CleanupVMCommand.class) { return execute((CleanupVMCommand) cmd); } else if (cmd instanceof StorageSubSystemCommand) { - if (cmd instanceof CreateObjectCommand) { - setTaskContext(cmdSequence, cmd, null); - } if (cmd instanceof AttachCommand) { - String vmName = ((AttachCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); - } else if (cmd instanceof DettachCommand) { - String vmName = ((DettachCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); - } checkStorageProcessorAndHandlerNfsVersionAttribute((StorageSubSystemCommand) cmd); return storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd); } else if (clz == ScaleVmCommand.class) { @@ -686,12 +647,8 @@ public Answer executeRequest(Command cmd) { logger.trace("Unable to register JMX monitoring due to exception " + ExceptionUtil.toString(e)); } } - } catch (InterruptedException e) { - logger.warn("Command execution interrupted: {}", cmd.getClass().getSimpleName()); - Thread.currentThread().interrupt(); // Restore interrupt status - return new Answer(cmd, false, "Command execution was cancelled: " + e.getMessage()); + } finally { - VmwareHelper.clearTaskContext(); recycleServiceContext(); ThreadContext.pop(); } @@ -702,23 +659,6 @@ public Answer executeRequest(Command cmd) { return answer; } - private void setTaskContext(long cmdSequence, Command cmd, String vmName) { - String commandType = cmd.getClass().getSimpleName(); - VmwareHelper.setTaskContext(cmdSequence, vmName, commandType); - } - - @Override - public boolean isRequestSequenceCancellable(long sequence) { - logger.info("Check if request sequence: {} is cancellable or not", sequence); - return VmwareHelper.isActiveVmTaskCancellable(sequence); - } - - @Override - public boolean cancelRequestSequence(long sequence) { - logger.info("Cancel request sequence: {}", sequence); - return VmwareHelper.cancelActiveVmTask(sequence); - } - private ExecutionResult getSystemVmVersionAndChecksum(String controlIp) { ExecutionResult result; try { @@ -2713,12 +2653,6 @@ protected StartAnswer execute(StartCommand cmd) { return startAnswer; } catch (Throwable e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - logger.warn("StartCommand interrupted"); - return new StartAnswer(cmd, "StartCommand cancelled: " + e.getMessage()); - } - StartAnswer startAnswer = new StartAnswer(cmd, createLogMessageException(e, cmd)); if (vmAlreadyExistsInVcenter) { startAnswer.setContextParam("stopRetry", "true"); @@ -2739,12 +2673,6 @@ protected StartAnswer execute(StartCommand cmd) { } } - private void checkCancellation() throws InterruptedException { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException("Command cancelled"); - } - } - private Pair getVolumeDatastoreDetails(DiskTO vol, HashMap> dataStoresDetails) throws Exception { boolean managed = false; String iScsiName = null; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index 540169151f97..d2d319ed9d03 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -27,7 +27,6 @@ import javax.inject.Inject; import com.cloud.agent.api.to.DiskTO; -import com.cloud.exception.OperationCancelledException; import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; @@ -381,9 +380,6 @@ private Answer migrateVmWithVolumesAcrossCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); - } catch (OperationCancelledException e) { - logger.error("Error while migrating vm " + vm + " to host " + destHost, e); - throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } @@ -417,9 +413,6 @@ private Answer migrateVmWithVolumesWithinCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); - } catch (OperationCancelledException e) { - logger.error("Error while migrating vm " + vm + " to host " + destHost, e); - throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java index 9373dfb04599..a29ac2aee76b 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/ha/XenServerFencer.java @@ -28,7 +28,6 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -84,11 +83,6 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { logger.debug("Moving on to the next host because " + h.toString() + " is unavailable", e); } continue; - } catch (OperationCancelledException e) { - if (logger.isDebugEnabled()) { - logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); - } - break; } if (answer != null && answer.getResult()) { return true; diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java index 3f555b4e9ec6..6d298629e7ef 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/discoverer/XcpServerDiscoverer.java @@ -30,7 +30,6 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; -import com.cloud.exception.OperationCancelledException; import com.cloud.hypervisor.xenserver.resource.XcpServer83Resource; import com.cloud.hypervisor.xenserver.resource.Xenserver84Resource; import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs; @@ -635,8 +634,6 @@ public void processConnect(com.cloud.host.Host agent, StartupCommand cmd, boolea logger.warn("Unable to setup agent " + agentId + " because it became unavailable.", e); } catch (OperationTimedoutException e) { logger.warn("Unable to setup agent " + agentId + " because it timed out", e); - } catch (OperationCancelledException e) { - logger.warn("Unable to setup agent " + agentId + " because it is cancelled", e); } throw new ConnectionException(true, "Reinitialize agent after setup."); } diff --git a/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java b/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java index 68585580efda..76807b89d0f7 100644 --- a/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java +++ b/plugins/hypervisors/xenserver/src/main/java/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java @@ -25,7 +25,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -383,9 +382,6 @@ private Answer migrateVmWithVolumesAcrossCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); - } catch (OperationCancelledException e) { - logger.error("Error while migrating vm " + vm + " to host " + destHost, e); - throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } @@ -419,9 +415,6 @@ private Answer migrateVmWithVolumesWithinCluster(VMInstanceVO vm, VirtualMachine } catch (OperationTimedoutException e) { logger.error("Error while migrating vm " + vm + " to host " + destHost, e); throw new AgentUnavailableException("Operation timed out on storage motion for " + vm, destHost.getId()); - } catch (OperationCancelledException e) { - logger.error("Error while migrating vm " + vm + " to host " + destHost, e); - throw new AgentUnavailableException("Operation cancelled on storage motion for " + vm, destHost.getId()); } } diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java index 9b899b7d39dc..2107850c36be 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/ManagementServerMock.java @@ -53,7 +53,6 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; -import com.cloud.exception.OperationCancelledException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -248,8 +247,6 @@ private void sendCommands(Answer callback) { logger.warn("no agent running", e); } catch (OperationTimedoutException e) { logger.warn("agent not responding (in time)", e); - } catch (OperationCancelledException e) { - logger.warn("cancelled", e); } } diff --git a/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java b/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java index 67b7ff00566f..804f29c01b1e 100644 --- a/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java +++ b/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java @@ -26,7 +26,6 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.MessageSubscriber; @@ -232,7 +231,7 @@ private void handleCreateTunnelAnswer(Answer[] answers) { } private String getGreEndpointIP(Host host, Network nw) - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { String endpointIp = null; // Fetch fefault name for network label from configuration String physNetLabel = _configDao.getValue(Config.OvsTunnelNetworkDefaultLabel.key()); @@ -384,8 +383,6 @@ protected void checkAndCreateTunnel(Network nw, Host host) { } catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) { // I really thing we should do a better handling of these exceptions logger.warn("Ovs Tunnel network created tunnel failed", e); - } catch (OperationCancelledException e) { - logger.warn("Ovs Tunnel network created tunnel cancelled", e); } } @@ -564,8 +561,6 @@ protected void checkAndCreateVpcTunnelNetworks(Host host, long vpcId) { handleSetupBridgeAnswer(answers); } catch (OperationTimedoutException | AgentUnavailableException e) { logger.warn("Ovs Tunnel network created bridge failed", e); - } catch (OperationCancelledException e) { - logger.warn("Ovs Tunnel network created bridge cancelled", e); } // now that bridge is setup, populate network acl's before the VM gets created @@ -649,8 +644,6 @@ protected void checkAndCreateVpcTunnelNetworks(Host host, long vpcId) { } catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) { // I really thing we should do a better handling of these exceptions logger.warn("Ovs Tunnel network created tunnel failed", e); - } catch (OperationCancelledException e) { - logger.warn("Ovs Tunnel network created tunnel cancelled", e); } } } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java index d1f410a65b04..0dd0de70464f 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.storage.motion; -import com.cloud.exception.OperationCancelledException; import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRcList; @@ -454,9 +453,6 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } catch (final OperationTimedoutException e) { errMsg = String.format("Operation timed out due to %s", e.getMessage()); throw new AgentUnavailableException(errMsg, destHost.getId()); - } catch (final OperationCancelledException e) { - errMsg = String.format("Operation cancelled due to %s", e.getMessage()); - throw new AgentUnavailableException(errMsg, destHost.getId()); } VMInstanceVO vm = _vmDao.findById(vmTO.getId()); @@ -489,7 +485,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException(errMsg); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException | CloudRuntimeException ex) { + } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { errMsg = String.format( "Copy volume(s) of VM [%s] to storage(s) [%s] and VM to host [%s] failed in LinstorDataMotionStrategy.copyAsync. Error message: [%s].", vmTO, srcHost, destHost, ex.getMessage()); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java index 6b48b04e99ef..cb111e9ccac5 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java @@ -23,7 +23,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.framework.config.ConfigKey; @@ -217,7 +216,7 @@ private String prepareSDCOnHost(Host host, DataStore dataStore, String systemId, PrepareStorageClientAnswer prepareStorageClientAnswer; try { prepareStorageClientAnswer = (PrepareStorageClientAnswer) agentManager.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { String err = String.format("Failed to prepare SDC on the host %s, due to: %s", host, e.getMessage()); logger.error(err); throw new CloudRuntimeException(err); @@ -334,7 +333,7 @@ private boolean unprepareSDCOnHost(Host host, DataStore dataStore, String mdms) Answer unprepareStorageClientAnswer; try { unprepareStorageClientAnswer = agentManager.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Failed to unprepare SDC on the host {} due to: {}", host, e.getMessage()); return false; } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java index 88d87136aafb..8d3dc868276c 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java @@ -33,7 +33,6 @@ import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -355,9 +354,6 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach } catch (final OperationTimedoutException e) { errMsg = String.format("Operation timed out due to %s", e.getMessage()); throw new AgentUnavailableException(errMsg, destHost.getId()); - } catch (final OperationCancelledException e) { - errMsg = String.format("Operation cancelled due to %s", e.getMessage()); - throw new AgentUnavailableException(errMsg, destHost.getId()); } VMInstanceVO vm = _vmDao.findById(vmTO.getId()); @@ -390,7 +386,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException(errMsg); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException | CloudRuntimeException ex) { + } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { errMsg = String.format( "Copy volume(s) of VM [%s] to storage(s) [%s] and VM to host [%s] failed in StorPoolDataMotionStrategy.copyAsync. Error message: [%s].", diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index caa83fd3a277..934600eb2b61 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -32,8 +32,6 @@ import com.cloud.cpu.CPU; import com.cloud.storage.GuestOSVO; -import com.cloud.utils.StringUtils; -import com.cloud.utils.UuidUtils; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.affinity.AffinityGroup; @@ -940,10 +938,6 @@ public static VMInstanceVO findVMInstanceById(long vmId) { return s_vmDao.findByIdIncludingRemoved(vmId); } - public static VMInstanceVO findVMInstanceByUuid(String vmUuid) { - return s_vmDao.findByUuidIncludingRemoved(vmUuid); - } - public static long getStorageCapacitybyPool(Long poolId, short capacityType) { // TODO: This method is for the API only, but it has configuration values (ramSize for system vms) // so if this Utils class can have some kind of config rather than a static initializer (maybe from @@ -1112,10 +1106,6 @@ public static Account findAccountByIdIncludingRemoved(Long accountId) { return s_accountDao.findByIdIncludingRemoved(accountId); } - public static Account findAccountByUuid(String accountUuid) { - return s_accountDao.findByUuidIncludingRemoved(accountUuid); - } - public static Account findAccountByNameDomain(String accountName, Long domainId) { return s_accountDao.findActiveAccount(accountName, domainId); } @@ -1196,10 +1186,6 @@ public static HostVO findHostById(Long hostId) { return s_hostDao.findByIdIncludingRemoved(hostId); } - public static HostVO findHostByUuid(String hostUuid) { - return s_hostDao.findByUuidIncludingRemoved(hostUuid); - } - public static HostVO findHostByTypeNameAndZoneId(Long zoneId, String name, Host.Type type) { return s_hostDao.findByTypeNameAndZoneId(zoneId, name, type); } @@ -1208,10 +1194,6 @@ public static IPAddressVO findIpAddressById(long addressId) { return s_ipAddressDao.findById(addressId); } - public static IPAddressVO findIpAddressByUuid(String addressUuid) { - return s_ipAddressDao.findByUuidIncludingRemoved(addressUuid); - } - public static GuestOSCategoryVO getHostGuestOSCategory(long hostId) { Long guestOSCategoryID = s_resourceMgr.getGuestOSCategoryId(hostId); @@ -1238,10 +1220,6 @@ public static SecurityGroup findSecurityGroupById(Long groupId) { return s_securityGroupDao.findById(groupId); } - public static SecurityGroup findSecurityGroupByUuid(String groupUuid) { - return s_securityGroupDao.findByUuidIncludingRemoved(groupUuid); - } - public static HostPodVO findPodById(Long podId) { return s_podDao.findById(podId); } @@ -1271,18 +1249,10 @@ public static Snapshot findSnapshotById(long snapshotId) { return s_snapshotDao.findByIdIncludingRemoved(snapshotId); } - public static Snapshot findSnapshotByUuid(String snapshotUuid) { - return s_snapshotDao.findByUuidIncludingRemoved(snapshotUuid); - } - public static StoragePoolVO findStoragePoolById(Long storagePoolId) { return s_storagePoolDao.findByIdIncludingRemoved(storagePoolId); } - public static StoragePoolVO findStoragePoolByUuid(String storagePoolUuid) { - return s_storagePoolDao.findByUuidIncludingRemoved(storagePoolUuid); - } - public static VMTemplateVO findTemplateById(Long templateId) { VMTemplateVO template = s_templateDao.findByIdIncludingRemoved(templateId); if (template != null) { @@ -1294,10 +1264,6 @@ public static VMTemplateVO findTemplateById(Long templateId) { return template; } - public static VMTemplateVO findTemplateByUuid(String templateUuid) { - return s_templateDao.findByUuidIncludingRemoved(templateUuid); - } - public static UploadVO findUploadById(Long id) { return s_uploadDao.findById(id); } @@ -1306,10 +1272,6 @@ public static User findUserById(Long userId) { return s_userDao.findById(userId); } - public static User findUserByUuid(String userUuid) { - return s_userDao.findByUuidIncludingRemoved(userUuid); - } - public static UserAccountJoinVO findUserAccountById(Long id) { return s_userAccountJoinDao.findById(id); } @@ -1326,10 +1288,6 @@ public static VolumeVO findVolumeById(Long volumeId) { return s_volumeDao.findByIdIncludingRemoved(volumeId); } - public static VolumeVO findVolumeByUuid(String volumeUuid) { - return s_volumeDao.findByUuidIncludingRemoved(volumeUuid); - } - public static Site2SiteVpnGatewayVO findVpnGatewayById(Long vpnGatewayId) { return s_site2SiteVpnGatewayDao.findById(vpnGatewayId); } @@ -1487,26 +1445,14 @@ public static PhysicalNetworkVO findPhysicalNetworkById(long id) { return s_physicalNetworkDao.findById(id); } - public static PhysicalNetworkVO findPhysicalNetworkByUuid(String uuid) { - return s_physicalNetworkDao.findByUuidIncludingRemoved(uuid); - } - public static PhysicalNetworkTrafficTypeVO findPhysicalNetworkTrafficTypeById(long id) { return s_physicalNetworkTrafficTypeDao.findById(id); } - public static PhysicalNetworkTrafficTypeVO findPhysicalNetworkTrafficTypeByUuid(String uuid) { - return s_physicalNetworkTrafficTypeDao.findByUuidIncludingRemoved(uuid); - } - public static NetworkVO findNetworkById(long id) { return s_networkDao.findByIdIncludingRemoved(id); } - public static NetworkVO findNetworkByUuid(String uuid) { - return s_networkDao.findByUuidIncludingRemoved(uuid); - } - public static Map> getNetworkCapabilities(long networkId, long zoneId) { return s_networkModel.getNetworkCapabilities(networkId); } @@ -1736,74 +1682,38 @@ public static CounterVO getCounter(long counterId) { return s_counterDao.findById(counterId); } - public static CounterVO getCounterByUuid(String counterUuid) { - return s_counterDao.findByUuidIncludingRemoved(counterUuid); - } - public static ConditionVO findConditionById(long conditionId) { return s_asConditionDao.findById(conditionId); } - public static ConditionVO findConditionByUuid(String conditionUuid) { - return s_asConditionDao.findByUuidIncludingRemoved(conditionUuid); - } - public static PhysicalNetworkServiceProviderVO findPhysicalNetworkServiceProviderById(long providerId) { return s_physicalNetworkServiceProviderDao.findById(providerId); } - public static PhysicalNetworkServiceProviderVO findPhysicalNetworkServiceProviderByUuid(String providerUuid) { - return s_physicalNetworkServiceProviderDao.findByUuidIncludingRemoved(providerUuid); - } - public static FirewallRuleVO findFirewallRuleById(long ruleId) { return s_firewallRuleDao.findById(ruleId); } - public static FirewallRuleVO findFirewallRuleByUuid(String ruleUuid) { - return s_firewallRuleDao.findByUuidIncludingRemoved(ruleUuid); - } - public static StaticRouteVO findStaticRouteById(long routeId) { return s_staticRouteDao.findById(routeId); } - public static StaticRouteVO findStaticRouteByUuid(String routeUuid) { - return s_staticRouteDao.findByUuidIncludingRemoved(routeUuid); - } - public static VpcGatewayVO findVpcGatewayById(long gatewayId) { return s_vpcGatewayDao.findById(gatewayId); } - public static VpcGatewayVO findVpcGatewayByUuid(String gatewayUUid) { - return s_vpcGatewayDao.findByUuidIncludingRemoved(gatewayUUid); - } - public static AutoScalePolicyVO findAutoScalePolicyById(long policyId) { return s_asPolicyDao.findById(policyId); } - public static AutoScalePolicyVO findAutoScalePolicyByUuid(String policyUuid) { - return s_asPolicyDao.findByUuidIncludingRemoved(policyUuid); - } - public static AutoScaleVmProfileVO findAutoScaleVmProfileById(long profileId) { return s_asVmProfileDao.findById(profileId); } - public static AutoScaleVmProfileVO findAutoScaleVmProfileByUuid(String profileUuid) { - return s_asVmProfileDao.findByUuidIncludingRemoved(profileUuid); - } - public static AutoScaleVmGroupVO findAutoScaleVmGroupById(long groupId) { return s_asVmGroupDao.findById(groupId); } - public static AutoScaleVmGroupVO findAutoScaleVmGroupByUuid(String groupUuid) { - return s_asVmGroupDao.findByUuidIncludingRemoved(groupUuid); - } - public static int countAvailableVmsByGroupId(long groupId) { return s_autoScaleVmGroupVmMapDao.countAvailableVmsByGroup(groupId); } @@ -2005,131 +1915,6 @@ public static String findJobInstanceUuid(AsyncJob job) { return jobInstanceId; } - public static Long findResourceId(ApiCommandResourceType resourceType, String resourceUuid) { - if (StringUtils.isBlank(resourceUuid) - || !UuidUtils.isUuid(resourceUuid) - || resourceType == ApiCommandResourceType.None) { - return null; - } - - Long resourceId = null; - - if (resourceType == ApiCommandResourceType.Volume) { - VolumeVO volume = ApiDBUtils.findVolumeByUuid(resourceUuid); - if (volume != null) { - resourceId = volume.getId(); - } - } else if (resourceType == ApiCommandResourceType.Template || resourceType == ApiCommandResourceType.Iso) { - VMTemplateVO template = ApiDBUtils.findTemplateByUuid(resourceUuid); - if (template != null) { - resourceId = template.getId(); - } - } else if (resourceType == ApiCommandResourceType.VirtualMachine || resourceType == ApiCommandResourceType.ConsoleProxy || - resourceType == ApiCommandResourceType.SystemVm || resourceType == ApiCommandResourceType.DomainRouter) { - VMInstanceVO vm = ApiDBUtils.findVMInstanceByUuid(resourceUuid); - if (vm != null) { - resourceId = vm.getId(); - } - } else if (resourceType == ApiCommandResourceType.Snapshot) { - Snapshot snapshot = ApiDBUtils.findSnapshotByUuid(resourceUuid); - if (snapshot != null) { - resourceId = snapshot.getId(); - } - } else if (resourceType == ApiCommandResourceType.Host) { - Host host = ApiDBUtils.findHostByUuid(resourceUuid); - if (host != null) { - resourceId = host.getId(); - } - } else if (resourceType == ApiCommandResourceType.StoragePool) { - StoragePoolVO spool = ApiDBUtils.findStoragePoolByUuid(resourceUuid); - if (spool != null) { - resourceId = spool.getId(); - } - } else if (resourceType == ApiCommandResourceType.IpAddress) { - IPAddressVO ip = ApiDBUtils.findIpAddressByUuid(resourceUuid); - if (ip != null) { - resourceId = ip.getId(); - } - } else if (resourceType == ApiCommandResourceType.SecurityGroup) { - SecurityGroup sg = ApiDBUtils.findSecurityGroupByUuid(resourceUuid); - if (sg != null) { - resourceId = sg.getId(); - } - } else if (resourceType == ApiCommandResourceType.PhysicalNetwork) { - PhysicalNetworkVO pnet = ApiDBUtils.findPhysicalNetworkByUuid(resourceUuid); - if (pnet != null) { - resourceId = pnet.getId(); - } - } else if (resourceType == ApiCommandResourceType.TrafficType) { - PhysicalNetworkTrafficTypeVO trafficType = ApiDBUtils.findPhysicalNetworkTrafficTypeByUuid(resourceUuid); - if (trafficType != null) { - resourceId = trafficType.getId(); - } - } else if (resourceType == ApiCommandResourceType.PhysicalNetworkServiceProvider) { - PhysicalNetworkServiceProvider sp = ApiDBUtils.findPhysicalNetworkServiceProviderByUuid(resourceUuid); - if (sp != null) { - resourceId = sp.getId(); - } - } else if (resourceType == ApiCommandResourceType.FirewallRule) { - FirewallRuleVO fw = ApiDBUtils.findFirewallRuleByUuid(resourceUuid); - if (fw != null) { - resourceId = fw.getId(); - } - } else if (resourceType == ApiCommandResourceType.Account) { - Account acct = ApiDBUtils.findAccountByUuid(resourceUuid); - if (acct != null) { - resourceId = acct.getId(); - } - } else if (resourceType == ApiCommandResourceType.User) { - User usr = ApiDBUtils.findUserByUuid(resourceUuid); - if (usr != null) { - resourceId = usr.getId(); - } - } else if (resourceType == ApiCommandResourceType.StaticRoute) { - StaticRouteVO route = ApiDBUtils.findStaticRouteByUuid(resourceUuid); - if (route != null) { - resourceId = route.getId(); - } - } else if (resourceType == ApiCommandResourceType.PrivateGateway) { - VpcGatewayVO gateway = ApiDBUtils.findVpcGatewayByUuid(resourceUuid); - if (gateway != null) { - resourceId = gateway.getId(); - } - } else if (resourceType == ApiCommandResourceType.Counter) { - CounterVO counter = ApiDBUtils.getCounterByUuid(resourceUuid); - if (counter != null) { - resourceId = counter.getId(); - } - } else if (resourceType == ApiCommandResourceType.Condition) { - ConditionVO condition = ApiDBUtils.findConditionByUuid(resourceUuid); - if (condition != null) { - resourceId = condition.getId(); - } - } else if (resourceType == ApiCommandResourceType.AutoScalePolicy) { - AutoScalePolicyVO policy = ApiDBUtils.findAutoScalePolicyByUuid(resourceUuid); - if (policy != null) { - resourceId = policy.getId(); - } - } else if (resourceType == ApiCommandResourceType.AutoScaleVmProfile) { - AutoScaleVmProfileVO profile = ApiDBUtils.findAutoScaleVmProfileByUuid(resourceUuid); - if (profile != null) { - resourceId = profile.getId(); - } - } else if (resourceType == ApiCommandResourceType.AutoScaleVmGroup) { - AutoScaleVmGroupVO group = ApiDBUtils.findAutoScaleVmGroupByUuid(resourceUuid); - if (group != null) { - resourceId = group.getId(); - } - } else if (resourceType == ApiCommandResourceType.Network) { - NetworkVO networkVO = ApiDBUtils.findNetworkByUuid(resourceUuid); - if(networkVO != null) { - resourceId = networkVO.getId(); - } - } - - return resourceId; - } - /////////////////////////////////////////////////////////////////////// // Newly Added Utility Methods for List API refactoring // /////////////////////////////////////////////////////////////////////// diff --git a/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java b/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java index a25451e57b21..4ee6adeab603 100644 --- a/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java +++ b/server/src/main/java/com/cloud/ha/AbstractInvestigatorImpl.java @@ -28,7 +28,6 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.PingTestCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; @@ -116,11 +115,6 @@ protected Status testIpAddress(Long hostId, String testHostIp) { logger.debug("host (" + testHostIp + "): " + e.getLocalizedMessage() + ", trapped OperationTimedoutException returning Unknown state"); } return Status.Unknown; - } catch (OperationCancelledException e) { - if (logger.isDebugEnabled()) { - logger.debug("host (" + testHostIp + "): " + e.getLocalizedMessage() + ", trapped OperationCancelledException returning Unknown state"); - } - return Status.Unknown; } } } diff --git a/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java b/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java index 4ccc43c3c4fb..e0dbc8fcbe11 100644 --- a/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java +++ b/server/src/main/java/com/cloud/ha/CheckOnAgentInvestigator.java @@ -23,7 +23,6 @@ import com.cloud.agent.api.CheckVirtualMachineAnswer; import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.Status; @@ -61,9 +60,6 @@ public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM { } catch (OperationTimedoutException e) { logger.debug("Operation timed out for " + vm.toString() + ": " + e.getMessage()); throw new UnknownVM(); - } catch (OperationCancelledException e) { - logger.debug("Operation cancelled for " + vm.toString() + ": " + e.getMessage()); - throw new UnknownVM(); } } } diff --git a/server/src/main/java/com/cloud/ha/KVMFencer.java b/server/src/main/java/com/cloud/ha/KVMFencer.java index b071338cf137..4a6606b09cc3 100644 --- a/server/src/main/java/com/cloud/ha/KVMFencer.java +++ b/server/src/main/java/com/cloud/ha/KVMFencer.java @@ -28,7 +28,6 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -101,10 +100,6 @@ public Boolean fenceOff(VirtualMachine vm, Host host) { } catch (AgentUnavailableException | OperationTimedoutException e) { logger.info("Moving on to the next host because {} is unavailable", h.toString(), e); continue; - } catch(OperationCancelledException e) { - logger.debug("Not moving on to the next host because " + h.toString() + " is unavailable, and operation cancelled", e); - break; - } if (answer != null && answer.getResult()) { return true; diff --git a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java index a93a13e6104f..1e6ab9d0ffa2 100644 --- a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java @@ -29,7 +29,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.command.admin.cluster.UpdateClusterCmd; @@ -500,7 +499,7 @@ private boolean isMaintenanceScriptDefinedOnHost(Host host, List ho try { RollingMaintenanceAnswer answer = (RollingMaintenanceAnswer) agentManager.send(host.getId(), new RollingMaintenanceCommand(true)); return answer.isMaintenaceScriptDefined(); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { String msg = String.format("Could not check for maintenance script on %s due to: %s", host, e.getMessage()); logger.error(msg, e); return false; @@ -545,7 +544,7 @@ private Ternary sendRollingMaintenanceCommandToHost(Ho while (!completed && timeSpent < timeout * 1000L) { try { answer = agentManager.send(host.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { // Agent may be restarted on the scripts - continue polling until it is up String msg = String.format("Cannot send command to %s, waiting %sms - %s", host, pingInterval, e.getMessage()); logger.warn(msg, e); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index d7b32e77316d..69f04393174c 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -200,7 +200,6 @@ import com.cloud.exception.DiscoveryException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceInUseException; @@ -2064,7 +2063,7 @@ public Pair sendToPool(StoragePool pool, long[] hostIdsToTryFirs answers.add(_agentMgr.send(targetHostId, cmd)); } return new Pair<>(hostId, answers.toArray(new Answer[answers.size()])); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.debug("Unable to send storage pool command to {} via {}", pool::toString, () -> _hostDao.findById(hostId), () -> e); } } @@ -3664,7 +3663,7 @@ protected Answer getCheckDatastorePolicyComplianceAnswer(String storagePolicyId, } catch (AgentUnavailableException e) { logger.debug("Unable to send storage pool command to " + pool + " via " + hostIds.get(0), e); throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId()); - } catch (OperationTimedoutException | OperationCancelledException e) { + } catch (OperationTimedoutException e) { logger.debug("Failed to process storage pool command to " + pool + " via " + hostIds.get(0), e); throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId()); } diff --git a/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java b/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java index b21284ef69a1..febef471e683 100644 --- a/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java +++ b/server/src/main/java/com/cloud/storage/clvm/ClvmPoolManager.java @@ -28,7 +28,6 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -289,7 +288,7 @@ private Boolean querySingleHostLockState(Long hostId, String lvPath, String volu logger.debug("Host {} reports volume {} active={} (attr={})", hostId, volumeUuid, queryAnswer.isActive(), queryAnswer.getLvAttributes()); return queryAnswer.isActive(); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.debug("Could not query host {} for lock state: {}", hostId, e.getMessage()); return null; } @@ -364,7 +363,7 @@ private Long tryActivateExclusivelyOnHost(Long volumeId, String volumeUuid, Stri logger.warn("Recovery activation of volume {} on host {} failed: {}", volumeUuid, targetHostId, activateAnswer != null ? activateAnswer.getDetails() : "null answer"); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.warn("Recovery activation of volume {} on host {} threw exception: {}", volumeUuid, targetHostId, e.getMessage()); } @@ -466,7 +465,7 @@ public boolean transferClvmVolumeLock(String volumeUuid, Long volumeId, String v return true; - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Exception during CLVM lock transfer for volume {}: {}", volumeUuid, e.getMessage(), e); return false; } diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index 8f46a19b4eb4..ea90fc6b4b53 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -33,7 +33,6 @@ import javax.inject.Inject; -import com.cloud.exception.OperationCancelledException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import org.apache.cloudstack.api.command.admin.backup.CreateImageTransferCmd; @@ -330,7 +329,7 @@ public Backup startBackup(StartBackupCmd cmd) { StartBackupAnswer answer; try { answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { removeFailedBackup(backup); logger.error("Failed to communicate with agent on {} for {} start", host, backup, e); throw new CloudRuntimeException("Failed to communicate with agent", e); @@ -408,7 +407,7 @@ public Backup finalizeBackup(FinalizeBackupCmd cmd) { StopBackupAnswer answer; try { answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { removeFailedBackup(backup); throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -494,7 +493,7 @@ private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volu imageTransfer = imageTransferDao.persist(imageTransfer); return imageTransfer; - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } } @@ -539,7 +538,7 @@ private void startNBDServer(String transferId, String direction, Long hostId, St ); try { nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(hostId, nbdServerCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!nbdServerAnswer.getResult()) { @@ -628,7 +627,7 @@ private ImageTransferVO createUploadImageTransfer(VolumeVO volume, ImageTransfer CreateImageTransferAnswer transferAnswer; try { transferAnswer = (CreateImageTransferAnswer) agentManager.send(imageTransfer.getHostId(), transferCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -722,7 +721,7 @@ private void finalizeDownloadImageTransfer(ImageTransferVO imageTransfer) { Answer answer; try { answer = agentManager.send(backup.getHostId(), finalizeCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } @@ -746,7 +745,7 @@ private boolean stopNBDServer(ImageTransferVO imageTransfer) { Answer answer; try { answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Failed to stop NBD server on image transfer finalization", e); return false; } @@ -764,7 +763,7 @@ private String finalizeUploadImageTransferWithResult(ImageTransferVO imageTransf Answer answer; try { answer = agentManager.send(imageTransfer.getHostId(), finalizeCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { return "Failed to communicate with agent"; } @@ -882,7 +881,7 @@ private void sendDeleteCheckpointCommand(VMInstanceVO vm, String checkpointId) { Answer answer; try { answer = agentManager.send(hostId, deleteCmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Failed to communicate with agent to delete checkpoint for VM {}", vm.getId(), e); throw new CloudRuntimeException("Failed to communicate with agent", e); } diff --git a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java index 5aa322fe8f90..237135b5176b 100644 --- a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java @@ -31,7 +31,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.command.user.consoleproxy.ConsoleEndpoint; @@ -744,7 +743,7 @@ private String acquireVncTicketForVmwareVm(VirtualMachine vm) { logger.info("VNC ticket could not be acquired correctly: " + ans.getDetails()); } return ans.getTicket(); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Error acquiring ticket", e); return null; } diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index 3cdd474e0fc4..3c9d355cd89f 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -42,7 +42,6 @@ import javax.naming.ConfigurationException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationCancelledException; import com.cloud.user.Account; import com.cloud.utils.Pair; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; @@ -764,7 +763,7 @@ protected Pair revokeCertificateAliasFromHost(String alias, Hos try { Answer answer = agentManager.send(host.getId(), cmd); return new Pair<>(answer != null && answer.getResult(), answer != null ? answer.getDetails() : ""); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Error revoking certificate {} from host {}", alias, host, e); return new Pair<>(false, e.getMessage()); } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 4a1c199cc8c8..840a1de45962 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -61,7 +61,6 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientVirtualNetworkCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; @@ -2112,7 +2111,7 @@ private CheckConvertInstanceAnswer checkConversionSupportOnHost(HostVO convertHo CheckConvertInstanceAnswer checkConvertInstanceAnswer; try { checkConvertInstanceAnswer = (CheckConvertInstanceAnswer) agentManager.send(convertHost.getId(), cmd); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { String err = String.format("Failed to check %s conversion support on the host %s for converting instance %s from VMware to KVM due to: %s", checkWindowsGuestConversionSupport? "windows guest" : "", convertHost, sourceVM, e.getMessage()); logger.error(err); @@ -2203,7 +2202,7 @@ private UnmanagedInstanceTO convertAndImportToKVM(ConvertInstanceCommand convert Answer convertAnswer; try { convertAnswer = agentManager.send(convertHost.getId(), convertInstanceCommand); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { String err = String.format("Could not send the convert instance command to host %s due to: %s", convertHost, e.getMessage()); logger.error(err, e); @@ -2235,7 +2234,7 @@ private UnmanagedInstanceTO convertAndImportToKVM(ConvertInstanceCommand convert logger.error(err); throw new CloudRuntimeException(err); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { cleanupConvertedDisks = true; String err = String.format( "Could not send the import converted instance command to host %s due to: %s", @@ -2261,7 +2260,7 @@ private void cleanupConvertedDisks(String sourceVM, HostVO convertHost, DataStor logger.warn("Failed to cleanup the converted disks for the VM {} through " + "the conversion host {}: {}", sourceVM, convertHost.getName(), cleanupAnswer.getDetails()); } - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { logger.error("Error cleaning up converted disks for VM {} through the conversion host {}", sourceVM, convertHost.getName(), e); } diff --git a/server/src/test/java/com/cloud/ha/KVMFencerTest.java b/server/src/test/java/com/cloud/ha/KVMFencerTest.java index 75fdc1a51111..c4b5666c0206 100644 --- a/server/src/test/java/com/cloud/ha/KVMFencerTest.java +++ b/server/src/test/java/com/cloud/ha/KVMFencerTest.java @@ -23,7 +23,6 @@ import com.cloud.agent.api.FenceCommand; import com.cloud.alert.AlertManager; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -97,7 +96,7 @@ public void testWithSingleHostDown() { } @Test - public void testWithHosts() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testWithHosts() throws AgentUnavailableException, OperationTimedoutException { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); @@ -125,7 +124,7 @@ public void testWithHosts() throws AgentUnavailableException, OperationTimedoutE } @Test - public void testWithFailingFence() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testWithFailingFence() throws AgentUnavailableException, OperationTimedoutException { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); @@ -152,7 +151,7 @@ public void testWithFailingFence() throws AgentUnavailableException, OperationTi } @Test - public void testWithTimeoutingFence() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testWithTimeoutingFence() throws AgentUnavailableException, OperationTimedoutException { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); diff --git a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java index ad7899cd12f2..ff1a2f284666 100644 --- a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java @@ -43,7 +43,6 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; -import com.cloud.exception.OperationCancelledException; import org.apache.cloudstack.affinity.AffinityGroupVO; import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.annotation.AnnotationService; @@ -1933,7 +1932,7 @@ public void getPublicIpAndPortFail2() { } @Test - public void checkNetScalerAsGroup() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void checkNetScalerAsGroup() throws OperationTimedoutException, AgentUnavailableException { when(asVmGroupMock.getId()).thenReturn(vmGroupId); AutoScaleVmGroupTO groupTO = Mockito.mock(AutoScaleVmGroupTO.class); when(lbRulesMgr.toAutoScaleVmGroupTO(asVmGroupMock)).thenReturn(groupTO); diff --git a/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java b/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java index 22bcdf04bbc0..e4773d01de92 100644 --- a/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java +++ b/server/src/test/java/com/cloud/network/router/NetworkHelperImplTest.java @@ -30,7 +30,6 @@ import com.cloud.deploy.DeployDestination; import com.cloud.exception.InsufficientAddressCapacityException; -import com.cloud.exception.OperationCancelledException; import com.cloud.network.Ipv6Service; import com.cloud.network.Network; import com.cloud.network.Networks; @@ -149,7 +148,7 @@ public void setUp() { @Test(expected=ResourceUnavailableException.class) public void testSendCommandsToRouterWrongRouterVersion() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); @@ -164,7 +163,7 @@ public void testSendCommandsToRouterWrongRouterVersion() @Test public void testSendCommandsToRouter() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); @@ -202,7 +201,7 @@ public void testSendCommandsToRouter() */ @Test public void testSendCommandsToRouterWithTrueResult() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); @@ -240,7 +239,7 @@ public void testSendCommandsToRouterWithTrueResult() */ @Test public void testSendCommandsToRouterWithNoAnswers() - throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException, ResourceUnavailableException { // Prepare NetworkHelperImpl nwHelperUT = networkHelperSpy; VirtualRouter vr = mock(VirtualRouter.class); diff --git a/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java b/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java index cb62193b072d..1678b3760919 100644 --- a/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java +++ b/server/src/test/java/com/cloud/storage/ClvmPoolManagerTest.java @@ -21,7 +21,6 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -163,7 +162,7 @@ public void testClearClvmLockHostDetail_NoDetail() { } @Test - public void testTransferClvmVolumeLock_Success() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testTransferClvmVolumeLock_Success() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn("/" + VG_NAME); @@ -193,7 +192,7 @@ public void testTransferClvmVolumeLock_NullPool() { } @Test - public void testTransferClvmVolumeLock_SameHost() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testTransferClvmVolumeLock_SameHost() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn("/" + VG_NAME); @@ -208,7 +207,7 @@ public void testTransferClvmVolumeLock_SameHost() throws AgentUnavailableExcepti } @Test - public void testTransferClvmVolumeLock_ActivationFails() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testTransferClvmVolumeLock_ActivationFails() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn(VG_NAME); @@ -222,7 +221,7 @@ public void testTransferClvmVolumeLock_ActivationFails() throws AgentUnavailable } @Test - public void testTransferClvmVolumeLock_AgentUnavailable() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testTransferClvmVolumeLock_AgentUnavailable() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn(VG_NAME); @@ -262,7 +261,7 @@ public void testQueryCurrentLockHolder_NoHostsInCluster() { } @Test - public void testQueryCurrentLockHolder_ZoneScopedPool() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_ZoneScopedPool() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(null); when(pool.getDataCenterId()).thenReturn(1L); @@ -285,7 +284,7 @@ public void testQueryCurrentLockHolder_ZoneScopedPool() throws AgentUnavailableE } @Test - public void testQueryCurrentLockHolder_SuccessfulQuery() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_SuccessfulQuery() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -307,7 +306,7 @@ public void testQueryCurrentLockHolder_SuccessfulQuery() throws AgentUnavailable } @Test - public void testQueryCurrentLockHolder_VolumeNotLocked() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_VolumeNotLocked() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -332,7 +331,7 @@ public void testQueryCurrentLockHolder_VolumeNotLocked() throws AgentUnavailable } @Test - public void testQueryCurrentLockHolder_EmptyHostname() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_EmptyHostname() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -357,7 +356,7 @@ public void testQueryCurrentLockHolder_EmptyHostname() throws AgentUnavailableEx } @Test - public void testQueryCurrentLockHolder_HostnameNotResolved() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_HostnameNotResolved() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -378,7 +377,7 @@ public void testQueryCurrentLockHolder_HostnameNotResolved() throws AgentUnavail } @Test - public void testQueryCurrentLockHolder_QueryFails() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_QueryFails() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -398,7 +397,7 @@ public void testQueryCurrentLockHolder_QueryFails() throws AgentUnavailableExcep } @Test - public void testQueryCurrentLockHolder_NullAnswer() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_NullAnswer() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -417,7 +416,7 @@ public void testQueryCurrentLockHolder_NullAnswer() throws AgentUnavailableExcep } @Test - public void testQueryCurrentLockHolder_AgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_AgentUnavailableException() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -437,7 +436,7 @@ public void testQueryCurrentLockHolder_AgentUnavailableException() throws AgentU } @Test - public void testQueryCurrentLockHolder_OperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_OperationTimedoutException() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -457,7 +456,7 @@ public void testQueryCurrentLockHolder_OperationTimedoutException() throws Agent } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_MatchingValue() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_UpdateDatabase_MatchingValue() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); Mockito.lenient().when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -485,7 +484,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_MatchingValue() throws Age } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_DifferentValue() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_UpdateDatabase_DifferentValue() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -518,7 +517,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_DifferentValue() throws Ag } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_NoExistingDetail() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_UpdateDatabase_NoExistingDetail() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -541,7 +540,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_NoExistingDetail() throws } @Test - public void testQueryCurrentLockHolder_UpdateDatabase_RemoveDetailWhenUnlocked() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_UpdateDatabase_RemoveDetailWhenUnlocked() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -580,7 +579,7 @@ public void testQueryCurrentLockHolder_UpdateDatabase_RemoveDetailWhenUnlocked() } @Test - public void testQueryCurrentLockHolder_SkipsNonKVMHosts() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_SkipsNonKVMHosts() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -603,7 +602,7 @@ public void testQueryCurrentLockHolder_SkipsNonKVMHosts() throws AgentUnavailabl } @Test - public void testQueryCurrentLockHolder_SkipsDownHosts() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_SkipsDownHosts() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -626,7 +625,7 @@ public void testQueryCurrentLockHolder_SkipsDownHosts() throws AgentUnavailableE } @Test - public void testQueryCurrentLockHolder_PathWithLeadingSlash() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_PathWithLeadingSlash() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -650,7 +649,7 @@ public void testQueryCurrentLockHolder_PathWithLeadingSlash() throws AgentUnavai * No fan-out should occur. */ @Test - public void testQueryCurrentLockHolder_FastPath_HitOnDbHost() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_FastPath_HitOnDbHost() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getPath()).thenReturn(VG_NAME); @@ -678,7 +677,7 @@ public void testQueryCurrentLockHolder_FastPath_HitOnDbHost() throws AgentUnavai * HOST_ID_1 should NOT be queried again during fan-out. */ @Test - public void testQueryCurrentLockHolder_FastPath_MissDbHost_FanOutFindsOther() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_FastPath_MissDbHost_FanOutFindsOther() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); when(pool.getPath()).thenReturn(VG_NAME); @@ -711,7 +710,7 @@ public void testQueryCurrentLockHolder_FastPath_MissDbHost_FanOutFindsOther() th * Fast path skip: DB host is DOWN. Fan-out proceeds to all UP hosts. */ @Test - public void testQueryCurrentLockHolder_FastPath_DbHostDown_FanOut() throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + public void testQueryCurrentLockHolder_FastPath_DbHostDown_FanOut() throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); when(pool.getPath()).thenReturn(VG_NAME); @@ -742,7 +741,7 @@ public void testQueryCurrentLockHolder_FastPath_DbHostDown_FanOut() throws Agent */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnDbHost() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -786,7 +785,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnDbHost() */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnClusterHostWhenNoDbRecord() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -826,7 +825,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_ActivatesOnClusterHost */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_SkipsDownDbHost_ActivatesOnClusterHost() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -869,7 +868,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_SkipsDownDbHost_Activa */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_ActivationThrows_ReturnsNull() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); @@ -906,7 +905,7 @@ public void testQueryCurrentLockHolder_InactiveEverywhere_ActivationThrows_Retur */ @Test public void testQueryCurrentLockHolder_InactiveEverywhere_NoEligibleHost_ReturnsNull() - throws AgentUnavailableException, OperationTimedoutException, OperationCancelledException { + throws AgentUnavailableException, OperationTimedoutException { StoragePoolVO pool = Mockito.mock(StoragePoolVO.class); when(pool.getClusterId()).thenReturn(10L); Mockito.lenient().when(pool.getName()).thenReturn("cluster-pool"); diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 35dfde864d94..f46699f88cfb 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -26,7 +26,6 @@ import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.HostPodDao; -import com.cloud.exception.OperationCancelledException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.resource.ResourceManager; @@ -507,14 +506,14 @@ public void testGetCheckDatastorePolicyComplianceAnswerAgentException() throws S try { Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(Command.class))) .thenThrow(AgentUnavailableException.class); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { Assert.fail(e.getMessage()); } storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool); try { Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(Command.class))) .thenThrow(OperationTimedoutException.class); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { Assert.fail(e.getMessage()); } storageManagerImpl.getCheckDatastorePolicyComplianceAnswer("1", pool); @@ -536,7 +535,7 @@ public void testGetCheckDatastorePolicyComplianceAnswerSuccess() throws StorageU Mockito.any(CheckDataStoreStoragePolicyComplianceCommand.class))) .thenReturn(new com.cloud.agent.api.Answer( Mockito.mock(CheckDataStoreStoragePolicyComplianceCommand.class))); - } catch (AgentUnavailableException | OperationTimedoutException | OperationCancelledException e) { + } catch (AgentUnavailableException | OperationTimedoutException e) { Assert.fail(e.getMessage()); } com.cloud.agent.api.Answer answer = diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index b14a05add794..dbc05ec99f39 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -109,7 +109,6 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.OperationCancelledException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.PermissionDeniedException; @@ -746,7 +745,7 @@ private enum VcenterParameter { } private void baseTestImportVmFromVmwareToKvm(VcenterParameter vcenterParameter, boolean selectConvertHost, - boolean selectTemporaryStorage) throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + boolean selectTemporaryStorage) throws OperationTimedoutException, AgentUnavailableException { long clusterId = 1L; long zoneId = 1L; long existingDatacenterId = 1L; @@ -994,47 +993,47 @@ private void importFromDisk(String source) throws InsufficientServerCapacityExce } @Test - public void testImportVmFromVmwareToKvmExistingVcenter() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenter() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING, false, false); } @Test - public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHost() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHost() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING, true, false); } @Test - public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHostAndTemporaryStorage() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterSetConvertHostAndTemporaryStorage() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING, true, true); } @Test(expected = ServerApiException.class) - public void testImportVmFromVmwareToKvmExistingVcenterExclusiveParameters() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterExclusiveParameters() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.BOTH, false, false); } @Test(expected = ServerApiException.class) - public void testImportVmFromVmwareToKvmExistingVcenterMissingParameters() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterMissingParameters() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.NONE, false, false); } @Test(expected = CloudRuntimeException.class) - public void testImportVmFromVmwareToKvmExistingVcenterInvalid() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterInvalid() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.EXISTING_INVALID, false, false); } @Test(expected = CloudRuntimeException.class) - public void testImportVmFromVmwareToKvmExistingVcenterAgentUnavailable() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterAgentUnavailable() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.AGENT_UNAVAILABLE, false, false); } @Test(expected = CloudRuntimeException.class) - public void testImportVmFromVmwareToKvmExistingVcenterConvertFailure() throws OperationTimedoutException, AgentUnavailableException, OperationCancelledException { + public void testImportVmFromVmwareToKvmExistingVcenterConvertFailure() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.CONVERT_FAILURE, false, false); } @Test(expected = ServerApiException.class) - public void testImportVmFromVmwareToKvmForceMsMutuallyExclusiveWithUseVddk() throws OperationCancelledException, OperationTimedoutException, AgentUnavailableException { + public void testImportVmFromVmwareToKvmForceMsMutuallyExclusiveWithUseVddk() throws OperationTimedoutException, AgentUnavailableException { baseTestImportVmFromVmwareToKvm(VcenterParameter.FORCE_MS_AND_USE_VDDK, false, false); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 2cf34dbbb706..288a108249e1 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -226,11 +226,6 @@ public boolean powerOn() throws Exception { return true; ManagedObjectReference morTask = _context.getService().powerOnVMTask(_mor, null); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("powerOn", morTask, _context); - } - // Monitor VM questions final Boolean[] flags = {false}; final VirtualMachineMO vmMo = this; @@ -298,24 +293,9 @@ public void run() { } else { logger.error("VMware powerOnVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("powerOn interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel powerOn task: " + ex.getMessage()); - } - throw e; } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } // make sure to let VM question monitor exit flags[0] = true; - if (future != null) { - future.cancel(true); - } } return false; @@ -365,15 +345,10 @@ public boolean safePowerOff(int shutdownWaitMs) throws Exception { private boolean powerOffNoCheck() throws Exception { ManagedObjectReference morTask = _context.getService().powerOffVMTask(_mor); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("powerOff", morTask, _context); - } - try { - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); // It seems that even if a power-off task is returned done, VM state may still not be marked, // wait up to 5 seconds to make sure to avoid race conditioning for immediate following on operations @@ -394,24 +369,10 @@ private boolean powerOffNoCheck() throws Exception { return true; } - logger.error("VMware powerOffVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - return false; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("powerOff interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel powerOff task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } + logger.error("VMware powerOffVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } + + return false; } public VirtualMachinePowerState getResetSafePowerState() throws Exception { @@ -447,34 +408,15 @@ public VirtualMachinePowerState getPowerState() throws Exception { public boolean reset() throws Exception { ManagedObjectReference morTask = _context.getService().resetVMTask(_mor); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("reset", morTask, _context); - } - try { - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware resetVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - return false; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("resetVM interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel resetVM task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware resetVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } + return false; } public void shutdown() throws Exception { @@ -500,35 +442,16 @@ public boolean isTemplate() throws Exception { public boolean migrate(ManagedObjectReference morRp, ManagedObjectReference morTargetHost) throws Exception { ManagedObjectReference morTask = _context.getService().migrateVMTask(_mor, morRp, morTargetHost, VirtualMachineMovePriority.DEFAULT_PRIORITY, null); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("migrateVM", morTask, _context); - } - try { - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware migrateVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - return false; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("migrateVM interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel migrateVM task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware migrateVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } + + return false; } public boolean changeDatastore(VirtualMachineRelocateSpec relocateSpec) throws Exception { @@ -605,63 +528,44 @@ public ManagedObjectReference createSnapshotGetReference(String snapshotName, St long apiTimeout = _context.getVimClient().getVcenterSessionTimeout(); ManagedObjectReference morTask = _context.getService().createSnapshotTask(_mor, snapshotName, snapshotDescription, dumpMemory, quiesce); boolean result = _context.getVimClient().waitForTask(morTask); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("createSnapshot", morTask, _context); - } - - try { - if (result) { - _context.waitForTaskProgressDone(morTask); - ManagedObjectReference morSnapshot = null; + if (result) { + _context.waitForTaskProgressDone(morTask); - // We still need to wait until the object appear in vCenter - long startTick = System.currentTimeMillis(); + ManagedObjectReference morSnapshot = null; - while (System.currentTimeMillis() - startTick < apiTimeout) { - morSnapshot = getSnapshotMor(snapshotName); + // We still need to wait until the object appear in vCenter + long startTick = System.currentTimeMillis(); - if (morSnapshot != null) { - break; - } + while (System.currentTimeMillis() - startTick < apiTimeout) { + morSnapshot = getSnapshotMor(snapshotName); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.debug("[ignored] interrupted while waiting for snapshot to be done."); - } + if (morSnapshot != null) { + break; } - if (morSnapshot == null) { - logger.error("We've been waiting for over " + apiTimeout + " milli seconds for snapshot MOR to be appearing in vCenter after CreateSnapshot task is done, " + - "but it is still not there?!"); - - return null; + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.debug("[ignored] interrupted while waiting for snapshot to be done."); } + } - logger.debug("Waited for " + (System.currentTimeMillis() - startTick) + " seconds for snapshot object [" + snapshotName + "] to appear in vCenter."); + if (morSnapshot == null) { + logger.error("We've been waiting for over " + apiTimeout + " milli seconds for snapshot MOR to be appearing in vCenter after CreateSnapshot task is done, " + + "but it is still not there?!"); - return morSnapshot; - } else { - logger.error("VMware createSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + return null; } - return null; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("createSnapshot interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel createSnapshot task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } + logger.debug("Waited for " + (System.currentTimeMillis() - startTick) + " seconds for snapshot object [" + snapshotName + "] to appear in vCenter."); + + return morSnapshot; + } else { + logger.error("VMware createSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } + + return null; } public boolean removeSnapshot(String snapshotName, boolean removeChildren) throws Exception { @@ -672,35 +576,15 @@ public boolean removeSnapshot(String snapshotName, boolean removeChildren) throw } ManagedObjectReference morTask = _context.getService().removeSnapshotTask(morSnapshot, removeChildren, true); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("removeSnapshot", morTask, _context); + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware removeSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } - try { - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware removeSnapshot_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - return false; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("removeSnapshot interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel removeSnapshot task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } - } + return false; } public boolean revertToSnapshot(String snapshotName) throws Exception { @@ -710,35 +594,15 @@ public boolean revertToSnapshot(String snapshotName) throws Exception { return false; } ManagedObjectReference morTask = _context.getService().revertToSnapshotTask(morSnapshot, _mor, null); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("revertToSnapshot", morTask, _context); + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + logger.error("VMware revert to snapshot failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } - try { - boolean result = _context.getVimClient().waitForTask(morTask); - if (result) { - _context.waitForTaskProgressDone(morTask); - return true; - } else { - logger.error("VMware revert to snapshot failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - return false; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("revertToSnapshot interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel revertToSnapshot task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } - } + return false; } /** @@ -1621,38 +1485,18 @@ public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference m reConfigSpec.getDeviceChange().add(deviceConfigSpec); ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("reconfigVM/attachDisk", morTask, _context); - } - - try { - boolean result = _context.getVimClient().waitForTask(morTask); - - if (!result) { - if (logger.isTraceEnabled()) - logger.trace("vCenter API trace - attachDisk() done(failed)"); - throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } + boolean result = _context.getVimClient().waitForTask(morTask); - _context.waitForTaskProgressDone(morTask); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("reconfigVM/attachDisk interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel reconfigVM/attachDisk task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } + if (!result) { + if (logger.isTraceEnabled()) + logger.trace("vCenter API trace - attachDisk() done(failed)"); + throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } + + _context.waitForTaskProgressDone(morTask); } - if (logger.isTraceEnabled()) + if(logger.isTraceEnabled()) logger.trace("vCenter API trace - attachDisk() done(successfully)"); } @@ -1705,32 +1549,12 @@ public List> detachDisk(String vmdkDatastor reConfigSpec.getDeviceChange().add(deviceConfigSpec); ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - VmwareHelper.TaskContext taskCtx = VmwareHelper.getCurrentTaskContext(); - if (taskCtx != null) { - taskCtx.registerTask("reconfigVM/detachDisk", morTask, _context); - } - - try { - boolean result = _context.getVimClient().waitForTask(morTask); + boolean result = _context.getVimClient().waitForTask(morTask); - if (!result) { - throw new CloudRuntimeException(String.format("Failed to detach disk from instance [%s] due to [%s].", getVmName(), TaskMO.getTaskFailureInfo(_context, morTask))); - } - _context.waitForTaskProgressDone(morTask); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("reconfigVM/detachDisk interrupted for VM: " + getVmName() + ", cancelling task"); - try { - _context.getVimClient().cancelTask(morTask); - } catch (Exception ex) { - logger.warn("Failed to cancel reconfigVM/detachDisk task: " + ex.getMessage()); - } - throw e; - } finally { - if (taskCtx != null) { - taskCtx.unregisterTask(); - } + if (!result) { + throw new CloudRuntimeException(String.format("Failed to detach disk from instance [%s] due to [%s].", getVmName(), TaskMO.getTaskFailureInfo(_context, morTask))); } + _context.waitForTaskProgressDone(morTask); // VMware does not update snapshot references to the detached disk, we have to work around it SnapshotDescriptor snapshotDescriptor = null; @@ -4032,13 +3856,9 @@ public void cancelPendingTasks() throws Exception { vmTasks++; if (!(info.getState().equals(TaskInfoState.SUCCESS) || info.getState().equals(TaskInfoState.ERROR))) { String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown"; - if (_context.getVimClient().isTaskCancellable(task)) { - logger.debug(taskName + " task pending for the Instance: " + vmName + ", cancelling it"); - vmPendingTasks++; - _context.getVimClient().cancelTask(task); - } else { - logger.debug(taskName + " task pending for the Instance: " + vmName + ", but not cancellable"); - } + logger.debug(taskName + " task pending for the Instance: " + vmName + ", cancelling it"); + vmPendingTasks++; + _context.getVimClient().cancelTask(task); } } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index b30d7f505fb4..0e46f2e03cd6 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -37,7 +37,6 @@ import javax.xml.ws.handler.PortInfo; -import com.cloud.utils.Pair; import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; @@ -416,28 +415,10 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa boolean retVal = false; - boolean cancelled = false; - if (Thread.currentThread().isInterrupted()) { - Thread.currentThread().interrupt(); // Restore interrupt status - try { - Pair status = cancelTask(task); - cancelled = status.first(); - } catch (Exception e) { - LOGGER.warn("Failed to cancel task on interruption: " + task.getValue(), e); - } - - if (cancelled) { - throw new InterruptedException("Task wait cancelled for task: " + task.getValue()); - } else { - // Clear interrupt flag to continue execution - Thread.interrupted(); - } - } - try { // info has a property - state for state of the task Object[] result = waitForValues(task, new String[] { "info.state", "info.error" }, new String[] { "state" }, new Object[][] { new Object[] { - TaskInfoState.SUCCESS, TaskInfoState.ERROR }}, false); + TaskInfoState.SUCCESS, TaskInfoState.ERROR } }); if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { @@ -461,7 +442,7 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa // Since task cancellation is asynchronous, wait for the task to be cancelled Object[] result = waitForValues(task, new String[] {"info.state", "info.error"}, new String[] {"state"}, - new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}, false); + new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}); if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { @@ -501,8 +482,8 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa * @throws InvalidPropertyFaultMsg * @throws InvalidCollectorVersionFaultMsg */ - private synchronized Object[] waitForValues(ManagedObjectReference objmor, String[] filterProps, String[] endWaitProps, Object[][] expectedVals, boolean isCancelTask) throws InvalidPropertyFaultMsg, - RuntimeFaultFaultMsg, InvalidCollectorVersionFaultMsg, InterruptedException { + private synchronized Object[] waitForValues(ManagedObjectReference objmor, String[] filterProps, String[] endWaitProps, Object[][] expectedVals) throws InvalidPropertyFaultMsg, + RuntimeFaultFaultMsg, InvalidCollectorVersionFaultMsg { // version string is initially null String version = ""; Object[] endVals = new Object[endWaitProps.length]; @@ -530,32 +511,6 @@ private synchronized Object[] waitForValues(ManagedObjectReference objmor, Strin List objupary = null; List propchgary = null; while (!reached) { - if (!isCancelTask && Thread.currentThread().isInterrupted()) { - boolean cancelled = false; - if (objmor.getType().equals("Task")) { - try { - Pair status = cancelTask(objmor); - cancelled = status.first(); - } catch (Exception e) { - LOGGER.warn("Failed to cancel task on interruption: " + objmor.getValue(), e); - } - } else { - Thread.currentThread().interrupt(); - throw new InterruptedException("Task wait cancelled for object: " + objmor.getValue()); - } - - if (cancelled) { - try { - vimPort.destroyPropertyFilter(filterSpecRef); - } catch (Exception e) { - LOGGER.warn("Failed to destroy property filter on interruption: " + e.getMessage()); - } - Thread.currentThread().interrupt(); - throw new InterruptedException("Task wait cancelled for object: " + objmor.getValue()); - } else { - Thread.interrupted(); - } - } updateset = vimPort.waitForUpdatesEx(propertyCollector, version, new WaitOptions()); if (updateset == null || updateset.getFilterSet() == null) { continue; @@ -825,32 +780,11 @@ public int getVcenterSessionTimeout() { return vCenterSessionTimeout; } - public boolean isTaskCancellable(ManagedObjectReference task) throws Exception { - TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); - if (info == null) { - LOGGER.warn("Unable to get the task info for task: " + task.getValue()); - return false; - } - - // Task must be in a pending state and be cancelable - TaskInfoState state = info.getState(); - if (state == null) { - return false; - } - - // Only pending tasks can be cancelled - if (state.equals(TaskInfoState.SUCCESS) || state.equals(TaskInfoState.ERROR)) { - return false; - } - - return info.isCancelable(); - } - - public Pair cancelTask(ManagedObjectReference task) throws Exception { + public void cancelTask(ManagedObjectReference task) throws Exception { TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); if (info == null) { - LOGGER.warn("Unable to get the VM task info, so couldn't cancel the task"); - return new Pair<>(false, "Unable to get the VM task info, so couldn't cancel the task"); + LOGGER.warn("Unable to get the task info, so couldn't cancel the task"); + return; } String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown"; @@ -859,44 +793,41 @@ public Pair cancelTask(ManagedObjectReference task) throws Exce String entityName = StringUtils.isNotBlank(info.getEntityName()) ? info.getEntityName() : ""; if (info.getState().equals(TaskInfoState.SUCCESS)) { - LOGGER.debug("VM task " + " successfully completed for the entity " + entityName + ", can't cancel it"); - return new Pair<>(false, "VM task successfully completed for the entity " + entityName); + LOGGER.debug(taskName + " task successfully completed for the entity " + entityName + ", can't cancel it"); + return; } if (info.getState().equals(TaskInfoState.ERROR)) { - LOGGER.debug("VM task " + taskName + " execution failed for the entity " + entityName + ", can't cancel it"); - return new Pair<>(false, "VM task execution failed for the entity " + entityName); + LOGGER.debug(taskName + " task execution failed for the entity " + entityName + ", can't cancel it"); + return; } - LOGGER.debug("VM task " + taskName + " pending for the entity " + entityName + ", trying to cancel"); + LOGGER.debug(taskName + " task pending for the entity " + entityName + ", trying to cancel"); if (!info.isCancelable()) { - LOGGER.warn("VM task " + taskName + " will continue to run on vCenter because it can't be cancelled"); - return new Pair<>(false, "VM task will continue to run because it can't be cancelled" + entityName); + LOGGER.warn(taskName + " task will continue to run on vCenter because it can't be cancelled"); + return; } - LOGGER.debug("Cancelling VM task " + taskName + " of the entity " + entityName); + LOGGER.debug("Cancelling task " + taskName + " of the entity " + entityName); getService().cancelTask(task); // Since task cancellation is asynchronous, wait for the task to be cancelled Object[] result = waitForValues(task, new String[] {"info.state", "info.error"}, new String[] {"state"}, - new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}, true); + new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}); if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error if (result[0].equals(TaskInfoState.SUCCESS)) { - LOGGER.warn("Failed to cancel VM task " + taskName + " of the entity " + entityName + ", the task successfully completed"); - return new Pair<>(false, "VM task successfully completed for the entity " + entityName); + LOGGER.warn("Failed to cancel" + taskName + " task of the entity " + entityName + ", the task successfully completed"); } if (result[1] instanceof LocalizedMethodFault) { MethodFault fault = ((LocalizedMethodFault)result[1]).getFault(); if (fault instanceof RequestCanceled) { - LOGGER.debug("VM task " + taskName + " of the entity " + entityName + " was successfully cancelled"); - return new Pair<>(true, "VM task successfully cancelled"); + LOGGER.debug(taskName + " task of the entity " + entityName + " was successfully cancelled"); } } else { - LOGGER.warn("Couldn't cancel VM task " + taskName + " of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage()); + LOGGER.warn("Couldn't cancel " + taskName + " task of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage()); } } - return new Pair<>(false, "Unable to cancel VM task " + taskName + " of the entity " + entityName); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index 6bef56f25bc9..c2b34aea5d93 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -345,8 +345,6 @@ public String getDatastoreNameFromPath(String inventoryPath) throws Exception { public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Exception { while (true) { - checkThreadInterrupted(morTask); - TaskInfo tinfo = (TaskInfo)_vimClient.getDynamicProperty(morTask, "info"); Integer progress = tinfo.getProgress(); if (progress == null) @@ -355,18 +353,10 @@ public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Excep if (progress.intValue() >= 100) break; - checkThreadInterrupted(morTask); Thread.sleep(1000); } } - private void checkThreadInterrupted(ManagedObjectReference morTask) throws InterruptedException { - if (Thread.currentThread().isInterrupted()) { - Thread.currentThread().interrupt(); - throw new InterruptedException("Task progress wait cancelled for task: " + morTask.getValue()); - } - } - public void getFile(String urlString, String localFileFullName) throws Exception { HttpURLConnection conn = getHTTPConnection(urlString); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 169eae68fa3a..89f6d7abd7d0 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -33,9 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Random; -import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; import javax.xml.datatype.DatatypeConfigurationException; @@ -137,186 +135,6 @@ public class VmwareHelper { public static final String MIN_VERSION_UEFI_LEGACY = "5.5"; public static final String MIN_VERSION_VMFS6 = "6.5"; - private static final ConcurrentHashMap activeVmTasks = new ConcurrentHashMap<>(); - private static final ConcurrentHashMap> vmNameToTaskSequences = new ConcurrentHashMap<>(); - private static final ThreadLocal currentTaskContext = new ThreadLocal<>(); - - public static class ActiveVmTaskInfo { - final long cmdSequence; - final String vmName; - final String commandType; - final ManagedObjectReference taskMor; - final VmwareContext context; - final long startTime; - - ActiveVmTaskInfo(long cmdSequence, String vmName, String commandType, - ManagedObjectReference taskMor, VmwareContext context) { - this.cmdSequence = cmdSequence; - this.vmName = vmName; - this.commandType = commandType; - this.taskMor = taskMor; - this.context = context; - this.startTime = System.currentTimeMillis(); - } - } - - private static void registerActiveVmTask(String taskName, long cmdSequence, String vmName, String commandType, - ManagedObjectReference taskMor, VmwareContext context) { - if (StringUtils.isBlank(vmName)) { - LOGGER.debug("No active VM to registered VM task for sequence={}, command={}, task={}", cmdSequence, commandType, taskMor); - return; - } - - ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, taskMor, context); - activeVmTasks.put(cmdSequence, taskInfo); - vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); - - LOGGER.debug("Registered active VM task: {} - sequence={}, vm={}, command={}, task={}", - taskName, cmdSequence, vmName, commandType, taskMor.getValue()); - } - - private static void unregisterActiveVmTask(long cmdSequence) { - ActiveVmTaskInfo taskInfo = activeVmTasks.remove(cmdSequence); - if (taskInfo != null) { - Set sequences = vmNameToTaskSequences.get(taskInfo.vmName); - if (sequences != null) { - sequences.remove(cmdSequence); - if (sequences.isEmpty()) { - vmNameToTaskSequences.remove(taskInfo.vmName); - } - } - LOGGER.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={} ms", - cmdSequence, taskInfo.vmName, taskInfo.commandType, - System.currentTimeMillis() - taskInfo.startTime); - } - } - - public static int cancelActiveVmTasks(String vmName) { - Set sequences = vmNameToTaskSequences.get(vmName); - if (sequences == null || sequences.isEmpty()) { - return 0; - } - - int cancelled = 0; - for (Long seq : sequences) { - if (cancelActiveVmTask(seq)) { - cancelled++; - } - } - return cancelled; - } - - public static boolean isActiveVmTaskCancellable(long cmdSequence) { - LOGGER.info("Check if active vm task for sequence: {} is cancellable or not", cmdSequence); - ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); - if (taskInfo == null) { - LOGGER.debug("No active vm task registered for sequence: {}", cmdSequence); - return true; - } - - try { - boolean cancellable = taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor); - LOGGER.debug("Active vm task registered for sequence: {} is ", cmdSequence, cancellable ? "cancellable" : "not cancellable"); - return cancellable; - } catch (Exception e) { - LOGGER.warn("Failed to verify vCenter task cancellability for sequence={}, vm={}, error={}", - cmdSequence, taskInfo.vmName, e.getMessage(), e); - return false; - } - } - - public static boolean cancelActiveVmTask(long cmdSequence) { - LOGGER.info("Cancel active vm tasks for sequence: {}", cmdSequence); - ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); - if (taskInfo == null) { - LOGGER.debug("No active vm tasks for sequence: {}", cmdSequence); - return true; - } - - try { - if (!taskInfo.context.getVimClient().isTaskCancellable(taskInfo.taskMor)) { - LOGGER.debug("Active vCenter VM task is not cancellable: sequence={}, vm={}, command={}", - cmdSequence, taskInfo.vmName, taskInfo.commandType); - return false; - } - - Pair cancellationResult = taskInfo.context.getVimClient().cancelTask(taskInfo.taskMor); - if (!cancellationResult.first()) { - LOGGER.debug("Unable to cancel vCenter VM task: sequence={}, vm={}, reason={}", - cmdSequence, taskInfo.vmName, cancellationResult.second()); - return false; - } - - LOGGER.info("Cancelled vCenter VM task: sequence={}, vm={}, command={}, task={}", - cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.taskMor.getValue()); - - // Unregister the task - unregisterActiveVmTask(cmdSequence); - return true; - } catch (Exception e) { - LOGGER.warn("Failed to cancel vCenter VM task: sequence={}, vm={}, error={}", - cmdSequence, taskInfo.vmName, e.getMessage(), e); - return false; - } - } - - public static List getActiveVmTasks(String vmName) { - Set sequences = vmNameToTaskSequences.get(vmName); - if (sequences == null || sequences.isEmpty()) { - return Collections.emptyList(); - } - - List tasks = new ArrayList<>(); - for (Long seq : sequences) { - ActiveVmTaskInfo taskInfo = activeVmTasks.get(seq); - if (taskInfo != null) { - tasks.add(taskInfo); - } - } - return tasks; - } - - public static List getAllActiveVmTasks() { - return new ArrayList<>(activeVmTasks.values()); - } - - public static class TaskContext { - private final long cmdSequence; - private final String vmName; - private final String commandType; - - public TaskContext(long cmdSequence, String vmName, String commandType) { - this.cmdSequence = cmdSequence; - this.vmName = vmName; - this.commandType = commandType; - } - - public void registerTask(String taskName, ManagedObjectReference taskMor, VmwareContext context) { - registerActiveVmTask(taskName, cmdSequence, vmName, commandType, taskMor, context); - } - - public void unregisterTask() { - unregisterActiveVmTask(cmdSequence); - } - } - - public static void setTaskContext(long cmdSequence, String vmName, String commandType) { - LOGGER.debug("Setting task context: sequence={}, vm={}, command={}", cmdSequence, vmName, commandType); - currentTaskContext.set(new TaskContext(cmdSequence, vmName, commandType)); - } - - public static void clearTaskContext() { - TaskContext taskContext = currentTaskContext.get(); - if (taskContext != null) { - LOGGER.debug("Clearing task context: sequence={}, vm={}, command={}", taskContext.cmdSequence, taskContext.vmName, taskContext.commandType); - } - currentTaskContext.remove(); - } - - public static TaskContext getCurrentTaskContext() { - return currentTaskContext.get(); - } - public static boolean isReservedScsiDeviceNumber(int deviceNumber) { // The SCSI controller is assigned to virtual device node (z:7), so that device node is unavailable for hard disks or SCSI devices. return (deviceNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER) == 7; From cd3f18558cd4ec1df2a8b8d878525a375c782227 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 10:48:38 +0530 Subject: [PATCH 16/29] VMware resource changes --- .../vmware/resource/VmwareResource.java | 32 +++ .../vmware/resource/VmwareResourceTest.java | 8 + .../hypervisor/vmware/util/VmwareClient.java | 72 +++++-- .../vmware/util/VmwareTaskRegistry.java | 183 ++++++++++++++++++ .../vmware/util/VmwareTaskRegistryTest.java | 161 +++++++++++++++ 5 files changed, 437 insertions(+), 19 deletions(-) create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java create mode 100644 vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 89b5b111e14f..9eabcb74eb2f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -267,11 +267,13 @@ import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.hypervisor.vmware.util.VmwareContextPool; import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.hypervisor.vmware.util.VmwareTaskRegistry; import com.cloud.network.Networks; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.TrafficType; import com.cloud.network.VmwareTrafficLabel; import com.cloud.network.router.VirtualRouterAutoScale; +import com.cloud.resource.RequestExecutionContext; import com.cloud.resource.ServerResource; import com.cloud.resource.ServerResourceBase; import com.cloud.serializer.GsonHelper; @@ -429,6 +431,10 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes protected volatile long _cmdSequence = 1; + // One registry per resource, so tasks are tracked per host and two hosts cannot collide on a + // request sequence number. + private final VmwareTaskRegistry taskRegistry = new VmwareTaskRegistry(); + protected StorageSubsystemCommandHandler storageHandler; private VmwareStorageProcessor _storageProcessor; @@ -467,6 +473,32 @@ private String getCommandLogTitle(Command cmd) { @Override public Answer executeRequest(Command cmd) { + final Long requestSequence = RequestExecutionContext.getRequestSequence(); + taskRegistry.beginRequest(requestSequence); + try { + Answer answer = executeRequestInternal(cmd); + // The command handlers report a cancelled vCenter task as an ordinary failure; mark it so + // the agent layer can tell a cancellation from a failure. + if (answer != null && !answer.getResult() && taskRegistry.wasCancelRequested(requestSequence)) { + answer.setCancelled(true); + } + return answer; + } finally { + taskRegistry.endRequest(requestSequence); + } + } + + @Override + public boolean isRequestSequenceCancellable(long sequence) { + return taskRegistry.isCancellable(sequence); + } + + @Override + public boolean cancelRequestSequence(long sequence) { + return taskRegistry.cancel(sequence); + } + + private Answer executeRequestInternal(Command cmd) { logCommand(cmd); Answer answer; ThreadContext.push(getCommandLogTitle(cmd)); diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java index be7ee46f6dda..79d10539c29d 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java @@ -874,4 +874,12 @@ public void testRemoveVirtualTPMDevice() throws Exception { Mockito.verify(vmwareResource, Mockito.times(1)).removeVirtualTPMDevice(vmConfigSpec, tpm); Mockito.verify(deviceChanges, Mockito.times(1)).add(any(VirtualDeviceConfigSpec.class)); } + + @Test + public void testUnknownRequestSequenceIsNotCancellable() { + // a resource that is not tracking a request must not claim it can cancel it, or the job is + // marked cancelled while the hypervisor finishes the operation + assertFalse(vmwareResource.isRequestSequenceCancellable(4242L)); + assertFalse(vmwareResource.cancelRequestSequence(4242L)); + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 0e46f2e03cd6..92b405c6a920 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; +import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import org.w3c.dom.Element; @@ -148,6 +149,8 @@ private static void trustAllHttpsCertificates() throws Exception { private String serviceCookie; private final static String SVC_INST_NAME = "ServiceInstance"; private int vCenterSessionTimeout = 1200000; // Timeout in milliseconds + private static final long CANCEL_TASK_WAIT_MS = 30000; + private static final long CANCEL_TASK_POLL_MS = 500; private boolean isConnected = false; @@ -415,6 +418,14 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa boolean retVal = false; + // Register with the request this thread is executing, so the request can be cancelled by + // cancelling this task. If the request was already cancelled before the task was created, + // cancel it now rather than let it run. + if (VmwareTaskRegistry.taskStarted(task, this)) { + LOGGER.info("Request executing on this thread was already cancelled, cancelling newly created task {}", task.getValue()); + cancelTask(task); + } + try { // info has a property - state for state of the task Object[] result = waitForValues(task, new String[] { "info.state", "info.error" }, new String[] { "state" }, new Object[][] { new Object[] { @@ -460,6 +471,8 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa throw new RuntimeException(((LocalizedMethodFault)result[1]).getLocalizedMessage()); } } + } finally { + VmwareTaskRegistry.taskFinished(task); } return retVal; } @@ -780,11 +793,26 @@ public int getVcenterSessionTimeout() { return vCenterSessionTimeout; } - public void cancelTask(ManagedObjectReference task) throws Exception { - TaskInfo info = (TaskInfo)(getDynamicProperty(task, "info")); + public boolean isTaskCancellable(ManagedObjectReference task) throws Exception { + TaskInfo info = getDynamicProperty(task, "info"); + if (info == null || info.getState() == null) { + return false; + } + if (info.getState().equals(TaskInfoState.SUCCESS) || info.getState().equals(TaskInfoState.ERROR)) { + return false; + } + return info.isCancelable(); + } + + /** + * Asks vCenter to cancel the task and waits, bounded, for it to reach a terminal state. + * Returns whether the task ended cancelled, with the reason when it did not. + */ + public Pair cancelTask(ManagedObjectReference task) throws Exception { + TaskInfo info = getDynamicProperty(task, "info"); if (info == null) { LOGGER.warn("Unable to get the task info, so couldn't cancel the task"); - return; + return new Pair<>(false, "unable to get the task info"); } String taskName = StringUtils.isNotBlank(info.getName()) ? info.getName() : "Unknown"; @@ -794,40 +822,46 @@ public void cancelTask(ManagedObjectReference task) throws Exception { if (info.getState().equals(TaskInfoState.SUCCESS)) { LOGGER.debug(taskName + " task successfully completed for the entity " + entityName + ", can't cancel it"); - return; + return new Pair<>(false, "task " + taskName + " already completed"); } if (info.getState().equals(TaskInfoState.ERROR)) { LOGGER.debug(taskName + " task execution failed for the entity " + entityName + ", can't cancel it"); - return; + return new Pair<>(false, "task " + taskName + " already failed"); } LOGGER.debug(taskName + " task pending for the entity " + entityName + ", trying to cancel"); if (!info.isCancelable()) { LOGGER.warn(taskName + " task will continue to run on vCenter because it can't be cancelled"); - return; + return new Pair<>(false, "task " + taskName + " is not cancelable on vCenter"); } LOGGER.debug("Cancelling task " + taskName + " of the entity " + entityName); getService().cancelTask(task); - // Since task cancellation is asynchronous, wait for the task to be cancelled - Object[] result = waitForValues(task, new String[] {"info.state", "info.error"}, new String[] {"state"}, - new Object[][] {new Object[] {TaskInfoState.SUCCESS, TaskInfoState.ERROR}}); - - if (result != null && result.length == 2) { //result for 2 properties: info.state, info.error - if (result[0].equals(TaskInfoState.SUCCESS)) { - LOGGER.warn("Failed to cancel" + taskName + " task of the entity " + entityName + ", the task successfully completed"); + // Cancellation is asynchronous. Poll for the outcome rather than going through waitForValues: + // that method is synchronized and the thread waiting on this very task holds its monitor. + final long deadline = System.currentTimeMillis() + CANCEL_TASK_WAIT_MS; + while (System.currentTimeMillis() < deadline) { + info = getDynamicProperty(task, "info"); + TaskInfoState state = info != null ? info.getState() : null; + if (TaskInfoState.SUCCESS.equals(state)) { + LOGGER.warn("Failed to cancel " + taskName + " task of the entity " + entityName + ", the task successfully completed"); + return new Pair<>(false, "task " + taskName + " completed before it could be cancelled"); } - - if (result[1] instanceof LocalizedMethodFault) { - MethodFault fault = ((LocalizedMethodFault)result[1]).getFault(); - if (fault instanceof RequestCanceled) { + if (TaskInfoState.ERROR.equals(state)) { + LocalizedMethodFault error = info.getError(); + if (error != null && error.getFault() instanceof RequestCanceled) { LOGGER.debug(taskName + " task of the entity " + entityName + " was successfully cancelled"); + return new Pair<>(true, "task " + taskName + " cancelled"); } - } else { - LOGGER.warn("Couldn't cancel " + taskName + " task of the entity " + entityName + " due to " + ((LocalizedMethodFault)result[1]).getLocalizedMessage()); + String reason = error != null ? error.getLocalizedMessage() : "unknown error"; + LOGGER.warn("Couldn't cancel " + taskName + " task of the entity " + entityName + " due to " + reason); + return new Pair<>(false, "task " + taskName + " failed: " + reason); } + Thread.sleep(CANCEL_TASK_POLL_MS); } + LOGGER.warn(taskName + " task of the entity " + entityName + " is still running after the cancel request"); + return new Pair<>(false, "task " + taskName + " did not stop within " + CANCEL_TASK_WAIT_MS / 1000 + " seconds of the cancel request"); } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java new file mode 100644 index 000000000000..50982d2ee03b --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.util; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.Pair; +import com.vmware.vim25.ManagedObjectReference; + +/** + * Tracks the vCenter tasks a resource is waiting on, per agent request sequence, so a request can be + * cancelled by cancelling the tasks it created. + * + * One instance per resource (one per host), so two hosts cannot collide on a sequence number. The + * resource opens a request scope for the executing thread; {@link VmwareClient#waitForTask} then reports + * every task it waits on through that scope. Registering at that single chokepoint is what gives complete + * coverage: every task the resource ever waits on passes through it. + */ +public class VmwareTaskRegistry { + private static final Logger LOGGER = LogManager.getLogger(VmwareTaskRegistry.class); + + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + private final Map requests = new ConcurrentHashMap<>(); + + private static final class ActiveTask { + final ManagedObjectReference mor; + final VmwareClient client; + + ActiveTask(final ManagedObjectReference mor, final VmwareClient client) { + this.mor = mor; + this.client = client; + } + } + + private static final class RequestState { + final Map tasks = new ConcurrentHashMap<>(); + volatile boolean cancelRequested; + } + + private static final class RequestScope { + final VmwareTaskRegistry registry; + final long sequence; + + RequestScope(final VmwareTaskRegistry registry, final long sequence) { + this.registry = registry; + this.sequence = sequence; + } + } + + /** + * Marks the current thread as executing the given request. A null sequence (a command that did not + * arrive through the agent layer) leaves the thread unscoped, so nothing is tracked for it. + */ + public void beginRequest(final Long sequence) { + if (sequence == null) { + return; + } + requests.putIfAbsent(sequence, new RequestState()); + CURRENT.set(new RequestScope(this, sequence)); + } + + public void endRequest(final Long sequence) { + CURRENT.remove(); + if (sequence != null) { + requests.remove(sequence); + } + } + + public boolean wasCancelRequested(final Long sequence) { + if (sequence == null) { + return false; + } + final RequestState state = requests.get(sequence); + return state != null && state.cancelRequested; + } + + /** + * Called by the client when it starts waiting on a task. Returns true when the request this thread is + * executing has already been cancelled, so the caller can cancel the newly created task straight away + * instead of letting it run. + */ + static boolean taskStarted(final ManagedObjectReference mor, final VmwareClient client) { + final RequestScope scope = CURRENT.get(); + if (scope == null || mor == null) { + return false; + } + final RequestState state = scope.registry.requests.get(scope.sequence); + if (state == null) { + return false; + } + state.tasks.put(mor.getValue(), new ActiveTask(mor, client)); + return state.cancelRequested; + } + + static void taskFinished(final ManagedObjectReference mor) { + final RequestScope scope = CURRENT.get(); + if (scope == null || mor == null) { + return; + } + final RequestState state = scope.registry.requests.get(scope.sequence); + if (state != null) { + state.tasks.remove(mor.getValue()); + } + } + + /** + * Whether the request can be stopped. Unknown sequences are not cancellable: we have nothing to stop + * and no way to know what the backend is doing. A request with no task in flight can be stopped by + * interrupting its thread; one with tasks in flight only if vCenter will cancel every one of them. + */ + public boolean isCancellable(final long sequence) { + final RequestState state = requests.get(sequence); + if (state == null) { + return false; + } + for (final ActiveTask task : state.tasks.values()) { + try { + if (!task.client.isTaskCancellable(task.mor)) { + LOGGER.debug("vCenter task {} of request sequence {} is not cancellable", task.mor.getValue(), sequence); + return false; + } + } catch (final Exception e) { + LOGGER.warn("Unable to check whether vCenter task {} of request sequence {} is cancellable", task.mor.getValue(), sequence, e); + return false; + } + } + return true; + } + + /** + * Cancels every task the request has in flight and marks the request so that any task it creates + * afterwards is cancelled as soon as the client starts waiting on it. Returns true only if every + * in-flight task was actually cancelled. + */ + public boolean cancel(final long sequence) { + final RequestState state = requests.get(sequence); + if (state == null) { + return false; + } + state.cancelRequested = true; + + boolean allCancelled = true; + for (final ActiveTask task : state.tasks.values()) { + try { + final Pair result = task.client.cancelTask(task.mor); + if (result.first()) { + LOGGER.info("Cancelled vCenter task {} of request sequence {}", task.mor.getValue(), sequence); + } else { + LOGGER.info("Could not cancel vCenter task {} of request sequence {}: {}", task.mor.getValue(), sequence, result.second()); + allCancelled = false; + } + } catch (final Exception e) { + LOGGER.warn("Failed to cancel vCenter task {} of request sequence {}", task.mor.getValue(), sequence, e); + allCancelled = false; + } + } + return allCancelled; + } + + int activeTaskCount(final long sequence) { + final RequestState state = requests.get(sequence); + return state == null ? 0 : state.tasks.size(); + } +} diff --git a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java new file mode 100644 index 000000000000..6fa64b619e01 --- /dev/null +++ b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.util; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.Pair; +import com.vmware.vim25.ManagedObjectReference; + +@RunWith(MockitoJUnitRunner.class) +public class VmwareTaskRegistryTest { + + private static final long SEQ = 42L; + + @Mock + private VmwareClient client; + + private final VmwareTaskRegistry registry = new VmwareTaskRegistry(); + + @After + public void tearDown() { + registry.endRequest(SEQ); + } + + @Test + public void unknownSequenceIsNeitherCancellableNorCancelled() { + Assert.assertFalse(registry.isCancellable(999L)); + Assert.assertFalse(registry.cancel(999L)); + Assert.assertFalse(registry.wasCancelRequested(999L)); + } + + @Test + public void requestWithNoTaskInFlightIsCancellableByInterruptAlone() throws Exception { + registry.beginRequest(SEQ); + + Assert.assertTrue(registry.isCancellable(SEQ)); + Assert.assertTrue(registry.cancel(SEQ)); + Assert.assertTrue(registry.wasCancelRequested(SEQ)); + Mockito.verify(client, Mockito.never()).cancelTask(Mockito.any()); + } + + @Test + public void taskCreatedAfterCancelIsReportedSoTheClientCancelsItImmediately() { + registry.beginRequest(SEQ); + registry.cancel(SEQ); + + Assert.assertTrue(VmwareTaskRegistry.taskStarted(mor("task-late"), client)); + } + + @Test + public void cancellableTaskIsCancelledThroughItsClient() throws Exception { + final ManagedObjectReference mor = mor("task-1"); + Mockito.when(client.isTaskCancellable(mor)).thenReturn(true); + Mockito.when(client.cancelTask(mor)).thenReturn(new Pair<>(true, "cancelled")); + + registry.beginRequest(SEQ); + Assert.assertFalse(VmwareTaskRegistry.taskStarted(mor, client)); + + Assert.assertTrue(registry.isCancellable(SEQ)); + Assert.assertTrue(registry.cancel(SEQ)); + Mockito.verify(client).cancelTask(mor); + } + + @Test + public void nonCancellableTaskRefusesAndIsLeftAlone() throws Exception { + final ManagedObjectReference mor = mor("task-2"); + Mockito.when(client.isTaskCancellable(mor)).thenReturn(false); + + registry.beginRequest(SEQ); + VmwareTaskRegistry.taskStarted(mor, client); + + Assert.assertFalse(registry.isCancellable(SEQ)); + Mockito.verify(client, Mockito.never()).cancelTask(Mockito.any()); + } + + @Test + public void cancelReportsFailureWhenVCenterDoesNotCancel() throws Exception { + final ManagedObjectReference mor = mor("task-3"); + Mockito.when(client.cancelTask(mor)).thenReturn(new Pair<>(false, "completed first")); + + registry.beginRequest(SEQ); + VmwareTaskRegistry.taskStarted(mor, client); + + Assert.assertFalse(registry.cancel(SEQ)); + } + + @Test + public void finishedTaskIsNoLongerTracked() throws Exception { + final ManagedObjectReference mor = mor("task-4"); + registry.beginRequest(SEQ); + VmwareTaskRegistry.taskStarted(mor, client); + Assert.assertEquals(1, registry.activeTaskCount(SEQ)); + + VmwareTaskRegistry.taskFinished(mor); + + Assert.assertEquals(0, registry.activeTaskCount(SEQ)); + Assert.assertTrue(registry.cancel(SEQ)); + Mockito.verify(client, Mockito.never()).cancelTask(Mockito.any()); + } + + @Test + public void endRequestForgetsTheSequence() { + registry.beginRequest(SEQ); + registry.endRequest(SEQ); + + Assert.assertFalse(registry.isCancellable(SEQ)); + Assert.assertFalse(VmwareTaskRegistry.taskStarted(mor("task-5"), client)); + Assert.assertEquals(0, registry.activeTaskCount(SEQ)); + } + + @Test + public void unscopedThreadRegistersNothing() throws Exception { + registry.beginRequest(SEQ); + final AtomicBoolean registered = new AtomicBoolean(true); + + // the scope is per thread: a task waited on from another thread belongs to no request + final Thread other = new Thread(() -> registered.set(VmwareTaskRegistry.taskStarted(mor("task-6"), client))); + other.start(); + other.join(); + + Assert.assertFalse(registered.get()); + Assert.assertEquals(0, registry.activeTaskCount(SEQ)); + } + + @Test + public void nullSequenceLeavesTheThreadUnscoped() { + registry.beginRequest(null); + + Assert.assertFalse(VmwareTaskRegistry.taskStarted(mor("task-7"), client)); + registry.endRequest(null); + } + + private static ManagedObjectReference mor(final String value) { + final ManagedObjectReference mor = new ManagedObjectReference(); + mor.setType("Task"); + mor.setValue(value); + return mor; + } +} From e58f7b4bedf927d66671796674b1751604478e03 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 10:56:05 +0530 Subject: [PATCH 17/29] XenServer resource changes --- .../xenserver/resource/CitrixHelper.java | 218 ---------------- .../resource/CitrixResourceBase.java | 157 +++++------- .../resource/XenServerTaskRegistry.java | 235 ++++++++++++++++++ .../resource/CitrixResourceBaseTest.java | 9 + .../resource/XenServerTaskRegistryTest.java | 178 +++++++++++++ 5 files changed, 478 insertions(+), 319 deletions(-) create mode 100644 plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java create mode 100644 plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java index 3ec483d3f0a6..a9ee1dd76082 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixHelper.java @@ -17,231 +17,13 @@ package com.cloud.hypervisor.xenserver.resource; import com.cloud.storage.Storage; -import com.xensource.xenapi.Connection; import com.xensource.xenapi.Host; -import com.xensource.xenapi.Task; -import com.xensource.xenapi.Types; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * Reduce bloat inside CitrixResourceBase * */ public class CitrixHelper { - protected static Logger LOGGER = LogManager.getLogger(CitrixHelper.class); - - /** - * Tracks active VM tasks (startVM, shutdownVM, migrateVM) by command sequence. - * Key: command sequence, Value: ActiveVmTaskInfo - */ - private static final ConcurrentHashMap activeVmTasks = new ConcurrentHashMap<>(); - - /** - * Tracks active VM tasks by VM name for quick lookup. - * Key: VM name, Value: Set of command sequences - */ - private static final ConcurrentHashMap> vmNameToTaskSequences = new ConcurrentHashMap<>(); - - /** - * ThreadLocal to store current command context for task registration. - */ - private static final ThreadLocal currentTaskContext = new ThreadLocal<>(); - - public static class ActiveVmTaskInfo { - final Task task; - final Connection connection; - final long cmdSequence; - final String vmName; - final String commandType; - final long startTime; - - ActiveVmTaskInfo(long cmdSequence, String vmName, String commandType, - Task task, Connection connection) { - this.cmdSequence = cmdSequence; - this.vmName = vmName; - this.commandType = commandType; - this.task = task; - this.connection = connection; - this.startTime = System.currentTimeMillis(); - } - } - - /** - * Registers an active XenServer task for tracking. - */ - private static void registerActiveVmTask(long cmdSequence, String vmName, String commandType, - Task task, Connection connection) { - if (StringUtils.isBlank(vmName)) { - LOGGER.debug("No active VM to registered VM task for sequence={}, command={}, task={}", cmdSequence, commandType, task); - return; - } - - ActiveVmTaskInfo taskInfo = new ActiveVmTaskInfo(cmdSequence, vmName, commandType, task, connection); - activeVmTasks.put(cmdSequence, taskInfo); - - vmNameToTaskSequences.computeIfAbsent(vmName, k -> ConcurrentHashMap.newKeySet()).add(cmdSequence); - - LOGGER.debug("Registered active VM task: sequence={}, vm={}, command={}, task={}", cmdSequence, vmName, commandType, task); - } - - /** - * Unregisters an active XenServer task. - */ - private static void unregisterActiveVmTask(long cmdSequence) { - ActiveVmTaskInfo taskInfo = activeVmTasks.remove(cmdSequence); - if (taskInfo != null) { - Set sequences = vmNameToTaskSequences.get(taskInfo.vmName); - if (sequences != null) { - sequences.remove(cmdSequence); - if (sequences.isEmpty()) { - vmNameToTaskSequences.remove(taskInfo.vmName); - } - } - LOGGER.debug("Unregistered active VM task: sequence={}, vm={}, command={}, duration={} ms", - cmdSequence, taskInfo.vmName, taskInfo.commandType, - System.currentTimeMillis() - taskInfo.startTime); - } - } - - /** - * Cancels all active XenServer tasks for a given VM name. - */ - public static int cancelActiveVmTasks(String vmName) { - Set sequences = vmNameToTaskSequences.get(vmName); - if (sequences == null || sequences.isEmpty()) { - return 0; - } - - int cancelled = 0; - for (Long seq : sequences) { - if (cancelActiveVmTask(seq)) { - cancelled++; - } - } - return cancelled; - } - - /** - * Checks whether an active XenServer task is Cancellable or not. - */ - public static boolean isActiveVmTaskCancellable(long cmdSequence) { - ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); - if (taskInfo == null) { - return true; - } - - try { - return taskInfo.task.getStatus(taskInfo.connection) == Types.TaskStatusType.PENDING; - } catch (Exception e) { - LOGGER.warn("Failed to verify XenServer task cancellability for sequence={}, vm={}, error={}", - cmdSequence, taskInfo.vmName, e.getMessage(), e); - return false; - } - } - - /** - * Cancels an active XenServer task by command sequence. - */ - public static boolean cancelActiveVmTask(long cmdSequence) { - ActiveVmTaskInfo taskInfo = activeVmTasks.get(cmdSequence); - if (taskInfo == null) { - return true; - } - - try { - if (taskInfo.task.getStatus(taskInfo.connection) != Types.TaskStatusType.PENDING) { - LOGGER.info("XenServer VM task is not cancellable: sequence={}, vm={}, command={}", - cmdSequence, taskInfo.vmName, taskInfo.commandType); - return false; - } - - // Cancel the XenServer task - taskInfo.task.cancel(taskInfo.connection); - LOGGER.info("Cancelled XenServer VM task: sequence={}, vm={}, command={}, task={}", - cmdSequence, taskInfo.vmName, taskInfo.commandType, taskInfo.task.getUuid(taskInfo.connection)); - - // Unregister the task - unregisterActiveVmTask(cmdSequence); - return true; - } catch (Exception e) { - LOGGER.warn("Failed to cancel XenServer VM task: sequence={}, vm={}, error={}", - cmdSequence, taskInfo.vmName, e.getMessage(), e); - return false; - } - } - - /** - * Gets all active XenServer tasks for a given VM name. - */ - public static List getActiveVmTasks(String vmName) { - Set sequences = vmNameToTaskSequences.get(vmName); - if (sequences == null || sequences.isEmpty()) { - return Collections.emptyList(); - } - - List tasks = new ArrayList<>(); - for (Long seq : sequences) { - ActiveVmTaskInfo taskInfo = activeVmTasks.get(seq); - if (taskInfo != null) { - tasks.add(taskInfo); - } - } - return tasks; - } - - /** - * Gets all active XenServer tasks. - */ - public static List getAllActiveVmTasks() { - return new ArrayList<>(activeVmTasks.values()); - } - - public static class TaskContext { - private final CitrixResourceBase resource; - private final long cmdSequence; - private final String vmName; - private final String commandType; - - public TaskContext(CitrixResourceBase resource, long cmdSequence, String vmName, String commandType) { - this.resource = resource; - this.cmdSequence = cmdSequence; - this.vmName = vmName; - this.commandType = commandType; - } - - public void registerTask(Task task, Connection connection) { - registerActiveVmTask(cmdSequence, vmName, commandType, task, connection); - } - - public void unregisterTask() { - unregisterActiveVmTask(cmdSequence); - } - } - - public static void setTaskContext(CitrixResourceBase resource, long cmdSequence, String vmName, String commandType) { - LOGGER.debug("Setting task context: sequence={}, vm={}, command={}", cmdSequence, vmName, commandType); - currentTaskContext.set(new TaskContext(resource, cmdSequence, vmName, commandType)); - } - - public static void clearTaskContext() { - TaskContext taskContext = currentTaskContext.get(); - if (taskContext != null) { - LOGGER.debug("Clearing task context: sequence={}, vm={}, command={}", taskContext.cmdSequence, taskContext.vmName, taskContext.commandType); - } - currentTaskContext.remove(); - } - - public static TaskContext getCurrentTaskContext() { - return currentTaskContext.get(); - } public static String getProductVersion(final Host.Record record) { String prodVersion = record.softwareVersion.get("product_version"); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 39f543650592..6a193d7bc352 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -51,7 +51,6 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.agent.api.MigrateCommand; import com.xensource.xenapi.VTPM; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageAnswer; @@ -127,6 +126,7 @@ import com.cloud.network.Networks; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.TrafficType; +import com.cloud.resource.RequestExecutionContext; import com.cloud.resource.ServerResource; import com.cloud.resource.ServerResourceBase; import com.cloud.resource.hypervisor.HypervisorResource; @@ -306,10 +306,12 @@ private static boolean isAlienVm(final VM vm, final Connection conn) throws XenA protected String _configDriveSRName = "ConfigDriveISOs"; public String _attachIsoDeviceNum = "3"; - protected volatile long _cmdSequence = 1; - protected XenServerUtilitiesHelper xenServerUtilitiesHelper = new XenServerUtilitiesHelper(); + // One registry per resource, so tasks are tracked per host and two hosts cannot collide on a + // request sequence number. + private final XenServerTaskRegistry taskRegistry = new XenServerTaskRegistry(); + protected int _wait; // Hypervisor specific params with generic value, may need to be overridden // for specific versions @@ -1778,46 +1780,40 @@ public ExecutionResult executeInVR(final String routerIP, final String script, f @Override public Answer executeRequest(final Command cmd) { - if (Thread.currentThread().isInterrupted()) { - String msg = "Command " + cmd.getClass().getSimpleName() + " was cancelled before execution"; - logger.warn(msg); - return new Answer(cmd, false, msg); + final Long requestSequence = RequestExecutionContext.getRequestSequence(); + taskRegistry.beginRequest(requestSequence); + try { + final Answer answer = executeRequestInternal(cmd); + // The command wrappers report a cancelled XenAPI task as an ordinary failure; mark it so + // the agent layer can tell a cancellation from a failure. + if (answer != null && !answer.getResult() && taskRegistry.wasCancelRequested(requestSequence)) { + answer.setCancelled(true); + } + return answer; + } finally { + taskRegistry.endRequest(requestSequence); } + } + + @Override + public boolean isRequestSequenceCancellable(final long sequence) { + return taskRegistry.isCancellable(sequence); + } + @Override + public boolean cancelRequestSequence(final long sequence) { + return taskRegistry.cancel(sequence); + } + + private Answer executeRequestInternal(final Command cmd) { final CitrixRequestWrapper wrapper = CitrixRequestWrapper.getInstance(); try { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException("Command execution cancelled"); - } - long cmdSequence = _cmdSequence++; - Class clz = cmd.getClass(); - if (clz == StartCommand.class) { - String vmName = ((StartCommand) cmd).getVirtualMachine().getName(); - setTaskContext(cmdSequence, cmd, vmName); - } else if (clz == StopCommand.class) { - String vmName = ((StopCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); - } else if (clz == RebootCommand.class) { - String vmName = ((RebootCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); - } else if (clz == MigrateCommand.class) { - String vmName = ((MigrateCommand) cmd).getVmName(); - setTaskContext(cmdSequence, cmd, vmName); - } - return wrapper.execute(cmd, this); } catch (final Exception e) { return Answer.createUnsupportedCommandAnswer(cmd); - } finally { - CitrixHelper.clearTaskContext(); } } - private void setTaskContext(long cmdSequence, Command cmd, String vmName) { - String commandType = cmd.getClass().getSimpleName(); - CitrixHelper.setTaskContext(this, cmdSequence, vmName, commandType); - } - protected void fillHostInfo(final Connection conn, final StartupRoutingCommand cmd) { final StringBuilder caps = new StringBuilder(); try { @@ -3983,14 +3979,10 @@ protected String logX(final XenAPIObject obj, final String msg) { public void migrateVM(final Connection conn, final Host destHost, final VM vm, final String vmName) throws Exception { Task task = null; - CitrixHelper.TaskContext taskContext = CitrixHelper.getCurrentTaskContext(); try { final Map other = new HashMap<>(); other.put("live", "true"); task = vm.poolMigrateAsync(conn, destHost, other); - if (taskContext != null) { - taskContext.registerTask(task, conn); - } try { // poll every 1 seconds final long timeout = _migratewait * 1000L; @@ -4002,10 +3994,6 @@ public void migrateVM(final Connection conn, final Host destHost, final VM vm, f return; } throw new CloudRuntimeException("migrate VM catch HandleInvalid and VM is not running on dest host"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("migrate VM interrupted for VM: " + vmName); - throw e; } } catch (final XenAPIException e) { final String msg = "Unable to migrate VM(" + vmName + ") from host(" + _host.getUuid() + ")"; @@ -4019,10 +4007,6 @@ public void migrateVM(final Connection conn, final Host destHost, final VM vm, f logger.debug("unable to destroy task(" + task + ") on host(" + _host.getUuid() + ") due to " + e1); } } - - if (taskContext != null) { - taskContext.unregisterTask(); - } } } @@ -5090,9 +5074,8 @@ public synchronized Network setupvSwitchNetwork(final Connection conn) { return null; } - public void shutdownVM(final Connection conn, final VM vm, final String vmName, final boolean forcedStop) throws Exception { + public void shutdownVM(final Connection conn, final VM vm, final String vmName, final boolean forcedStop) throws XmlRpcException { Task task = null; - CitrixHelper.TaskContext taskContext = CitrixHelper.getCurrentTaskContext(); try { if (forcedStop) { task = vm.hardShutdownAsync(conn); @@ -5100,10 +5083,6 @@ public void shutdownVM(final Connection conn, final VM vm, final String vmName, task = vm.cleanShutdownAsync(conn); } - if (taskContext != null) { - taskContext.registerTask(task, conn); - } - try { // poll every 1 seconds , timeout after 10 minutes waitForTask(conn, task, 1000, 10 * 60 * 1000); @@ -5114,10 +5093,6 @@ public void shutdownVM(final Connection conn, final VM vm, final String vmName, return; } throw new CloudRuntimeException("Shutdown VM catch HandleInvalid and VM is not in HALTED state"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Shutdown VM interrupted for VM: " + vmName); - throw e; } } catch (final XenAPIException e) { logger.debug("Unable to shutdown VM(" + vmName + ") with force=" + forcedStop + " on host(" + _host.getUuid() + ") due to " + e); @@ -5153,10 +5128,6 @@ public void shutdownVM(final Connection conn, final VM vm, final String vmName, logger.debug("unable to destroy task(" + task + ") on host(" + _host.getUuid() + ") due to " + e1); } } - - if (taskContext != null) { - taskContext.unregisterTask(); - } } } @@ -5167,12 +5138,8 @@ public boolean start() { public void startVM(final Connection conn, final Host host, final VM vm, final String vmName) throws Exception { Task task = null; - CitrixHelper.TaskContext taskContext = CitrixHelper.getCurrentTaskContext(); try { task = vm.startOnAsync(conn, host, false, true); - if (taskContext != null) { - taskContext.registerTask(task, conn); - } try { // poll every 1 seconds , timeout after 10 minutes waitForTask(conn, task, 1000, 10 * 60 * 1000); @@ -5191,10 +5158,6 @@ public void startVM(final Connection conn, final Host host, final VM vm, final S return; } throw new CloudRuntimeException("Start VM " + vmName + " catch BadAsyncResult and VM is not in RUNNING state"); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Start VM interrupted for VM: " + vmName); - throw e; } } catch (final XenAPIException e) { final String msg = "Unable to start VM(" + vmName + ") on host(" + _host.getUuid() + ") due to " + e; @@ -5208,9 +5171,6 @@ public void startVM(final Connection conn, final Host host, final VM vm, final S logger.debug("unable to destroy task(" + task + ") on host(" + _host.getUuid() + ") due to " + e1); } } - if (taskContext != null) { - taskContext.unregisterTask(); - } } } @@ -5349,46 +5309,41 @@ public String upgradeSnapshot(final Connection conn, final String templatePath, } } - public void waitForTask(final Connection c, final Task task, final long pollInterval, final long timeout) throws XenAPIException, XmlRpcException, TimeoutException, InterruptedException { + public void waitForTask(final Connection c, final Task task, final long pollInterval, final long timeout) throws XenAPIException, XmlRpcException, TimeoutException { final long beginTime = System.currentTimeMillis(); if (logger.isTraceEnabled()) { logger.trace("Task " + task.getNameLabel(c) + " (" + task.getUuid(c) + ") sent to " + c.getSessionReference() + " is pending completion with a " + timeout + "ms timeout"); } - while (task.getStatus(c) == Types.TaskStatusType.PENDING) { - // Check for thread interruption before each iteration - if (Thread.currentThread().isInterrupted()) { - Thread.currentThread().interrupt(); // Restore interrupt status + // Register with the request this thread is executing, so the request can be cancelled by + // cancelling this task. If the request was already cancelled before the task was created, + // cancel it now rather than let it run. + if (XenServerTaskRegistry.taskStarted(task, c)) { + logger.info("Request executing on this thread was already cancelled, cancelling newly created task " + task); + XenServerTaskRegistry.cancelTask(task, c); + } + try { + // Keep waiting through CANCELLING as well: the task is still winding down on the host, and + // callers read its final status right after this returns. + Types.TaskStatusType status = task.getStatus(c); + while (status == Types.TaskStatusType.PENDING || status == Types.TaskStatusType.CANCELLING) { try { - task.cancel(c); - logger.info("Cancelled XenServer task: " + task.getUuid(c) + " due to interruption"); - } catch (Exception e) { - logger.warn("Failed to cancel task: " + task.getUuid(c), e); - } - throw new InterruptedException("Task wait cancelled for task: " + task.getUuid(c)); - } - - try { - if (logger.isTraceEnabled()) { - logger.trace("Task " + task.getNameLabel(c) + " (" + task.getUuid(c) + ") is pending, sleeping for " + pollInterval + "ms"); + if (logger.isTraceEnabled()) { + logger.trace("Task " + task.getNameLabel(c) + " (" + task.getUuid(c) + ") is " + status + ", sleeping for " + pollInterval + "ms"); + } + Thread.sleep(pollInterval); + } catch (final InterruptedException ignored) { } - Thread.sleep(pollInterval); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); // Restore interrupt status - try { + if (System.currentTimeMillis() - beginTime > timeout) { + final String msg = "Async " + timeout / 1000 + " seconds timeout for task " + task; + logger.warn(msg); task.cancel(c); - logger.info("Cancelled XenServer task: " + task.getUuid(c) + " due to interruption"); - } catch (Exception ex) { - logger.warn("Failed to cancel task: " + task.getUuid(c), ex); + task.destroy(c); + throw new TimeoutException(msg); } - throw new InterruptedException("Task wait interrupted for task: " + task.getUuid(c)); - } - if (System.currentTimeMillis() - beginTime > timeout) { - final String msg = "Async " + timeout / 1000 + " seconds timeout for task " + task; - logger.warn(msg); - task.cancel(c); - task.destroy(c); - throw new TimeoutException(msg); + status = task.getStatus(c); } + } finally { + XenServerTaskRegistry.taskFinished(task); } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java new file mode 100644 index 000000000000..d40e9f27e18c --- /dev/null +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.xenserver.resource; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.Pair; +import com.xensource.xenapi.Connection; +import com.xensource.xenapi.Task; +import com.xensource.xenapi.Types; + +/** + * Tracks the XenAPI tasks a resource is waiting on, per agent request sequence, so a request can be + * cancelled by cancelling the tasks it created. + * + * One instance per resource (one per host). The resource opens a request scope for the executing + * thread; {@link CitrixResourceBase#waitForTask} then reports every task it waits on through that + * scope. Registering at that single chokepoint is what gives complete coverage: the storage processor + * and the command wrappers all wait on their tasks through the resource. + */ +public class XenServerTaskRegistry { + private static final Logger LOGGER = LogManager.getLogger(XenServerTaskRegistry.class); + + private static final long CANCEL_TASK_WAIT_MS = 30000; + private static final long CANCEL_TASK_POLL_MS = 500; + + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + private final Map requests = new ConcurrentHashMap<>(); + + private static final class ActiveTask { + final Task task; + final Connection connection; + + ActiveTask(final Task task, final Connection connection) { + this.task = task; + this.connection = connection; + } + } + + private static final class RequestState { + final Map tasks = new ConcurrentHashMap<>(); + volatile boolean cancelRequested; + } + + private static final class RequestScope { + final XenServerTaskRegistry registry; + final long sequence; + + RequestScope(final XenServerTaskRegistry registry, final long sequence) { + this.registry = registry; + this.sequence = sequence; + } + } + + /** + * Marks the current thread as executing the given request. A null sequence (a command that did not + * arrive through the agent layer) leaves the thread unscoped, so nothing is tracked for it. + */ + public void beginRequest(final Long sequence) { + if (sequence == null) { + return; + } + requests.putIfAbsent(sequence, new RequestState()); + CURRENT.set(new RequestScope(this, sequence)); + } + + public void endRequest(final Long sequence) { + CURRENT.remove(); + if (sequence != null) { + requests.remove(sequence); + } + } + + public boolean wasCancelRequested(final Long sequence) { + if (sequence == null) { + return false; + } + final RequestState state = requests.get(sequence); + return state != null && state.cancelRequested; + } + + /** + * Called by the resource when it starts waiting on a task. Returns true when the request this thread + * is executing has already been cancelled, so the caller can cancel the newly created task straight + * away instead of letting it run. + */ + static boolean taskStarted(final Task task, final Connection connection) { + final RequestScope scope = CURRENT.get(); + if (scope == null || task == null) { + return false; + } + final RequestState state = scope.registry.requests.get(scope.sequence); + if (state == null) { + return false; + } + state.tasks.put(refOf(task), new ActiveTask(task, connection)); + return state.cancelRequested; + } + + static void taskFinished(final Task task) { + final RequestScope scope = CURRENT.get(); + if (scope == null || task == null) { + return; + } + final RequestState state = scope.registry.requests.get(scope.sequence); + if (state != null) { + state.tasks.remove(refOf(task)); + } + } + + /** + * Whether the request can be stopped. Unknown sequences are not cancellable: we have nothing to stop + * and no way to know what the backend is doing. A request with no task in flight can be stopped by + * interrupting its thread; one with tasks in flight only if every one of them is still pending -- + * XenAPI has no cancelable flag, a task that is no longer pending simply cannot be cancelled. + */ + public boolean isCancellable(final long sequence) { + final RequestState state = requests.get(sequence); + if (state == null) { + return false; + } + for (final ActiveTask active : state.tasks.values()) { + try { + if (active.task.getStatus(active.connection) != Types.TaskStatusType.PENDING) { + LOGGER.debug("XenAPI task {} of request sequence {} is no longer pending, not cancellable", refOf(active.task), sequence); + return false; + } + } catch (final Exception e) { + LOGGER.warn("Unable to check the status of XenAPI task {} of request sequence {}", refOf(active.task), sequence, e); + return false; + } + } + return true; + } + + /** + * Cancels every task the request has in flight and marks the request so that any task it creates + * afterwards is cancelled as soon as the resource starts waiting on it. Returns true only if every + * in-flight task was actually cancelled. + */ + public boolean cancel(final long sequence) { + final RequestState state = requests.get(sequence); + if (state == null) { + return false; + } + state.cancelRequested = true; + + boolean allCancelled = true; + for (final ActiveTask active : state.tasks.values()) { + final Pair result = cancelTask(active.task, active.connection); + if (result.first()) { + LOGGER.info("Cancelled XenAPI task {} of request sequence {}", refOf(active.task), sequence); + } else { + LOGGER.info("Could not cancel XenAPI task {} of request sequence {}: {}", refOf(active.task), sequence, result.second()); + allCancelled = false; + } + } + return allCancelled; + } + + /** + * Asks XenServer to cancel the task and waits, bounded, for it to reach a terminal state. + * + * Uses the asynchronous cancel: the synchronous form would hold the cancelling thread on a XenServer + * round trip, and this is called from the agent layer while it holds nothing but the request's + * bookkeeping. Returns whether the task ended cancelled, with the reason when it did not. + */ + static Pair cancelTask(final Task task, final Connection connection) { + final String ref = refOf(task); + try { + final Types.TaskStatusType status = task.getStatus(connection); + if (status != Types.TaskStatusType.PENDING && status != Types.TaskStatusType.CANCELLING) { + return new Pair<>(false, "task " + ref + " is already " + status); + } + if (status == Types.TaskStatusType.PENDING) { + task.cancelAsync(connection); + } + + final long deadline = System.currentTimeMillis() + CANCEL_TASK_WAIT_MS; + while (System.currentTimeMillis() < deadline) { + final Types.TaskStatusType now = task.getStatus(connection); + if (now == Types.TaskStatusType.CANCELLED) { + return new Pair<>(true, "task " + ref + " cancelled"); + } + if (now == Types.TaskStatusType.SUCCESS) { + return new Pair<>(false, "task " + ref + " completed before it could be cancelled"); + } + if (now == Types.TaskStatusType.FAILURE) { + return new Pair<>(false, "task " + ref + " failed before it could be cancelled"); + } + Thread.sleep(CANCEL_TASK_POLL_MS); + } + return new Pair<>(false, "task " + ref + " did not stop within " + CANCEL_TASK_WAIT_MS / 1000 + " seconds of the cancel request"); + } catch (final Types.OperationNotAllowed e) { + return new Pair<>(false, "XenServer does not allow cancelling task " + ref); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return new Pair<>(false, "interrupted while waiting for task " + ref + " to cancel"); + } catch (final Exception e) { + LOGGER.warn("Failed to cancel XenAPI task {}", ref, e); + return new Pair<>(false, "failed to cancel task " + ref + ": " + e.getMessage()); + } + } + + // A task's opaque ref is the natural key, but a task that has not been round-tripped to the + // server (or a test double) has none; fall back to identity so the map never sees a null key. + private static String refOf(final Task task) { + final String ref = task.toWireString(); + return ref != null ? ref : "task@" + Integer.toHexString(System.identityHashCode(task)); + } + + int activeTaskCount(final long sequence) { + final RequestState state = requests.get(sequence); + return state == null ? 0 : state.tasks.size(); + } +} diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java index aac865622196..cf872889f4c0 100644 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java @@ -68,6 +68,7 @@ import static com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.PLATFORM_CORES_PER_SOCKET_KEY; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.mockito.Mockito.doReturn; @RunWith(MockitoJUnitRunner.class) @@ -616,4 +617,12 @@ public void testListFilesAtPathWithFile() throws IOException, XmlRpcException { Assert.assertEquals(1024L, listAnswer.getSizes().get(0).longValue()); Assert.assertEquals(123456789000L, listAnswer.getLastModified().get(0).longValue()); } + + @Test + public void testUnknownRequestSequenceIsNotCancellable() { + // a resource that is not tracking a request must not claim it can cancel it, or the job is + // marked cancelled while the hypervisor finishes the operation + assertFalse(citrixResourceBase.isRequestSequenceCancellable(4242L)); + assertFalse(citrixResourceBase.cancelRequestSequence(4242L)); + } } diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java new file mode 100644 index 000000000000..56ecee4fa8f8 --- /dev/null +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.xenserver.resource; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.Pair; +import com.xensource.xenapi.Connection; +import com.xensource.xenapi.Task; +import com.xensource.xenapi.Types; + +@RunWith(MockitoJUnitRunner.class) +public class XenServerTaskRegistryTest { + + private static final long SEQ = 42L; + + @Mock + private Connection connection; + + private final XenServerTaskRegistry registry = new XenServerTaskRegistry(); + + @After + public void tearDown() { + registry.endRequest(SEQ); + } + + @Test + public void unknownSequenceIsNeitherCancellableNorCancelled() { + Assert.assertFalse(registry.isCancellable(999L)); + Assert.assertFalse(registry.cancel(999L)); + Assert.assertFalse(registry.wasCancelRequested(999L)); + } + + @Test + public void requestWithNoTaskInFlightIsCancellableByInterruptAlone() { + registry.beginRequest(SEQ); + + Assert.assertTrue(registry.isCancellable(SEQ)); + Assert.assertTrue(registry.cancel(SEQ)); + Assert.assertTrue(registry.wasCancelRequested(SEQ)); + } + + @Test + public void taskCreatedAfterCancelIsReportedSoTheResourceCancelsItImmediately() throws Exception { + registry.beginRequest(SEQ); + registry.cancel(SEQ); + + Assert.assertTrue(XenServerTaskRegistry.taskStarted(task("OpaqueRef:late", Types.TaskStatusType.PENDING), connection)); + } + + @Test + public void pendingTaskIsCancelledAsynchronouslyAndConfirmed() throws Exception { + final Task task = task("OpaqueRef:1", Types.TaskStatusType.PENDING, Types.TaskStatusType.PENDING, + Types.TaskStatusType.CANCELLING, Types.TaskStatusType.CANCELLED); + + registry.beginRequest(SEQ); + Assert.assertFalse(XenServerTaskRegistry.taskStarted(task, connection)); + + Assert.assertTrue(registry.isCancellable(SEQ)); + Assert.assertTrue(registry.cancel(SEQ)); + Mockito.verify(task).cancelAsync(connection); + Mockito.verify(task, Mockito.never()).cancel(Mockito.any()); + } + + @Test + public void taskNoLongerPendingRefusesAndIsLeftAlone() throws Exception { + final Task task = task("OpaqueRef:2", Types.TaskStatusType.SUCCESS); + + registry.beginRequest(SEQ); + XenServerTaskRegistry.taskStarted(task, connection); + + Assert.assertFalse(registry.isCancellable(SEQ)); + Assert.assertFalse(registry.cancel(SEQ)); + Mockito.verify(task, Mockito.never()).cancelAsync(Mockito.any()); + } + + @Test + public void cancelReportsFailureWhenTheTaskCompletesFirst() throws Exception { + final Task task = task("OpaqueRef:3", Types.TaskStatusType.PENDING, Types.TaskStatusType.PENDING, Types.TaskStatusType.SUCCESS); + + registry.beginRequest(SEQ); + XenServerTaskRegistry.taskStarted(task, connection); + + Assert.assertFalse(registry.cancel(SEQ)); + } + + @Test + public void cancelReportsFailureWhenXenServerDoesNotAllowIt() throws Exception { + final Task task = task("OpaqueRef:4", Types.TaskStatusType.PENDING); + Mockito.doThrow(new Types.OperationNotAllowed("not allowed")).when(task).cancelAsync(connection); + + final Pair result = XenServerTaskRegistry.cancelTask(task, connection); + + Assert.assertFalse(result.first()); + Assert.assertTrue(result.second(), result.second().contains("does not allow")); + } + + @Test + public void finishedTaskIsNoLongerTracked() throws Exception { + final Task task = task("OpaqueRef:5", Types.TaskStatusType.PENDING); + registry.beginRequest(SEQ); + XenServerTaskRegistry.taskStarted(task, connection); + Assert.assertEquals(1, registry.activeTaskCount(SEQ)); + + XenServerTaskRegistry.taskFinished(task); + + Assert.assertEquals(0, registry.activeTaskCount(SEQ)); + Assert.assertTrue(registry.cancel(SEQ)); + Mockito.verify(task, Mockito.never()).cancelAsync(Mockito.any()); + } + + @Test + public void endRequestForgetsTheSequence() throws Exception { + registry.beginRequest(SEQ); + registry.endRequest(SEQ); + + Assert.assertFalse(registry.isCancellable(SEQ)); + Assert.assertFalse(XenServerTaskRegistry.taskStarted(task("OpaqueRef:6", Types.TaskStatusType.PENDING), connection)); + Assert.assertEquals(0, registry.activeTaskCount(SEQ)); + } + + @Test + public void unscopedThreadRegistersNothing() throws Exception { + registry.beginRequest(SEQ); + final Task task = task("OpaqueRef:7", Types.TaskStatusType.PENDING); + final AtomicBoolean registered = new AtomicBoolean(true); + + // the scope is per thread: a task waited on from another thread belongs to no request + final Thread other = new Thread(() -> registered.set(XenServerTaskRegistry.taskStarted(task, connection))); + other.start(); + other.join(); + + Assert.assertFalse(registered.get()); + Assert.assertEquals(0, registry.activeTaskCount(SEQ)); + } + + @Test + public void taskWithoutAReferenceIsStillTracked() throws Exception { + // a mocked or not-yet-round-tripped task has no opaque ref; it must not blow up the map + final Task task = Mockito.mock(Task.class); + Mockito.when(task.getStatus(connection)).thenReturn(Types.TaskStatusType.PENDING); + + registry.beginRequest(SEQ); + XenServerTaskRegistry.taskStarted(task, connection); + + Assert.assertEquals(1, registry.activeTaskCount(SEQ)); + Assert.assertTrue(registry.isCancellable(SEQ)); + } + + private Task task(final String ref, final Types.TaskStatusType first, final Types.TaskStatusType... rest) throws Exception { + final Task task = Mockito.mock(Task.class); + Mockito.when(task.toWireString()).thenReturn(ref); + Mockito.when(task.getStatus(connection)).thenReturn(first, rest); + return task; + } +} From bd3db61676feec838ca47f17af964fe2b75998c1 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 11:21:01 +0530 Subject: [PATCH 18/29] indirect agent changes --- .../src/main/java/com/cloud/agent/Agent.java | 8 +-- .../com/cloud/agent/api/CancelCommand.java | 18 +++++ .../api/agent/test/CancelCommandTest.java | 6 ++ .../com/cloud/agent/manager/AgentAttache.java | 44 +++++++++--- .../cloud/agent/manager/AgentManagerImpl.java | 4 +- .../agent/manager/ConnectedAgentAttache.java | 42 ++++++++++++ .../agent/manager/DirectAgentAttache.java | 16 ++--- .../manager/ConnectedAgentAttacheTest.java | 68 +++++++++++++++++++ .../agent/manager/DirectAgentAttacheTest.java | 6 +- 9 files changed, 185 insertions(+), 27 deletions(-) diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 275fd41edc34..db5ded18ab1e 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -807,9 +807,7 @@ protected void processRequest(final Request request, final Link link) { } commandsInProgress.incrementAndGet(); try { - if (cmd.isReconcile()) { - cmd.setRequestSequence(request.getSequence()); - } + cmd.setRequestSequence(request.getSequence()); answer = serverResource.executeRequest(cmd); } finally { commandsInProgress.decrementAndGet(); @@ -1103,9 +1101,7 @@ public void processOtherTask(final Task task) { Answer answer = null; commandsInProgress.incrementAndGet(); try { - if (command.isReconcile()) { - command.setRequestSequence(req.getSequence()); - } + command.setRequestSequence(req.getSequence()); answer = serverResource.executeRequest(command); } finally { commandsInProgress.decrementAndGet(); diff --git a/core/src/main/java/com/cloud/agent/api/CancelCommand.java b/core/src/main/java/com/cloud/agent/api/CancelCommand.java index 5adace2d334e..f2955c535c1d 100644 --- a/core/src/main/java/com/cloud/agent/api/CancelCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CancelCommand.java @@ -19,16 +19,30 @@ package com.cloud.agent.api; +/** + * Asks whoever is executing a request sequence to stop it. + * + * Between management servers this travels as a control request and drops the peer's bookkeeping for + * the sequence. Sent to an agent as an ordinary command, it asks the agent to stop the backend work + * the sequence has in flight; with checkOnly set it only asks whether that would be possible, which is + * what lets the job layer refuse a cancellation it cannot honour instead of recording it. + */ public class CancelCommand extends Command { protected long sequence; protected String reason; + protected boolean checkOnly; protected CancelCommand() { } public CancelCommand(long sequence, String reason) { + this(sequence, reason, false); + } + + public CancelCommand(long sequence, String reason, boolean checkOnly) { this.sequence = sequence; this.reason = reason; + this.checkOnly = checkOnly; } public long getSequence() { @@ -39,6 +53,10 @@ public String getReason() { return reason; } + public boolean isCheckOnly() { + return checkOnly; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/CancelCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/CancelCommandTest.java index 65fc5bc9e069..2aad6ee56427 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/CancelCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/CancelCommandTest.java @@ -41,6 +41,12 @@ public void testGetReason() { assertTrue(r.equals("goodreason")); } + @Test + public void testCheckOnlyDefaultsToFalse() { + assertFalse(cc.isCheckOnly()); + assertTrue(new CancelCommand(1L, "probe", true).isCheckOnly()); + } + @Test public void testExecuteInSequence() { boolean b = cc.executeInSequence(); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 9a99771dd636..31456dcc0baa 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -129,6 +129,9 @@ public int compare(final Object o1, final Object o2) { protected long _nextSequence; protected AgentManagerImpl _agentMgr; + // Requests cancelled because their job was cancelled, as opposed to timed out. A waiting sender + // consults this so it reports a cancellation and not a timeout when the answer never comes. + private final Set _cancelledSequences = ConcurrentHashMap.newKeySet(); public final static String[] s_commandsAllowedInMaintenanceMode = new String[] { MaintainCommand.class.toString(), MigrateCommand.class.toString(), StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(), @@ -232,8 +235,33 @@ protected synchronized void cancel(final long seq) { } } + /** + * Whether the resource can stop what it is doing for this request. Attaches that cannot ask their + * resource must answer false: claiming a cancellation that did not happen is the very mismatch + * this exists to prevent. + */ protected boolean isExecutionCancellable(final long seq) { - return true; + return false; + } + + /** + * Stops the resource's work for an executing request. Returns true only when it was stopped (or + * had nothing left to stop). Attaches that cannot reach their resource answer false. + */ + protected boolean cancelRunning(final long seq) { + return false; + } + + /** + * Cancels a request because the job it belongs to was cancelled: stops the resource's work for it, + * then drops the request. Distinct from {@link #cancel(long)}, which is also the timeout path and + * must never reach into the hypervisor. + */ + public boolean cancelExecution(final long seq) { + _cancelledSequences.add(seq); + final boolean stopped = cancelRunning(seq); + cancel(seq); + return stopped; } protected synchronized int findRequest(final Request req) { @@ -416,14 +444,13 @@ public void send(final Request req, final Listener listener) throws AgentUnavail public Answer[] send(final Request req, final int wait) throws AgentUnavailableException, OperationTimedoutException { SynchronousListener sl = new SynchronousListener(null); - final Long jobId = _agentMgr.getAsyncJobId(); long seq = req.getSequence(); send(req, sl); try { for (int i = 0; i < 2; i++) { Answer[] answers = null; - if (isJobCancelled(jobId) && isExecutionCancellable(seq)) { + if (_cancelledSequences.contains(seq)) { throw new OperationCancelledException(req.getCommands(), _id, seq, wait, false); } Command[] cmds = req.getCommands(); @@ -437,7 +464,7 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE } catch (final InterruptedException e) { logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Interrupted while waiting for the answer"); Thread.currentThread().interrupt(); - if (isJobCancelled(jobId) && isExecutionCancellable(seq)) { + if (_cancelledSequences.contains(seq)) { throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, "Cancelled while waiting for the answer"); } } @@ -453,6 +480,10 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE return answers; } + if (_cancelledSequences.contains(seq)) { + throw new OperationCancelledException(req.getCommands(), _id, seq, wait, true, "Cancelled while waiting for the answer"); + } + answers = sl.getAnswers(); // Try it again. if (answers != null) { new Response(req, answers).logD("Received after timeout: ", true); @@ -499,14 +530,11 @@ public Answer[] send(final Request req, final int wait) throws AgentUnavailableE _agentMgr.updateReconcileCommandsIfNeeded(req.getSequence(), req.getCommands(), Command.State.TIMED_OUT); throw new OperationTimedoutException(req.getCommands(), _id, seq, wait, false); } finally { + _cancelledSequences.remove(seq); unregisterListener(seq); } } - private boolean isJobCancelled(final Long jobId) { - return jobId != null && _agentMgr.isJobCancelled(jobId); - } - private Answer[] waitForAnswerOfReconcileCommand(SynchronousListener sl, final long seq, final Command command, final int wait) { Answer[] answers = null; int waitTimeLeft = wait; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 129251ab3789..2b8fa58982dc 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -2197,7 +2197,9 @@ public boolean cancelJobExecution(final long jobId, final String reason) { continue; } logger.debug("Cancelling sequence {} on host {} for job-{} ({})", request.second(), request.first(), jobId, reason); - attache.cancel(request.second()); + if (!attache.cancelExecution(request.second())) { + allCancelled = false; + } } return allCancelled; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java index f4efaaa34a42..20fbbcf4cf4e 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java @@ -19,8 +19,12 @@ import java.nio.channels.ClosedChannelException; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CancelCommand; +import com.cloud.agent.api.UnsupportedAnswer; import com.cloud.agent.transport.Request; import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.nio.Link; @@ -31,6 +35,7 @@ public class ConnectedAgentAttache extends AgentAttache { protected Link _link; + private static final int CANCEL_COMMAND_WAIT_SECONDS = 60; public ConnectedAgentAttache(final AgentManagerImpl agentMgr, final long id, final String uuid, final String name, final Hypervisor.HypervisorType hypervisorType, final Link link, final boolean maintenance) { super(agentMgr, id, uuid, name, hypervisorType, maintenance); @@ -51,6 +56,43 @@ public synchronized boolean isClosed() { return _link == null; } + /** + * The command is executing on a remote agent, so only the agent can say whether its backend work + * can be stopped; ask it. An agent that does not understand the question (an older one, or a + * resource with nothing to stop) answers unsupported, which is a no. + */ + @Override + protected boolean isExecutionCancellable(final long seq) { + return askAgentToCancel(seq, true); + } + + @Override + protected boolean cancelRunning(final long seq) { + return askAgentToCancel(seq, false); + } + + private boolean askAgentToCancel(final long seq, final boolean checkOnly) { + if (_agentMgr == null) { + return false; + } + final CancelCommand cmd = new CancelCommand(seq, "Job cancelled", checkOnly); + cmd.setWait(CANCEL_COMMAND_WAIT_SECONDS); + try { + final Answer answer = _agentMgr.send(_id, cmd); + if (answer == null || answer instanceof UnsupportedAnswer) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Agent does not support cancellation, treating as not cancellable"); + return false; + } + if (!answer.getResult()) { + logger.debug(LOG_SEQ_FORMATTED_STRING, seq, (checkOnly ? "Not cancellable: " : "Not cancelled: ") + answer.getDetails()); + } + return answer.getResult(); + } catch (final AgentUnavailableException | OperationTimedoutException e) { + logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Unable to ask the agent about cancellation: " + e.getMessage()); + return false; + } + } + @Override public void disconnect(final Status state) { synchronized (this) { diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index bb1701943dc2..92501add6775 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -132,29 +132,28 @@ protected boolean isExecutionCancellable(final long seq) { } @Override - protected void cancel(final long seq) { + protected boolean cancelRunning(final long seq) { if (removeQueuedTask(seq)) { logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelled queued task."); - super.cancel(seq); - return; + return true; } final Future future = _taskFutures.get(seq); if (future == null) { - super.cancel(seq); - return; + // Neither queued nor running any more: it finished between the check and now, nothing to stop. + return true; } final ServerResource resource = _resource; if (resource == null || !resource.isRequestSequenceCancellable(seq)) { logger.info(LOG_SEQ_FORMATTED_STRING, seq, "Cancellation requested but the command is not cancellable, letting it run to completion."); - return; + return false; } logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Cancelling request sequence at the resource."); if (!resource.cancelRequestSequence(seq)) { logger.info(LOG_SEQ_FORMATTED_STRING, seq, "Resource could not cancel the command, letting it run to completion."); - return; + return false; } final Request request = _taskRequests.get(seq); @@ -163,8 +162,7 @@ protected void cancel(final long seq) { } final boolean cancelled = future.cancel(true); logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Running task " + (cancelled ? "cancelled" : "not cancelled")); - - super.cancel(seq); + return true; } private synchronized boolean removeQueuedTask(final long seq) { diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java index 72b217d0fdd9..84ca66187006 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java @@ -19,8 +19,18 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CancelCommand; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.UnsupportedAnswer; +import com.cloud.exception.AgentUnavailableException; import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.nio.Link; @@ -80,4 +90,62 @@ public void testEqualsFalseDiffClass() throws Exception { assertFalse(agentAttache1.equals("abc")); } + + @Test + public void isExecutionCancellableAsksTheAgentWithoutCancelling() throws Exception { + AgentManagerImpl agentMgr = mock(AgentManagerImpl.class); + ConnectedAgentAttache attache = new ConnectedAgentAttache(agentMgr, 5L, "uuid", "host", Hypervisor.HypervisorType.KVM, mock(Link.class), false); + when(agentMgr.send(Mockito.eq(5L), Mockito.any(Command.class))).thenReturn(new Answer(null, true, "can")); + + assertTrue(attache.isExecutionCancellable(77L)); + + ArgumentCaptor sent = ArgumentCaptor.forClass(Command.class); + verify(agentMgr).send(Mockito.eq(5L), sent.capture()); + CancelCommand cancel = (CancelCommand) sent.getValue(); + assertTrue(cancel.isCheckOnly()); + assertTrue(77L == cancel.getSequence()); + assertFalse(cancel.executeInSequence()); + } + + @Test + public void cancelRunningSendsARealCancelAndReportsTheAgentsAnswer() throws Exception { + AgentManagerImpl agentMgr = mock(AgentManagerImpl.class); + ConnectedAgentAttache attache = new ConnectedAgentAttache(agentMgr, 5L, "uuid", "host", Hypervisor.HypervisorType.KVM, mock(Link.class), false); + when(agentMgr.send(Mockito.eq(5L), Mockito.any(Command.class))).thenReturn(new Answer(null, false, "nothing to stop")); + + assertFalse(attache.cancelRunning(77L)); + + ArgumentCaptor sent = ArgumentCaptor.forClass(Command.class); + verify(agentMgr).send(Mockito.eq(5L), sent.capture()); + assertFalse(((CancelCommand) sent.getValue()).isCheckOnly()); + } + + @Test + public void anAgentThatDoesNotUnderstandCancellationIsNotCancellable() throws Exception { + // an older agent, or a resource with nothing to stop, answers unsupported; claiming it could + // be cancelled would record a cancellation that never happened + AgentManagerImpl agentMgr = mock(AgentManagerImpl.class); + ConnectedAgentAttache attache = new ConnectedAgentAttache(agentMgr, 5L, "uuid", "host", Hypervisor.HypervisorType.KVM, mock(Link.class), false); + when(agentMgr.send(Mockito.eq(5L), Mockito.any(Command.class))).thenReturn(new UnsupportedAnswer(null, "unsupported")); + + assertFalse(attache.isExecutionCancellable(77L)); + assertFalse(attache.cancelRunning(77L)); + } + + @Test + public void anUnreachableAgentIsNotCancellable() throws Exception { + AgentManagerImpl agentMgr = mock(AgentManagerImpl.class); + ConnectedAgentAttache attache = new ConnectedAgentAttache(agentMgr, 5L, "uuid", "host", Hypervisor.HypervisorType.KVM, mock(Link.class), false); + when(agentMgr.send(Mockito.eq(5L), Mockito.any(Command.class))).thenThrow(new AgentUnavailableException(5L)); + + assertFalse(attache.isExecutionCancellable(77L)); + } + + @Test + public void withoutAnAgentManagerNothingIsAsked() { + ConnectedAgentAttache attache = new ConnectedAgentAttache(null, 5L, "uuid", "host", Hypervisor.HypervisorType.KVM, mock(Link.class), false); + + assertFalse(attache.isExecutionCancellable(77L)); + assertFalse(attache.cancelRunning(77L)); + } } diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java index 7b3cc8ce7264..5b45baebe4d0 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/DirectAgentAttacheTest.java @@ -76,7 +76,7 @@ public void testCancelQueuedTaskDropsItBeforeItReachesTheResource() throws Excep Assert.assertEquals(1, directAgentAttache.tasks.size()); Assert.assertTrue(directAgentAttache.isExecutionCancellable(202L)); - directAgentAttache.cancel(202L); + directAgentAttache.cancelExecution(202L); Assert.assertTrue(request.isCancelled()); Assert.assertEquals(0, directAgentAttache.tasks.size()); @@ -90,7 +90,7 @@ public void testCancelRunningTaskStopsTheResourceThenTheThread() throws Exceptio Mockito.doReturn(true).when(_resource).cancelRequestSequence(101L); Assert.assertTrue(directAgentAttache.isExecutionCancellable(101L)); - directAgentAttache.cancel(101L); + directAgentAttache.cancelExecution(101L); Mockito.verify(_resource).cancelRequestSequence(101L); Mockito.verify(runningTask).cancel(true); @@ -105,7 +105,7 @@ public void testCancelLeavesNonCancellableRunningTaskAlone() throws Exception { Mockito.doReturn(false).when(_resource).isRequestSequenceCancellable(303L); Assert.assertFalse(directAgentAttache.isExecutionCancellable(303L)); - directAgentAttache.cancel(303L); + directAgentAttache.cancelExecution(303L); Mockito.verify(_resource, Mockito.never()).cancelRequestSequence(Mockito.anyLong()); Mockito.verify(runningTask, Mockito.never()).cancel(Mockito.anyBoolean()); From 856076434f91249c1496211e0ae13313550ebaf2 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 11:21:34 +0530 Subject: [PATCH 19/29] KVM changes --- .../kvm/resource/KvmCancellableRequests.java | 164 ++++++++++++++++++ .../resource/LibvirtComputingResource.java | 32 ++++ .../wrapper/LibvirtCancelCommandWrapper.java | 48 +++++ .../resource/KvmCancellableRequestsTest.java | 131 ++++++++++++++ .../LibvirtCancelCommandWrapperTest.java | 77 ++++++++ 5 files changed, 452 insertions(+) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapperTest.java diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java new file mode 100644 index 000000000000..cba81a92111c --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.agent.api.Command; +import com.cloud.hypervisor.kvm.resource.disconnecthook.DisconnectHook; + +/** + * Tracks, per agent request sequence, the backend work the resource could stop for it. + * + * The command wrappers already register a {@link DisconnectHook} for every long-running libvirt job + * they start, so that an agent disconnect aborts it. Those hooks are exactly what a cancellation needs + * to run; this keeps them keyed by the request that created them. A request with no hook has nothing + * that can be stopped -- the agent cannot interrupt a blocking libvirt call or a child process from + * outside -- and is reported as not cancellable rather than cancelled in name only. + */ +public class KvmCancellableRequests { + private static final Logger LOGGER = LogManager.getLogger(KvmCancellableRequests.class); + + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + private final Map requests = new ConcurrentHashMap<>(); + + private static final class RequestState { + final Command command; + final List hooks = new CopyOnWriteArrayList<>(); + volatile boolean cancelRequested; + + RequestState(final Command command) { + this.command = command; + } + } + + /** + * Marks the current thread as executing the given request. A sequence of zero (a command that did + * not arrive through the agent layer) leaves the thread unscoped, so nothing is tracked for it. + */ + public void begin(final long sequence, final Command command) { + if (sequence <= 0) { + return; + } + requests.putIfAbsent(sequence, new RequestState(command)); + CURRENT.set(sequence); + } + + public void end(final long sequence) { + CURRENT.remove(); + if (sequence > 0) { + requests.remove(sequence); + } + } + + public boolean wasCancelRequested(final long sequence) { + final RequestState state = requests.get(sequence); + return state != null && state.cancelRequested; + } + + /** Associates a hook with the request the current thread is executing, if any. */ + public void attach(final DisconnectHook hook) { + final Long sequence = CURRENT.get(); + if (sequence == null || hook == null) { + return; + } + final RequestState state = requests.get(sequence); + if (state != null) { + state.hooks.add(hook); + } + } + + public void detach(final DisconnectHook hook) { + if (hook == null) { + return; + } + final Long sequence = CURRENT.get(); + final RequestState scoped = sequence != null ? requests.get(sequence) : null; + if (scoped != null && scoped.hooks.remove(hook)) { + return; + } + for (final RequestState state : requests.values()) { + state.hooks.remove(hook); + } + } + + /** A request can be stopped only while it has a hook that knows how to stop it. */ + public boolean isCancellable(final long sequence) { + final RequestState state = requests.get(sequence); + return state != null && !state.hooks.isEmpty(); + } + + /** + * Runs the request's hooks, each once and bounded by its own timeout, and hands every hook that ran + * to the caller so it can be dropped from the disconnect list -- a Thread runs only once, and the + * disconnect path would otherwise try to start it again. Returns true only if the request was known, + * had something to stop, and every hook ran to completion. + */ + public boolean cancel(final long sequence, final Consumer onHookRan) { + final RequestState state = requests.get(sequence); + if (state == null) { + return false; + } + state.cancelRequested = true; + if (state.hooks.isEmpty()) { + LOGGER.debug("Request sequence {} ({}) has nothing that can be stopped", sequence, describe(state.command)); + return false; + } + + boolean allRan = true; + for (final DisconnectHook hook : state.hooks) { + try { + if (hook.getState() == Thread.State.NEW) { + hook.start(); + } + hook.join(hook.getTimeoutMs()); + if (hook.isAlive()) { + LOGGER.warn("Cancel hook {} for request sequence {} did not finish within {} ms", hook.getName(), sequence, hook.getTimeoutMs()); + allRan = false; + } else { + LOGGER.info("Ran cancel hook {} for request sequence {} ({})", hook.getName(), sequence, describe(state.command)); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + allRan = false; + } catch (final Exception e) { + LOGGER.warn("Cancel hook {} for request sequence {} failed", hook.getName(), sequence, e); + allRan = false; + } finally { + onHookRan.accept(hook); + } + } + return allRan; + } + + int hookCount(final long sequence) { + final RequestState state = requests.get(sequence); + return state == null ? 0 : state.hooks.size(); + } + + private static String describe(final Command command) { + return command == null ? "unknown command" : command.getClass().getSimpleName(); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 9009ec629ca3..d990a2a67cea 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -654,6 +654,8 @@ public synchronized void registerStatusUpdater(AgentStatusUpdater updater) { } protected List _disconnectHooks = new CopyOnWriteArrayList<>(); + // The same hooks, keyed by the request that registered them, so a cancellation can run them. + private final KvmCancellableRequests cancellableRequests = new KvmCancellableRequests(); @Override public ExecutionResult executeInVR(final String routerIp, final String script, final String args) { @@ -2403,6 +2405,34 @@ public boolean stop() { */ @Override public Answer executeRequest(final Command cmd) { + // The agent stamps the request sequence on every command it dispatches; a command that did + // not come that way carries zero and is not tracked. + final long requestSequence = cmd.getRequestSequence(); + cancellableRequests.begin(requestSequence, cmd); + try { + final Answer answer = executeRequestInternal(cmd); + // A wrapper whose libvirt job was aborted reports an ordinary failure; mark it so the + // management server can tell a cancellation from a failure. + if (answer != null && !answer.getResult() && cancellableRequests.wasCancelRequested(requestSequence)) { + answer.setCancelled(true); + } + return answer; + } finally { + cancellableRequests.end(requestSequence); + } + } + + @Override + public boolean isRequestSequenceCancellable(final long sequence) { + return cancellableRequests.isCancellable(sequence); + } + + @Override + public boolean cancelRequestSequence(final long sequence) { + return cancellableRequests.cancel(sequence, this::removeDisconnectHook); + } + + private Answer executeRequestInternal(final Command cmd) { if (isReconcileCommandsEnabled) { ReconcileCommandUtils.updateLogFileForCommand(COMMANDS_LOG_PATH, cmd, Command.State.STARTED); } @@ -6921,10 +6951,12 @@ public void disconnected() { public void addDisconnectHook(DisconnectHook hook) { LOGGER.debug("Adding disconnect hook " + hook); _disconnectHooks.add(hook); + cancellableRequests.attach(hook); } public void removeDisconnectHook(DisconnectHook hook) { LOGGER.debug("Removing disconnect hook " + hook); + cancellableRequests.detach(hook); if (_disconnectHooks.contains(hook)) { LOGGER.debug("Removing disconnect hook " + hook); _disconnectHooks.remove(hook); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java new file mode 100644 index 000000000000..22a852202a05 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CancelCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; + +/** + * Answers the management server's question about another request in flight on this agent: can it be + * stopped (checkOnly), or stop it. Runs out of sequence so it is not queued behind the very request it + * is asking about. + */ +@ResourceWrapper(handles = CancelCommand.class) +public final class LibvirtCancelCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(final CancelCommand command, final LibvirtComputingResource libvirtComputingResource) { + final long sequence = command.getSequence(); + if (command.isCheckOnly()) { + final boolean cancellable = libvirtComputingResource.isRequestSequenceCancellable(sequence); + return new Answer(command, cancellable, cancellable + ? "request sequence " + sequence + " can be cancelled" + : "request sequence " + sequence + " has no work that can be stopped on this host"); + } + + final boolean cancelled = libvirtComputingResource.cancelRequestSequence(sequence); + return new Answer(command, cancelled, cancelled + ? "request sequence " + sequence + " cancelled (" + command.getReason() + ")" + : "request sequence " + sequence + " could not be cancelled on this host"); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java new file mode 100644 index 000000000000..e445181190d7 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import com.cloud.agent.api.CheckHealthCommand; +import com.cloud.hypervisor.kvm.resource.disconnecthook.DisconnectHook; + +public class KvmCancellableRequestsTest { + + private static final long SEQ = 42L; + + private final KvmCancellableRequests requests = new KvmCancellableRequests(); + private final List dropped = new ArrayList<>(); + + private static final class CountingHook extends DisconnectHook { + final AtomicInteger runs = new AtomicInteger(); + + CountingHook() { + super("counting", 2000); + } + + @Override + public void run() { + runs.incrementAndGet(); + } + } + + @After + public void tearDown() { + requests.end(SEQ); + } + + @Test + public void unknownSequenceIsNeitherCancellableNorCancelled() { + Assert.assertFalse(requests.isCancellable(999L)); + Assert.assertFalse(requests.cancel(999L, dropped::add)); + Assert.assertFalse(requests.wasCancelRequested(999L)); + } + + @Test + public void requestWithoutAHookHasNothingToStop() { + // the agent cannot interrupt a blocking libvirt call or a child process from outside, so a + // request that registered no hook must be refused, not recorded as cancelled + requests.begin(SEQ, new CheckHealthCommand()); + + Assert.assertFalse(requests.isCancellable(SEQ)); + Assert.assertFalse(requests.cancel(SEQ, dropped::add)); + Assert.assertTrue(requests.wasCancelRequested(SEQ)); + } + + @Test + public void hookRegisteredOnTheExecutingThreadIsRunOnceOnCancel() { + final CountingHook hook = new CountingHook(); + requests.begin(SEQ, new CheckHealthCommand()); + requests.attach(hook); + + Assert.assertTrue(requests.isCancellable(SEQ)); + Assert.assertTrue(requests.cancel(SEQ, dropped::add)); + Assert.assertEquals(1, hook.runs.get()); + Assert.assertEquals(1, dropped.size()); + + // a Thread runs only once; a second cancel must not try to start it again + Assert.assertTrue(requests.cancel(SEQ, dropped::add)); + Assert.assertEquals(1, hook.runs.get()); + } + + @Test + public void detachedHookIsNotRun() { + final CountingHook hook = new CountingHook(); + requests.begin(SEQ, new CheckHealthCommand()); + requests.attach(hook); + requests.detach(hook); + + Assert.assertEquals(0, requests.hookCount(SEQ)); + Assert.assertFalse(requests.isCancellable(SEQ)); + Assert.assertEquals(0, hook.runs.get()); + } + + @Test + public void hookAttachedFromAnUnscopedThreadIsIgnored() throws Exception { + final CountingHook hook = new CountingHook(); + requests.begin(SEQ, new CheckHealthCommand()); + + final Thread other = new Thread(() -> requests.attach(hook)); + other.start(); + other.join(); + + Assert.assertEquals(0, requests.hookCount(SEQ)); + } + + @Test + public void zeroSequenceLeavesTheThreadUnscoped() { + requests.begin(0L, new CheckHealthCommand()); + requests.attach(new CountingHook()); + + Assert.assertFalse(requests.isCancellable(0L)); + requests.end(0L); + } + + @Test + public void endForgetsTheSequence() { + requests.begin(SEQ, new CheckHealthCommand()); + requests.attach(new CountingHook()); + requests.end(SEQ); + + Assert.assertFalse(requests.isCancellable(SEQ)); + Assert.assertEquals(0, requests.hookCount(SEQ)); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapperTest.java new file mode 100644 index 000000000000..9284c71af898 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapperTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CancelCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCancelCommandWrapperTest { + + @Mock + private LibvirtComputingResource resource; + + private final LibvirtCancelCommandWrapper wrapper = new LibvirtCancelCommandWrapper(); + + @Test + public void checkOnlyAsksWithoutCancelling() { + Mockito.when(resource.isRequestSequenceCancellable(7L)).thenReturn(true); + + final Answer answer = wrapper.execute(new CancelCommand(7L, "probe", true), resource); + + Assert.assertTrue(answer.getResult()); + Mockito.verify(resource, Mockito.never()).cancelRequestSequence(Mockito.anyLong()); + } + + @Test + public void checkOnlyReportsNothingToStop() { + Mockito.when(resource.isRequestSequenceCancellable(7L)).thenReturn(false); + + final Answer answer = wrapper.execute(new CancelCommand(7L, "probe", true), resource); + + Assert.assertFalse(answer.getResult()); + Assert.assertTrue(answer.getDetails(), answer.getDetails().contains("no work that can be stopped")); + } + + @Test + public void cancelDelegatesAndReportsTheOutcome() { + Mockito.when(resource.cancelRequestSequence(7L)).thenReturn(true); + + final Answer answer = wrapper.execute(new CancelCommand(7L, "job cancelled"), resource); + + Assert.assertTrue(answer.getResult()); + Assert.assertTrue(answer.getDetails(), answer.getDetails().contains("job cancelled")); + Mockito.verify(resource, Mockito.never()).isRequestSequenceCancellable(Mockito.anyLong()); + } + + @Test + public void failedCancelIsReportedAsSuch() { + Mockito.when(resource.cancelRequestSequence(7L)).thenReturn(false); + + final Answer answer = wrapper.execute(new CancelCommand(7L, "job cancelled"), resource); + + Assert.assertFalse(answer.getResult()); + } +} From ce651b2b2098927033f44d908b6e5dabbe1d4986 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 11:49:54 +0530 Subject: [PATCH 20/29] cleanup / refactor --- .../main/java/com/cloud/agent/api/Answer.java | 3 +- .../OperationCancelledException.java | 9 +--- .../jobs/JobCancellationHandler.java | 22 ++------- .../com/cloud/agent/api/CancelCommand.java | 9 +--- .../resource/RequestExecutionContext.java | 9 +--- .../com/cloud/resource/ServerResource.java | 11 +---- .../java/com/cloud/agent/AgentManager.java | 5 +-- .../com/cloud/agent/manager/AgentAttache.java | 20 ++------- .../cloud/agent/manager/AgentManagerImpl.java | 18 ++------ .../agent/manager/ConnectedAgentAttache.java | 6 +-- .../agent/manager/DirectAgentAttache.java | 11 +---- .../cloud/vm/VirtualMachineManagerImpl.java | 4 +- .../manager/ConnectedAgentAttacheTest.java | 2 - .../agent/manager/DirectAgentAttacheTest.java | 2 - .../framework/jobs/AsyncJobManager.java | 6 +-- .../jobs/impl/AsyncJobManagerImpl.java | 21 +++------ .../kvm/resource/KvmCancellableRequests.java | 22 ++------- .../resource/LibvirtComputingResource.java | 7 +-- .../wrapper/LibvirtCancelCommandWrapper.java | 6 +-- .../resource/KvmCancellableRequestsTest.java | 4 +- .../vmware/resource/VmwareResource.java | 6 +-- .../vmware/resource/VmwareResourceTest.java | 2 - .../resource/CitrixResourceBase.java | 13 ++---- .../resource/XenServerTaskRegistry.java | 45 +++---------------- .../resource/CitrixResourceBaseTest.java | 2 - .../resource/XenServerTaskRegistryTest.java | 3 +- .../com/cloud/api/ApiAsyncJobDispatcher.java | 3 +- .../com/cloud/api/query/QueryManagerImpl.java | 6 +-- .../com/cloud/event/ActionEventUtils.java | 6 +-- .../com/cloud/usage/UsageServiceImpl.java | 3 +- .../hypervisor/vmware/util/VmwareClient.java | 12 ++--- .../vmware/util/VmwareTaskRegistry.java | 33 +++----------- .../vmware/util/VmwareTaskRegistryTest.java | 2 +- 33 files changed, 63 insertions(+), 270 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/Answer.java b/api/src/main/java/com/cloud/agent/api/Answer.java index e6df7f64b4cc..861a38d79da6 100644 --- a/api/src/main/java/com/cloud/agent/api/Answer.java +++ b/api/src/main/java/com/cloud/agent/api/Answer.java @@ -21,8 +21,7 @@ public class Answer extends Command { protected boolean result; protected String details; - // Set when the command was stopped because its async job was cancelled, so callers can tell a - // cancellation from an ordinary failure without matching on the detail text. + // set when the command was stopped because its async job was cancelled protected boolean cancelled; protected Answer() { diff --git a/api/src/main/java/com/cloud/exception/OperationCancelledException.java b/api/src/main/java/com/cloud/exception/OperationCancelledException.java index 25b81d6177f1..9bcbe5603332 100644 --- a/api/src/main/java/com/cloud/exception/OperationCancelledException.java +++ b/api/src/main/java/com/cloud/exception/OperationCancelledException.java @@ -22,14 +22,7 @@ import java.util.Arrays; -/** - * Thrown when an operation was stopped because its async job was cancelled. - * - * Unchecked on purpose: a cancellation is not an alternative outcome that every caller between the - * agent layer and the job layer should have to name and handle. Almost all of them would only - * re-label it as a timeout or an unavailable agent, which is exactly what hides the cancellation - * from the job result. Callers that genuinely need to distinguish it still can. - */ +/** Thrown when an operation was stopped because its async job was cancelled. Unchecked: intermediate callers must not relabel it as a timeout. */ public class OperationCancelledException extends CloudRuntimeException { private static final long serialVersionUID = SerialVersionUID.OperationCancelledException; long _agentId; diff --git a/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java b/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java index c51da8035d29..0a3d8f5b87df 100644 --- a/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java +++ b/api/src/main/java/org/apache/cloudstack/jobs/JobCancellationHandler.java @@ -16,28 +16,12 @@ // under the License. package org.apache.cloudstack.jobs; -/** - * Lets the job layer ask whoever is running a job's backend work whether that work can be stopped, - * and then stop it. - * - * This is an interface rather than a direct call to the agent manager because the job framework is - * built before the agent layer and the agent layer already depends on it; calling the other way - * round would close the cycle. - */ +/** Lets the job layer ask whoever runs a job's backend work whether it can be stopped, and stop it. An interface, not AgentManager: the job framework is built first. */ public interface JobCancellationHandler { - /** - * Whether everything this job currently has in flight can be stopped. - * - * Asked before the job is marked cancelled. A job whose backend work cannot be stopped is - * refused outright, rather than recorded as cancelled while the operation runs to completion -- - * that divergence between what CloudStack believes and what the hypervisor did is the whole - * reason this check exists. - */ + /** Asked before the job is marked cancelled; a job whose work cannot be stopped is refused, not recorded as cancelled. */ boolean isJobExecutionCancellable(long jobId); - /** - * Stops whatever this job has in flight. Returns true only if all of it was actually stopped. - */ + /** Returns true only if everything in flight was actually stopped. */ boolean cancelJobExecution(long jobId, String reason); } diff --git a/core/src/main/java/com/cloud/agent/api/CancelCommand.java b/core/src/main/java/com/cloud/agent/api/CancelCommand.java index f2955c535c1d..ce63e4bce403 100644 --- a/core/src/main/java/com/cloud/agent/api/CancelCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CancelCommand.java @@ -19,14 +19,7 @@ package com.cloud.agent.api; -/** - * Asks whoever is executing a request sequence to stop it. - * - * Between management servers this travels as a control request and drops the peer's bookkeeping for - * the sequence. Sent to an agent as an ordinary command, it asks the agent to stop the backend work - * the sequence has in flight; with checkOnly set it only asks whether that would be possible, which is - * what lets the job layer refuse a cancellation it cannot honour instead of recording it. - */ +/** Asks the executor of a request sequence to stop it; between management servers as a control request, to an agent as an ordinary command (checkOnly = just ask). */ public class CancelCommand extends Command { protected long sequence; protected String reason; diff --git a/core/src/main/java/com/cloud/resource/RequestExecutionContext.java b/core/src/main/java/com/cloud/resource/RequestExecutionContext.java index 925a06f0f4c1..0f0cc4d442c4 100644 --- a/core/src/main/java/com/cloud/resource/RequestExecutionContext.java +++ b/core/src/main/java/com/cloud/resource/RequestExecutionContext.java @@ -16,14 +16,7 @@ // under the License. package com.cloud.resource; -/** - * The agent request sequence the current thread is executing a command for. - * - * Set by the direct-agent attache around each {@link ServerResource#executeRequest} call and read by - * resources that track backend work per request, so that a cancellation aimed at a request sequence can - * find the hypervisor task that request created. Resources used to number commands with their own - * counter, which never matched the sequence the agent layer cancels by. - */ +/** The agent request sequence the current thread executes a command for; set by the direct-agent attache so resources can key backend work by it. */ public final class RequestExecutionContext { private static final ThreadLocal REQUEST_SEQUENCE = new ThreadLocal<>(); diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 92d768a4f292..1dad2a0b7cf3 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -93,19 +93,12 @@ default boolean isAppendAgentNameToLogs() { default void processPingAnswer(PingAnswer answer) {}; - /** - * Indicates whether a currently executing request sequence can be cancelled at the resource layer. - * The default is false: a resource that has not implemented cancellation must not claim it, - * or the job is marked cancelled while the backend operation runs to completion. - */ + /** Defaults to false: a resource that has not implemented cancellation must not claim it. */ default boolean isRequestSequenceCancellable(long sequence) { return false; } - /** - * Attempts to cancel a currently executing request sequence at the resource layer. - * Returns true only when the backend operation was actually stopped. - */ + /** Returns true only when the backend operation was actually stopped. */ default boolean cancelRequestSequence(long sequence) { return false; } diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index e94d40a1bf6e..ce776f836f9d 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -183,10 +183,7 @@ enum TapAgentsAction { Long getAsyncJobId(); - /** - * True when the given job has been cancelled. Answered from an in-memory view refreshed by the - * cancelled-jobs poller, so it is cheap enough to consult on the command path. - */ + /** Answered from an in-memory view kept by the cancelled-jobs poller. */ boolean isJobCancelled(Long jobId); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 31456dcc0baa..6e9062a1d2f7 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -129,8 +129,7 @@ public int compare(final Object o1, final Object o2) { protected long _nextSequence; protected AgentManagerImpl _agentMgr; - // Requests cancelled because their job was cancelled, as opposed to timed out. A waiting sender - // consults this so it reports a cancellation and not a timeout when the answer never comes. + // cancelled because the job was cancelled, as opposed to timed out private final Set _cancelledSequences = ConcurrentHashMap.newKeySet(); public final static String[] s_commandsAllowedInMaintenanceMode = new String[] { MaintainCommand.class.toString(), MigrateCommand.class.toString(), @@ -235,28 +234,17 @@ protected synchronized void cancel(final long seq) { } } - /** - * Whether the resource can stop what it is doing for this request. Attaches that cannot ask their - * resource must answer false: claiming a cancellation that did not happen is the very mismatch - * this exists to prevent. - */ + /** Default false: an attache that cannot ask its resource must not claim it. */ protected boolean isExecutionCancellable(final long seq) { return false; } - /** - * Stops the resource's work for an executing request. Returns true only when it was stopped (or - * had nothing left to stop). Attaches that cannot reach their resource answer false. - */ + /** Returns true only when the resource's work was stopped (or nothing was left to stop). */ protected boolean cancelRunning(final long seq) { return false; } - /** - * Cancels a request because the job it belongs to was cancelled: stops the resource's work for it, - * then drops the request. Distinct from {@link #cancel(long)}, which is also the timeout path and - * must never reach into the hypervisor. - */ + /** Job-cancel path; distinct from cancel(seq), which is also the timeout path and never reaches the hypervisor. */ public boolean cancelExecution(final long seq) { _cancelledSequences.add(seq); final boolean stopped = cancelRunning(seq); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 2b8fa58982dc..6605b61638c6 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -165,12 +165,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected List _loadingAgents = new ArrayList<>(); protected Map _commandTimeouts = new HashMap<>(); private int _monitorId = 0; - // A job can have several commands in flight, on more than one host, so this maps to a set. - // Written from every worker thread in send() and read by the cancelled-jobs poller. + // a job can have several commands in flight on several hosts protected final Map>> _jobToHostIdAndReqSequenceMap = new ConcurrentHashMap<>(); - // Jobs known to be cancelled, keyed by id with the time first seen. Deliberately sticky rather - // than a mirror of the database: a worker thread keeps consulting this after the job row has - // been finalised, and the row then no longer shows up as cancelled-and-unacknowledged. + // sticky on purpose: workers consult it after the job row has been finalised protected final Map _cancelledJobs = new ConcurrentHashMap<>(); private static final long CANCELLED_JOB_MEMORY_MS = TimeUnit.HOURS.toMillis(1); @@ -2112,12 +2109,7 @@ protected List findAgentsBehindOnPing() { } } - /** - * Refreshes the local view of cancelled jobs and stops whatever they still have in flight. - * - * Scoped to jobs executing on this management server: another server holds the agent connection - * for its own jobs, and acting on them here would race it for work we cannot reach. - */ + /** Stops in-flight work of jobs cancelled on this management server and acknowledges them. */ protected class CancelledJobsCheckTask extends ManagedContextRunnable { @Override protected void runInContext() { @@ -2151,7 +2143,6 @@ public boolean isJobCancelled(final Long jobId) { public boolean isJobExecutionCancellable(final long jobId) { final Set> inFlight = _jobToHostIdAndReqSequenceMap.get(jobId); if (CollectionUtils.isEmpty(inFlight)) { - // Nothing is running on an agent for this job, so cancelling it strands no backend work. return true; } @@ -2173,8 +2164,7 @@ public boolean isJobExecutionCancellable(final long jobId) { @Override public boolean cancelJobExecution(final long jobId, final String reason) { - // Mark it before touching the agents, so a worker thread that catches the interruption and - // carries on to its next command is refused there. + // mark first, so a worker that carries on to its next command is refused there _cancelledJobs.putIfAbsent(jobId, System.currentTimeMillis()); final Set> inFlight = _jobToHostIdAndReqSequenceMap.get(jobId); if (CollectionUtils.isEmpty(inFlight)) { diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java index 20fbbcf4cf4e..f8570e3f66b0 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ConnectedAgentAttache.java @@ -56,11 +56,7 @@ public synchronized boolean isClosed() { return _link == null; } - /** - * The command is executing on a remote agent, so only the agent can say whether its backend work - * can be stopped; ask it. An agent that does not understand the question (an older one, or a - * resource with nothing to stop) answers unsupported, which is a no. - */ + /** Only the agent knows whether its backend work can be stopped; an agent that does not understand the question answers unsupported. */ @Override protected boolean isExecutionCancellable(final long seq) { return askAgentToCancel(seq, true); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index 92501add6775..4139d8bc6ae3 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -114,20 +114,16 @@ private synchronized boolean isQueued(final long seq) { @Override protected boolean isExecutionCancellable(final long seq) { - // Still queued: it has not reached the resource, so dropping it strands nothing. if (isQueued(seq)) { return true; } if (!_taskFutures.containsKey(seq)) { - // Neither queued nor running here. We have nothing to stop and no way to know what the - // backend is doing, so do not claim it can be cancelled. return false; } final ServerResource resource = _resource; - // Deliberately not holding the attache lock: asking the resource means a call out to the - // hypervisor, and the send path needs the lock to keep queueing work. + // not under the attache lock: this call goes out to the hypervisor return resource != null && resource.isRequestSequenceCancellable(seq); } @@ -140,7 +136,6 @@ protected boolean cancelRunning(final long seq) { final Future future = _taskFutures.get(seq); if (future == null) { - // Neither queued nor running any more: it finished between the check and now, nothing to stop. return true; } @@ -434,8 +429,7 @@ protected void runInContext() { answer = new Answer(cmds[i], false, "Resource returned null answer"); } - // If cancellation was requested while the current command was running, - // stop scheduling further commands for cancellable sequences. + // cancelled mid-request: do not start the remaining commands if ((Thread.currentThread().isInterrupted() || _req.isCancelled()) && resource.isRequestSequenceCancellable(seq)) { answers.add(answer); @@ -468,7 +462,6 @@ protected void runInContext() { } catch (InterruptedException e) { logger.warn(LOG_SEQ_FORMATTED_STRING, seq, "Task interrupted"); Thread.currentThread().interrupt(); - // Send cancellation response handleCancellation(seq, "Task interrupted: " + e.getMessage()); } catch (Throwable t) { // This is pretty serious as processAnswers might not be called and the calling process is stuck waiting for the full timeout diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index f54ddd879e12..374c661945ca 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1677,9 +1677,7 @@ public void orchestrateStart(final String vmUuid, final Map listCancelledJobsExecutingOn(long msid); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index e4e7b5d4e48c..d1b3ab5e5863 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -185,8 +185,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private NetworkOrchestrationService networkOrchestrationService; @Inject private ReconcileCommandService reconcileCommandService; - // A list so that the job framework still starts in contexts that have no agent layer; - // Spring leaves it empty rather than failing to wire. + // a list: Spring leaves it empty when no agent layer is present @Inject private List jobCancellationHandlers; @@ -1478,11 +1477,7 @@ public boolean stop() { return true; } - /** - * A queued child (VM work) job whose parent has already finished must not run: the parent was - * cancelled or failed while this child waited its turn. Completes the child with the parent's - * status and drops it from the queue. - */ + /** A queued VM work job whose parent already finished is completed with the parent's status instead of run. */ private boolean isChildOfFinishedJob(final SyncQueueItemVO item) { if (!SyncQueueItem.AsyncJobContentType.equalsIgnoreCase(item.getContentType())) { return false; @@ -1522,8 +1517,6 @@ public String cancelAsyncJob(final long jobId, final String reason) { return errMessage; } - // A job that is still queued can always be dropped. One that is executing -- here or on - // another management server -- only if its command opted in. if (job.getExecutingMsid() != null || isActiveJob(jobId)) { try { final Class cmdClass = Class.forName(job.getCmd()); @@ -1547,9 +1540,7 @@ public String cancelAsyncJob(final long jobId, final String reason) { return errMessage; } - // Ask before changing anything. If the backend work cannot be stopped, refusing is the - // honest answer: recording the job as cancelled while the hypervisor finishes the operation - // is the divergence this whole feature exists to prevent. + // ask before changing state: a job whose work cannot be stopped is refused, not recorded as cancelled for (final JobCancellationHandler handler : getJobCancellationHandlers()) { if (!handler.isJobExecutionCancellable(jobId)) { errMessage = "Cannot cancel job-" + jobId + ", the operation it is running cannot be stopped at this point."; @@ -1568,9 +1559,7 @@ public String cancelAsyncJob(final long jobId, final String reason) { } } - // If no other management server is executing the job there is nobody left to - // acknowledge the cancellation, so finish the row here. Otherwise keep executing_msid - // so that server's poller can find the job, stop its in-flight commands and finish it. + // finalise here unless another management server is executing the job; its poller acknowledges final Long executingMsid = job.getExecutingMsid(); final boolean finalizeNow = executingMsid == null || executingMsid == getMsid(); completeAsyncJob(jobId, JobInfo.Status.CANCELLED, 0, "Job is cancelled due to " + reason, finalizeNow); @@ -1606,7 +1595,7 @@ private List getJobCancellationHandlers() { } private boolean isActiveJob(final long jobId) { - // a VM work child runs on behalf of its API parent, so check whichever of the two is active + // a VM work child runs on behalf of its API parent final AsyncJobVO relatedJob = _jobDao.getRelatedJob(String.valueOf(jobId)); if (relatedJob != null) { return _jobMonitor.isActiveJob(relatedJob.getId()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java index cba81a92111c..dc182081bedf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequests.java @@ -28,15 +28,7 @@ import com.cloud.agent.api.Command; import com.cloud.hypervisor.kvm.resource.disconnecthook.DisconnectHook; -/** - * Tracks, per agent request sequence, the backend work the resource could stop for it. - * - * The command wrappers already register a {@link DisconnectHook} for every long-running libvirt job - * they start, so that an agent disconnect aborts it. Those hooks are exactly what a cancellation needs - * to run; this keeps them keyed by the request that created them. A request with no hook has nothing - * that can be stopped -- the agent cannot interrupt a blocking libvirt call or a child process from - * outside -- and is reported as not cancellable rather than cancelled in name only. - */ +/** Per agent request sequence, the DisconnectHooks the wrappers registered for its libvirt jobs: they are what a cancellation runs. A request without a hook has nothing that can be stopped. */ public class KvmCancellableRequests { private static final Logger LOGGER = LogManager.getLogger(KvmCancellableRequests.class); @@ -54,10 +46,7 @@ private static final class RequestState { } } - /** - * Marks the current thread as executing the given request. A sequence of zero (a command that did - * not arrive through the agent layer) leaves the thread unscoped, so nothing is tracked for it. - */ + /** A sequence of zero (not from the agent layer) leaves the thread unscoped. */ public void begin(final long sequence, final Command command) { if (sequence <= 0) { return; @@ -110,12 +99,7 @@ public boolean isCancellable(final long sequence) { return state != null && !state.hooks.isEmpty(); } - /** - * Runs the request's hooks, each once and bounded by its own timeout, and hands every hook that ran - * to the caller so it can be dropped from the disconnect list -- a Thread runs only once, and the - * disconnect path would otherwise try to start it again. Returns true only if the request was known, - * had something to stop, and every hook ran to completion. - */ + /** Runs each hook once, bounded by its timeout; a Thread cannot be started twice, so ran hooks are handed back for removal from the disconnect list. */ public boolean cancel(final long sequence, final Consumer onHookRan) { final RequestState state = requests.get(sequence); if (state == null) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index d990a2a67cea..42f5955245ef 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -654,7 +654,7 @@ public synchronized void registerStatusUpdater(AgentStatusUpdater updater) { } protected List _disconnectHooks = new CopyOnWriteArrayList<>(); - // The same hooks, keyed by the request that registered them, so a cancellation can run them. + // the same hooks keyed by the request that registered them private final KvmCancellableRequests cancellableRequests = new KvmCancellableRequests(); @Override @@ -2405,14 +2405,11 @@ public boolean stop() { */ @Override public Answer executeRequest(final Command cmd) { - // The agent stamps the request sequence on every command it dispatches; a command that did - // not come that way carries zero and is not tracked. final long requestSequence = cmd.getRequestSequence(); cancellableRequests.begin(requestSequence, cmd); try { final Answer answer = executeRequestInternal(cmd); - // A wrapper whose libvirt job was aborted reports an ordinary failure; mark it so the - // management server can tell a cancellation from a failure. + // an aborted libvirt job surfaces as an ordinary failure; flag it as cancelled if (answer != null && !answer.getResult() && cancellableRequests.wasCancelRequested(requestSequence)) { answer.setCancelled(true); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java index 22a852202a05..d77b935d7a91 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCancelCommandWrapper.java @@ -22,11 +22,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -/** - * Answers the management server's question about another request in flight on this agent: can it be - * stopped (checkOnly), or stop it. Runs out of sequence so it is not queued behind the very request it - * is asking about. - */ +/** Can another request in flight on this agent be stopped (checkOnly), or stop it. Out of sequence, so it is not queued behind that request. */ @ResourceWrapper(handles = CancelCommand.class) public final class LibvirtCancelCommandWrapper extends CommandWrapper { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java index e445181190d7..75de371f7a92 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/KvmCancellableRequestsTest.java @@ -61,8 +61,6 @@ public void unknownSequenceIsNeitherCancellableNorCancelled() { @Test public void requestWithoutAHookHasNothingToStop() { - // the agent cannot interrupt a blocking libvirt call or a child process from outside, so a - // request that registered no hook must be refused, not recorded as cancelled requests.begin(SEQ, new CheckHealthCommand()); Assert.assertFalse(requests.isCancellable(SEQ)); @@ -81,7 +79,7 @@ public void hookRegisteredOnTheExecutingThreadIsRunOnceOnCancel() { Assert.assertEquals(1, hook.runs.get()); Assert.assertEquals(1, dropped.size()); - // a Thread runs only once; a second cancel must not try to start it again + // a Thread runs only once Assert.assertTrue(requests.cancel(SEQ, dropped::add)); Assert.assertEquals(1, hook.runs.get()); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 9eabcb74eb2f..7756ba414f1b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -431,8 +431,7 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes protected volatile long _cmdSequence = 1; - // One registry per resource, so tasks are tracked per host and two hosts cannot collide on a - // request sequence number. + // per resource, so two hosts cannot collide on a request sequence private final VmwareTaskRegistry taskRegistry = new VmwareTaskRegistry(); protected StorageSubsystemCommandHandler storageHandler; @@ -477,8 +476,7 @@ public Answer executeRequest(Command cmd) { taskRegistry.beginRequest(requestSequence); try { Answer answer = executeRequestInternal(cmd); - // The command handlers report a cancelled vCenter task as an ordinary failure; mark it so - // the agent layer can tell a cancellation from a failure. + // a cancelled vCenter task surfaces as an ordinary failure; flag it as cancelled if (answer != null && !answer.getResult() && taskRegistry.wasCancelRequested(requestSequence)) { answer.setCancelled(true); } diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java index 79d10539c29d..3ea921e560b6 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java @@ -877,8 +877,6 @@ public void testRemoveVirtualTPMDevice() throws Exception { @Test public void testUnknownRequestSequenceIsNotCancellable() { - // a resource that is not tracking a request must not claim it can cancel it, or the job is - // marked cancelled while the hypervisor finishes the operation assertFalse(vmwareResource.isRequestSequenceCancellable(4242L)); assertFalse(vmwareResource.cancelRequestSequence(4242L)); } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 6a193d7bc352..aa171e617513 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -308,8 +308,7 @@ private static boolean isAlienVm(final VM vm, final Connection conn) throws XenA protected XenServerUtilitiesHelper xenServerUtilitiesHelper = new XenServerUtilitiesHelper(); - // One registry per resource, so tasks are tracked per host and two hosts cannot collide on a - // request sequence number. + // per resource, so two hosts cannot collide on a request sequence private final XenServerTaskRegistry taskRegistry = new XenServerTaskRegistry(); protected int _wait; @@ -1784,8 +1783,7 @@ public Answer executeRequest(final Command cmd) { taskRegistry.beginRequest(requestSequence); try { final Answer answer = executeRequestInternal(cmd); - // The command wrappers report a cancelled XenAPI task as an ordinary failure; mark it so - // the agent layer can tell a cancellation from a failure. + // a cancelled XenAPI task surfaces as an ordinary failure; flag it as cancelled if (answer != null && !answer.getResult() && taskRegistry.wasCancelRequested(requestSequence)) { answer.setCancelled(true); } @@ -5314,16 +5312,13 @@ public void waitForTask(final Connection c, final Task task, final long pollInte if (logger.isTraceEnabled()) { logger.trace("Task " + task.getNameLabel(c) + " (" + task.getUuid(c) + ") sent to " + c.getSessionReference() + " is pending completion with a " + timeout + "ms timeout"); } - // Register with the request this thread is executing, so the request can be cancelled by - // cancelling this task. If the request was already cancelled before the task was created, - // cancel it now rather than let it run. + // register with the executing request; a request already cancelled cancels the new task at once if (XenServerTaskRegistry.taskStarted(task, c)) { logger.info("Request executing on this thread was already cancelled, cancelling newly created task " + task); XenServerTaskRegistry.cancelTask(task, c); } try { - // Keep waiting through CANCELLING as well: the task is still winding down on the host, and - // callers read its final status right after this returns. + // wait through CANCELLING too: callers read the final status right after this returns Types.TaskStatusType status = task.getStatus(c); while (status == Types.TaskStatusType.PENDING || status == Types.TaskStatusType.CANCELLING) { try { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java index d40e9f27e18c..15655f87e540 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistry.java @@ -27,15 +27,7 @@ import com.xensource.xenapi.Task; import com.xensource.xenapi.Types; -/** - * Tracks the XenAPI tasks a resource is waiting on, per agent request sequence, so a request can be - * cancelled by cancelling the tasks it created. - * - * One instance per resource (one per host). The resource opens a request scope for the executing - * thread; {@link CitrixResourceBase#waitForTask} then reports every task it waits on through that - * scope. Registering at that single chokepoint is what gives complete coverage: the storage processor - * and the command wrappers all wait on their tasks through the resource. - */ +/** Per agent request sequence, the XenAPI tasks a resource is waiting on; one instance per host. waitForTask is the single registration point. */ public class XenServerTaskRegistry { private static final Logger LOGGER = LogManager.getLogger(XenServerTaskRegistry.class); @@ -71,10 +63,7 @@ private static final class RequestScope { } } - /** - * Marks the current thread as executing the given request. A null sequence (a command that did not - * arrive through the agent layer) leaves the thread unscoped, so nothing is tracked for it. - */ + /** A null sequence (not from the agent layer) leaves the thread unscoped. */ public void beginRequest(final Long sequence) { if (sequence == null) { return; @@ -98,11 +87,7 @@ public boolean wasCancelRequested(final Long sequence) { return state != null && state.cancelRequested; } - /** - * Called by the resource when it starts waiting on a task. Returns true when the request this thread - * is executing has already been cancelled, so the caller can cancel the newly created task straight - * away instead of letting it run. - */ + /** Returns true when the request was already cancelled, so the caller cancels the new task at once. */ static boolean taskStarted(final Task task, final Connection connection) { final RequestScope scope = CURRENT.get(); if (scope == null || task == null) { @@ -127,12 +112,7 @@ static void taskFinished(final Task task) { } } - /** - * Whether the request can be stopped. Unknown sequences are not cancellable: we have nothing to stop - * and no way to know what the backend is doing. A request with no task in flight can be stopped by - * interrupting its thread; one with tasks in flight only if every one of them is still pending -- - * XenAPI has no cancelable flag, a task that is no longer pending simply cannot be cancelled. - */ + /** Unknown sequences are not cancellable; XenAPI has no cancelable flag, so tasks must still be pending. */ public boolean isCancellable(final long sequence) { final RequestState state = requests.get(sequence); if (state == null) { @@ -152,11 +132,7 @@ public boolean isCancellable(final long sequence) { return true; } - /** - * Cancels every task the request has in flight and marks the request so that any task it creates - * afterwards is cancelled as soon as the resource starts waiting on it. Returns true only if every - * in-flight task was actually cancelled. - */ + /** Cancels the in-flight tasks and marks the request so later tasks are cancelled on arrival. */ public boolean cancel(final long sequence) { final RequestState state = requests.get(sequence); if (state == null) { @@ -177,13 +153,7 @@ public boolean cancel(final long sequence) { return allCancelled; } - /** - * Asks XenServer to cancel the task and waits, bounded, for it to reach a terminal state. - * - * Uses the asynchronous cancel: the synchronous form would hold the cancelling thread on a XenServer - * round trip, and this is called from the agent layer while it holds nothing but the request's - * bookkeeping. Returns whether the task ended cancelled, with the reason when it did not. - */ + /** cancelAsync, then poll to a terminal state: the synchronous cancel would hold the caller on a XenServer round trip. */ static Pair cancelTask(final Task task, final Connection connection) { final String ref = refOf(task); try { @@ -221,8 +191,7 @@ static Pair cancelTask(final Task task, final Connection connec } } - // A task's opaque ref is the natural key, but a task that has not been round-tripped to the - // server (or a test double) has none; fall back to identity so the map never sees a null key. + // no opaque ref before a round trip (or on a test double): fall back to identity private static String refOf(final Task task) { final String ref = task.toWireString(); return ref != null ? ref : "task@" + Integer.toHexString(System.identityHashCode(task)); diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java index cf872889f4c0..67d5d3ac3a79 100644 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java @@ -620,8 +620,6 @@ public void testListFilesAtPathWithFile() throws IOException, XmlRpcException { @Test public void testUnknownRequestSequenceIsNotCancellable() { - // a resource that is not tracking a request must not claim it can cancel it, or the job is - // marked cancelled while the hypervisor finishes the operation assertFalse(citrixResourceBase.isRequestSequenceCancellable(4242L)); assertFalse(citrixResourceBase.cancelRequestSequence(4242L)); } diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java index 56ecee4fa8f8..29b359dae635 100644 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/XenServerTaskRegistryTest.java @@ -147,7 +147,7 @@ public void unscopedThreadRegistersNothing() throws Exception { final Task task = task("OpaqueRef:7", Types.TaskStatusType.PENDING); final AtomicBoolean registered = new AtomicBoolean(true); - // the scope is per thread: a task waited on from another thread belongs to no request + // scope is per thread final Thread other = new Thread(() -> registered.set(XenServerTaskRegistry.taskStarted(task, connection))); other.start(); other.join(); @@ -158,7 +158,6 @@ public void unscopedThreadRegistersNothing() throws Exception { @Test public void taskWithoutAReferenceIsStillTracked() throws Exception { - // a mocked or not-yet-round-tripped task has no opaque ref; it must not blow up the map final Task task = Mockito.mock(Task.class); Mockito.when(task.getStatus(connection)).thenReturn(Types.TaskStatusType.PENDING); diff --git a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java index 3f6e82f54ce1..8eabd6b62457 100644 --- a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java @@ -118,8 +118,7 @@ public void runJob(final AsyncJob job) { CallContext.unregister(); } } catch (Throwable e) { - // A job cancelled mid-execution has already reached a terminal state; re-reading it here - // stops the failure that cancellation caused from overwriting the CANCELLED result. + // a job cancelled mid-execution is already terminal; do not overwrite CANCELLED with the failure it caused AsyncJobVO jobFromDb = _asyncJobMgr.getAsyncJob(job.getId()); if (jobFromDb != null && jobFromDb.getStatus().done()) { logger.debug("Not recording failure for job-{}, it is already in {}", job.getId(), jobFromDb.getStatus()); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 41a406ba044b..eb1e517c7f7c 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -3225,8 +3225,7 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync Boolean isRecursive = domainIdRecursiveListProject.second(); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); - // A job is soft-deleted the moment it completes, so anything but the default pending-only - // listing has to reach into removed rows or it can never return a finished job. + // completed jobs are soft-deleted; anything but the pending-only default must include removed rows final boolean filterByStatus = cmd.getJobStatuses() != null; final boolean includeRemoved = filterByStatus || cmd.getEndDate() != null || cmd.getDuration() != null; @@ -3299,8 +3298,7 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync } if (cmd.getDuration() != null) { - // Rows are written with DateUtil.currentGMTTime(); anchoring the window in the system - // zone would shift it by the offset on any management server that is not on UTC. + // rows are written in GMT Date lastDate = new Date(DateUtil.currentGMTTime().getTime() - TimeUnit.HOURS.toMillis(cmd.getDuration())); SearchCriteria scc = _jobJoinDao.createSearchCriteria(); diff --git a/server/src/main/java/com/cloud/event/ActionEventUtils.java b/server/src/main/java/com/cloud/event/ActionEventUtils.java index 7fa2800f36b5..6031b110684d 100644 --- a/server/src/main/java/com/cloud/event/ActionEventUtils.java +++ b/server/src/main/java/com/cloud/event/ActionEventUtils.java @@ -209,11 +209,7 @@ private static Event persistActionEvent(Long userId, Long accountId, Long domain return event; } - /** - * The API job on whose behalf this thread is working, or null outside a job. VM work jobs run on - * behalf of an API job (their "related" id); that parent is what the user can see and list, so it - * is the one recorded. - */ + /** The API job this thread works for (a VM work job's parent), or null outside a job. */ private static Long currentJobId() { final AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrent(); if (context == null || context.getJob() == null) { diff --git a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java index 3d57d7831790..15e37851943f 100644 --- a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java +++ b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java @@ -525,8 +525,7 @@ public ListResponse getUsageJobs(ListUsageJobsCmd cmd) { } if (cmd.getDuration() != null) { - // Usage job timestamps are GMT; anchoring the window in the system zone would shift it - // by the offset on any management server that is not on UTC. + // usage job timestamps are GMT Date lastDate = new Date(DateUtil.currentGMTTime().getTime() - TimeUnit.HOURS.toMillis(cmd.getDuration())); SearchCriteria scc = _usageJobDao.createSearchCriteria(); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 92b405c6a920..3f47d1e44702 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -418,9 +418,7 @@ public boolean waitForTask(ManagedObjectReference task) throws InvalidPropertyFa boolean retVal = false; - // Register with the request this thread is executing, so the request can be cancelled by - // cancelling this task. If the request was already cancelled before the task was created, - // cancel it now rather than let it run. + // register with the executing request; a request already cancelled cancels the new task at once if (VmwareTaskRegistry.taskStarted(task, this)) { LOGGER.info("Request executing on this thread was already cancelled, cancelling newly created task {}", task.getValue()); cancelTask(task); @@ -804,10 +802,7 @@ public boolean isTaskCancellable(ManagedObjectReference task) throws Exception { return info.isCancelable(); } - /** - * Asks vCenter to cancel the task and waits, bounded, for it to reach a terminal state. - * Returns whether the task ended cancelled, with the reason when it did not. - */ + /** Asks vCenter to cancel and waits, bounded, for a terminal state. */ public Pair cancelTask(ManagedObjectReference task) throws Exception { TaskInfo info = getDynamicProperty(task, "info"); if (info == null) { @@ -839,8 +834,7 @@ public Pair cancelTask(ManagedObjectReference task) throws Exce LOGGER.debug("Cancelling task " + taskName + " of the entity " + entityName); getService().cancelTask(task); - // Cancellation is asynchronous. Poll for the outcome rather than going through waitForValues: - // that method is synchronized and the thread waiting on this very task holds its monitor. + // poll rather than waitForValues: it is synchronized and the thread waiting on this task holds its monitor final long deadline = System.currentTimeMillis() + CANCEL_TASK_WAIT_MS; while (System.currentTimeMillis() < deadline) { info = getDynamicProperty(task, "info"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java index 50982d2ee03b..a9f25d6e891a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistry.java @@ -25,15 +25,7 @@ import com.cloud.utils.Pair; import com.vmware.vim25.ManagedObjectReference; -/** - * Tracks the vCenter tasks a resource is waiting on, per agent request sequence, so a request can be - * cancelled by cancelling the tasks it created. - * - * One instance per resource (one per host), so two hosts cannot collide on a sequence number. The - * resource opens a request scope for the executing thread; {@link VmwareClient#waitForTask} then reports - * every task it waits on through that scope. Registering at that single chokepoint is what gives complete - * coverage: every task the resource ever waits on passes through it. - */ +/** Per agent request sequence, the vCenter tasks a resource is waiting on; one instance per host. VmwareClient.waitForTask is the single registration point. */ public class VmwareTaskRegistry { private static final Logger LOGGER = LogManager.getLogger(VmwareTaskRegistry.class); @@ -66,10 +58,7 @@ private static final class RequestScope { } } - /** - * Marks the current thread as executing the given request. A null sequence (a command that did not - * arrive through the agent layer) leaves the thread unscoped, so nothing is tracked for it. - */ + /** A null sequence (not from the agent layer) leaves the thread unscoped. */ public void beginRequest(final Long sequence) { if (sequence == null) { return; @@ -93,11 +82,7 @@ public boolean wasCancelRequested(final Long sequence) { return state != null && state.cancelRequested; } - /** - * Called by the client when it starts waiting on a task. Returns true when the request this thread is - * executing has already been cancelled, so the caller can cancel the newly created task straight away - * instead of letting it run. - */ + /** Returns true when the request was already cancelled, so the caller cancels the new task at once. */ static boolean taskStarted(final ManagedObjectReference mor, final VmwareClient client) { final RequestScope scope = CURRENT.get(); if (scope == null || mor == null) { @@ -122,11 +107,7 @@ static void taskFinished(final ManagedObjectReference mor) { } } - /** - * Whether the request can be stopped. Unknown sequences are not cancellable: we have nothing to stop - * and no way to know what the backend is doing. A request with no task in flight can be stopped by - * interrupting its thread; one with tasks in flight only if vCenter will cancel every one of them. - */ + /** Unknown sequences are not cancellable; with tasks in flight, every one must be cancelable on vCenter. */ public boolean isCancellable(final long sequence) { final RequestState state = requests.get(sequence); if (state == null) { @@ -146,11 +127,7 @@ public boolean isCancellable(final long sequence) { return true; } - /** - * Cancels every task the request has in flight and marks the request so that any task it creates - * afterwards is cancelled as soon as the client starts waiting on it. Returns true only if every - * in-flight task was actually cancelled. - */ + /** Cancels the in-flight tasks and marks the request so later tasks are cancelled on arrival. */ public boolean cancel(final long sequence) { final RequestState state = requests.get(sequence); if (state == null) { diff --git a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java index 6fa64b619e01..4445dfdd8293 100644 --- a/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java +++ b/vmware-base/src/test/java/com/cloud/hypervisor/vmware/util/VmwareTaskRegistryTest.java @@ -135,7 +135,7 @@ public void unscopedThreadRegistersNothing() throws Exception { registry.beginRequest(SEQ); final AtomicBoolean registered = new AtomicBoolean(true); - // the scope is per thread: a task waited on from another thread belongs to no request + // scope is per thread final Thread other = new Thread(() -> registered.set(VmwareTaskRegistry.taskStarted(mor("task-6"), client))); other.start(); other.join(); From 0dea6926d172adf438e13b6c45daa52e484f0424 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 11:50:58 +0530 Subject: [PATCH 21/29] UI changes --- ui/public/locales/en.json | 19 +- .../config/section/infra/managementServers.js | 10 +- ui/src/config/section/storage.js | 6 + ui/src/views/compute/InstanceTab.vue | 5 + ui/src/views/infra/AsyncJobsTab.vue | 315 ++++++++++++++++-- ui/src/views/infra/UsageJobsTab.vue | 188 +++++++++++ 6 files changed, 506 insertions(+), 37 deletions(-) create mode 100644 ui/src/views/infra/UsageJobsTab.vue diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 99bf2cf7aef9..201418064bcd 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -4425,5 +4425,22 @@ "GB*Month": "GB * Month", "IP*Month": "IP * Month", "Policy*Month": "Policy * Month", -"message.kms.key.optional": "Optional: Select a KMS key for encryption. If not selected, legacy passphrase encryption will be used." +"message.kms.key.optional": "Optional: Select a KMS key for encryption. If not selected, legacy passphrase encryption will be used.", +"label.jobs": "Jobs", +"label.usage.jobs": "Usage Jobs", +"label.jobresult": "Result", +"label.pending": "Pending", +"label.cancelled": "Cancelled", +"label.last.hour": "Last hour", +"label.last.24.hours": "Last 24 hours", +"label.last.7.days": "Last 7 days", +"label.usage.server": "Usage server", +"label.job.type": "Job type", +"label.execution.time": "Execution time (ms)", +"label.heartbeat": "Heartbeat", +"label.recurring": "Recurring", +"label.single": "Single", +"label.cancel.job": "Cancel job", +"message.confirm.cancel.job": "Please confirm that you want to cancel this job. Where the hypervisor supports it, the operation in progress is stopped as well; if it cannot be stopped, the request is refused and the job is left to finish.", +"message.cancel.job.success": "Job cancelled" } diff --git a/ui/src/config/section/infra/managementServers.js b/ui/src/config/section/infra/managementServers.js index 4bf54943fd85..70d2d06b375b 100644 --- a/ui/src/config/section/infra/managementServers.js +++ b/ui/src/config/section/infra/managementServers.js @@ -48,8 +48,14 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/views/infra/ManagementServerPeerTab.vue'))) }, { - name: 'pending.jobs', - component: shallowRef(defineAsyncComponent(() => import('@/views/infra/AsyncJobsTab.vue'))) + name: 'jobs', + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/AsyncJobsTab.vue'))), + show: () => { return 'listAsyncJobs' in store.getters.apis } + }, + { + name: 'usage.jobs', + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/UsageJobsTab.vue'))), + show: () => { return 'listUsageJobs' in store.getters.apis } }, { name: 'connected.agents', diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 75bdfd4d5fa6..0342fb6d8402 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -87,6 +87,12 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/components/view/EventsTab.vue'))), show: () => { return 'listEvents' in store.getters.apis } }, + { + name: 'jobs', + resourceType: 'Volume', + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/AsyncJobsTab.vue'))), + show: () => { return 'listAsyncJobs' in store.getters.apis } + }, { name: 'comments', component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) diff --git a/ui/src/views/compute/InstanceTab.vue b/ui/src/views/compute/InstanceTab.vue index d125995e3e1c..f840c10c1c6d 100644 --- a/ui/src/views/compute/InstanceTab.vue +++ b/ui/src/views/compute/InstanceTab.vue @@ -110,6 +110,9 @@ + + + - - - - - +
+ + + + + {{ option.label }} + + + + + + + {{ option.label }} + + + + + + + {{ $t('label.refresh') }} + + + + + + + + + + + + + + + +
+ + diff --git a/ui/src/views/infra/UsageJobsTab.vue b/ui/src/views/infra/UsageJobsTab.vue new file mode 100644 index 000000000000..b0d99dbc129c --- /dev/null +++ b/ui/src/views/infra/UsageJobsTab.vue @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + From 9ef78fed7ea49e8ca0301254a0c911064d808820 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 12:04:33 +0530 Subject: [PATCH 22/29] fixes for api, job framework changes A completed job keeps the server that completed it: async_job_view exposes job_complete_msid, the response falls back to it once executing_msid is cleared, and the managementserverid filter matches either column. Without this a completed job had no management server in the API and a server's finished jobs could not be listed. --- .../resources/META-INF/db/views/cloud.async_job_view.sql | 1 + .../main/java/com/cloud/api/query/QueryManagerImpl.java | 6 +++++- .../java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java | 6 ++++-- .../main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java | 7 +++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql index 80c45b55f69e..0ac08bf40a63 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql @@ -44,6 +44,7 @@ select async_job.instance_type, async_job.instance_id, async_job.job_executing_msid, + async_job.job_complete_msid, CASE WHEN async_job.instance_type = 'Volume' THEN volumes.uuid WHEN diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index eb1e517c7f7c..0ed7b93f0061 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -3258,7 +3258,10 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync } if (cmd.getManagementServerId() != null) { - sb.and("executingMsid", sb.entity().getExecutingMsid(), SearchCriteria.Op.EQ); + // a completed job keeps only the server that completed it + sb.and().op("executingMsid", sb.entity().getExecutingMsid(), SearchCriteria.Op.EQ); + sb.or("completeMsid", sb.entity().getCompleteMsid(), SearchCriteria.Op.EQ); + sb.cp(); } Object keyword = cmd.getKeyword(); @@ -3325,6 +3328,7 @@ private Pair, Integer> searchForAsyncJobsInternal(ListAsync throw new InvalidParameterValueException("Unable to find a management server with the specified id"); } sc.setParameters("executingMsid", msHost.getMsid()); + sc.setParameters("completeMsid", msHost.getMsid()); } return _jobJoinDao.searchAndCount(sc, searchFilter, includeRemoved); diff --git a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java index 93af9a04e144..c200ba2e1534 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java @@ -69,8 +69,10 @@ public AsyncJobResponse newAsyncJobResponse(final AsyncJobJoinVO job) { jobResponse.setJobId(job.getUuid()); jobResponse.setJobStatus(job.getStatus()); jobResponse.setJobProcStatus(job.getProcessStatus()); - if (job.getExecutingMsid() != null) { - ManagementServerHostVO managementServer = managementServerHostDao.findByMsid(job.getExecutingMsid()); + // a finished job no longer has an executing server; the one that completed it is the answer then + final Long msid = job.getExecutingMsid() != null ? job.getExecutingMsid() : job.getCompleteMsid(); + if (msid != null) { + ManagementServerHostVO managementServer = managementServerHostDao.findByMsid(msid); if (managementServer != null) { jobResponse.setManagementServerId(managementServer.getUuid()); jobResponse.setManagementServerName(managementServer.getName()); diff --git a/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java index cb6e949b88c4..7fea7eb1a25a 100644 --- a/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/AsyncJobJoinVO.java @@ -79,6 +79,9 @@ public class AsyncJobJoinVO extends BaseViewVO implements ControlledViewEntity { @Column(name = "job_executing_msid") private Long executingMsid; + @Column(name = "job_complete_msid") + private Long completeMsid; + @Column(name = "job_status") private int status; @@ -229,6 +232,10 @@ public Long getExecutingMsid() { return executingMsid; } + public Long getCompleteMsid() { + return completeMsid; + } + @Override public String getProjectUuid() { // TODO Auto-generated method stub From 0ba370ad99f2b41070e7cbacf60e30876a6e277d Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 12:05:19 +0530 Subject: [PATCH 23/29] fixes for UI changes cancelAsyncJob takes jobid, not id. --- ui/src/views/infra/AsyncJobsTab.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/infra/AsyncJobsTab.vue b/ui/src/views/infra/AsyncJobsTab.vue index e8730315b885..42236cae0d22 100644 --- a/ui/src/views/infra/AsyncJobsTab.vue +++ b/ui/src/views/infra/AsyncJobsTab.vue @@ -291,7 +291,7 @@ export default { }, cancelJob (record) { this.cancelling = record.jobid - postAPI('cancelAsyncJob', { id: record.jobid }).then(() => { + postAPI('cancelAsyncJob', { jobid: record.jobid }).then(() => { this.$message.success(this.$t('message.cancel.job.success')) }).catch(error => { // a refusal (the operation cannot be stopped) is expected; show the server's reason From ff1155482ce0768215ec1cc3872951e7e4d2ef91 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 12:06:04 +0530 Subject: [PATCH 24/29] marvin test for async jobs --- .github/workflows/ci.yml | 1 + test/integration/smoke/test_async_jobs.py | 243 ++++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 test/integration/smoke/test_async_jobs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f6645f4af4d..0ee24ed4c300 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,7 @@ jobs: smoke/test_affinity_groups_projects smoke/test_annotations smoke/test_async_job + smoke/test_async_jobs smoke/test_attach_multiple_volumes smoke/test_backup_recovery_dummy smoke/test_certauthority_root diff --git a/test/integration/smoke/test_async_jobs.py b/test/integration/smoke/test_async_jobs.py new file mode 100644 index 000000000000..5fc57b9cd349 --- /dev/null +++ b/test/integration/smoke/test_async_jobs.py @@ -0,0 +1,243 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for listing async jobs by status, period, resource and management server, and for cancelling them """ + +import time +import uuid + +from marvin.cloudstackAPI import (cancelAsyncJob, + listAsyncJobs, + listEvents, + listUsageJobs, + migrateVirtualMachine, + queryAsyncJobResult) +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.codes import (JOB_CANCELLED, + JOB_FAILED, + JOB_INPROGRESS, + JOB_SUCCEEDED, + PASS) +from marvin.lib.base import (Account, + DiskOffering, + ServiceOffering, + VirtualMachine) +from marvin.lib.common import (get_domain, + get_test_template, + get_zone, + list_hosts) +from nose.plugins.attrib import attr + +ALL_STATUSES = "%d,%d,%d,%d" % (JOB_INPROGRESS, JOB_SUCCEEDED, JOB_FAILED, JOB_CANCELLED) +CANCELLABLE_HYPERVISORS = ["kvm", "vmware", "xenserver"] + + +class TestAsyncJobsManagement(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super(TestAsyncJobsManagement, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.testdata = cls.testClient.getParsedTestDataConfig() + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.template = get_test_template(cls.apiclient, cls.zone.id, cls.hypervisor) + cls._cleanup = [] + + cls.service_offering = ServiceOffering.create(cls.apiclient, cls.testdata["service_offering"]) + cls._cleanup.append(cls.service_offering) + cls.disk_offering = DiskOffering.create(cls.apiclient, cls.testdata["disk_offering"]) + cls._cleanup.append(cls.disk_offering) + cls.account = Account.create(cls.apiclient, cls.testdata["account"], domainid=cls.domain.id) + cls._cleanup.append(cls.account) + + cls.testdata["virtual_machine"]["zoneid"] = cls.zone.id + cls.testdata["virtual_machine"]["template"] = cls.template.id + cls.virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.testdata["virtual_machine"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.service_offering.id, + diskofferingid=cls.disk_offering.id, + hypervisor=cls.hypervisor + ) + cls._cleanup.append(cls.virtual_machine) + # the deploy job is the completed job every listing test looks for + cls.deploy_job_id = cls.virtual_machine.jobid + + @classmethod + def tearDownClass(cls): + super(TestAsyncJobsManagement, cls).tearDownClass() + + def setUp(self): + self.cleanup = [] + + def tearDown(self): + super(TestAsyncJobsManagement, self).tearDown() + + def list_jobs(self, **kwargs): + cmd = listAsyncJobs.listAsyncJobsCmd() + cmd.listall = True + for key, value in kwargs.items(): + setattr(cmd, key, value) + return self.apiclient.listAsyncJobs(cmd) or [] + + @staticmethod + def find_job(jobs, jobid): + return next((job for job in jobs if job.jobid == jobid), None) + + def query_job(self, jobid): + cmd = queryAsyncJobResult.queryAsyncJobResultCmd() + cmd.jobid = jobid + return self.apiclient.queryAsyncJobResult(cmd) + + def wait_for_job(self, jobid, timeout=300): + deadline = time.time() + timeout + while time.time() < deadline: + job = self.query_job(jobid) + if job.jobstatus != JOB_INPROGRESS: + return job + time.sleep(3) + self.fail("Job %s still in progress after %d seconds" % (jobid, timeout)) + + def cancel_job(self, jobid): + cmd = cancelAsyncJob.cancelAsyncJobCmd() + cmd.jobid = jobid + return self.apiclient.cancelAsyncJob(cmd) + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_01_default_listing_is_pending_only(self): + """Without a status filter only pending jobs are listed, so a completed job is absent""" + jobs = self.list_jobs() + self.assertIsNone(self.find_job(jobs, self.deploy_job_id), + "Completed deploy job %s listed without a status filter" % self.deploy_job_id) + for job in jobs: + self.assertEqual(job.jobstatus, JOB_INPROGRESS, "Non-pending job %s in the default listing" % job.jobid) + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_02_list_completed_jobs_by_status_and_duration(self): + """A completed job is reachable by status, with the period, by ordinal or by name""" + job = self.find_job(self.list_jobs(jobstatus=JOB_SUCCEEDED, duration=1), self.deploy_job_id) + self.assertIsNotNone(job, "Deploy job %s not listed with jobstatus=%d duration=1" % (self.deploy_job_id, JOB_SUCCEEDED)) + self.assertEqual(job.jobstatus, JOB_SUCCEEDED) + self.assertTrue(job.completed, "Completed job has no completion time") + self.assertIn("DeployVMCmd", job.cmd) + self.assertEqual(job.jobinstanceid, self.virtual_machine.id) + + by_name = self.find_job(self.list_jobs(jobstatus="SUCCEEDED", duration=1), self.deploy_job_id) + self.assertIsNotNone(by_name, "Status filter by name did not match the deploy job") + + failed = self.find_job(self.list_jobs(jobstatus=JOB_FAILED, duration=1), self.deploy_job_id) + self.assertIsNone(failed, "Succeeded deploy job listed under the failed status") + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_03_list_jobs_by_resource(self): + """Jobs are reachable from the resource they acted on""" + jobs = self.list_jobs(resourcetype="VirtualMachine", resourceid=self.virtual_machine.id, + jobstatus=ALL_STATUSES, duration=1) + self.assertIsNotNone(self.find_job(jobs, self.deploy_job_id), "Deploy job not listed for its instance") + for job in jobs: + self.assertEqual(job.jobinstanceid, self.virtual_machine.id, "Job %s for another resource listed" % job.jobid) + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_04_list_jobs_by_management_server(self): + """Jobs are reachable from the management server that ran them""" + job = self.find_job(self.list_jobs(jobstatus=JOB_SUCCEEDED, duration=1), self.deploy_job_id) + self.assertIsNotNone(job) + msid = getattr(job, "managementserverid", None) + if not msid: + self.skipTest("Deploy job carries no management server id") + + jobs = self.list_jobs(managementserverid=msid, jobstatus=JOB_SUCCEEDED, duration=1) + self.assertIsNotNone(self.find_job(jobs, self.deploy_job_id), "Deploy job not listed for its management server") + for entry in jobs: + self.assertEqual(entry.managementserverid, msid, "Job %s of another management server listed" % entry.jobid) + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_05_events_link_to_job(self): + """The events a job raised are reachable by its id""" + cmd = listEvents.listEventsCmd() + cmd.listall = True + cmd.jobid = self.deploy_job_id + events = self.apiclient.listEvents(cmd) or [] + self.assertTrue(events, "No events listed for deploy job %s" % self.deploy_job_id) + self.assertTrue(any(event.type.startswith("VM.CREATE") for event in events), "No VM.CREATE event for the deploy job") + for event in events: + self.assertEqual(event.jobid, self.deploy_job_id, "Event %s of another job listed" % event.id) + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_06_cancel_completed_job_is_refused(self): + """A finished job cannot be cancelled and is left as it was""" + with self.assertRaises(Exception): + self.cancel_job(self.deploy_job_id) + self.assertEqual(self.query_job(self.deploy_job_id).jobstatus, JOB_SUCCEEDED, "Refused cancel changed the job") + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_07_cancel_unknown_job_is_refused(self): + with self.assertRaises(Exception): + self.cancel_job(str(uuid.uuid4())) + + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") + def test_08_cancel_running_migration(self): + """Cancelling a live migration either stops it or is refused; the instance ends Running either way""" + if self.hypervisor.lower() not in CANCELLABLE_HYPERVISORS: + self.skipTest("Cancellation of hypervisor jobs is not implemented for %s" % self.hypervisor) + + vm = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)[0] + source_host = list_hosts(self.apiclient, id=vm.hostid)[0] + targets = [h for h in (list_hosts(self.apiclient, clusterid=source_host.clusterid, type="Routing", + state="Up", resourcestate="Enabled") or []) if h.id != source_host.id] + if not targets: + self.skipTest("Need a second host in cluster %s to migrate to" % source_host.clusterid) + + cmd = migrateVirtualMachine.migrateVirtualMachineCmd() + cmd.virtualmachineid = vm.id + cmd.hostid = targets[0].id + cmd.isAsync = "false" + jobid = self.apiclient.migrateVirtualMachine(cmd, method="GET").jobid + + cancelled = True + try: + self.cancel_job(jobid) + except Exception as e: + # the migration finished, or the hypervisor would not stop it: a refusal, not a failure + self.debug("Cancel of migration job %s refused: %s" % (jobid, e)) + cancelled = False + + job = self.wait_for_job(jobid) + if cancelled: + self.assertEqual(job.jobstatus, JOB_CANCELLED, "Accepted cancel did not end the job as cancelled") + else: + self.assertIn(job.jobstatus, [JOB_SUCCEEDED, JOB_FAILED], "Refused cancel left the job in an odd state") + + response = self.virtual_machine.getState(self.apiclient, VirtualMachine.RUNNING) + self.assertEqual(response[0], PASS, response[1]) + if cancelled: + vm = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)[0] + self.assertEqual(vm.hostid, source_host.id, "Cancelled migration left the instance on the target host") + + @attr(tags=["advanced", "advancedns", "smoke", "basic"], required_hardware="false") + def test_09_list_usage_jobs(self): + """Usage server jobs list without error, and filter by usage server""" + cmd = listUsageJobs.listUsageJobsCmd() + cmd.duration = 24 + self.apiclient.listUsageJobs(cmd) + + cmd = listUsageJobs.listUsageJobsCmd() + cmd.usageserver = "no-such-usage-server" + self.assertFalse(self.apiclient.listUsageJobs(cmd), "Jobs listed for a usage server that does not exist") From 325abee8a65050ba00ade17d4c73f32a3618d760 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Sep 2026 13:28:02 +0530 Subject: [PATCH 25/29] allow scroll to view job entries --- ui/src/views/infra/AsyncJobsTab.vue | 1 + ui/src/views/infra/UsageJobsTab.vue | 1 + 2 files changed, 2 insertions(+) diff --git a/ui/src/views/infra/AsyncJobsTab.vue b/ui/src/views/infra/AsyncJobsTab.vue index 42236cae0d22..4f7e3b01ae63 100644 --- a/ui/src/views/infra/AsyncJobsTab.vue +++ b/ui/src/views/infra/AsyncJobsTab.vue @@ -52,6 +52,7 @@ :columns="columns" :dataSource="jobs" :rowKey="item => item.jobid" + :scroll="{ x: 'max-content' }" :pagination="false" >