From f3f2adb33425db3f1444971f4b8de42e117a83df Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 20 Jun 2018 12:00:21 +0200 Subject: [PATCH 01/14] initial commit: api, service, db access --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/response/AsyncJobResponse.java | 8 +++++++ .../org/apache/cloudstack/jobs/JobInfo.java | 2 ++ .../framework/jobs/dao/AsyncJobDaoImpl.java | 11 +++++---- .../jobs/impl/AsyncJobManagerImpl.java | 23 +++++++++++-------- .../framework/jobs/impl/AsyncJobVO.java | 10 ++++++++ .../java/com/cloud/api/ApiResponseHelper.java | 10 ++++---- .../api/query/dao/AsyncJobJoinDaoImpl.java | 11 +++++---- 8 files changed, 51 insertions(+), 25 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 504b2149837e..30766be59cab 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -548,6 +548,7 @@ public class ApiConstants { public static final String IPSEC_PSK = "ipsecpsk"; public static final String GUEST_IP = "guestip"; public static final String REMOVED = "removed"; + public static final String END_TIME = "endtime"; public static final String IKE_POLICY = "ikepolicy"; public static final String ESP_POLICY = "esppolicy"; public static final String IKE_LIFETIME = "ikelifetime"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java index 70bbeee30c3f..641ab2c48f26 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java @@ -75,6 +75,10 @@ public class AsyncJobResponse extends BaseResponse { @Param(description = " the created date of the job") private Date created; + @SerializedName(ApiConstants.END_TIME) + @Param(description = " the removed date of the job") + private Date removed; + public void setAccountId(String accountId) { this.accountId = accountId; } @@ -119,4 +123,8 @@ public void setJobInstanceId(String jobInstanceId) { public void setCreated(Date created) { this.created = created; } + + public void setRemoved(final Date removed) { + this.removed = removed; + } } diff --git a/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java b/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java index c7c9b96547d6..5b63e627d622 100644 --- a/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java +++ b/api/src/main/java/org/apache/cloudstack/jobs/JobInfo.java @@ -68,6 +68,8 @@ public boolean done() { Date getCreated(); + Date getRemoved(); + Date getLastUpdated(); Date getLastPolled(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 0ccd4add19a1..ef992ff5cc02 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -21,6 +21,7 @@ import java.util.Date; import java.util.List; +import org.apache.cloudstack.api.ApiConstants; import org.apache.log4j.Logger; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; @@ -71,7 +72,7 @@ public AsyncJobDaoImpl() { expiringUnfinishedAsyncJobSearch.done(); expiringCompletedAsyncJobSearch = createSearchBuilder(); - expiringCompletedAsyncJobSearch.and("created", expiringCompletedAsyncJobSearch.entity().getCreated(), SearchCriteria.Op.LTEQ); + expiringCompletedAsyncJobSearch.and(ApiConstants.REMOVED, expiringCompletedAsyncJobSearch.entity().getRemoved(), SearchCriteria.Op.LTEQ); expiringCompletedAsyncJobSearch.and("completeMsId", expiringCompletedAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NNULL); expiringCompletedAsyncJobSearch.and("jobStatus", expiringCompletedAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.NEQ); expiringCompletedAsyncJobSearch.done(); @@ -168,11 +169,11 @@ public List getExpiredUnfinishedJobs(Date cutTime, int limit) { } @Override - public List getExpiredCompletedJobs(Date cutTime, int limit) { - SearchCriteria sc = expiringCompletedAsyncJobSearch.create(); - sc.setParameters("created", cutTime); + public List getExpiredCompletedJobs(final Date cutTime, final int limit) { + final SearchCriteria sc = expiringCompletedAsyncJobSearch.create(); + sc.setParameters(ApiConstants.REMOVED, cutTime); sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS); - Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit); + final Filter filter = new Filter(AsyncJobVO.class, ApiConstants.REMOVED, true, 0L, (long)limit); return listIncludingRemovedBy(sc, filter); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 174f1f300cf4..2095cc041fb3 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -286,9 +286,9 @@ public void completeAsyncJob(final long jobId, final Status jobStatus, final int if (s_logger.isDebugEnabled()) { s_logger.debug("Wake up jobs related to job-" + jobId); } - List wakeupList = Transaction.execute(new TransactionCallback>() { + final List wakeupList = Transaction.execute(new TransactionCallback>() { @Override - public List doInTransaction(TransactionStatus status) { + public List doInTransaction(final TransactionStatus status) { if (s_logger.isDebugEnabled()) { s_logger.debug("Update db status for job-" + jobId); } @@ -302,14 +302,16 @@ public List doInTransaction(TransactionStatus status) { job.setResult(null); } - job.setLastUpdated(DateUtil.currentGMTTime()); + final Date currentGMTTime = DateUtil.currentGMTTime(); + job.setLastUpdated(currentGMTTime); + job.setRemoved(currentGMTTime); job.setExecutingMsid(null); _jobDao.update(jobId, job); if (s_logger.isDebugEnabled()) { s_logger.debug("Wake up jobs joined with job-" + jobId + " and disjoin all subjobs created from job- " + jobId); } - List wakeupList = wakeupByJoinedJobCompletion(jobId); + final List wakeupList = wakeupByJoinedJobCompletion(jobId); _joinMapDao.disjoinAllJobs(jobId); // purge the job sync item from queue @@ -445,8 +447,8 @@ public void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObj } @Override - public AsyncJob queryJob(long jobId, boolean updatePollTime) { - AsyncJobVO job = _jobDao.findById(jobId); + public AsyncJob queryJob(final long jobId, final boolean updatePollTime) { + final AsyncJobVO job = _jobDao.findByIdIncludingRemoved(jobId); if (updatePollTime) { job.setLastPolled(DateUtil.currentGMTTime()); @@ -1025,8 +1027,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { // purge sync queue item running on this ms node _queueMgr.cleanupActiveQueueItems(msid, true); // reset job status for all jobs running on this ms node - List jobs = _jobDao.getResetJobs(msid); - for (AsyncJobVO job : jobs) { + final List jobs = _jobDao.getResetJobs(msid); + for (final AsyncJobVO job : jobs) { if (s_logger.isDebugEnabled()) { s_logger.debug("Cancel left-over job-" + job.getId()); } @@ -1034,6 +1036,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode()); job.setResult("job cancelled because of management server restart or shutdown"); job.setCompleteMsid(msid); + job.setRemoved(DateUtil.currentGMTTime()); _jobDao.update(job.getId(), job); if (s_logger.isDebugEnabled()) { s_logger.debug("Purge queue item for cancelled job-" + job.getId()); @@ -1049,8 +1052,8 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } } } - List snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false); - for (SnapshotDetailsVO snapshotDetailsVO : snapshotList) { + final List snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false); + for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) { SnapshotInfo snapshot = snapshotFactory.getSnapshot(snapshotDetailsVO.getResourceId(), DataStoreRole.Primary); snapshotSrv.processEventOnSnapshotObject(snapshot, Snapshot.Event.OperationFailed); _snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(), AsyncJob.Constants.MS_ID); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java index 0ca9ed512546..9d30c2c87b95 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java @@ -372,6 +372,15 @@ public void setUuid(String uuid) { this.uuid = uuid; } + @Override + public Date getRemoved() { + return removed; + } + + public void setRemoved(final Date removed) { + this.removed = removed; + } + @Override public String toString() { StringBuffer sb = new StringBuffer(); @@ -392,6 +401,7 @@ public String toString() { sb.append(", lastUpdated: ").append(getLastUpdated()); sb.append(", lastPolled: ").append(getLastPolled()); sb.append(", created: ").append(getCreated()); + sb.append(", removed: ").append(getRemoved()); sb.append("}"); return sb.toString(); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 4d7de2a1a3f5..4177223a05dd 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -1808,16 +1808,16 @@ public TemplatePermissionsResponse createTemplatePermissionsResponse(ResponseVie } @Override - public AsyncJobResponse queryJobResult(QueryAsyncJobResultCmd cmd) { - Account caller = CallContext.current().getCallingAccount(); + public AsyncJobResponse queryJobResult(final QueryAsyncJobResultCmd cmd) { + final Account caller = CallContext.current().getCallingAccount(); - AsyncJob job = _entityMgr.findById(AsyncJob.class, cmd.getId()); + final AsyncJob job = _entityMgr.findByIdIncludingRemoved(AsyncJob.class, cmd.getId()); if (job == null) { throw new InvalidParameterValueException("Unable to find a job by id " + cmd.getId()); } - User userJobOwner = _accountMgr.getUserIncludingRemoved(job.getUserId()); - Account jobOwner = _accountMgr.getAccount(userJobOwner.getAccountId()); + final User userJobOwner = _accountMgr.getUserIncludingRemoved(job.getUserId()); + final Account jobOwner = _accountMgr.getAccount(userJobOwner.getAccountId()); //check permissions if (_accountMgr.isNormalUser(caller.getId())) { diff --git a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java index fefc89677dd6..bd110154e37c 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java @@ -50,12 +50,13 @@ protected AsyncJobJoinDaoImpl() { } @Override - public AsyncJobResponse newAsyncJobResponse(AsyncJobJoinVO job) { - AsyncJobResponse jobResponse = new AsyncJobResponse(); + public AsyncJobResponse newAsyncJobResponse(final AsyncJobJoinVO job) { + final AsyncJobResponse jobResponse = new AsyncJobResponse(); jobResponse.setAccountId(job.getAccountUuid()); jobResponse.setUserId(job.getUserUuid()); jobResponse.setCmd(job.getCmd()); jobResponse.setCreated(job.getCreated()); + jobResponse.setRemoved(job.getRemoved()); jobResponse.setJobId(job.getUuid()); jobResponse.setJobStatus(job.getStatus()); jobResponse.setJobProcStatus(job.getProcessStatus()); @@ -68,15 +69,15 @@ public AsyncJobResponse newAsyncJobResponse(AsyncJobJoinVO job) { } jobResponse.setJobResultCode(job.getResultCode()); - boolean savedValue = SerializationContext.current().getUuidTranslation(); + final boolean savedValue = SerializationContext.current().getUuidTranslation(); SerializationContext.current().setUuidTranslation(false); - Object resultObject = ApiSerializerHelper.fromSerializedString(job.getResult()); + final Object resultObject = ApiSerializerHelper.fromSerializedString(job.getResult()); jobResponse.setJobResult((ResponseObject)resultObject); SerializationContext.current().setUuidTranslation(savedValue); if (resultObject != null) { - Class clz = resultObject.getClass(); + final Class clz = resultObject.getClass(); if (clz.isPrimitive() || clz.getSuperclass() == Number.class || clz == String.class || clz == Date.class) { jobResponse.setJobResultType("text"); } else { From c81989df8547770073da19c7719fb29aba02d353 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 20 Jun 2018 16:31:52 +0200 Subject: [PATCH 02/14] add AsyncJobJoinDaoTest --- .../storage/dao/AsyncJobJoinDaoTest.java | 74 +++++++++++++++++++ .../resources/AsyncJobJoinDaoTestContext.xml | 19 +++++ 2 files changed, 93 insertions(+) create mode 100644 server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java create mode 100644 server/src/test/resources/AsyncJobJoinDaoTestContext.xml diff --git a/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java b/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java new file mode 100644 index 000000000000..b53daa084381 --- /dev/null +++ b/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java @@ -0,0 +1,74 @@ +package com.cloud.storage.dao; + +import com.cloud.api.query.dao.AsyncJobJoinDaoImpl; +import com.cloud.api.query.vo.AsyncJobJoinVO; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import javax.inject.Inject; +import java.util.Date; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = "classpath:/AsyncJobJoinDaoTestContext.xml") +public class AsyncJobJoinDaoTest { + + @Inject + AsyncJobJoinDaoImpl dao; + + @Test + public void testNewAsyncJobResponseValidValues() { + + final AsyncJobJoinVO job = new AsyncJobJoinVO(); + ReflectionTestUtils.setField(job,"uuid","a2b22932-1b61-4406-8e89-4ae19968e8d3"); + ReflectionTestUtils.setField(job,"accountUuid","4dea2836-72cc-11e8-b2de-107b4429825a"); + ReflectionTestUtils.setField(job,"domainUuid","4dea136b-72cc-11e8-b2de-107b4429825a"); + ReflectionTestUtils.setField(job,"userUuid","4decc724-72cc-11e8-b2de-107b4429825a"); + ReflectionTestUtils.setField(job,"cmd","org.apache.cloudstack.api.command.admin.vm.StartVMCmdByAdmin"); + ReflectionTestUtils.setField(job,"status",0); + ReflectionTestUtils.setField(job,"resultCode",0); + ReflectionTestUtils.setField(job,"result",null); + ReflectionTestUtils.setField(job,"created",new Date()); + ReflectionTestUtils.setField(job,"removed",new Date()); + ReflectionTestUtils.setField(job,"instanceType",ApiCommandJobType.VirtualMachine); + ReflectionTestUtils.setField(job,"instanceId",3L); + final AsyncJobResponse response = dao.newAsyncJobResponse(job); + Assert.assertEquals(job.getUuid(),response.getJobId()); + Assert.assertEquals(job.getAccountUuid(), ReflectionTestUtils.getField(response, "accountId")); + Assert.assertEquals(job.getUserUuid(), ReflectionTestUtils.getField(response, "userId")); + Assert.assertEquals(job.getCmd(), ReflectionTestUtils.getField(response, "cmd")); + Assert.assertEquals(job.getStatus(), ReflectionTestUtils.getField(response, "jobStatus")); + Assert.assertEquals(job.getStatus(), ReflectionTestUtils.getField(response, "jobProcStatus")); + Assert.assertEquals(job.getResultCode(), ReflectionTestUtils.getField(response, "jobResultCode")); + Assert.assertEquals(null, ReflectionTestUtils.getField(response, "jobResultType")); + Assert.assertEquals(job.getResult(), ReflectionTestUtils.getField(response, "jobResult")); + Assert.assertEquals(job.getInstanceType().toString(), ReflectionTestUtils.getField(response, "jobInstanceType")); + Assert.assertEquals(job.getInstanceUuid(), ReflectionTestUtils.getField(response, "jobInstanceId")); + Assert.assertEquals(job.getCreated(), ReflectionTestUtils.getField(response, "created")); + Assert.assertEquals(job.getRemoved(), ReflectionTestUtils.getField(response, "removed")); + } + + @Test + public void testNewAsyncJobResponseNullValues() { + final AsyncJobJoinVO job = new AsyncJobJoinVO(); + final AsyncJobResponse response = dao.newAsyncJobResponse(job); + Assert.assertEquals(job.getUuid(),response.getJobId()); + Assert.assertEquals(job.getAccountUuid(), ReflectionTestUtils.getField(response, "accountId")); + Assert.assertEquals(job.getUserUuid(), ReflectionTestUtils.getField(response, "userId")); + Assert.assertEquals(job.getCmd(), ReflectionTestUtils.getField(response, "cmd")); + Assert.assertEquals(job.getStatus(), ReflectionTestUtils.getField(response, "jobStatus")); + Assert.assertEquals(job.getStatus(), ReflectionTestUtils.getField(response, "jobProcStatus")); + Assert.assertEquals(job.getResultCode(), ReflectionTestUtils.getField(response, "jobResultCode")); + Assert.assertEquals(null, ReflectionTestUtils.getField(response, "jobResultType")); + Assert.assertEquals(job.getResult(), ReflectionTestUtils.getField(response, "jobResult")); + Assert.assertEquals(job.getInstanceType(), ReflectionTestUtils.getField(response, "jobInstanceType")); + Assert.assertEquals(job.getInstanceUuid(), ReflectionTestUtils.getField(response, "jobInstanceId")); + Assert.assertEquals(job.getCreated(), ReflectionTestUtils.getField(response, "created")); + Assert.assertEquals(job.getRemoved(), ReflectionTestUtils.getField(response, "removed")); + } +} diff --git a/server/src/test/resources/AsyncJobJoinDaoTestContext.xml b/server/src/test/resources/AsyncJobJoinDaoTestContext.xml new file mode 100644 index 000000000000..f99f12aa1748 --- /dev/null +++ b/server/src/test/resources/AsyncJobJoinDaoTestContext.xml @@ -0,0 +1,19 @@ + + + + + \ No newline at end of file From 85146020b6a3b5c0b82e08bb0a9a1cb086b76d4c Mon Sep 17 00:00:00 2001 From: ernjvr Date: Thu, 21 Jun 2018 09:54:15 +0200 Subject: [PATCH 03/14] added licence to AsyncJobJoinDaoTest --- .../storage/dao/AsyncJobJoinDaoTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java b/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java index b53daa084381..b8dc3252bb61 100644 --- a/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java +++ b/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package com.cloud.storage.dao; import com.cloud.api.query.dao.AsyncJobJoinDaoImpl; @@ -23,7 +41,6 @@ public class AsyncJobJoinDaoTest { @Test public void testNewAsyncJobResponseValidValues() { - final AsyncJobJoinVO job = new AsyncJobJoinVO(); ReflectionTestUtils.setField(job,"uuid","a2b22932-1b61-4406-8e89-4ae19968e8d3"); ReflectionTestUtils.setField(job,"accountUuid","4dea2836-72cc-11e8-b2de-107b4429825a"); From 74b30a8267262847bd389de3fe2263743aac0de3 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Fri, 22 Jun 2018 10:18:22 +0200 Subject: [PATCH 04/14] refactor AsyncJobJoinDaoTest --- .../jobs/AsyncJobManagerImplTest.java | 79 +++++++++++++++++++ .../storage/dao/AsyncJobJoinDaoTest.java | 10 +-- .../resources/AsyncJobJoinDaoTestContext.xml | 19 ----- 3 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java delete mode 100644 server/src/test/resources/AsyncJobJoinDaoTestContext.xml diff --git a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java new file mode 100644 index 000000000000..d88556492378 --- /dev/null +++ b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import com.cloud.storage.dao.SnapshotDetailsDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; +import org.apache.cloudstack.framework.jobs.impl.SyncQueueManager; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.powermock.modules.junit4.PowerMockRunner; + +import static org.junit.Assert.assertFalse; + +@RunWith(PowerMockRunner.class) +public class AsyncJobManagerImplTest { + + @InjectMocks + private AsyncJobManagerImpl asyncJobManager; + + @Mock + private SyncQueueManager _queueMgr; + + @Mock + private AsyncJobDao _jobDao; + + @Mock + private VolumeDetailsDao _volumeDetailsDao; + + @Mock + private VolumeDao _volsDao; + + @Mock + private SnapshotDetailsDao _snapshotDetailsDao; + + @Mock + private SnapshotDataFactory snapshotFactory; + + @Mock + private SnapshotService snapshotSrv; + + @Mock + private Logger s_logger; + +// @Before +// public void setup() { +// asyncJobManager = spy(AsyncJobManagerImpl.class); +// } + + @Test + public void testcleanupLeftOverJobs() throws Exception { +// Whitebox.invokeMethod(asyncJobManager, "cleanupLeftOverJobs", 1L); +// verify(_queueMgr).cleanupActiveQueueItems(1L, true); +// boolean result = Whitebox.invokeMethod(asyncJobManager, "stop"); + assertFalse(false); + } +} diff --git a/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java b/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java index b8dc3252bb61..ed936985a21e 100644 --- a/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java +++ b/server/src/test/java/com/cloud/storage/dao/AsyncJobJoinDaoTest.java @@ -25,18 +25,16 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.mockito.InjectMocks; +import org.mockito.runners.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; -import javax.inject.Inject; import java.util.Date; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = "classpath:/AsyncJobJoinDaoTestContext.xml") +@RunWith(MockitoJUnitRunner.class) public class AsyncJobJoinDaoTest { - @Inject + @InjectMocks AsyncJobJoinDaoImpl dao; @Test diff --git a/server/src/test/resources/AsyncJobJoinDaoTestContext.xml b/server/src/test/resources/AsyncJobJoinDaoTestContext.xml deleted file mode 100644 index f99f12aa1748..000000000000 --- a/server/src/test/resources/AsyncJobJoinDaoTestContext.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - \ No newline at end of file From 9c6ff0dde9f7f1faefb08b699d29003c0dceff86 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Mon, 25 Jun 2018 16:53:37 +0200 Subject: [PATCH 05/14] add marvin test --- .../jobs/AsyncJobManagerImplTest.java | 79 --------- test/integration/smoke/test_async_job.py | 160 ++++++++++++++++++ 2 files changed, 160 insertions(+), 79 deletions(-) delete mode 100644 framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java create mode 100644 test/integration/smoke/test_async_job.py diff --git a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java deleted file mode 100644 index d88556492378..000000000000 --- a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerImplTest.java +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.framework.jobs; - -import com.cloud.storage.dao.SnapshotDetailsDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.storage.dao.VolumeDetailsDao; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; -import org.apache.cloudstack.framework.jobs.impl.SyncQueueManager; -import org.apache.log4j.Logger; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.powermock.modules.junit4.PowerMockRunner; - -import static org.junit.Assert.assertFalse; - -@RunWith(PowerMockRunner.class) -public class AsyncJobManagerImplTest { - - @InjectMocks - private AsyncJobManagerImpl asyncJobManager; - - @Mock - private SyncQueueManager _queueMgr; - - @Mock - private AsyncJobDao _jobDao; - - @Mock - private VolumeDetailsDao _volumeDetailsDao; - - @Mock - private VolumeDao _volsDao; - - @Mock - private SnapshotDetailsDao _snapshotDetailsDao; - - @Mock - private SnapshotDataFactory snapshotFactory; - - @Mock - private SnapshotService snapshotSrv; - - @Mock - private Logger s_logger; - -// @Before -// public void setup() { -// asyncJobManager = spy(AsyncJobManagerImpl.class); -// } - - @Test - public void testcleanupLeftOverJobs() throws Exception { -// Whitebox.invokeMethod(asyncJobManager, "cleanupLeftOverJobs", 1L); -// verify(_queueMgr).cleanupActiveQueueItems(1L, true); -// boolean result = Whitebox.invokeMethod(asyncJobManager, "stop"); - assertFalse(false); - } -} diff --git a/test/integration/smoke/test_async_job.py b/test/integration/smoke/test_async_job.py new file mode 100644 index 000000000000..36fb327203a0 --- /dev/null +++ b/test/integration/smoke/test_async_job.py @@ -0,0 +1,160 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +from marvin.cloudstackTestCase import * +from marvin.lib.utils import * +from marvin.lib.base import * +from marvin.lib.common import * + + +class TestAsyncJob(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super(TestAsyncJob, cls).getClsTestClient() + cls.api_client = cls.testClient.getApiClient() + + cls.testdata = cls.testClient.getParsedTestDataConfig() + # Get Zone, Domain and templates + cls.domain = get_domain(cls.api_client) + cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.hypervisor = cls.testClient.getHypervisorInfo() + + cls.template = get_test_template( + cls.api_client, + cls.zone.id, + cls.hypervisor + ) + + # Create service, disk offerings etc + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.testdata["service_offering"] + ) + + cls.disk_offering = DiskOffering.create( + cls.api_client, + cls.testdata["disk_offering"] + ) + + cls._cleanup = [ + cls.service_offering, + cls.disk_offering + ] + return + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.hypervisor = self.testClient.getHypervisorInfo() + self.testdata["virtual_machine"]["zoneid"] = self.zone.id + self.testdata["virtual_machine"]["template"] = self.template.id + self.testdata["iso"]["zoneid"] = self.zone.id + self.account = Account.create( + self.apiclient, + self.testdata["account"], + domainid=self.domain.id + ) + self.cleanup = [self.account] + return + + def tearDown(self): + try: + self.debug("Cleaning up the resources") + cleanup_resources(self.apiclient, self.cleanup) + self.debug("Cleanup complete!") + except Exception as e: + self.debug("Warning! Exception in tearDown: %s" % e) + + @attr( + tags=[ + "advanced", + "eip", + "advancedns", + "basic", + "sg"], + required_hardware="true") + def test_queryAsyncJobResult(self): + """ + Test queryAsyncJobResult API for expected values + """ + self.debug("Deploying instance in the account: %s" % + self.account.name) + self.virtual_machine = VirtualMachine.create( + self.apiclient, + self.testdata["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + diskofferingid=self.disk_offering.id, + hypervisor=self.hypervisor + ) + + response = self.virtual_machine.getState( + self.apiclient, + VirtualMachine.RUNNING) + self.assertEqual(response[0], PASS, response[1]) + + cmd = queryAsyncJobResult.queryAsyncJobResultCmd() + cmd.jobid = self.virtual_machine.jobid + cmd_response = self.apiclient.queryAsyncJobResult(cmd) + endtime = cmd_response.endtime + self.assertIsNotNone(endtime, "Expected 'endtime' field of queryAsyncJobResult to have a timestamp of when the job finished.") + return + + @attr( + tags=[ + "advanced", + "eip", + "advancedns", + "basic", + "sg"], + required_hardware="true") + def test_async_job_table_values(self): + """ + Test async job db table for valid values + """ + self.debug("Deploying instance in the account: %s" % + self.account.name) + self.virtual_machine = VirtualMachine.create( + self.apiclient, + self.testdata["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + diskofferingid=self.disk_offering.id, + hypervisor=self.hypervisor + ) + + response = self.virtual_machine.getState( + self.apiclient, + VirtualMachine.RUNNING) + self.assertEqual(response[0], PASS, response[1]) + + result = self.dbclient.execute("select * from async_job where uuid='%s'" % self.virtual_machine.jobid) + removed = result[0][17] + self.assertIsNotNone(removed, "Expected 'removed' column of async_job table to return a timestamp of when the job finished.") + return + From 98ddc26f93d4a17049d68348a10e86a09ad1fec9 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Tue, 3 Jul 2018 11:05:15 +0200 Subject: [PATCH 06/14] include removed in findById search --- .../cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 2095cc041fb3..527da2b0b027 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -161,7 +161,7 @@ public ConfigKey[] getConfigKeys() { @Override public AsyncJobVO getAsyncJob(long jobId) { - return _jobDao.findById(jobId); + return _jobDao.findByIdIncludingRemoved(jobId); } @Override From f685024f4211023901bd31498e8dfb45aed031ef Mon Sep 17 00:00:00 2001 From: ernjvr Date: Tue, 3 Jul 2018 17:08:59 +0200 Subject: [PATCH 07/14] Override findById to exclude 'removed IS NULL' sql clause. Rename API response tag name. --- .../apache/cloudstack/api/ApiConstants.java | 2 +- .../api/response/AsyncJobResponse.java | 2 +- .../framework/jobs/dao/AsyncJobDao.java | 4 ++++ .../framework/jobs/dao/AsyncJobDaoImpl.java | 24 +++++++++++++++++++ .../jobs/impl/AsyncJobManagerImpl.java | 4 ++-- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 30766be59cab..75254ca5a7fc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -548,7 +548,7 @@ public class ApiConstants { public static final String IPSEC_PSK = "ipsecpsk"; public static final String GUEST_IP = "guestip"; public static final String REMOVED = "removed"; - public static final String END_TIME = "endtime"; + public static final String COMPLETED = "completed"; public static final String IKE_POLICY = "ikepolicy"; public static final String ESP_POLICY = "esppolicy"; public static final String IKE_LIFETIME = "ikelifetime"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java index 641ab2c48f26..99da99c2999b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java @@ -75,7 +75,7 @@ public class AsyncJobResponse extends BaseResponse { @Param(description = " the created date of the job") private Date created; - @SerializedName(ApiConstants.END_TIME) + @SerializedName(ApiConstants.COMPLETED) @Param(description = " the removed date of the job") private Date removed; diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 8778bef7b7b9..cbbcfe8e7d86 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -24,6 +24,10 @@ import com.cloud.utils.db.GenericDao; public interface AsyncJobDao extends GenericDao { + + @Override + AsyncJobVO findById(Long id); + AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId); List findInstancePendingAsyncJobs(String instanceType, Long accountId); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index ef992ff5cc02..37a123e79282 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -17,10 +17,12 @@ package org.apache.cloudstack.framework.jobs.dao; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Date; import java.util.List; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ApiConstants; import org.apache.log4j.Logger; @@ -35,6 +37,8 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; +import javax.persistence.EmbeddedId; + public class AsyncJobDaoImpl extends GenericDaoBase implements AsyncJobDao { private static final Logger s_logger = Logger.getLogger(AsyncJobDaoImpl.class.getName()); @@ -96,6 +100,26 @@ public AsyncJobDaoImpl() { } + @Override + public AsyncJobVO findById(Long id) { + final StringBuilder sql = new StringBuilder(_selectByIdSql); + final TransactionLegacy txn = TransactionLegacy.currentTxn(); + + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql.toString()); + + if (_idField.getAnnotation(EmbeddedId.class) == null) { + prepareAttribute(1, pstmt, _idAttributes.get(_table)[0], id); + } + + final ResultSet rs = pstmt.executeQuery(); + return rs.next() ? toEntityBean(rs, true) : null; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + pstmt, e); + } + } + @Override public AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId) { SearchCriteria sc = pendingAsyncJobSearch.create(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 527da2b0b027..48995fa3a483 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -161,7 +161,7 @@ public ConfigKey[] getConfigKeys() { @Override public AsyncJobVO getAsyncJob(long jobId) { - return _jobDao.findByIdIncludingRemoved(jobId); + return _jobDao.findById(jobId); } @Override @@ -448,7 +448,7 @@ public void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObj @Override public AsyncJob queryJob(final long jobId, final boolean updatePollTime) { - final AsyncJobVO job = _jobDao.findByIdIncludingRemoved(jobId); + final AsyncJobVO job = _jobDao.findById(jobId); if (updatePollTime) { job.setLastPolled(DateUtil.currentGMTTime()); From 58044ee9ee66846352ce974bcab6eba4428665cd Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 4 Jul 2018 10:19:35 +0200 Subject: [PATCH 08/14] revert to findById --- server/src/main/java/com/cloud/api/ApiResponseHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 4177223a05dd..a4d39f724267 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -1811,7 +1811,7 @@ public TemplatePermissionsResponse createTemplatePermissionsResponse(ResponseVie public AsyncJobResponse queryJobResult(final QueryAsyncJobResultCmd cmd) { final Account caller = CallContext.current().getCallingAccount(); - final AsyncJob job = _entityMgr.findByIdIncludingRemoved(AsyncJob.class, cmd.getId()); + final AsyncJob job = _entityMgr.findById(AsyncJob.class, cmd.getId()); if (job == null) { throw new InvalidParameterValueException("Unable to find a job by id " + cmd.getId()); } From a12b5ff283543ad934ef16daf87a544cafd3c3ae Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 4 Jul 2018 16:30:01 +0200 Subject: [PATCH 09/14] revert findById override; fix nullPointer during cleanupLeftOverJobs; fix API description --- .../api/response/AsyncJobResponse.java | 2 +- .../framework/jobs/dao/AsyncJobDao.java | 3 --- .../framework/jobs/dao/AsyncJobDaoImpl.java | 23 ------------------- .../jobs/impl/AsyncJobManagerImpl.java | 10 ++++---- .../java/com/cloud/api/ApiResponseHelper.java | 2 +- 5 files changed, 8 insertions(+), 32 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java index 99da99c2999b..eecd6be6c521 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java @@ -76,7 +76,7 @@ public class AsyncJobResponse extends BaseResponse { private Date created; @SerializedName(ApiConstants.COMPLETED) - @Param(description = " the removed date of the job") + @Param(description = " the completed date of the job") private Date removed; public void setAccountId(String accountId) { diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index cbbcfe8e7d86..b2b685d07e19 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -25,9 +25,6 @@ public interface AsyncJobDao extends GenericDao { - @Override - AsyncJobVO findById(Long id); - AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId); List findInstancePendingAsyncJobs(String instanceType, Long accountId); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 37a123e79282..2b591ec4aa62 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -17,12 +17,10 @@ package org.apache.cloudstack.framework.jobs.dao; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Date; import java.util.List; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ApiConstants; import org.apache.log4j.Logger; @@ -37,7 +35,6 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; -import javax.persistence.EmbeddedId; public class AsyncJobDaoImpl extends GenericDaoBase implements AsyncJobDao { private static final Logger s_logger = Logger.getLogger(AsyncJobDaoImpl.class.getName()); @@ -100,26 +97,6 @@ public AsyncJobDaoImpl() { } - @Override - public AsyncJobVO findById(Long id) { - final StringBuilder sql = new StringBuilder(_selectByIdSql); - final TransactionLegacy txn = TransactionLegacy.currentTxn(); - - PreparedStatement pstmt = null; - try { - pstmt = txn.prepareAutoCloseStatement(sql.toString()); - - if (_idField.getAnnotation(EmbeddedId.class) == null) { - prepareAttribute(1, pstmt, _idAttributes.get(_table)[0], id); - } - - final ResultSet rs = pstmt.executeQuery(); - return rs.next() ? toEntityBean(rs, true) : null; - } catch (SQLException e) { - throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } - } - @Override public AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId) { SearchCriteria sc = pendingAsyncJobSearch.create(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 48995fa3a483..7e7ffc059602 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -161,7 +161,7 @@ public ConfigKey[] getConfigKeys() { @Override public AsyncJobVO getAsyncJob(long jobId) { - return _jobDao.findById(jobId); + return _jobDao.findByIdIncludingRemoved(jobId); } @Override @@ -448,7 +448,7 @@ public void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObj @Override public AsyncJob queryJob(final long jobId, final boolean updatePollTime) { - final AsyncJobVO job = _jobDao.findById(jobId); + final AsyncJobVO job = _jobDao.findByIdIncludingRemoved(jobId); if (updatePollTime) { job.setLastPolled(DateUtil.currentGMTTime()); @@ -1036,13 +1036,15 @@ public void doInTransactionWithoutResult(TransactionStatus status) { job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode()); job.setResult("job cancelled because of management server restart or shutdown"); job.setCompleteMsid(msid); - job.setRemoved(DateUtil.currentGMTTime()); + final Date currentGMTTime = DateUtil.currentGMTTime(); + job.setLastUpdated(currentGMTTime); + job.setRemoved(currentGMTTime); _jobDao.update(job.getId(), job); if (s_logger.isDebugEnabled()) { s_logger.debug("Purge queue item for cancelled job-" + job.getId()); } _queueMgr.purgeAsyncJobQueueItemId(job.getId()); - if (job.getInstanceType().equals(ApiCommandJobType.Volume.toString())) { + if (job.getInstanceType() != null && job.getInstanceType().equals(ApiCommandJobType.Volume.toString())) { try { _volumeDetailsDao.removeDetail(job.getInstanceId(), "SNAPSHOT_ID"); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index a4d39f724267..4177223a05dd 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -1811,7 +1811,7 @@ public TemplatePermissionsResponse createTemplatePermissionsResponse(ResponseVie public AsyncJobResponse queryJobResult(final QueryAsyncJobResultCmd cmd) { final Account caller = CallContext.current().getCallingAccount(); - final AsyncJob job = _entityMgr.findById(AsyncJob.class, cmd.getId()); + final AsyncJob job = _entityMgr.findByIdIncludingRemoved(AsyncJob.class, cmd.getId()); if (job == null) { throw new InvalidParameterValueException("Unable to find a job by id " + cmd.getId()); } From 5de2bc60b5db6c5806a5673a59882dca32ad3465 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Thu, 5 Jul 2018 11:05:27 +0200 Subject: [PATCH 10/14] add sql update for 41110to41200 upgrade --- .../src/main/resources/META-INF/db/schema-41110to41200.sql | 4 +++- .../apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41110to41200.sql b/engine/schema/src/main/resources/META-INF/db/schema-41110to41200.sql index d5e6d61ea71d..de6865fab515 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41110to41200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41110to41200.sql @@ -32,4 +32,6 @@ ALTER TABLE `vlan` CHANGE `description` `ip4_range` varchar(255); -- We are only adding the permission to the default rules. Any custom rule must be configured by the root admin. INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) values (UUID(), 2, 'moveNetworkAclItem', 'ALLOW', 100) ON DUPLICATE KEY UPDATE rule=rule; INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) values (UUID(), 3, 'moveNetworkAclItem', 'ALLOW', 302) ON DUPLICATE KEY UPDATE rule=rule; -INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) values (UUID(), 4, 'moveNetworkAclItem', 'ALLOW', 260) ON DUPLICATE KEY UPDATE rule=rule; \ No newline at end of file +INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) values (UUID(), 4, 'moveNetworkAclItem', 'ALLOW', 260) ON DUPLICATE KEY UPDATE rule=rule; + +UPDATE `cloud`.`async_job` SET `removed` = now() WHERE `removed` IS NULL; \ No newline at end of file diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 2b591ec4aa62..ef992ff5cc02 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -35,7 +35,6 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; - public class AsyncJobDaoImpl extends GenericDaoBase implements AsyncJobDao { private static final Logger s_logger = Logger.getLogger(AsyncJobDaoImpl.class.getName()); From d8fca2d9aa46b31dd589e0285e98093cf9ccafc2 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Mon, 9 Jul 2018 09:20:21 +0200 Subject: [PATCH 11/14] refactored test case to compare api field value against db column value --- test/integration/smoke/test_async_job.py | 43 +++++------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/test/integration/smoke/test_async_job.py b/test/integration/smoke/test_async_job.py index 36fb327203a0..ba604e1a12ee 100644 --- a/test/integration/smoke/test_async_job.py +++ b/test/integration/smoke/test_async_job.py @@ -20,6 +20,7 @@ from marvin.lib.utils import * from marvin.lib.base import * from marvin.lib.common import * +from pytz import timezone class TestAsyncJob(cloudstackTestCase): @@ -117,44 +118,18 @@ def test_queryAsyncJobResult(self): VirtualMachine.RUNNING) self.assertEqual(response[0], PASS, response[1]) + result = self.dbclient.execute("select * from async_job where uuid='%s'" % self.virtual_machine.jobid) + cmd = queryAsyncJobResult.queryAsyncJobResultCmd() cmd.jobid = self.virtual_machine.jobid cmd_response = self.apiclient.queryAsyncJobResult(cmd) - endtime = cmd_response.endtime - self.assertIsNotNone(endtime, "Expected 'endtime' field of queryAsyncJobResult to have a timestamp of when the job finished.") - return - - @attr( - tags=[ - "advanced", - "eip", - "advancedns", - "basic", - "sg"], - required_hardware="true") - def test_async_job_table_values(self): - """ - Test async job db table for valid values - """ - self.debug("Deploying instance in the account: %s" % - self.account.name) - self.virtual_machine = VirtualMachine.create( - self.apiclient, - self.testdata["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - diskofferingid=self.disk_offering.id, - hypervisor=self.hypervisor - ) - response = self.virtual_machine.getState( - self.apiclient, - VirtualMachine.RUNNING) - self.assertEqual(response[0], PASS, response[1]) + # verify that 'completed' value from api equals 'removed' db column value + completed = cmd_response.completed + removed = timezone('UTC').localize(result[0][17]) + removed = removed.astimezone(timezone('CET')) + removed = removed.strftime("%Y-%m-%dT%H:%M:%S%z") + self.assertEqual(completed, removed, "Expected 'completed' tag value to be equal to 'removed' db column value.") - result = self.dbclient.execute("select * from async_job where uuid='%s'" % self.virtual_machine.jobid) - removed = result[0][17] - self.assertIsNotNone(removed, "Expected 'removed' column of async_job table to return a timestamp of when the job finished.") return From 3020cf3ef09cf520404f1cb8ba7267adc3b0b832 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Mon, 9 Jul 2018 10:01:26 +0200 Subject: [PATCH 12/14] job status check added --- test/integration/smoke/test_async_job.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/integration/smoke/test_async_job.py b/test/integration/smoke/test_async_job.py index ba604e1a12ee..16ae6c801dba 100644 --- a/test/integration/smoke/test_async_job.py +++ b/test/integration/smoke/test_async_job.py @@ -131,5 +131,10 @@ def test_queryAsyncJobResult(self): removed = removed.strftime("%Y-%m-%dT%H:%M:%S%z") self.assertEqual(completed, removed, "Expected 'completed' tag value to be equal to 'removed' db column value.") + # verify that api job_status value equals db job_status value + jobstatus_api = cmd_response.jobstatus + jobstatus_db = result[0][8] + self.assertEqual(jobstatus_api, jobstatus_db, "Expected 'jobstatus' tag value to be equal to 'job_status' db column value.") + return From 6a36f78f274b49415fc6055436ed871a6cb380a4 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 11 Jul 2018 11:45:57 +0200 Subject: [PATCH 13/14] formatting and wording changes --- test/integration/smoke/test_async_job.py | 26 +++++++----------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/test/integration/smoke/test_async_job.py b/test/integration/smoke/test_async_job.py index 16ae6c801dba..5c8b4e092f04 100644 --- a/test/integration/smoke/test_async_job.py +++ b/test/integration/smoke/test_async_job.py @@ -57,7 +57,6 @@ def setUpClass(cls): cls.service_offering, cls.disk_offering ] - return @classmethod def tearDownClass(cls): @@ -79,7 +78,6 @@ def setUp(self): domainid=self.domain.id ) self.cleanup = [self.account] - return def tearDown(self): try: @@ -89,14 +87,7 @@ def tearDown(self): except Exception as e: self.debug("Warning! Exception in tearDown: %s" % e) - @attr( - tags=[ - "advanced", - "eip", - "advancedns", - "basic", - "sg"], - required_hardware="true") + @attr(tags=["advanced", "eip", "advancedns", "basic", "sg"], required_hardware="false") def test_queryAsyncJobResult(self): """ Test queryAsyncJobResult API for expected values @@ -118,23 +109,20 @@ def test_queryAsyncJobResult(self): VirtualMachine.RUNNING) self.assertEqual(response[0], PASS, response[1]) - result = self.dbclient.execute("select * from async_job where uuid='%s'" % self.virtual_machine.jobid) - cmd = queryAsyncJobResult.queryAsyncJobResultCmd() cmd.jobid = self.virtual_machine.jobid cmd_response = self.apiclient.queryAsyncJobResult(cmd) + db_result = self.dbclient.execute("select * from async_job where uuid='%s'" % self.virtual_machine.jobid) + # verify that 'completed' value from api equals 'removed' db column value completed = cmd_response.completed - removed = timezone('UTC').localize(result[0][17]) + removed = timezone('UTC').localize(db_result[0][17]) removed = removed.astimezone(timezone('CET')) removed = removed.strftime("%Y-%m-%dT%H:%M:%S%z") - self.assertEqual(completed, removed, "Expected 'completed' tag value to be equal to 'removed' db column value.") + self.assertEqual(completed, removed, "Expected 'completed' timestamp value to be equal to 'removed' db column value.") # verify that api job_status value equals db job_status value + jobstatus_db = db_result[0][8] jobstatus_api = cmd_response.jobstatus - jobstatus_db = result[0][8] - self.assertEqual(jobstatus_api, jobstatus_db, "Expected 'jobstatus' tag value to be equal to 'job_status' db column value.") - - return - + self.assertEqual(jobstatus_api, jobstatus_db, "Expected 'jobstatus' api value to be equal to 'job_status' db column value.") From 4ac57961f2e85cea6ad755c02bf3bd7996f213b1 Mon Sep 17 00:00:00 2001 From: ernjvr Date: Wed, 11 Jul 2018 13:22:20 +0200 Subject: [PATCH 14/14] pylint changes --- test/integration/smoke/test_async_job.py | 43 +++++++++++++++--------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/test/integration/smoke/test_async_job.py b/test/integration/smoke/test_async_job.py index 5c8b4e092f04..bfd0dd4749d8 100644 --- a/test/integration/smoke/test_async_job.py +++ b/test/integration/smoke/test_async_job.py @@ -1,3 +1,6 @@ +""" +Integration Test +""" # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,15 +19,18 @@ # under the License. from nose.plugins.attrib import attr -from marvin.cloudstackTestCase import * -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import ServiceOffering, DiskOffering, Account, VirtualMachine,\ + queryAsyncJobResult, PASS +from marvin.lib.common import get_domain, get_zone, get_test_template from pytz import timezone class TestAsyncJob(cloudstackTestCase): - + """ + Test queryAsyncJobResult + """ @classmethod def setUpClass(cls): cls.testClient = super(TestAsyncJob, cls).getClsTestClient() @@ -62,8 +68,8 @@ def setUpClass(cls): def tearDownClass(cls): try: cleanup_resources(cls.api_client, cls._cleanup) - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) + except Exception as exception: + raise Exception("Warning: Exception during cleanup : %s" % exception) def setUp(self): self.apiclient = self.testClient.getApiClient() @@ -84,17 +90,17 @@ def tearDown(self): self.debug("Cleaning up the resources") cleanup_resources(self.apiclient, self.cleanup) self.debug("Cleanup complete!") - except Exception as e: - self.debug("Warning! Exception in tearDown: %s" % e) + except Exception as exception: + self.debug("Warning! Exception in tearDown: %s" % exception) @attr(tags=["advanced", "eip", "advancedns", "basic", "sg"], required_hardware="false") - def test_queryAsyncJobResult(self): + def test_query_async_job_result(self): """ Test queryAsyncJobResult API for expected values """ self.debug("Deploying instance in the account: %s" % self.account.name) - self.virtual_machine = VirtualMachine.create( + virtual_machine = VirtualMachine.create( self.apiclient, self.testdata["virtual_machine"], accountid=self.account.name, @@ -104,25 +110,30 @@ def test_queryAsyncJobResult(self): hypervisor=self.hypervisor ) - response = self.virtual_machine.getState( + response = virtual_machine.getState( self.apiclient, VirtualMachine.RUNNING) self.assertEqual(response[0], PASS, response[1]) cmd = queryAsyncJobResult.queryAsyncJobResultCmd() - cmd.jobid = self.virtual_machine.jobid + cmd.jobid = virtual_machine.jobid cmd_response = self.apiclient.queryAsyncJobResult(cmd) - db_result = self.dbclient.execute("select * from async_job where uuid='%s'" % self.virtual_machine.jobid) + db_result = self.dbclient.execute("select * from async_job where uuid='%s'" % + virtual_machine.jobid) # verify that 'completed' value from api equals 'removed' db column value completed = cmd_response.completed removed = timezone('UTC').localize(db_result[0][17]) removed = removed.astimezone(timezone('CET')) removed = removed.strftime("%Y-%m-%dT%H:%M:%S%z") - self.assertEqual(completed, removed, "Expected 'completed' timestamp value to be equal to 'removed' db column value.") + self.assertEqual(completed, removed, + "Expected 'completed' timestamp value to be equal to " + "'removed' db column value.") # verify that api job_status value equals db job_status value jobstatus_db = db_result[0][8] jobstatus_api = cmd_response.jobstatus - self.assertEqual(jobstatus_api, jobstatus_db, "Expected 'jobstatus' api value to be equal to 'job_status' db column value.") + self.assertEqual(jobstatus_api, jobstatus_db, + "Expected 'jobstatus' api value to be equal to " + "'job_status' db column value.")