diff --git a/docs/development/extensions-core/k8s-jobs.md b/docs/development/extensions-core/k8s-jobs.md index a9c32370d609..d9c9b85ed579 100644 --- a/docs/development/extensions-core/k8s-jobs.md +++ b/docs/development/extensions-core/k8s-jobs.md @@ -85,7 +85,9 @@ To use these APIs, ensure you have read and write permissions for the CONFIG res #### Get dynamic configuration -Retrieves the current dynamic execution config for the Kubernetes task runner. +> Prior to Druid 37.0.0, this API will return an empty value when the dynamic config has not been updated via the POST method below. This has since changed to always reflect the dynamic config that will be used by the task runner to create K8s jobs. + +Retrieve the current execution config used by the Kubernetes task runner. Returns a JSON object with the dynamic configuration properties. ##### URL diff --git a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/DefaultKubernetesTaskRunnerDynamicConfig.java b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/DefaultKubernetesTaskRunnerDynamicConfig.java index fde4dcc7b0bf..cb6ea848fffb 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/DefaultKubernetesTaskRunnerDynamicConfig.java +++ b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/DefaultKubernetesTaskRunnerDynamicConfig.java @@ -35,7 +35,9 @@ public class DefaultKubernetesTaskRunnerDynamicConfig implements KubernetesTaskR @JsonCreator public DefaultKubernetesTaskRunnerDynamicConfig( + @Nullable @JsonProperty("podTemplateSelectStrategy") PodTemplateSelectStrategy podTemplateSelectStrategy, + @Nullable @JsonProperty("capacity") Integer capacity ) { @@ -43,6 +45,7 @@ public DefaultKubernetesTaskRunnerDynamicConfig( this.capacity = capacity; } + @Nullable @Override @JsonProperty public PodTemplateSelectStrategy getPodTemplateSelectStrategy() @@ -50,6 +53,7 @@ public PodTemplateSelectStrategy getPodTemplateSelectStrategy() return podTemplateSelectStrategy; } + @Nullable @Override @JsonProperty public Integer getCapacity() diff --git a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResource.java b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResource.java index 432a41933ede..d8e16d1dd2b0 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResource.java +++ b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResource.java @@ -27,6 +27,7 @@ import org.apache.druid.common.config.JacksonConfigManager; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.k8s.overlord.KubernetesTaskRunnerEffectiveConfig; import org.apache.druid.server.http.security.ConfigResourceFilter; import org.apache.druid.server.security.AuthorizationUtils; import org.joda.time.Interval; @@ -43,7 +44,6 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; /** * Resource that manages Kubernetes-specific execution configurations for running tasks. @@ -57,16 +57,18 @@ public class KubernetesTaskExecutionConfigResource private static final Logger log = new Logger(KubernetesTaskExecutionConfigResource.class); private final JacksonConfigManager configManager; private final AuditManager auditManager; - private AtomicReference dynamicConfigRef = null; + private final KubernetesTaskRunnerEffectiveConfig effectiveConfig; @Inject public KubernetesTaskExecutionConfigResource( final JacksonConfigManager configManager, - final AuditManager auditManager + final AuditManager auditManager, + final KubernetesTaskRunnerEffectiveConfig effectiveConfig ) { this.configManager = configManager; this.auditManager = auditManager; + this.effectiveConfig = effectiveConfig; } /** @@ -84,12 +86,9 @@ public Response setExecutionConfig( @Context final HttpServletRequest req ) { - KubernetesTaskRunnerDynamicConfig currentConfig = getDynamicConfig(); - KubernetesTaskRunnerDynamicConfig mergedConfig = dynamicConfig; + KubernetesTaskRunnerDynamicConfig currentConfig = getCurrentConfiguration(); + KubernetesTaskRunnerDynamicConfig mergedConfig = currentConfig.merge(dynamicConfig); - if (currentConfig != null) { - mergedConfig = currentConfig.merge(dynamicConfig); - } final ConfigManager.SetResult setResult = configManager.set( KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, mergedConfig, @@ -154,14 +153,14 @@ public Response getExecutionConfigHistory( @ResourceFilters(ConfigResourceFilter.class) public Response getExecutionConfig() { - return Response.ok(getDynamicConfig()).build(); + return Response.ok(getCurrentConfiguration()).build(); } - private KubernetesTaskRunnerDynamicConfig getDynamicConfig() + private KubernetesTaskRunnerDynamicConfig getCurrentConfiguration() { - if (dynamicConfigRef == null) { - dynamicConfigRef = configManager.watch(KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, KubernetesTaskRunnerDynamicConfig.class); - } - return dynamicConfigRef.get(); + return new DefaultKubernetesTaskRunnerDynamicConfig( + effectiveConfig.getPodTemplateSelectStrategy(), + effectiveConfig.getCapacity() + ); } } diff --git a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResourceTest.java b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResourceTest.java index c06056f01133..6f572effa4ec 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResourceTest.java +++ b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/execution/KubernetesTaskExecutionConfigResourceTest.java @@ -19,27 +19,46 @@ package org.apache.druid.k8s.overlord.execution; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import org.apache.druid.audit.AuditEntry; +import org.apache.druid.audit.AuditInfo; import org.apache.druid.audit.AuditManager; import org.apache.druid.common.config.ConfigManager; import org.apache.druid.common.config.JacksonConfigManager; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.k8s.overlord.KubernetesTaskRunnerEffectiveConfig; +import org.apache.druid.k8s.overlord.KubernetesTaskRunnerStaticConfig; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthorizationUtils; import org.easymock.EasyMock; +import org.joda.time.Interval; import org.junit.Before; import org.junit.Test; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; -import java.util.concurrent.atomic.AtomicReference; +import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class KubernetesTaskExecutionConfigResourceTest { private JacksonConfigManager configManager; private AuditManager auditManager; private HttpServletRequest req; - private KubernetesTaskRunnerDynamicConfig dynamicConfig; + private static final KubernetesTaskRunnerEffectiveConfig DEFAULT_CONFIG = new KubernetesTaskRunnerEffectiveConfig( + new KubernetesTaskRunnerStaticConfig(), + null + ); + private static final KubernetesTaskRunnerDynamicConfig DEFAULT_DYNAMIC_CONFIG = + new DefaultKubernetesTaskRunnerDynamicConfig( + DEFAULT_CONFIG.getPodTemplateSelectStrategy(), + DEFAULT_CONFIG.getCapacity() + ); @Before public void setUp() @@ -47,20 +66,25 @@ public void setUp() configManager = EasyMock.createMock(JacksonConfigManager.class); auditManager = EasyMock.createMock(AuditManager.class); req = EasyMock.createMock(HttpServletRequest.class); - dynamicConfig = EasyMock.createMock(KubernetesTaskRunnerDynamicConfig.class); } @Test public void setExecutionConfigSuccessfulUpdate() { - EasyMock.expect(configManager.watch( - KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - KubernetesTaskRunnerDynamicConfig.class - )).andReturn(new AtomicReference<>(null)); KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( configManager, - auditManager + auditManager, + DEFAULT_CONFIG + ); + + KubernetesTaskRunnerDynamicConfig inputConfig = new DefaultKubernetesTaskRunnerDynamicConfig( + new TaskTypePodTemplateSelectStrategy(), 10 + ); + + KubernetesTaskRunnerDynamicConfig expectedMerged = new DefaultKubernetesTaskRunnerDynamicConfig( + new TaskTypePodTemplateSelectStrategy(), 10 ); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn(null).anyTimes(); EasyMock.expect(req.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn(null).anyTimes(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(null).anyTimes(); @@ -68,26 +92,32 @@ public void setExecutionConfigSuccessfulUpdate() EasyMock.replay(req); EasyMock.expect(configManager.set( KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - dynamicConfig, + expectedMerged, AuthorizationUtils.buildAuditInfo(req) )).andReturn(ConfigManager.SetResult.ok()); - EasyMock.replay(configManager, auditManager, dynamicConfig); + EasyMock.replay(configManager, auditManager); - Response result = testedResource.setExecutionConfig(dynamicConfig, req); + Response result = testedResource.setExecutionConfig(inputConfig, req); assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); } @Test public void setExecutionConfigFailedUpdate() { - EasyMock.expect(configManager.watch( - KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - KubernetesTaskRunnerDynamicConfig.class - )).andReturn(new AtomicReference<>(null)); KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( configManager, - auditManager + auditManager, + DEFAULT_CONFIG + ); + + KubernetesTaskRunnerDynamicConfig inputConfig = new DefaultKubernetesTaskRunnerDynamicConfig( + new TaskTypePodTemplateSelectStrategy(), 10 ); + + KubernetesTaskRunnerDynamicConfig expectedMerged = new DefaultKubernetesTaskRunnerDynamicConfig( + new TaskTypePodTemplateSelectStrategy(), 10 + ); + EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn(null).anyTimes(); EasyMock.expect(req.getHeader(AuditManager.X_DRUID_COMMENT)).andReturn(null).anyTimes(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(null).anyTimes(); @@ -95,33 +125,35 @@ public void setExecutionConfigFailedUpdate() EasyMock.replay(req); EasyMock.expect(configManager.set( KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - dynamicConfig, + expectedMerged, AuthorizationUtils.buildAuditInfo(req) )).andReturn(ConfigManager.SetResult.failure(new RuntimeException())); - EasyMock.replay(configManager, auditManager, dynamicConfig); + EasyMock.replay(configManager, auditManager); - Response result = testedResource.setExecutionConfig(dynamicConfig, req); + Response result = testedResource.setExecutionConfig(inputConfig, req); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), result.getStatus()); } @Test public void setExecutionConfig_MergeUsesCurrentCapacityWhenRequestCapacityNull() { + PodTemplateSelectStrategy currentStrategy = new TaskTypePodTemplateSelectStrategy(); + KubernetesTaskRunnerDynamicConfig currentDynamic = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 5); + KubernetesTaskRunnerEffectiveConfig effectiveConfig = new KubernetesTaskRunnerEffectiveConfig( + new KubernetesTaskRunnerStaticConfig(), + Suppliers.ofInstance(currentDynamic) + ); + KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( configManager, - auditManager + auditManager, + effectiveConfig ); - PodTemplateSelectStrategy currentStrategy = new TaskTypePodTemplateSelectStrategy(); - KubernetesTaskRunnerDynamicConfig currentConfig = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 5); - EasyMock.expect(configManager.watch( - KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - KubernetesTaskRunnerDynamicConfig.class - )).andReturn(new AtomicReference<>(currentConfig)); - PodTemplateSelectStrategy requestStrategy = new TaskTypePodTemplateSelectStrategy(); KubernetesTaskRunnerDynamicConfig requestConfig = new DefaultKubernetesTaskRunnerDynamicConfig(requestStrategy, null); + // Effective current: (TaskType, 5). Request: (TaskType, null). Merged: (TaskType, 5). KubernetesTaskRunnerDynamicConfig expectedMergedConfig = new DefaultKubernetesTaskRunnerDynamicConfig(requestStrategy, 5); EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn(null).anyTimes(); @@ -145,20 +177,22 @@ public void setExecutionConfig_MergeUsesCurrentCapacityWhenRequestCapacityNull() @Test public void setExecutionConfig_MergeUsesCurrentStrategyWhenRequestStrategyNull() { + PodTemplateSelectStrategy currentStrategy = new TaskTypePodTemplateSelectStrategy(); + KubernetesTaskRunnerDynamicConfig currentDynamic = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 2); + KubernetesTaskRunnerEffectiveConfig effectiveConfig = new KubernetesTaskRunnerEffectiveConfig( + new KubernetesTaskRunnerStaticConfig(), + Suppliers.ofInstance(currentDynamic) + ); + KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( configManager, - auditManager + auditManager, + effectiveConfig ); - PodTemplateSelectStrategy currentStrategy = new TaskTypePodTemplateSelectStrategy(); - KubernetesTaskRunnerDynamicConfig currentConfig = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 2); - EasyMock.expect(configManager.watch( - KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - KubernetesTaskRunnerDynamicConfig.class - )).andReturn(new AtomicReference<>(currentConfig)); - KubernetesTaskRunnerDynamicConfig requestConfig = new DefaultKubernetesTaskRunnerDynamicConfig(null, 7); + // Effective current: (TaskType, 2). Request: (null, 7). Merged: (TaskType, 7). KubernetesTaskRunnerDynamicConfig expectedMergedConfig = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 7); EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn(null).anyTimes(); @@ -182,20 +216,22 @@ public void setExecutionConfig_MergeUsesCurrentStrategyWhenRequestStrategyNull() @Test public void setExecutionConfig_MergeUsesCurrentWhenBothRequestFieldsNull() { + PodTemplateSelectStrategy currentStrategy = new TaskTypePodTemplateSelectStrategy(); + KubernetesTaskRunnerDynamicConfig currentDynamic = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 9); + KubernetesTaskRunnerEffectiveConfig effectiveConfig = new KubernetesTaskRunnerEffectiveConfig( + new KubernetesTaskRunnerStaticConfig(), + Suppliers.ofInstance(currentDynamic) + ); + KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( configManager, - auditManager + auditManager, + effectiveConfig ); - PodTemplateSelectStrategy currentStrategy = new TaskTypePodTemplateSelectStrategy(); - KubernetesTaskRunnerDynamicConfig currentConfig = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 9); - EasyMock.expect(configManager.watch( - KubernetesTaskRunnerDynamicConfig.CONFIG_KEY, - KubernetesTaskRunnerDynamicConfig.class - )).andReturn(new AtomicReference<>(currentConfig)); - KubernetesTaskRunnerDynamicConfig requestConfig = new DefaultKubernetesTaskRunnerDynamicConfig(null, null); + // Effective current: (TaskType, 9). Request: (null, null). Merged: (TaskType, 9). KubernetesTaskRunnerDynamicConfig expectedMergedConfig = new DefaultKubernetesTaskRunnerDynamicConfig(currentStrategy, 9); EasyMock.expect(req.getHeader(AuditManager.X_DRUID_AUTHOR)).andReturn(null).anyTimes(); @@ -215,4 +251,145 @@ public void setExecutionConfig_MergeUsesCurrentWhenBothRequestFieldsNull() Response result = testedResource.setExecutionConfig(requestConfig, req); assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); } + + @Test + public void getExecutionConfig_ReturnsDefaultWhenNoConfigSet() + { + EasyMock.replay(configManager, auditManager); + + KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( + configManager, + auditManager, + DEFAULT_CONFIG + ); + + Response result = testedResource.getExecutionConfig(); + assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); + + KubernetesTaskRunnerDynamicConfig returnedConfig = (KubernetesTaskRunnerDynamicConfig) result.getEntity(); + assertNotNull(returnedConfig); + assertEquals(DEFAULT_DYNAMIC_CONFIG, returnedConfig); + } + + @Test + @SuppressWarnings("unchecked") + public void getExecutionConfigHistory_SanityCheck() + { + AuditInfo admin = new AuditInfo("admin", "crewmate", "initial setup", "10.0.0.1"); + AuditInfo operator = new AuditInfo("operator", "imposter", "scaled up capacity", "10.0.0.2"); + AuditInfo paranoidUser = new AuditInfo("paranoid-user", "crewmate", "rollback to safe config", "10.0.0.3"); + + String configKey = KubernetesTaskRunnerDynamicConfig.CONFIG_KEY; + + AuditEntry entry1 = AuditEntry.builder() + .key(configKey) + .type(configKey) + .auditInfo(admin) + .serializedPayload("{\"type\":\"default\",\"podTemplateSelectStrategy\":{\"type\":\"taskType\"},\"capacity\":5}") + .auditTime(DateTimes.of("2024-06-01T10:00:00Z")) + .build(); + AuditEntry entry2 = AuditEntry.builder() + .key(configKey) + .type(configKey) + .auditInfo(operator) + .serializedPayload("{\"type\":\"default\",\"podTemplateSelectStrategy\":{\"type\":\"taskType\"},\"capacity\":20}") + .auditTime(DateTimes.of("2024-09-15T14:30:00Z")) + .build(); + AuditEntry entry3 = AuditEntry.builder() + .key(configKey) + .type(configKey) + .auditInfo(paranoidUser) + .serializedPayload("{\"type\":\"default\",\"podTemplateSelectStrategy\":{\"type\":\"taskType\"},\"capacity\":10}") + .auditTime(DateTimes.of("2024-11-20T08:00:00Z")) + .build(); + + List fullHistory = ImmutableList.of(entry3, entry2, entry1); + List lastTwo = ImmutableList.of(entry3, entry2); + String intervalStr = "2024-06-01/2024-10-01"; + Interval interval = Intervals.of(intervalStr); + List intervalFiltered = ImmutableList.of(entry2, entry1); + + // Query by count: returns the last 2 entries + auditManager = EasyMock.createMock(AuditManager.class); + EasyMock.expect(auditManager.fetchAuditHistory(configKey, configKey, 2)).andReturn(lastTwo); + EasyMock.replay(configManager, auditManager); + + KubernetesTaskExecutionConfigResource testedResource = new KubernetesTaskExecutionConfigResource( + configManager, + auditManager, + DEFAULT_CONFIG + ); + Response result = testedResource.getExecutionConfigHistory(null, 2); + assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); + List resultEntries = (List) result.getEntity(); + assertEquals(2, resultEntries.size()); + assertEquals(lastTwo, resultEntries); + EasyMock.verify(auditManager); + + // Query by interval: returns entries within the interval + EasyMock.reset(configManager, auditManager); + EasyMock.expect(auditManager.fetchAuditHistory(configKey, configKey, interval)).andReturn(intervalFiltered); + EasyMock.replay(configManager, auditManager); + + testedResource = new KubernetesTaskExecutionConfigResource( + configManager, + auditManager, + DEFAULT_CONFIG + ); + result = testedResource.getExecutionConfigHistory(intervalStr, null); + assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); + resultEntries = (List) result.getEntity(); + assertEquals(2, resultEntries.size()); + assertEquals(intervalFiltered, resultEntries); + EasyMock.verify(auditManager); + + // Both interval and count provided: interval takes precedence + EasyMock.reset(configManager, auditManager); + EasyMock.expect(auditManager.fetchAuditHistory(configKey, configKey, interval)).andReturn(intervalFiltered); + EasyMock.replay(configManager, auditManager); + + testedResource = new KubernetesTaskExecutionConfigResource( + configManager, + auditManager, + DEFAULT_CONFIG + ); + result = testedResource.getExecutionConfigHistory(intervalStr, 99); + assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); + assertEquals(intervalFiltered, result.getEntity()); + EasyMock.verify(auditManager); + + // Neither interval nor count: falls through to interval-based fetch with null + EasyMock.reset(configManager, auditManager); + EasyMock.expect(auditManager.fetchAuditHistory(configKey, configKey, null)).andReturn(fullHistory); + EasyMock.replay(configManager, auditManager); + + testedResource = new KubernetesTaskExecutionConfigResource( + configManager, + auditManager, + DEFAULT_CONFIG + ); + result = testedResource.getExecutionConfigHistory(null, null); + assertEquals(Response.Status.OK.getStatusCode(), result.getStatus()); + resultEntries = (List) result.getEntity(); + assertEquals(3, resultEntries.size()); + assertEquals(fullHistory, resultEntries); + EasyMock.verify(auditManager); + + // Invalid count: returns BAD_REQUEST with an error message + EasyMock.reset(configManager, auditManager); + EasyMock.expect(auditManager.fetchAuditHistory(configKey, configKey, -1)) + .andThrow(new IllegalArgumentException("count must be positive")); + EasyMock.replay(configManager, auditManager); + + testedResource = new KubernetesTaskExecutionConfigResource( + configManager, + auditManager, + DEFAULT_CONFIG + ); + result = testedResource.getExecutionConfigHistory(null, -1); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), result.getStatus()); + Map errorEntity = (Map) result.getEntity(); + assertEquals("count must be positive", errorEntity.get("error")); + EasyMock.verify(auditManager); + } }