diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index a14adc80ddec..a01855d1b63e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -126,6 +126,21 @@ public static void unregister() { @Metric(about = "resolveBucketLink latency in listKeys") private MutableRate listKeysResolveBucketLatencyNs; + @Metric(about = "deleteKeyFailure latency in nano seconds") + private MutableRate deleteKeyFailureLatencyNs; + + @Metric(about = "deleteKeySuccess latency in nano seconds") + private MutableRate deleteKeySuccessLatencyNs; + + @Metric(about = "resolveBucketLink latency in deleteKeys") + private MutableRate deleteKeysResolveBucketLatencyNs; + + @Metric(about = "ACLs check latency in deleteKeys") + private MutableRate deleteKeysAclCheckLatencyNs; + + @Metric(about = "resolveBucketLink and ACLs check latency in deleteKey") + private MutableRate deleteKeyResolveBucketAndAclCheckLatencyNs; + @Metric(about = "readFromRockDb latency in listKeys") private MutableRate listKeysReadFromRocksDbLatencyNs; @@ -242,6 +257,26 @@ MutableRate getListKeysResolveBucketLatencyNs() { return listKeysResolveBucketLatencyNs; } + public void setDeleteKeyFailureLatencyNs(long latencyInNs) { + deleteKeyFailureLatencyNs.add(latencyInNs); + } + + public void setDeleteKeySuccessLatencyNs(long latencyInNs) { + deleteKeySuccessLatencyNs.add(latencyInNs); + } + + public void setDeleteKeysResolveBucketLatencyNs(long latencyInNs) { + deleteKeysResolveBucketLatencyNs.add(latencyInNs); + } + + public void setDeleteKeysAclCheckLatencyNs(long latencyInNs) { + deleteKeysAclCheckLatencyNs.add(latencyInNs); + } + + public MutableRate getDeleteKeyResolveBucketAndAclCheckLatencyNs() { + return deleteKeyResolveBucketAndAclCheckLatencyNs; + } + public void addListKeysReadFromRocksDbLatencyNs(long latencyInNs) { listKeysReadFromRocksDbLatencyNs.add(latencyInNs); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 61e5976f8052..5739ee5096ee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -22,6 +22,7 @@ import java.nio.file.InvalidPathException; import java.util.Map; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.server.protocol.TermIndex; @@ -60,6 +61,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; +import static org.apache.hadoop.util.MetricUtil.captureLatencyNs; /** * Handles DeleteKey request. @@ -98,8 +100,9 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { protected KeyArgs resolveBucketAndCheckAcls(OzoneManager ozoneManager, KeyArgs.Builder newKeyArgs) throws IOException { - return resolveBucketAndCheckKeyAcls(newKeyArgs.build(), ozoneManager, - ACLType.DELETE); + return captureLatencyNs( + ozoneManager.getPerfMetrics().getDeleteKeyResolveBucketAndAclCheckLatencyNs(), + () -> resolveBucketAndCheckKeyAcls(newKeyArgs.build(), ozoneManager, ACLType.DELETE)); } @Override @@ -117,7 +120,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OMMetrics omMetrics = ozoneManager.getMetrics(); omMetrics.incNumKeyDeletes(); - + OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics(); AuditLogger auditLogger = ozoneManager.getAuditLogger(); OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); @@ -128,7 +131,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn boolean acquiredLock = false; OMClientResponse omClientResponse = null; Result result = null; - + long startNanos = Time.monotonicNowNanos(); try { String objectKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); @@ -183,12 +186,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn omBucketInfo.copyObject(), dbOpenKey); result = Result.SUCCESS; + long endNanosDeleteKeySuccessLatencyNs = Time.monotonicNowNanos(); + perfMetrics.setDeleteKeySuccessLatencyNs(endNanosDeleteKeySuccessLatencyNs - startNanos); } catch (IOException | InvalidPathException ex) { result = Result.FAILURE; exception = ex; omClientResponse = new OMKeyDeleteResponse(createErrorOMResponse(omResponse, exception), getBucketLayout()); + long endNanosDeleteKeyFailureLatencyNs = Time.monotonicNowNanos(); + perfMetrics.setDeleteKeyFailureLatencyNs(endNanosDeleteKeyFailureLatencyNs - startNanos); } finally { if (acquiredLock) { mergeOmLockDetails(omMetadataManager.getLock() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 19e9ed716e4a..fe66cb4e79bd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -24,9 +24,11 @@ import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.util.Time; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -101,6 +103,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OMMetrics omMetrics = ozoneManager.getMetrics(); omMetrics.incNumKeyDeletes(); + OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics(); String volumeName = deleteKeyArgs.getVolumeName(); String bucketName = deleteKeyArgs.getBucketName(); Map auditMap = new LinkedHashMap<>(); @@ -126,9 +129,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .setVolumeName(volumeName).setBucketName(bucketName); boolean deleteStatus = true; + long startNanos = Time.monotonicNowNanos(); try { - ResolvedBucket bucket = - ozoneManager.resolveBucketLink(Pair.of(volumeName, bucketName), this); + long startNanosDeleteKeysResolveBucketLatency = Time.monotonicNowNanos(); + ResolvedBucket bucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, bucketName), this); + perfMetrics.setDeleteKeysResolveBucketLatencyNs( + Time.monotonicNowNanos() - startNanosDeleteKeysResolveBucketLatency); bucket.audit(auditMap); volumeName = bucket.realVolume(); bucketName = bucket.realBucket(); @@ -159,9 +165,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn try { // check Acl + long startNanosDeleteKeysAclCheckLatency = Time.monotonicNowNanos(); checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY, volumeOwner); + perfMetrics.setDeleteKeysAclCheckLatencyNs(Time.monotonicNowNanos() - startNanosDeleteKeysAclCheckLatency); OzoneFileStatus fileStatus = getOzoneKeyStatus( ozoneManager, omMetadataManager, volumeName, bucketName, keyName); addKeyToAppropriateList(omKeyInfoList, omKeyInfo, dirList, @@ -193,7 +201,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, dbOpenKeys); result = Result.SUCCESS; - + long endNanosDeleteKeySuccessLatencyNs = Time.monotonicNowNanos(); + perfMetrics.setDeleteKeySuccessLatencyNs(endNanosDeleteKeySuccessLatencyNs - startNanos); } catch (IOException | InvalidPathException ex) { result = Result.FAILURE; exception = ex; @@ -213,7 +222,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .setUnDeletedKeys(unDeletedKeys).build()).build(); omClientResponse = new OMKeysDeleteResponse(omResponse.build(), getBucketLayout()); - + long endNanosDeleteKeyFailureLatencyNs = Time.monotonicNowNanos(); + perfMetrics.setDeleteKeyFailureLatencyNs(endNanosDeleteKeyFailureLatencyNs - startNanos); } finally { if (acquiredLock) { mergeOmLockDetails(omMetadataManager.getLock() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index fde83d7b7697..05316b613025 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -121,6 +121,7 @@ public class TestOMKeyRequest { public void setup() throws Exception { ozoneManager = mock(OzoneManager.class); omMetrics = OMMetrics.create(); + metrics = OMPerformanceMetrics.register(); OzoneConfiguration ozoneConfiguration = getOzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); @@ -130,6 +131,7 @@ public void setup() throws Exception { omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); when(ozoneManager.getMetrics()).thenReturn(omMetrics); + when(ozoneManager.getPerfMetrics()).thenReturn(metrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);