From 6d312305e8a2fec5e1e9011c7d1c3f5d7ba00778 Mon Sep 17 00:00:00 2001 From: cxorm Date: Fri, 22 Nov 2019 12:03:41 +0800 Subject: [PATCH 01/18] Update error message in listAllVolumes --- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index bfe358b0f5f7..a315b9c89f5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1862,6 +1862,16 @@ public List listVolumeByUser(String userName, String prefix, @Override public List listAllVolumes(String prefix, String prevKey, int maxKeys) throws IOException { + if(isAclEnabled) { + if (!ozAdmins.contains(ProtobufRpcEngine.Server. + getRemoteUser().getUserName()) + && !ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD)) { + LOG.error("Only admin users are authorized to list " + + "Ozone volumes."); + throw new OMException("Only admin users are authorized to list " + + "Ozone volumes.", ResultCodes.PERMISSION_DENIED); + } + } boolean auditSuccess = true; Map auditMap = new LinkedHashMap<>(); auditMap.put(OzoneConsts.PREV_KEY, prevKey); From d6b23fb09e06d1bfc27bf7e8fc8be6307ea7d437 Mon Sep 17 00:00:00 2001 From: cxorm Date: Fri, 22 Nov 2019 12:19:03 +0800 Subject: [PATCH 02/18] Fix listVolumes to list all volumes --- .../ozone/om/OmMetadataManagerImpl.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index bdc31e5f155e..5c5c682e3d8c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -804,13 +804,23 @@ public List listTrash(String volumeName, String bucketName, return deletedKeys; } + /** + * @param userName volume owner, null for listing all volumes. + * @throws IOException + */ @Override public List listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException { + List result = Lists.newArrayList(); + List volumes = Lists.newArrayList(); if (StringUtil.isBlank(userName)) { - throw new OMException("User name is required to list Volumes.", - ResultCodes.USER_NOT_FOUND); + // null userName represents listing all volumes in cluster. + for (String user : getAllUsers()) { + volumes.add(getVolumesByUser(user)); + } + } else { + volumes.add(getVolumesByUser(userName)); } final List result = Lists.newArrayList(); @@ -842,7 +852,6 @@ public List listVolumes(String userName, throw new OMException("Volume info not found for " + volumeName, ResultCodes.VOLUME_NOT_FOUND); } - result.add(volumeArgs); } index++; } @@ -850,6 +859,18 @@ public List listVolumes(String userName, return result; } + private List getAllUsers() throws IOException { + TableIterator> iterator + = getUserTable().iterator(); + List allUsers = Lists.newArrayList(); + + while (iterator.hasNext()) { + allUsers.add(iterator.next().getKey()); + } + + return allUsers; + } + private UserVolumeInfo getVolumesByUser(String userNameKey) throws OMException { try { From da0fa7b37823841c25c94e2c9670e3d622bdfc0c Mon Sep 17 00:00:00 2001 From: cxorm Date: Fri, 22 Nov 2019 16:48:27 +0800 Subject: [PATCH 03/18] Update UT to test list all volumes --- .../ozone/om/TestOmMetadataManager.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 6802afcd620e..2fc264a89cf2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -89,6 +89,32 @@ public void testListVolumes() throws Exception { Assert.assertEquals(volumeList.size(), totalVol - startOrder - 1); } + @Test + public void testListVolumes() throws Exception { + OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder().setAdminName("admin") + .setOwnerName("ownerA").setVolume("volume").build(); + for (int i = 0; i < 50; i++) { + String volName = "vol" + i; + String ownerName = "owner" + i; + omVolumeArgs = OmVolumeArgs.newBuilder() + .setAdminName("admin") + .setOwnerName(ownerName) + .setVolume(volName) + .build(); + TestOMRequestUtils.addVolumeToOM(omMetadataManager, omVolumeArgs); + TestOMRequestUtils.addUserToDB(volName, ownerName, omMetadataManager); + } + + String prefix = ""; + String startKey = ""; + + // Test list all volumes + List volListA = omMetadataManager.listVolumes(null, + prefix, startKey, 1000); + Assert.assertEquals(volListA.size(), 50); + + } + @Test public void testListBuckets() throws Exception { From bc6a675580afb1c17046c3f938a598d8a5b3f47c Mon Sep 17 00:00:00 2001 From: cxorm Date: Sat, 23 Nov 2019 01:19:58 +0800 Subject: [PATCH 04/18] Revert "Fix listVolumes to list all volumes" This reverts commit 633523553db1d46bb6bca23879e66d71a046749c. --- .../ozone/om/OmMetadataManagerImpl.java | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 5c5c682e3d8c..ca83fb01a37e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -804,24 +804,16 @@ public List listTrash(String volumeName, String bucketName, return deletedKeys; } - /** - * @param userName volume owner, null for listing all volumes. - * @throws IOException - */ @Override public List listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException { List result = Lists.newArrayList(); - List volumes = Lists.newArrayList(); - + UserVolumeInfo volumes; if (StringUtil.isBlank(userName)) { - // null userName represents listing all volumes in cluster. - for (String user : getAllUsers()) { - volumes.add(getVolumesByUser(user)); - } - } else { - volumes.add(getVolumesByUser(userName)); + throw new OMException("User name is required to list Volumes.", + ResultCodes.USER_NOT_FOUND); } + volumes = getVolumesByUser(userName); final List result = Lists.newArrayList(); final List volumes = getVolumesByUser(userName) @@ -852,6 +844,7 @@ public List listVolumes(String userName, throw new OMException("Volume info not found for " + volumeName, ResultCodes.VOLUME_NOT_FOUND); } + result.add(volumeArgs); } index++; } @@ -859,18 +852,6 @@ public List listVolumes(String userName, return result; } - private List getAllUsers() throws IOException { - TableIterator> iterator - = getUserTable().iterator(); - List allUsers = Lists.newArrayList(); - - while (iterator.hasNext()) { - allUsers.add(iterator.next().getKey()); - } - - return allUsers; - } - private UserVolumeInfo getVolumesByUser(String userNameKey) throws OMException { try { From b20f1885459847eab38426249fa20ab3f9913c9c Mon Sep 17 00:00:00 2001 From: cxorm Date: Sat, 23 Nov 2019 02:59:26 +0800 Subject: [PATCH 05/18] Revert "Revert "Fix listVolumes to list all volumes"" This reverts commit 6a4ab501c9d36ef30baab52b1c89771b96583142. --- .../ozone/om/OmMetadataManagerImpl.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index ca83fb01a37e..5c5c682e3d8c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -804,16 +804,24 @@ public List listTrash(String volumeName, String bucketName, return deletedKeys; } + /** + * @param userName volume owner, null for listing all volumes. + * @throws IOException + */ @Override public List listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException { List result = Lists.newArrayList(); - UserVolumeInfo volumes; + List volumes = Lists.newArrayList(); + if (StringUtil.isBlank(userName)) { - throw new OMException("User name is required to list Volumes.", - ResultCodes.USER_NOT_FOUND); + // null userName represents listing all volumes in cluster. + for (String user : getAllUsers()) { + volumes.add(getVolumesByUser(user)); + } + } else { + volumes.add(getVolumesByUser(userName)); } - volumes = getVolumesByUser(userName); final List result = Lists.newArrayList(); final List volumes = getVolumesByUser(userName) @@ -844,7 +852,6 @@ public List listVolumes(String userName, throw new OMException("Volume info not found for " + volumeName, ResultCodes.VOLUME_NOT_FOUND); } - result.add(volumeArgs); } index++; } @@ -852,6 +859,18 @@ public List listVolumes(String userName, return result; } + private List getAllUsers() throws IOException { + TableIterator> iterator + = getUserTable().iterator(); + List allUsers = Lists.newArrayList(); + + while (iterator.hasNext()) { + allUsers.add(iterator.next().getKey()); + } + + return allUsers; + } + private UserVolumeInfo getVolumesByUser(String userNameKey) throws OMException { try { From 61e1785da2a417217e397d88cc376413dd7df7c7 Mon Sep 17 00:00:00 2001 From: cxorm Date: Sat, 23 Nov 2019 03:03:11 +0800 Subject: [PATCH 06/18] Extract error message --- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index a315b9c89f5e..c38a1defc7ae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1866,10 +1866,10 @@ public List listAllVolumes(String prefix, String prevKey, int if (!ozAdmins.contains(ProtobufRpcEngine.Server. getRemoteUser().getUserName()) && !ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD)) { - LOG.error("Only admin users are authorized to list " + - "Ozone volumes."); - throw new OMException("Only admin users are authorized to list " + - "Ozone volumes.", ResultCodes.PERMISSION_DENIED); + String errMsg = "Only admin users are authorized to" + + " list Ozone volumes."; + LOG.error(errMsg); + throw new OMException(errMsg, ResultCodes.PERMISSION_DENIED); } } boolean auditSuccess = true; From 8325bf2b3d9262aed3a7ec346f835f7d39922225 Mon Sep 17 00:00:00 2001 From: cxorm Date: Sat, 23 Nov 2019 18:55:33 +0800 Subject: [PATCH 07/18] Update the UT --- .../hadoop/ozone/om/TestOmMetadataManager.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 2fc264a89cf2..a36c3d663320 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -91,16 +91,15 @@ public void testListVolumes() throws Exception { @Test public void testListVolumes() throws Exception { - OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder().setAdminName("admin") - .setOwnerName("ownerA").setVolume("volume").build(); + OmVolumeArgs.Builder argsBuilder = + OmVolumeArgs.newBuilder().setAdminName("admin"); + String volName; + String ownerName; for (int i = 0; i < 50; i++) { - String volName = "vol" + i; - String ownerName = "owner" + i; - omVolumeArgs = OmVolumeArgs.newBuilder() - .setAdminName("admin") - .setOwnerName(ownerName) - .setVolume(volName) - .build(); + ownerName = "owner" + i; + volName = "vol" + i; + OmVolumeArgs omVolumeArgs = argsBuilder. + setOwnerName(ownerName).setVolume(volName).build(); TestOMRequestUtils.addVolumeToOM(omMetadataManager, omVolumeArgs); TestOMRequestUtils.addUserToDB(volName, ownerName, omMetadataManager); } From 005dce2c9333986371d62bea87012adfb4d1c55a Mon Sep 17 00:00:00 2001 From: cxorm Date: Sat, 23 Nov 2019 18:57:21 +0800 Subject: [PATCH 08/18] Update the getAllUsers with CacheTable --- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 5c5c682e3d8c..22932b6c7f8c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -860,12 +860,21 @@ public List listVolumes(String userName, } private List getAllUsers() throws IOException { - TableIterator> iterator + TableIterator> dbIterator = getUserTable().iterator(); + Iterator, CacheValue>> + cacheIterator = getUserTable().cacheIterator(); List allUsers = Lists.newArrayList(); - while (iterator.hasNext()) { - allUsers.add(iterator.next().getKey()); + // Get users from DB. + while (dbIterator.hasNext()) { + allUsers.add(dbIterator.next().getKey()); + } + // Get users from Cache. + while (cacheIterator.hasNext()) { + Map.Entry, CacheValue> entry = + cacheIterator.next(); + allUsers.add(entry.getKey().getCacheKey()); } return allUsers; From a9c22ce9316f6053d428e75273b0fb53741116f1 Mon Sep 17 00:00:00 2001 From: cxorm Date: Tue, 17 Dec 2019 02:28:56 +0800 Subject: [PATCH 09/18] Handle listing all volumes separately --- .../ozone/om/OmMetadataManagerImpl.java | 33 +++++++++---------- .../ozone/om/TestOmMetadataManager.java | 5 ++- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 22932b6c7f8c..242e6220d85e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -816,9 +816,7 @@ public List listVolumes(String userName, if (StringUtil.isBlank(userName)) { // null userName represents listing all volumes in cluster. - for (String user : getAllUsers()) { - volumes.add(getVolumesByUser(user)); - } + return listAllVolumes(maxKeys); } else { volumes.add(getVolumesByUser(userName)); } @@ -859,25 +857,24 @@ public List listVolumes(String userName, return result; } - private List getAllUsers() throws IOException { - TableIterator> dbIterator - = getUserTable().iterator(); - Iterator, CacheValue>> - cacheIterator = getUserTable().cacheIterator(); - List allUsers = Lists.newArrayList(); + /** + * @return list of all volumes. + * @throws IOException + */ + private List listAllVolumes(int maxKeys) throws IOException { + List result = Lists.newArrayList(); - // Get users from DB. - while (dbIterator.hasNext()) { - allUsers.add(dbIterator.next().getKey()); - } - // Get users from Cache. - while (cacheIterator.hasNext()) { - Map.Entry, CacheValue> entry = + /* volumeTable is full-cache, so we use cacheIterator. */ + Iterator, CacheValue>> + cacheIterator = getVolumeTable().cacheIterator(); + + while (cacheIterator.hasNext() && result.size() < maxKeys) { + Map.Entry, CacheValue> entry = cacheIterator.next(); - allUsers.add(entry.getKey().getCacheKey()); + result.add(entry.getValue().getCacheValue()); } - return allUsers; + return result; } private UserVolumeInfo getVolumesByUser(String userNameKey) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index a36c3d663320..66ddeae6617b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -108,10 +108,9 @@ public void testListVolumes() throws Exception { String startKey = ""; // Test list all volumes - List volListA = omMetadataManager.listVolumes(null, + List volList = omMetadataManager.listVolumes(null, prefix, startKey, 1000); - Assert.assertEquals(volListA.size(), 50); - + Assert.assertEquals(volList.size(), 50); } @Test From 4df522adffc3abf0ee0b1ea32b03f3da81d21eff Mon Sep 17 00:00:00 2001 From: cxorm Date: Thu, 19 Dec 2019 22:54:32 +0800 Subject: [PATCH 10/18] Leave original code to handle listVolumes for a particular user --- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 242e6220d85e..627a84966081 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -806,19 +806,16 @@ public List listTrash(String volumeName, String bucketName, /** * @param userName volume owner, null for listing all volumes. - * @throws IOException */ @Override public List listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException { - List result = Lists.newArrayList(); - List volumes = Lists.newArrayList(); + + UserVolumeInfo volumes; if (StringUtil.isBlank(userName)) { // null userName represents listing all volumes in cluster. return listAllVolumes(maxKeys); - } else { - volumes.add(getVolumesByUser(userName)); } final List result = Lists.newArrayList(); @@ -850,6 +847,7 @@ public List listVolumes(String userName, throw new OMException("Volume info not found for " + volumeName, ResultCodes.VOLUME_NOT_FOUND); } + result.add(volumeArgs); } index++; } @@ -859,9 +857,8 @@ public List listVolumes(String userName, /** * @return list of all volumes. - * @throws IOException */ - private List listAllVolumes(int maxKeys) throws IOException { + private List listAllVolumes(int maxKeys) { List result = Lists.newArrayList(); /* volumeTable is full-cache, so we use cacheIterator. */ From 56af25006410a9eaa4acd40f804113a47d7b6182 Mon Sep 17 00:00:00 2001 From: cxorm Date: Thu, 19 Dec 2019 23:50:56 +0800 Subject: [PATCH 11/18] Update listAllVolumes with prefix and startKey --- .../ozone/om/OmMetadataManagerImpl.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 627a84966081..3b8d7005ab8a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -815,7 +815,7 @@ public List listVolumes(String userName, if (StringUtil.isBlank(userName)) { // null userName represents listing all volumes in cluster. - return listAllVolumes(maxKeys); + return listAllVolumes(prefix, startKey, maxKeys); } final List result = Lists.newArrayList(); @@ -858,17 +858,36 @@ public List listVolumes(String userName, /** * @return list of all volumes. */ - private List listAllVolumes(int maxKeys) { + private List listAllVolumes(String prefix, String startKey, + int maxKeys) { List result = Lists.newArrayList(); /* volumeTable is full-cache, so we use cacheIterator. */ Iterator, CacheValue>> cacheIterator = getVolumeTable().cacheIterator(); + String volumeName; + OmVolumeArgs omVolumeArgs; + boolean prefixIsEmpty = Strings.isNullOrEmpty(prefix); + boolean startKeyFound = Strings.isNullOrEmpty(startKey); while (cacheIterator.hasNext() && result.size() < maxKeys) { Map.Entry, CacheValue> entry = cacheIterator.next(); - result.add(entry.getValue().getCacheValue()); + omVolumeArgs = entry.getValue().getCacheValue(); + volumeName = omVolumeArgs.getVolume(); + + if (!prefixIsEmpty) { + if (!volumeName.startsWith(prefix)) { + continue; + } + } + if (!startKeyFound && volumeName.equals(startKey)) { + result.add(0, omVolumeArgs); + startKeyFound = true; + continue; + } + + result.add(omVolumeArgs); } return result; From 6fcb09a2e4e56a695c9652666a7eae8f0409f7b7 Mon Sep 17 00:00:00 2001 From: cxorm Date: Thu, 19 Dec 2019 23:51:54 +0800 Subject: [PATCH 12/18] Update UT --- .../ozone/om/TestOmMetadataManager.java | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 66ddeae6617b..cd877e4bcc5e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -97,7 +97,15 @@ public void testListVolumes() throws Exception { String ownerName; for (int i = 0; i < 50; i++) { ownerName = "owner" + i; - volName = "vol" + i; + volName = "vola" + i; + OmVolumeArgs omVolumeArgs = argsBuilder. + setOwnerName(ownerName).setVolume(volName).build(); + TestOMRequestUtils.addVolumeToOM(omMetadataManager, omVolumeArgs); + TestOMRequestUtils.addUserToDB(volName, ownerName, omMetadataManager); + } + for (int i = 0; i < 50; i++) { + ownerName = "owner" + i; + volName = "volb" + i; OmVolumeArgs omVolumeArgs = argsBuilder. setOwnerName(ownerName).setVolume(volName).build(); TestOMRequestUtils.addVolumeToOM(omMetadataManager, omVolumeArgs); @@ -108,9 +116,22 @@ public void testListVolumes() throws Exception { String startKey = ""; // Test list all volumes - List volList = omMetadataManager.listVolumes(null, - prefix, startKey, 1000); - Assert.assertEquals(volList.size(), 50); + List volListA = omMetadataManager.listVolumes(null, + prefix, startKey, 1000); + Assert.assertEquals(volListA.size(), 100); + + // Test list all volumes with prefix + prefix = "volb"; + List volListB = omMetadataManager.listVolumes(null, + prefix, startKey, 1000); + Assert.assertEquals(volListB.size(), 50); + + // Test list all volumes with startKey + prefix = ""; + startKey = "vola1"; + List volListC = omMetadataManager.listVolumes(null, + prefix, startKey, 1000); + Assert.assertEquals(volListC.size(), 100); } @Test From 1babcd5702e6affc8ce24b1384c7e0ef7c8a772d Mon Sep 17 00:00:00 2001 From: cxorm Date: Sun, 19 Jan 2020 23:51:33 +0800 Subject: [PATCH 13/18] Fix compile error --- .../java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index cd877e4bcc5e..86260adcf9df 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -90,7 +90,7 @@ public void testListVolumes() throws Exception { } @Test - public void testListVolumes() throws Exception { + public void testListAllVolumes() throws Exception { OmVolumeArgs.Builder argsBuilder = OmVolumeArgs.newBuilder().setAdminName("admin"); String volName; From f1f9342a541f0d16bb69addc202ac3e9f6c6b84b Mon Sep 17 00:00:00 2001 From: cxorm Date: Mon, 20 Jan 2020 01:02:53 +0800 Subject: [PATCH 14/18] Fix listAllVolumes for --start --- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 3b8d7005ab8a..236188f5193a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -869,21 +869,21 @@ private List listAllVolumes(String prefix, String startKey, String volumeName; OmVolumeArgs omVolumeArgs; boolean prefixIsEmpty = Strings.isNullOrEmpty(prefix); - boolean startKeyFound = Strings.isNullOrEmpty(startKey); + boolean startKeyIsEmpty = Strings.isNullOrEmpty(startKey); while (cacheIterator.hasNext() && result.size() < maxKeys) { Map.Entry, CacheValue> entry = cacheIterator.next(); omVolumeArgs = entry.getValue().getCacheValue(); volumeName = omVolumeArgs.getVolume(); - if (!prefixIsEmpty) { - if (!volumeName.startsWith(prefix)) { - continue; - } + if (!prefixIsEmpty && !volumeName.startsWith(prefix)) { + continue; } - if (!startKeyFound && volumeName.equals(startKey)) { - result.add(0, omVolumeArgs); - startKeyFound = true; + + if (!startKeyIsEmpty) { + if (volumeName.equals(startKey)) { + startKeyIsEmpty = true; + } continue; } From af4698e31da2e30eca112c4db0ccd717eb7b7eaf Mon Sep 17 00:00:00 2001 From: cxorm Date: Mon, 20 Jan 2020 01:07:07 +0800 Subject: [PATCH 15/18] Update UT for --start --- .../apache/hadoop/ozone/om/TestOmMetadataManager.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 86260adcf9df..dc2187ccec87 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -126,12 +126,15 @@ public void testListAllVolumes() throws Exception { prefix, startKey, 1000); Assert.assertEquals(volListB.size(), 50); - // Test list all volumes with startKey + // Test list all volumes with setting startVolume + // that was not part of result. prefix = ""; - startKey = "vola1"; + int totalVol = volListB.size(); + int startOrder = 0; + startKey = "volb" + startOrder; List volListC = omMetadataManager.listVolumes(null, prefix, startKey, 1000); - Assert.assertEquals(volListC.size(), 100); + Assert.assertEquals(volListC.size(), totalVol - startOrder - 1); } @Test From 0abe6d1b856c8a3674e8e615a4a6acef3cc8ab95 Mon Sep 17 00:00:00 2001 From: cxorm Date: Mon, 9 Mar 2020 22:34:43 +0800 Subject: [PATCH 16/18] Remove redundant var --- .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 236188f5193a..9164d480d493 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -811,8 +811,6 @@ public List listTrash(String volumeName, String bucketName, public List listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException { - UserVolumeInfo volumes; - if (StringUtil.isBlank(userName)) { // null userName represents listing all volumes in cluster. return listAllVolumes(prefix, startKey, maxKeys); From a1c7cff3937700c2218519832e69580656d5ed08 Mon Sep 17 00:00:00 2001 From: cxorm Date: Mon, 9 Mar 2020 22:38:19 +0800 Subject: [PATCH 17/18] Rebase with seperating chechAdmin --- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index c38a1defc7ae..bfe358b0f5f7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1862,16 +1862,6 @@ public List listVolumeByUser(String userName, String prefix, @Override public List listAllVolumes(String prefix, String prevKey, int maxKeys) throws IOException { - if(isAclEnabled) { - if (!ozAdmins.contains(ProtobufRpcEngine.Server. - getRemoteUser().getUserName()) - && !ozAdmins.contains(OZONE_ADMINISTRATORS_WILDCARD)) { - String errMsg = "Only admin users are authorized to" + - " list Ozone volumes."; - LOG.error(errMsg); - throw new OMException(errMsg, ResultCodes.PERMISSION_DENIED); - } - } boolean auditSuccess = true; Map auditMap = new LinkedHashMap<>(); auditMap.put(OzoneConsts.PREV_KEY, prevKey); From c905785c4b4f755d6f82336bba43d37017f105fb Mon Sep 17 00:00:00 2001 From: cxorm Date: Tue, 10 Mar 2020 01:59:29 +0800 Subject: [PATCH 18/18] Trigger Actions