From 1fa533f3cff0dbc782be82f74b5b63d031c4e9ad Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 10 Jun 2025 13:39:46 -0700 Subject: [PATCH 1/4] HDDS-13234. Expired secret key can abort leader OM startup. Change-Id: I0643c8fa8cbdf0b4be37afcfc04b7893e0c02d9e --- .../OzoneDelegationTokenSecretManager.java | 10 ++++++- ...TestOzoneDelegationTokenSecretManager.java | 26 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 93ca46c36e87..7e243dd37d23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -513,7 +513,11 @@ private void loadTokenSecretState( LOG.info("Loading token state into token manager."); for (Map.Entry entry : state.getTokenState().entrySet()) { - addPersistedDelegationToken(entry.getKey(), entry.getValue()); + try { + addPersistedDelegationToken(entry.getKey(), entry.getValue()); + } catch (Exception e) { + LOG.error("exception while loading delegation token from DB... ignored to continue startup", e); + } } } @@ -528,6 +532,10 @@ private void addPersistedDelegationToken(OzoneTokenIdentifier identifier, byte[] password; if (StringUtils.isNotEmpty(identifier.getSecretKeyId())) { ManagedSecretKey signKey = secretKeyClient.getSecretKey(UUID.fromString(identifier.getSecretKeyId())); + if (signKey == null) { + throw new IOException("Secret key " + UUID.fromString(identifier.getSecretKeyId()) + + " not found for token " + formatTokenId(identifier)); + } password = signKey.sign(identifier.getBytes()); } else { if (LOG.isDebugEnabled()) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java index fe7b2d18e9d3..442fd43f23a3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java @@ -78,6 +78,11 @@ import org.junit.jupiter.api.io.TempDir; import org.slf4j.event.Level; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * Test class for {@link OzoneDelegationTokenSecretManager}. */ @@ -231,6 +236,27 @@ public void testCreateToken() throws Exception { validateHash(token.getPassword(), token.getIdentifier()); } + @Test + public void testExpiredSecretKey() throws Exception { + SecretKeyClient old = secretKeyClient; + secretKeyClient = spy(secretKeyClient); + doReturn(null).when(secretKeyClient).getSecretKey(any()); + + Text tester = new Text("tester"); + OzoneTokenIdentifier identifier = + new OzoneTokenIdentifier(tester, tester, tester); + identifier.setSecretKeyId(UUID.randomUUID().toString()); + + om.getMetadataManager().getDelegationTokenTable().put(identifier, 12345L); + try { + secretManager = createSecretManager(conf, TOKEN_MAX_LIFETIME, + expiryTime, TOKEN_REMOVER_SCAN_INTERVAL); + } finally { + verify(secretKeyClient, times(1)).getSecretKey(any()); + secretKeyClient = old; + } + } + private void restartSecretManager() throws IOException { secretManager.stop(); secretManager = null; From 6f2a6e792a3d4f8fac84238a497d6720b48d917b Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 10 Jun 2025 14:04:51 -0700 Subject: [PATCH 2/4] Checkstyle Change-Id: I962b2ca950779533bd4f69fa47158f59a7f1ae29 --- .../security/TestOzoneDelegationTokenSecretManager.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java index 442fd43f23a3..c97aa9d0a49b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java @@ -26,8 +26,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; @@ -78,11 +82,6 @@ import org.junit.jupiter.api.io.TempDir; import org.slf4j.event.Level; - -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; /** * Test class for {@link OzoneDelegationTokenSecretManager}. */ From 2257242d7c1b5bb0a26068a6fc9dd955305045ff Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 13 Jun 2025 15:23:59 -0700 Subject: [PATCH 3/4] Handle the corner case where delegation token expires. Change-Id: I41410e2b5bc3a47a373475d8f590afa2bcfceff4 --- .../OzoneDelegationTokenSecretManager.java | 7 +++++++ ...TestOzoneDelegationTokenSecretManager.java | 20 +++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 7e243dd37d23..7cd9b0c04f3c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -56,6 +56,7 @@ import org.apache.hadoop.security.HadoopKerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -533,6 +534,12 @@ private void addPersistedDelegationToken(OzoneTokenIdentifier identifier, if (StringUtils.isNotEmpty(identifier.getSecretKeyId())) { ManagedSecretKey signKey = secretKeyClient.getSecretKey(UUID.fromString(identifier.getSecretKeyId())); if (signKey == null) { + // if delegation token expired, remove it from the store. + if (renewDate < Time.now()) { + LOG.info("Removing expired persisted delegation token {} from DB", identifier); + this.store.removeToken(identifier); + } + throw new IOException("Secret key " + UUID.fromString(identifier.getSecretKeyId()) + " not found for token " + formatTokenId(identifier)); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java index c97aa9d0a49b..941167191193 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java @@ -245,13 +245,29 @@ public void testExpiredSecretKey() throws Exception { OzoneTokenIdentifier identifier = new OzoneTokenIdentifier(tester, tester, tester); identifier.setSecretKeyId(UUID.randomUUID().toString()); + identifier.setOmServiceId(OzoneConsts.OM_SERVICE_ID_DEFAULT); - om.getMetadataManager().getDelegationTokenTable().put(identifier, 12345L); + // case 1: Secret key not found, and delegation token is valid. + om.getMetadataManager().getDelegationTokenTable().put(identifier, Long.MAX_VALUE); try { secretManager = createSecretManager(conf, TOKEN_MAX_LIFETIME, expiryTime, TOKEN_REMOVER_SCAN_INTERVAL); + om.getMetadataManager().getDelegationTokenTable().delete(identifier); + + // case 2: Secret key not found, and delegation token is expired. + OzoneTokenIdentifier identifier2 = + new OzoneTokenIdentifier(tester, tester, tester); + identifier2.setSecretKeyId(UUID.randomUUID().toString()); + identifier2.setOmServiceId(OzoneConsts.OM_SERVICE_ID_DEFAULT); + + om.getMetadataManager().getDelegationTokenTable().put(identifier2, Time.now() - 1); + secretManager = createSecretManager(conf, TOKEN_MAX_LIFETIME, + expiryTime, TOKEN_REMOVER_SCAN_INTERVAL); + // expired token should be removed from the table. + assertFalse(om.getMetadataManager().getDelegationTokenTable().isExist(identifier2), + "Expired token " + identifier2 + " should be removed from the table"); } finally { - verify(secretKeyClient, times(1)).getSecretKey(any()); + verify(secretKeyClient, times(2)).getSecretKey(any()); secretKeyClient = old; } } From a6ab41e473c197626998ec13f4dc31a93d5ab55c Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 13 Jun 2025 23:40:48 -0700 Subject: [PATCH 4/4] Checkstyle Change-Id: I6e8ae80c3575c3b0d25167648ec2655c343ccc18 --- .../hadoop/ozone/security/OzoneDelegationTokenSecretManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 7cd9b0c04f3c..4d6928c53508 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -56,7 +56,6 @@ import org.apache.hadoop.security.HadoopKerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Daemon; -import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory;