From 258a4a505847ad9028bebbc30e75af67af8123c8 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 25 Feb 2025 07:36:48 +0100 Subject: [PATCH 1/5] TEZ-4600: Secret managers in Tez should respect the algorithm set by hadoop Change-Id: I236505c2439ba675f4a277aeb1f7b0faae2d1053 --- .../java/org/apache/tez/client/TezClient.java | 5 +- .../security/JobTokenSecretManager.java | 49 +++++++++++-------- .../apache/tez/client/TestTezClientUtils.java | 8 +-- .../org/apache/tez/dag/app/DAGAppMaster.java | 5 +- .../tez/dag/app/TezTaskCommunicatorImpl.java | 3 +- .../apache/tez/dag/app/TestDAGAppMaster.java | 6 +-- .../dag/app/TestTaskCommunicatorManager1.java | 4 +- .../dag/app/launcher/TestDeletionTracker.java | 3 +- .../tez/shufflehandler/ShuffleHandler.java | 2 +- .../processor/reduce/TestReduceProcessor.java | 2 +- .../tez/auxservices/ShuffleHandler.java | 2 +- .../tez/auxservices/TestShuffleHandler.java | 5 +- .../common/shuffle/impl/ShuffleManager.java | 2 +- .../orderedgrouped/ShuffleScheduler.java | 2 +- .../shuffle/orderedgrouped/TestShuffle.java | 2 +- ...huffleInputEventHandlerOrderedGrouped.java | 3 +- .../orderedgrouped/TestShuffleScheduler.java | 2 +- 17 files changed, 58 insertions(+), 47 deletions(-) diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index 937346cacd..41599f1d12 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -132,8 +132,7 @@ public class TezClient { Map cachedTezJarResources; boolean usingTezArchiveDeploy = false; private static final long SLEEP_FOR_READY = 500; - private JobTokenSecretManager jobTokenSecretManager = - new JobTokenSecretManager(); + private final JobTokenSecretManager jobTokenSecretManager; private final Map additionalLocalResources = Maps.newHashMap(); @VisibleForTesting final TezApiVersionInfo apiVersionInfo; @@ -206,6 +205,8 @@ protected TezClient(String name, TezConfiguration tezConf, boolean isSession, TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT); Limits.setConfiguration(tezConf); + this.jobTokenSecretManager = new JobTokenSecretManager(tezConf); + LOG.info("Tez Client Version: " + apiVersionInfo.toString()); } diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java index 785613e928..9c58987db2 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java @@ -28,6 +28,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; @@ -37,11 +39,31 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class JobTokenSecretManager extends SecretManager { - private static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1"; - private final SecretKey masterKey; private final Map currentJobTokens; private final Mac mac; + + /** + * @param conf a mandatory configuration for JobTokenSecretManager to prevent algorithm mismatch + */ + public JobTokenSecretManager(Configuration conf) { + this(null, conf); + } + + public JobTokenSecretManager(SecretKey key, Configuration conf) { + String algorithm = getAlgorithm(conf); + SecretKey masterKey = (key == null) ? generateSecret() : key; + this.currentJobTokens = new TreeMap<>(); + try { + mac = Mac.getInstance(algorithm); + mac.init(masterKey); + } catch (NoSuchAlgorithmException nsa) { + throw new IllegalArgumentException("Can't find " + algorithm + " algorithm.", nsa); + } catch (InvalidKeyException ike) { + throw new IllegalArgumentException("Invalid key to HMAC computation", ike); + } + } + /** * Convert the byte[] to a secret key * @param key the byte[] to create the secret key from @@ -72,27 +94,12 @@ public byte[] computeHash(byte[] msg) { } } - /** - * Default constructor - */ - public JobTokenSecretManager() { - this(null); - } - - public JobTokenSecretManager(SecretKey key) { - this.masterKey = (key == null) ? generateSecret() : key; - this.currentJobTokens = new TreeMap(); - try { - mac = Mac.getInstance(DEFAULT_HMAC_ALGORITHM); - mac.init(masterKey); - } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + " algorithm.", nsa); - } catch (InvalidKeyException ike) { - throw new IllegalArgumentException("Invalid key to HMAC computation", ike); - } + private String getAlgorithm(Configuration conf) { + return conf.get( + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY, + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT); } - /** * Create a new password/secret for the given job token identifier. * @param identifier the job token identifier diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index a2f1ce1175..a52cb3e499 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -359,7 +359,7 @@ public void testAppSubmissionContextForPriority() throws Exception { ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); TezClientUtils.createSessionToken(appId.toString(), - new JobTokenSecretManager(), credentials); + new JobTokenSecretManager(tezConf), credentials); tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); Map m = new HashMap(); tezConf.setInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, testpriority); @@ -413,7 +413,7 @@ public void testSessionTokenInAmClc() throws IOException, YarnException { .setTaskLaunchCmdOpts("initialLaunchOpts")); Credentials credentials = new Credentials(); - JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); + JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(tezConf); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); Token jobToken = TokenCache.getSessionToken(credentials); assertNotNull(jobToken); @@ -447,7 +447,7 @@ public void testAMLoggingOptsSimple() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); - JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); + JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(tezConf); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); DAG dag = DAG.create("DAG-testAMLoggingOptsSimple"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) @@ -488,7 +488,7 @@ public void testAMLoggingOptsPerLogger() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); - JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); + JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(tezConf); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); DAG dag = DAG.create("DAG-testAMLoggingOptsPerLogger"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 45b5266ff2..0b6723ef5d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -261,8 +261,7 @@ public class DAGAppMaster extends AbstractService { private ContainerHeartbeatHandler containerHeartbeatHandler; private TaskHeartbeatHandler taskHeartbeatHandler; private TaskCommunicatorManagerInterface taskCommunicatorManager; - private JobTokenSecretManager jobTokenSecretManager = - new JobTokenSecretManager(); + private JobTokenSecretManager jobTokenSecretManager; private Token sessionToken; private DagEventDispatcher dagEventDispatcher; private VertexEventDispatcher vertexEventDispatcher; @@ -520,6 +519,8 @@ protected void serviceInit(final Configuration conf) throws Exception { containerHeartbeatHandler = createContainerHeartbeatHandler(context, conf); addIfService(containerHeartbeatHandler, true); + jobTokenSecretManager = new JobTokenSecretManager(amConf); + sessionToken = TokenCache.getSessionToken(amCredentials); if (sessionToken == null) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java index 48aee3103b..369ee9be2a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java @@ -145,8 +145,7 @@ public void shutdown() { protected void startRpcServer() { try { - JobTokenSecretManager jobTokenSecretManager = - new JobTokenSecretManager(); + JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(conf); jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken); server = new RPC.Builder(conf) diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java index 46e8c98510..91c3dde34e 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java @@ -527,7 +527,7 @@ public void testBadProgress() throws Exception { // create some sample AM credentials Credentials amCreds = new Credentials(); - JobTokenSecretManager jtsm = new JobTokenSecretManager(); + JobTokenSecretManager jtsm = new JobTokenSecretManager(conf); JobTokenIdentifier identifier = new JobTokenIdentifier( new Text(appId.toString())); Token sessionToken = @@ -608,7 +608,7 @@ private void testDagCredentials(boolean doMerge) throws IOException { // create some sample AM credentials Credentials amCreds = new Credentials(); - JobTokenSecretManager jtsm = new JobTokenSecretManager(); + JobTokenSecretManager jtsm = new JobTokenSecretManager(conf); JobTokenIdentifier identifier = new JobTokenIdentifier( new Text(appId.toString())); Token sessionToken = @@ -764,7 +764,7 @@ public DAGAppMasterForTest(ApplicationAttemptId attemptId, boolean isSession) { public static Credentials createCredentials() { Credentials creds = new Credentials(); - JobTokenSecretManager jtsm = new JobTokenSecretManager(); + JobTokenSecretManager jtsm = new JobTokenSecretManager(new TezConfiguration()); JobTokenIdentifier jtid = new JobTokenIdentifier(new Text()); Token token = new Token(jtid, jtsm); TokenCache.setSessionToken(token, creds); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java index d7e62ee530..469d1ae993 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java @@ -360,7 +360,7 @@ public void testPortRange_NotSpecified() throws IOException, TezException { JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( "fakeIdentifier")); Token sessionToken = new Token(identifier, - new JobTokenSecretManager()); + new JobTokenSecretManager(conf)); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, credentials); UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf); @@ -381,7 +381,7 @@ private boolean testPortRange(int port) { JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( "fakeIdentifier")); Token sessionToken = new Token(identifier, - new JobTokenSecretManager()); + new JobTokenSecretManager(conf)); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, credentials); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java index fba35a69f2..c4b7edd7da 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.tez.common.security.JobTokenSecretManager; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.records.TezDAGID; import org.junit.Assert; import org.junit.Test; @@ -65,7 +66,7 @@ protected void build() { deletionTracker.addNodeShufflePort(nodeId, shufflePort); Assert.assertEquals("Unexpected number of entries in NodeIdShufflePortMap!", 1, deletionTracker.getNodeIdShufflePortMap().size()); - deletionTracker.dagComplete(new TezDAGID(), new JobTokenSecretManager()); + deletionTracker.dagComplete(new TezDAGID(), new JobTokenSecretManager(new TezConfiguration())); Assert.assertEquals("Unexpected number of entries in NodeIdShufflePortMap after dagComplete!", 1, deletionTracker.getNodeIdShufflePortMap().size()); } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java index 43f24ba654..a7d97aced6 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java @@ -206,7 +206,7 @@ public Thread newThread(Runnable r) { DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE)); userRsrc = new ConcurrentHashMap(); - secretManager = new JobTokenSecretManager(); + secretManager = new JobTokenSecretManager(conf); } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java index 0223482da6..c55bc7b63d 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java @@ -181,7 +181,7 @@ public void testReduceProcessor() throws Exception { LOG.info("Starting reduce..."); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(dagName)); - JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); + JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(jobConf); Token shuffleToken = new Token(identifier, jobTokenSecretManager); shuffleToken.setService(identifier.getJobId()); diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java index ce57978ca1..2020c48c54 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java @@ -560,7 +560,7 @@ public Thread newThread(Runnable r) { protected void serviceStart() throws Exception { Configuration conf = getConfig(); userRsrc = new ConcurrentHashMap(); - secretManager = new JobTokenSecretManager(); + secretManager = new JobTokenSecretManager(conf); recoverState(conf); ServerBootstrap bootstrap = new ServerBootstrap() .channel(NioServerSocketChannel.class) diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java index d32998afcc..09826c1ead 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java @@ -66,6 +66,7 @@ import org.apache.tez.runtime.library.common.security.SecureShuffleUtils; import org.apache.tez.common.security.JobTokenIdentifier; import org.apache.tez.common.security.JobTokenSecretManager; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.http.BaseHttpConnection; import org.apache.tez.http.HttpConnectionParams; import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; @@ -217,7 +218,7 @@ class MockShuffleHandlerWithFatalDiskError extends org.apache.tez.auxservices.Sh "Could not find application_1234/240/output/attempt_1234_0/file.out.index"; private JobTokenSecretManager secretManager = - new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(getSecret().getBytes())); + new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(getSecret().getBytes()), new TezConfiguration()); protected JobTokenSecretManager getSecretManager(){ return secretManager; @@ -1209,7 +1210,7 @@ private static int getShuffleResponseCode(ShuffleHandler shuffle, HttpURLConnection conn = (HttpURLConnection) url.openConnection(); String encHash = SecureShuffleUtils.hashFromString( SecureShuffleUtils.buildMsgFrom(url), - new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(jt.getPassword()))); + new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(jt.getPassword()), new TezConfiguration())); conn.addRequestProperty( SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java index 646194c6d7..456b3c1a75 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java @@ -309,7 +309,7 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu SecretKey shuffleSecret = ShuffleUtils .getJobTokenSecretFromTokenBytes(inputContext .getServiceConsumerMetaData(auxiliaryService)); - this.jobTokenSecretMgr = new JobTokenSecretManager(shuffleSecret); + this.jobTokenSecretMgr = new JobTokenSecretManager(shuffleSecret, conf); this.asyncHttp = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_USE_ASYNC_HTTP, false); httpConnectionParams = ShuffleUtils.getHttpConnectionParams(conf); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java index 3fc7d63059..de1ddf3b4d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java @@ -381,7 +381,7 @@ public ShuffleScheduler(InputContext inputContext, SecretKey jobTokenSecret = ShuffleUtils .getJobTokenSecretFromTokenBytes(inputContext .getServiceConsumerMetaData(auxiliaryService)); - this.jobTokenSecretManager = new JobTokenSecretManager(jobTokenSecret); + this.jobTokenSecretManager = new JobTokenSecretManager(jobTokenSecret, conf); final ExecutorService fetcherRawExecutor; if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCHER_USE_SHARED_POOL, diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java index 590affc078..7c703c26b3 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java @@ -152,7 +152,7 @@ private InputContext createTezInputContext() throws IOException { doReturn(shuffleBuffer).when(inputContext).getServiceProviderMetaData(anyString()); Token sessionToken = new Token(new JobTokenIdentifier(new Text("text")), - new JobTokenSecretManager()); + new JobTokenSecretManager(new TezConfiguration())); ByteBuffer tokenBuffer = TezCommonUtils.serializeServiceData(sessionToken); doReturn(tokenBuffer).when(inputContext).getServiceConsumerMetaData(anyString()); when(inputContext.createTezFrameworkExecutorService(anyInt(), anyString())).thenAnswer( diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java index 8da4adcf48..664a790a16 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java @@ -12,6 +12,7 @@ import org.apache.tez.common.counters.TezCounters; import org.apache.tez.common.security.JobTokenIdentifier; import org.apache.tez.common.security.JobTokenSecretManager; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.runtime.api.Event; import org.apache.tez.runtime.api.ExecutionContext; import org.apache.tez.runtime.api.InputContext; @@ -87,7 +88,7 @@ private InputContext createTezInputContext() throws IOException { ByteBuffer shuffleBuffer = ByteBuffer.allocate(4).putInt(0, 4); doReturn(shuffleBuffer).when(inputContext).getServiceProviderMetaData(anyString()); Token sessionToken = new Token(new JobTokenIdentifier(new Text("text")), - new JobTokenSecretManager()); + new JobTokenSecretManager(new TezConfiguration())); ByteBuffer tokenBuffer = TezCommonUtils.serializeServiceData(sessionToken); doReturn(tokenBuffer).when(inputContext).getServiceConsumerMetaData(anyString()); when(inputContext.createTezFrameworkExecutorService(anyInt(), anyString())).thenAnswer( diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java index 9df9aaf69b..1a8329678b 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java @@ -1008,7 +1008,7 @@ private InputContext createTezInputContext() throws IOException { doReturn(shuffleBuffer).when(inputContext).getServiceProviderMetaData(anyString()); Token sessionToken = new Token(new JobTokenIdentifier(new Text("text")), - new JobTokenSecretManager()); + new JobTokenSecretManager(new TezConfiguration())); ByteBuffer tokenBuffer = TezCommonUtils.serializeServiceData(sessionToken); doReturn(tokenBuffer).when(inputContext).getServiceConsumerMetaData(anyString()); when(inputContext.createTezFrameworkExecutorService(anyInt(), anyString())).thenAnswer( From b690de8bcbf2191c3abce0c1dc06a63efff5f12e Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 25 Feb 2025 11:36:19 +0100 Subject: [PATCH 2/5] PR comments + checkstyle --- .../org/apache/tez/common/security/JobTokenSecretManager.java | 2 +- .../java/org/apache/tez/auxservices/TestShuffleHandler.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java index 9c58987db2..6b24b1f7d1 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java @@ -60,7 +60,7 @@ public JobTokenSecretManager(SecretKey key, Configuration conf) { } catch (NoSuchAlgorithmException nsa) { throw new IllegalArgumentException("Can't find " + algorithm + " algorithm.", nsa); } catch (InvalidKeyException ike) { - throw new IllegalArgumentException("Invalid key to HMAC computation", ike); + throw new IllegalArgumentException("Invalid key to " + algorithm + " computation", ike); } } diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java index 09826c1ead..fd989f305d 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java @@ -218,7 +218,8 @@ class MockShuffleHandlerWithFatalDiskError extends org.apache.tez.auxservices.Sh "Could not find application_1234/240/output/attempt_1234_0/file.out.index"; private JobTokenSecretManager secretManager = - new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(getSecret().getBytes()), new TezConfiguration()); + new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(getSecret().getBytes()), + new TezConfiguration()); protected JobTokenSecretManager getSecretManager(){ return secretManager; From bcda616e27f56344014ec48e90b744ddcc1ef92a Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 26 Feb 2025 11:00:13 +0100 Subject: [PATCH 3/5] handle possible NPEs --- .../common/shuffle/impl/TestShuffleInputEventHandlerImpl.java | 2 +- .../runtime/library/common/shuffle/impl/TestShuffleManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java index f4ddf590ef..29bae5c92a 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java @@ -224,7 +224,7 @@ private ShuffleManager createShuffleManager(InputContext inputContext) throws IO DataOutputBuffer out = new DataOutputBuffer(); Token token = new Token(new JobTokenIdentifier(), - new JobTokenSecretManager(null)); + new JobTokenSecretManager(new TezConfiguration())); token.write(out); doReturn(ByteBuffer.wrap(out.getData())).when(inputContext).getServiceConsumerMetaData( conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java index ba854b9c14..92c37e47b7 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java @@ -289,7 +289,7 @@ private ShuffleManagerForTest createShuffleManager( DataOutputBuffer out = new DataOutputBuffer(); Token token = new Token(new JobTokenIdentifier(), - new JobTokenSecretManager(null)); + new JobTokenSecretManager(new TezConfiguration())); token.write(out); doReturn(ByteBuffer.wrap(out.getData())).when(inputContext). getServiceConsumerMetaData( From a060cc4cda8396392fd46ee0fe47b1099801f969 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 26 Feb 2025 11:04:32 +0100 Subject: [PATCH 4/5] hardcoded config --- .../apache/tez/common/security/JobTokenSecretManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java index 6b24b1f7d1..2789b024f6 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java @@ -95,9 +95,8 @@ public byte[] computeHash(byte[] msg) { } private String getAlgorithm(Configuration conf) { - return conf.get( - CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY, - CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT); + // TODO: TEZ-4607: replace with CommonConfigurationKeysPublic enum values + return conf.get("hadoop.security.secret-manager.key-generator.algorithm", "HmacSHA1"); } /** From 50ddea51247cdf718b6eec9d1993af49f53a9ec5 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 4 Mar 2025 08:23:53 +0100 Subject: [PATCH 5/5] removed unused import from JobTokenSecretManager --- .../org/apache/tez/common/security/JobTokenSecretManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java index 2789b024f6..d4cd2be83e 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java @@ -29,7 +29,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token;