From 7141a315059f810e675385884114f18c893c537c Mon Sep 17 00:00:00 2001 From: Lucas Capistrant Date: Tue, 28 Apr 2020 14:43:30 -0500 Subject: [PATCH] Ensure Krb auth before killing YARN apps in graceful shutdown --- .../src/main/java/org/apache/druid/indexer/JobHelper.java | 5 ++--- .../apache/druid/indexing/common/task/HadoopIndexTask.java | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java index 84f5c8f8127e..c047c2b34e45 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java @@ -97,9 +97,8 @@ public static Path distributedClassPath(Path base) * Dose authenticate against a secured hadoop cluster * In case of any bug fix make sure to fix the code at HdfsStorageAuthentication#authenticate as well. * - * @param config containing the principal name and keytab path. */ - public static void authenticate(HadoopDruidIndexerConfig config) + public static void authenticate() { String principal = HadoopDruidIndexerConfig.HADOOP_KERBEROS_CONFIG.getPrincipal(); String keytab = HadoopDruidIndexerConfig.HADOOP_KERBEROS_CONFIG.getKeytab(); @@ -348,7 +347,7 @@ public static Configuration injectSystemProperties(Configuration conf, HadoopDru public static void ensurePaths(HadoopDruidIndexerConfig config) { - authenticate(config); + authenticate(); // config.addInputPaths() can have side-effects ( boo! :( ), so this stuff needs to be done before anything else try { Job job = Job.getInstance( diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java index 21818ef66430..dc8d2104e0fa 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java @@ -35,6 +35,7 @@ import org.apache.druid.indexer.HadoopDruidIndexerJob; import org.apache.druid.indexer.HadoopIngestionSpec; import org.apache.druid.indexer.IngestionState; +import org.apache.druid.indexer.JobHelper; import org.apache.druid.indexer.MetadataStorageUpdaterJobHandler; import org.apache.druid.indexer.TaskMetricsGetter; import org.apache.druid.indexer.TaskMetricsUtils; @@ -765,6 +766,8 @@ public String[] runTask(String[] args) throws Exception } if (jobId != null) { + // This call to JobHelper#authenticate will be transparent if already authenticated or using inseucre Hadoop. + JobHelper.authenticate(); int res = ToolRunner.run(new JobClient(), new String[]{ "-kill", jobId