From 40af3119ea3a4d228fa5b1f7d24c49ac62d6400c Mon Sep 17 00:00:00 2001 From: "Lucas.Capistrant" Date: Mon, 6 Dec 2021 15:02:45 -0600 Subject: [PATCH 1/3] Add authentication call before cleaning up intermediate files in hadoop ingestions --- .../apache/druid/storage/hdfs/HdfsStorageAuthentication.java | 2 ++ .../src/main/java/org/apache/druid/indexer/JobHelper.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java index 096875fdaf44..c6b874c42d40 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java @@ -64,6 +64,8 @@ public void authenticate() || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) { log.info("Trying to authenticate user [%s] with keytab [%s]..", principal, keytab); UserGroupInformation.loginUserFromKeytab(principal, keytab); + } else { + log.debug("HdfsStorageAuthentication#authenticate() was called, but I am already authenticated!"); } } catch (IOException e) { diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java index acff75381551..e92eadb784ed 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java @@ -111,6 +111,8 @@ public static void authenticate() || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) { log.info("trying to authenticate user [%s] with keytab [%s]", principal, keytab); UserGroupInformation.loginUserFromKeytab(principal, keytab); + } else { + log.debug("JobHelper#authenticate() was called, but I am already authenticated!"); } } catch (IOException e) { @@ -409,6 +411,9 @@ public static void maybeDeleteIntermediatePath( boolean jobSucceeded, HadoopIngestionSpec indexerSchema) { + // Ensure we are authenticated before we try to delete intermediate paths! + authenticate(); + HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSpec(indexerSchema); final Configuration configuration = JobHelper.injectSystemProperties(new Configuration(), config); config.addJobProperties(configuration); From 6a6494a8d8780fc749bc8d3f4fd1fdab477ff63e Mon Sep 17 00:00:00 2001 From: "Lucas.Capistrant" Date: Tue, 7 Dec 2021 09:30:37 -0600 Subject: [PATCH 2/3] fix checkstyle --- .../apache/druid/storage/hdfs/HdfsStorageAuthentication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java index c6b874c42d40..e506c3f0a8d8 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java @@ -64,7 +64,7 @@ public void authenticate() || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) { log.info("Trying to authenticate user [%s] with keytab [%s]..", principal, keytab); UserGroupInformation.loginUserFromKeytab(principal, keytab); - } else { + } else { log.debug("HdfsStorageAuthentication#authenticate() was called, but I am already authenticated!"); } } From 054fb76101d9a4db3fa65b49907f0557c83339a7 Mon Sep 17 00:00:00 2001 From: "Lucas.Capistrant" Date: Wed, 8 Dec 2021 10:45:06 -0600 Subject: [PATCH 3/3] remove debug log --- .../apache/druid/storage/hdfs/HdfsStorageAuthentication.java | 4 +--- .../src/main/java/org/apache/druid/indexer/JobHelper.java | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java index e506c3f0a8d8..bb38b717e6bf 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java @@ -48,7 +48,7 @@ public HdfsStorageAuthentication(HdfsKerberosConfig hdfsKerberosConfig, @Hdfs Co } /** - * Dose authenticate against a secured hadoop cluster + * Does authenticate against a secured hadoop cluster * In case of any bug fix make sure to fix the code in JobHelper#authenticate as well. */ @LifecycleStart @@ -64,8 +64,6 @@ public void authenticate() || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) { log.info("Trying to authenticate user [%s] with keytab [%s]..", principal, keytab); UserGroupInformation.loginUserFromKeytab(principal, keytab); - } else { - log.debug("HdfsStorageAuthentication#authenticate() was called, but I am already authenticated!"); } } catch (IOException e) { diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java index e92eadb784ed..6b1b84d02d7a 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java @@ -94,7 +94,7 @@ public static Path distributedClassPath(Path base) public static final String INDEX_ZIP = "index.zip"; /** - * Dose authenticate against a secured hadoop cluster + * Does authenticate against a secured hadoop cluster * In case of any bug fix make sure to fix the code at HdfsStorageAuthentication#authenticate as well. * */ @@ -111,8 +111,6 @@ public static void authenticate() || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) { log.info("trying to authenticate user [%s] with keytab [%s]", principal, keytab); UserGroupInformation.loginUserFromKeytab(principal, keytab); - } else { - log.debug("JobHelper#authenticate() was called, but I am already authenticated!"); } } catch (IOException e) {