From d9b993f465fe0a4108c6266877e6d16ea974c30b Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 4 Feb 2021 11:33:08 +0100 Subject: [PATCH] TEZ-4281: dag_*_priority.dot files should go into a valid log directory Change-Id: I611098449116fe764b64888118bfd0ae3e17c084 --- tez-dag/findbugs-exclude.xml | 12 ++++++++++++ .../src/main/java/org/apache/tez/Utils.java | 18 +++--------------- .../org/apache/tez/dag/app/DAGAppMaster.java | 2 +- .../apache/tez/dag/app/dag/impl/DAGImpl.java | 18 +++++++++++++++++- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml index a6ce38053b..50422ff0e0 100644 --- a/tez-dag/findbugs-exclude.xml +++ b/tez-dag/findbugs-exclude.xml @@ -252,4 +252,16 @@ + + + + + + + + + + + + diff --git a/tez-dag/src/main/java/org/apache/tez/Utils.java b/tez-dag/src/main/java/org/apache/tez/Utils.java index dbde327107..b352334f44 100644 --- a/tez-dag/src/main/java/org/apache/tez/Utils.java +++ b/tez-dag/src/main/java/org/apache/tez/Utils.java @@ -16,9 +16,7 @@ import javax.annotation.Nullable; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.event.Event; -import org.apache.tez.common.TezCommonUtils; import org.apache.tez.dag.api.records.DAGProtos; import org.apache.tez.dag.app.AppContext; import org.apache.tez.dag.app.dag.DAG; @@ -109,19 +107,6 @@ public static void processNonFatalServiceErrorReport(String entityString, } } - /** - * Generate a visualization file. - * @param dag DAG. - * @param dagPB DAG plan. - * @param scheduler scheduler that provide the priorities of the vertexes. - */ - public static void generateDAGVizFile(final DAG dag, - final DAGProtos.DAGPlan dagPB, @Nullable final DAGScheduler scheduler) { - generateDAGVizFile(dag, dagPB, TezCommonUtils.getTrimmedStrings( - System.getenv(ApplicationConstants.Environment.LOG_DIRS.name())), - scheduler); - } - /** * Generate a visualization file. * @param dag DAG. @@ -217,6 +202,9 @@ public static void generateDAGVizFile(final DAG dag, if (logDirs != null && logDirs.length != 0) { outputFile += logDirs[0]; outputFile += File.separator; + } else { + LOG.warn("DAGVizFile will be created under current (.) directory: {}," + + " which is not expected and recommended", new File(".").getAbsolutePath()); } outputFile += dagId.toString(); // Means we have set the priorities diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 395e84a105..1352b68f26 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -1030,7 +1030,7 @@ DAGImpl createDAG(DAGPlan dagPB, TezDAGID dagId) { new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(), taskCommunicatorManager, dagCredentials, clock, appMasterUgi.getShortUserName(), - taskHeartbeatHandler, context); + taskHeartbeatHandler, context).setLogDirs(logDirs); try { if (LOG.isDebugEnabled()) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java index 09e9e71b92..026ca29b7a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java @@ -57,6 +57,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; @@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.tez.common.ATSConstants; import org.apache.tez.common.ReflectionUtils; +import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.counters.AggregateTezCounters; import org.apache.tez.common.counters.DAGCounter; import org.apache.tez.common.counters.TezCounters; @@ -225,6 +227,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, private TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption; private static final DagStateChangedCallback STATE_CHANGED_CALLBACK = new DagStateChangedCallback(); + private String[] logDirs; @VisibleForTesting Map> commitFutures @@ -1662,7 +1665,7 @@ DAGState initializeDAG() { // which didn't have the priorities if (getConf().getBoolean(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS, TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT)) { - Utils.generateDAGVizFile(this, jobPlan, dagScheduler); + Utils.generateDAGVizFile(this, jobPlan, logDirs, dagScheduler); } return DAGState.INITED; } @@ -2510,4 +2513,17 @@ public void onFailure(Throwable t) { eventHandler.handle(new DAGEventCommitCompleted(dagId, outputKey, false, t)); } } + + public String[] getLogDirs() { + if (logDirs == null) { + logDirs = TezCommonUtils + .getTrimmedStrings(System.getenv(ApplicationConstants.Environment.LOG_DIRS.name())); + } + return logDirs; + } + + public DAGImpl setLogDirs(String[] logDirs) { + this.logDirs = logDirs; + return this; + } }