From cf8b30f64f8855ea1574fa47955a7ce42c7d0703 Mon Sep 17 00:00:00 2001 From: ouyangxiaochen Date: Wed, 24 Oct 2018 14:48:24 +0800 Subject: [PATCH] WorkDirCleanup should only remove the directory at the beginning of the app- --- core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index d5ea2523c628b..a125b8c6e7e54 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -462,7 +462,7 @@ private[deploy] class Worker( // when cleaning up val appIdFromDir = dir.getName val isAppStillRunning = appIds.contains(appIdFromDir) - dir.isDirectory && !isAppStillRunning && + dir.isDirectory && !isAppStillRunning && appIdFromDir.startsWith("app-") && !Utils.doesDirectoryContainAnyNewFiles(dir, APP_DATA_RETENTION_SECONDS) }.foreach { dir => logInfo(s"Removing directory: ${dir.getPath}")