From 7377ed4fb8ae28c1ad1d05f487f0c261c66feb1f Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Mon, 24 May 2021 19:44:23 +0900
Subject: [PATCH 1/3] Add pulled image removal feature.

---
 .../apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index d6270313cabea..c4ee2871fd6fd 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -99,6 +99,8 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
   val connectionTimeout = timeout(5.minutes)
   val keepContainer =
     sys.props.getOrElse("spark.test.docker.keepContainer", "false").toBoolean
+  val removePulledImage =
+    sys.props.getOrElse("spark.test.docker.removePulledImage", "false").toBoolean
 
   private var docker: DockerClient = _
   // Configure networking (necessary for boot2docker / Docker Machine)
@@ -109,6 +111,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
     port
   }
   private var containerId: String = _
+  private var pulled: Boolean = false
   protected var jdbcUrl: String = _
 
   override def beforeAll(): Unit = {
@@ -130,6 +133,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
         case e: ImageNotFoundException =>
           log.warn(s"Docker image ${db.imageName} not found; pulling image from registry")
           docker.pull(db.imageName)
+          pulled = true
       }
       val hostConfigBuilder = HostConfig.builder()
         .privileged(db.privileged)
@@ -215,6 +219,9 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
           }
       } finally {
         docker.removeContainer(containerId)
+        if (removePulledImage && pulled) {
+          docker.removeImage(db.imageName)
+        }
       }
     }
   }

From 60ea4fd9d395ddfab6e98be02b95c62ec52df228 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Mon, 24 May 2021 21:11:38 +0900
Subject: [PATCH 2/3] Add the property to pom.xml.

---
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pom.xml b/pom.xml
index 6b1b63c21bb25..54f66eb4d3589 100644
--- a/pom.xml
+++ b/pom.xml
@@ -264,6 +264,7 @@
     <spark.test.home>${session.executionRootDirectory}</spark.test.home>
     <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
     <spark.test.docker.keepContainer>false</spark.test.docker.keepContainer>
+    <spark.test.docker.removePulledImage>false</spark.test.docker.removePulledImage>
 
     <CodeCacheSize>1g</CodeCacheSize>
     <!-- Needed for consistent times -->
@@ -2728,6 +2729,7 @@
               <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
               <spark.test.webdriver.chrome.driver>${spark.test.webdriver.chrome.driver}</spark.test.webdriver.chrome.driver>
               <spark.test.docker.keepContainer>${spark.test.docker.keepContainer}</spark.test.docker.keepContainer>
+              <spark.test.docker.removePulledImage>${spark.test.docker.removePulledImage}</spark.test.docker.removePulledImage>
               <!-- Needed by sql/hive tests. -->
               <test.src.tables>__not_used__</test.src.tables>
             </systemProperties>

From da66bb1a05d77665a352f395e81a3521febcd6e4 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 26 May 2021 10:43:15 +0900
Subject: [PATCH 3/3] Enabled the property by default.

---
 .../org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala  | 2 +-
 pom.xml                                                         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index c4ee2871fd6fd..2020c9b6c34af 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -100,7 +100,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu
   val keepContainer =
     sys.props.getOrElse("spark.test.docker.keepContainer", "false").toBoolean
   val removePulledImage =
-    sys.props.getOrElse("spark.test.docker.removePulledImage", "false").toBoolean
+    sys.props.getOrElse("spark.test.docker.removePulledImage", "true").toBoolean
 
   private var docker: DockerClient = _
   // Configure networking (necessary for boot2docker / Docker Machine)
diff --git a/pom.xml b/pom.xml
index ffd782fe99a30..796e0f2f647d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -264,7 +264,7 @@
     <spark.test.home>${session.executionRootDirectory}</spark.test.home>
     <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
     <spark.test.docker.keepContainer>false</spark.test.docker.keepContainer>
-    <spark.test.docker.removePulledImage>false</spark.test.docker.removePulledImage>
+    <spark.test.docker.removePulledImage>true</spark.test.docker.removePulledImage>
 
     <CodeCacheSize>1g</CodeCacheSize>
     <!-- Needed for consistent times -->