From 566a68a412667fe9b96a92e42fd4e0522b38a3ca Mon Sep 17 00:00:00 2001
From: Leonard Lausen <lausen@amazon.com>
Date: Mon, 27 Apr 2020 19:45:28 +0000
Subject: [PATCH] Enable docker cache build for images listed in
 docker-compose.yml

---
 ci/Jenkinsfile_docker_cache |  1 +
 ci/build.py                 | 10 +++++++---
 ci/docker_cache.py          |  2 +-
 tests/README.md             | 18 +++++++++++++-----
 4 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache
index f11dd60ab2cb..9d5022a8aec5 100644
--- a/ci/Jenkinsfile_docker_cache
+++ b/ci/Jenkinsfile_docker_cache
@@ -38,6 +38,7 @@ core_logic: {
         timeout(time: total_timeout, unit: 'MINUTES') {
           utils.init_git()
           sh "python3 ./ci/docker_cache.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}"
+          sh "cd ci && docker-compose -f docker/docker-compose.yml build --parallel && docker-compose -f docker/docker-compose.yml push "
         }
       }
     }
diff --git a/ci/build.py b/ci/build.py
index fada2f7cf772..323a4487fafc 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -47,17 +47,21 @@
                             'centos7_gpu_cu101', 'centos7_gpu_cu102', 'ubuntu_cpu',
                             'ubuntu_build_cuda', 'ubuntu_gpu_cu101', 'publish.test.centos7_cpu',
                             'publish.test.centos7_gpu')
+# Files for docker compose
+DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/publish.test.centos7'))
 
 
 def get_dockerfiles_path():
     return "docker"
 
 
-def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
+def get_platforms(path: str = get_dockerfiles_path(), legacy_only=False) -> List[str]:
     """Get a list of architectures given our dockerfiles"""
     dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
-    dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
-    files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
+    dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles))
+    files = set(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
+    if legacy_only:
+        files = files - DOCKER_COMPOSE_FILES
     platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
     return platforms
 
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index ec73e93fdb7c..14a0f467a302 100644
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -190,7 +190,7 @@ def script_name() -> str:
 
     args = parser.parse_args()
 
-    platforms = build_util.get_platforms()
+    platforms = build_util.get_platforms(legacy_only=True)
 
     secret_name = os.environ['DOCKERHUB_SECRET_NAME']
     endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL']
diff --git a/tests/README.md b/tests/README.md
index b59335ea1593..1d34f6446c33 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -62,13 +62,21 @@ Ninja is a build tool (like make) that prioritizes building speed. If you will b
     
 ## Runing Python Tests Within Docker
 
-1. To run tests inside docker run the following comamdn
-    ```
-    ci/build.py --platform {PLATFORM} /work/runtime_functions.sh {RUNTIME_FUNCTION}
-    ```
+To run tests inside docker, you first need to install `docker` and `docker-compose` on your machine.
+
+On Ubuntu you may install them via `sudo apt-get install docker.io docker-compose`
+and set them up via `sudo usermod $(whoami) -G docker -a`.
+
+Then, to run tests inside docker run the following command
+
+```
+ci/build.py --platform {PLATFORM} /work/runtime_functions.sh {RUNTIME_FUNCTION}
+```
+
 An example for running python tests would be
+
 ```
-ci/build.py --platform build_ubuntu_cpu_mkldnn /work/runtime_functions.sh unittest_ubuntu_python3_cpu PYTHONPATH=./python/ pytest tests/python/unittest
+ci/build.py --platform build_ubuntu_cpu_mkldnn /work/runtime_functions.sh unittest_ubuntu_python3_cpu
 ```