Removed makefile build for armv7

.gitignore

@@ -166,3 +166,7 @@ python/.eggs
 *DartConfiguration.tcl
 tests/Makefile
 tests/mxnet_unit_tests
+
+# generated wrappers for ccache
+cc
+cxx

CMakeLists.txt

@@ -321,14 +321,15 @@ endif()
 
 # ---[ OpenCV
 if(USE_OPENCV)
-  find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
+  find_package(OpenCV COMPONENTS core highgui imgproc imgcodecs)
   if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
+    message(STATUS "OpenCV imgcodecs missing")
     find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
   endif()
   include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
   list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS})
   message(STATUS " OpenCV_LIBS=${OpenCV_LIBS}")
-  message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
+  message(STATUS "OpenCV ${OpenCV_VERSION} found (${OpenCV_CONFIG_PATH})")
   add_definitions(-DMXNET_USE_OPENCV=1)
 else(USE_OPENCV)
   message(STATUS "OpenCV Disabled")
@@ -340,7 +341,11 @@ if(USE_OPENMP)
   find_package(OpenMP REQUIRED)
   # This should build on Windows, but there's some problem and I don't have a Windows box, so
   # could a Windows user please fix?
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT MSVC)
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt
+     AND SYSTEM_ARCHITECTURE STREQUAL "x86_64"
+     AND NOT MSVC
+     AND NOT CMAKE_CROSSCOMPILING)
+
     # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
     set(OPENMP_STANDALONE_BUILD TRUE)
     set(LIBOMP_ENABLE_SHARED TRUE)
@@ -648,7 +653,7 @@ if(USE_PLUGINS_WARPCTC)
 endif()
 
 
-if(USE_OPENCV)
+if(USE_OPENCV AND OpenCV_VERSION_MAJOR GREATER 2)
   add_executable(im2rec "tools/im2rec.cc")
   if(MSVC)
     target_link_libraries(im2rec mxnet)
@@ -662,6 +667,9 @@ if(USE_OPENCV)
     ${nnvm_LINKER_LIBS}
     ${pslite_LINKER_LIBS}
     )
+else()
+    message(WARNING "OpenCV_VERSION_MAJOR: ${OpenCV_VERSION_MAJOR}, version 3 with imgcodecs \
+    is required for im2rec, im2rec will not be available")
 endif()
 
 target_link_libraries(mxnet PUBLIC dmlc)

Makefile

@@ -477,7 +477,7 @@ endif
 $(PS_PATH)/build/libps.a: PSLITE
 
 PSLITE:
-	$(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps
+	$(MAKE) CXX="$(CXX)" DEPS_PATH="$(DEPS_PATH)" -C $(PS_PATH) ps
 
 $(DMLC_CORE)/libdmlc.a: DMLCCORE
 

ci/README.md

@@ -54,7 +54,7 @@ The artifacts are located in the build/ directory in the project root. In case
 
 ## Add a platform
 
-To add a platform, you should add the appropiate dockerfile in
+To add a platform, you should add the appropriate dockerfile in
 docker/Dockerfile.build.<platform> and add a shell function named
 build_<platform> to the file docker/runtime_functions.sh with build
 instructions for that platform.
@@ -63,3 +63,9 @@ instructions for that platform.
 Due to current limitations of the CMake build system creating artifacts in the
 source 3rdparty folder of the parent mxnet sources concurrent builds of
 different platforms is NOT SUPPORTED.
+
+## ccache
+For all builds a directory from the host system is mapped where ccache will store cached 
+compiled object files (defaults to /tmp/ci_ccache). This will speed up rebuilds 
+significantly. You can set this directory explicitly by setting CCACHE_DIR environment 
+variable. All ccache instances are currently set to be 10 Gigabytes max in size.

ci/build.py

@@ -33,13 +33,15 @@
 import shutil
 import subprocess
 import sys
+import tempfile
 from copy import deepcopy
 from itertools import chain
 from subprocess import call, check_call
 from typing import *
 
+CCACHE_MAXSIZE = '10G'
 
-def get_platforms(path: Optional[str]="docker"):
+def get_platforms(path: Optional[str] = "docker"):
     """Get a list of architectures given our dockerfiles"""
     dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
     dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
@@ -72,11 +74,11 @@ def build_docker(platform: str, docker_binary: str, registry: str) -> None:
     tag = get_docker_tag(platform=platform, registry=registry)
     logging.info("Building container tagged '%s' with %s", tag, docker_binary)
     cmd = [docker_binary, "build",
-        "-f", get_dockerfile(platform),
-        "--build-arg", "USER_ID={}".format(os.getuid()),
-        "--cache-from", tag,
-        "-t", tag,
-        "docker"]
+           "-f", get_dockerfile(platform),
+           "--build-arg", "USER_ID={}".format(os.getuid()),
+           "--cache-from", tag,
+           "-t", tag,
+           "docker"]
     logging.info("Running command: '%s'", ' '.join(cmd))
     check_call(cmd)
 
@@ -102,8 +104,10 @@ def _get_local_image_id(docker_binary, docker_tag):
 
 def get_mxnet_root() -> str:
     curpath = os.path.abspath(os.path.dirname(__file__))
+
     def is_mxnet_root(path: str) -> bool:
         return os.path.exists(os.path.join(path, ".mxnet_root"))
+
     while not is_mxnet_root(curpath):
         parent = os.path.abspath(os.path.join(curpath, os.pardir))
         if parent == curpath:
@@ -116,10 +120,20 @@ def buildir() -> str:
     return os.path.join(get_mxnet_root(), "build")
 
 
+def default_ccache_dir() -> str:
+    if 'CCACHE_DIR' in os.environ:
+        ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
+        os.makedirs(ccache_dir, exist_ok=True)
+        return ccache_dirpython
+    # Share ccache across containers
+    return os.path.join(tempfile.gettempdir(), "ci_ccache")
+
+
 def container_run(platform: str,
                   docker_binary: str,
                   docker_registry: str,
                   shared_memory_size: str,
+                  local_ccache_dir: str,
                   command: List[str],
                   dry_run: bool = False,
                   into_container: bool = False) -> str:
@@ -128,12 +142,17 @@ def container_run(platform: str,
     local_build_folder = buildir()
     # We need to create it first, otherwise it will be created by the docker daemon with root only permissions
     os.makedirs(local_build_folder, exist_ok=True)
+    os.makedirs(local_ccache_dir, exist_ok=True)
+    logging.info("Using ccache directory: %s", local_ccache_dir)
     runlist = [docker_binary, 'run', '--rm', '-t',
-        '--shm-size={}'.format(shared_memory_size),
-        '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
-        '-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
-        '-u', '{}:{}'.format(os.getuid(), os.getgid()),
-        tag]
+               '--shm-size={}'.format(shared_memory_size),
+               '-v', "{}:/work/mxnet".format(mx_root),  # mount mxnet root
+               '-v', "{}:/work/build".format(local_build_folder),  # mount mxnet/build for storing build artifacts
+               '-v', "{}:/work/ccache".format(local_ccache_dir),
+               '-u', '{}:{}'.format(os.getuid(), os.getgid()),
+               '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
+               '-e', "CCACHE_DIR=/work/ccache",  # this path is inside the container as /work/ccache is mounted
+               tag]
     runlist.extend(command)
     cmd = ' '.join(runlist)
     if not dry_run and not into_container:
@@ -160,19 +179,17 @@ def container_run(platform: str,
 def list_platforms() -> str:
     print("\nSupported platforms:\n{}".format('\n'.join(get_platforms())))
 
-
 def load_docker_cache(tag, docker_registry) -> None:
     if docker_registry:
         try:
             import docker_cache
-            logging.info('Docker cache download is enabled')
+            logging.info('Docker cache download is enabled from registry %s', docker_registry)
             docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag)
         except Exception:
             logging.exception('Unable to retrieve Docker cache. Continue without...')
     else:
         logging.info('Distributed docker cache disabled')
 
-
 def main() -> int:
     # We need to be in the same directory than the script so the commands in the dockerfiles work as
     # expected. But the script can be invoked from a different path
@@ -187,7 +204,7 @@ def script_name() -> str:
     logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
 
     parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker
-    containers""",epilog="")
+    containers""", epilog="")
     parser.add_argument("-p", "--platform",
                         help="platform",
                         type=str)
@@ -221,63 +238,75 @@ def script_name() -> str:
                         help="go in a shell inside the container",
                         action='store_true')
 
-    parser.add_argument("--docker-registry",
-                        help="Dockerhub registry name to retrieve cache from",
+    parser.add_argument("-d", "--docker-registry",
+                        help="Dockerhub registry name to retrieve cache from. Default is 'mxnetci'",
                         default='mxnetci',
                         type=str)
 
+    parser.add_argument("-c", "--cache", action="store_true",
+                        help="Enable docker registry cache")
+
     parser.add_argument("command",
                         help="command to run in the container",
                         nargs='*', action='append', type=str)
 
+    parser.add_argument("--ccache-dir",
+                        default=default_ccache_dir(),
+                        help="Ccache directory",
+                        type=str)
+
     args = parser.parse_args()
-    docker_registry = args.docker_registry
+    def use_cache():
+        return args.cache or 'JOB_NAME' in os.environ # we are in Jenkins
+
     command = list(chain(*args.command))
     docker_binary = get_docker_binary(args.nvidiadocker)
     shared_memory_size = args.shared_memory_size
 
-    print("into container: {}".format(args.into_container))
     if args.list:
         list_platforms()
     elif args.platform:
         platform = args.platform
-        tag = get_docker_tag(platform=platform, registry=docker_registry)
-        load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-        build_docker(platform, docker_binary, registry=docker_registry)
+        tag = get_docker_tag(platform=platform, registry=args.docker_registry)
+        if use_cache():
+            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+        build_docker(platform, docker_binary, registry=args.docker_registry)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
             return 0
 
         if command:
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=command, docker_registry=docker_registry)
+                          command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
         elif args.print_docker_run:
             print(container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                                command=[], dry_run=True, docker_registry=docker_registry))
+                                command=[], dry_run=True, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir))
         elif args.into_container:
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=[], dry_run=False, into_container=True, docker_registry=docker_registry)
+                          command=[], dry_run=False, into_container=True, docker_registry=args.docker_registry,
+                          local_ccache_dir=args.ccache_dir)
         else:
             cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
             logging.info("No command specified, trying default build: %s", ' '.join(cmd))
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=cmd, docker_registry=docker_registry)
+                          command=cmd, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
 
     elif args.all:
         platforms = get_platforms()
         logging.info("Building for all architectures: {}".format(platforms))
         logging.info("Artifacts will be produced in the build/ directory.")
         for platform in platforms:
-            tag = get_docker_tag(platform=platform, registry=docker_registry)
-            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-            build_docker(platform, docker_binary)
+            tag = get_docker_tag(platform=platform, registry=args.docker_registry)
+            if use_cache():
+                load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+            build_docker(platform, docker_binary, args.docker_registry)
             if args.build_only:
                 continue
             build_platform = "build_{}".format(platform)
             cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
             shutil.rmtree(buildir(), ignore_errors=True)
             container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
-                          command=cmd, docker_registry=docker_registry)
+                          command=cmd, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
             plat_buildir = os.path.join(get_mxnet_root(), build_platform)
             shutil.move(buildir(), plat_buildir)
             logging.info("Built files left in: %s", plat_buildir)

ci/docker/Dockerfile.build.android_arm64

@@ -18,13 +18,20 @@
 #
 # Dockerfile to build MXNet for Android ARM64/ARMv8
 
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
+
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
-# The cross-compiling emulator
+# extract ccache binary into latest context
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+
 RUN apt-get update && apt-get install -y \
-  qemu-user \
-  qemu-user-static \
   unzip
 
 ENV CROSS_TRIPLE=aarch64-linux-android

ci/docker/Dockerfile.build.android_armv7

@@ -18,9 +18,19 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
+
 FROM dockcross/base:latest
 MAINTAINER Pedro Larroy "pllarroy@amazon.com"
 
+# extract ccache binary into latest context
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+
 # The cross-compiling emulator
 RUN apt-get update && apt-get install -y \
   qemu-user \

ci/docker/Dockerfile.build.arm64

@@ -18,22 +18,34 @@
 #
 # Dockerfile to build MXNet for ARM64/ARMv8
 
+FROM ubuntu:16.04 as ccachebuilder
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_ccache.sh /work/
+RUN /work/ubuntu_ccache.sh
+
 # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
 #FROM dockcross/linux-arm64
 FROM mxnetci/dockcross-linux-arm64:05082018
 
+# extract ccache binary into latest context
+COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
+
 ENV ARCH aarch64
-ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
 ENV HOSTCC gcc
 ENV TARGET ARMV8
 
-WORKDIR /work
+WORKDIR /work/deps
+
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
 
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
-    cd OpenBLAS && \
-    make -j$(nproc) && \
-    PREFIX=${CROSS_ROOT} make install
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
 
 COPY runtime_functions.sh /work/
-WORKDIR /work/mxnet
+WORKDIR /work/build