apache · masahi · Nov 15, 2022 · Nov 11, 2022 · Nov 14, 2022 · Nov 14, 2022
diff --git a/.gitmodules b/.gitmodules
@@ -16,3 +16,6 @@
 [submodule "3rdparty/cutlass"]
 	path = 3rdparty/cutlass
 	url = https://github.com/NVIDIA/cutlass.git
+[submodule "3rdparty/OpenCL-Headers"]
+	path = 3rdparty/OpenCL-Headers
+	url = https://github.com/KhronosGroup/OpenCL-Headers.git
diff --git a/3rdparty/OpenCL-Headers b/3rdparty/OpenCL-Headers
diff --git a/LICENSE b/LICENSE
@@ -211,6 +211,7 @@ Apache Software Foundation License 2.0
 
 3rdparty/dlpack
 3rdparty/dmlc-core
+3rdparty/OpenCL-Headers
 
 
 BSD 2-clause License

diff --git a/apps/android_camera/app/src/main/jni/Android.mk b/apps/android_camera/app/src/main/jni/Android.mk
@@ -41,6 +41,7 @@ LOCAL_C_INCLUDES := $(ROOT_PATH)/include \
 					$(ROOT_PATH)/src/runtime/rpc \
                     $(ROOT_PATH)/3rdparty/dlpack/include \
                     $(ROOT_PATH)/3rdparty/dmlc-core/include \
+                    $(ROOT_PATH)/3rdparty/OpenCL-Headers \
                     $(MY_PATH)
 
 LOCAL_MODULE = tvm4j_runtime_packed

diff --git a/apps/android_camera/app/src/main/jni/make/config.mk b/apps/android_camera/app/src/main/jni/make/config.mk
@@ -34,7 +34,7 @@ APP_ABI = all
 APP_PLATFORM = android-24
 
 # whether enable OpenCL during compile
-USE_OPENCL = 0
+USE_OPENCL = 1
 
 # whether to enable Vulkan during compile
 USE_VULKAN = 0

diff --git a/apps/android_camera/app/src/main/jni/tvm_runtime.h b/apps/android_camera/app/src/main/jni/tvm_runtime.h
@@ -62,6 +62,8 @@
 #ifdef TVM_OPENCL_RUNTIME
 #include "../src/runtime/opencl/opencl_device_api.cc"
 #include "../src/runtime/opencl/opencl_module.cc"
+#include "../src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc"
+#include "../src/runtime/opencl/texture_pool.cc"
 #include "../src/runtime/source_utils.cc"
 #endif
 

diff --git a/apps/android_deploy/README.md b/apps/android_deploy/README.md
@@ -21,7 +21,7 @@ This folder contains Android Demo app that allows us to show how to deploy model
 
 You will need [JDK](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html), [Android SDK](https://developer.android.com/studio/index.html), [Android NDK](https://developer.android.com/ndk) and an Android device to use this. Make sure the `ANDROID_HOME` variable already points to your Android SDK folder or set it using `export ANDROID_HOME=[Path to your Android SDK, e.g., ~/Android/sdk]`. We use [Gradle](https://gradle.org) to build. Please follow [the installation instruction](https://gradle.org/install) for your operating system.
 
-Alternatively, you may execute Docker image we provide which contains the required packages. Use the command below to build the image and enter interactive session. Note, that building with OpenCL was not tested from Docker.
+Alternatively, you may execute Docker image we provide which contains the required packages. Use the command below to build the image and enter interactive session.
 
 ```bash
 ./docker/build.sh demo_android -it bash
@@ -50,7 +50,7 @@ dependencies {
 }
 ```
 
-Application default has CPU version TVM runtime flavor and follow below instruction to setup.
+Application default has CPU and GPU (OpenCL) versions TVM runtime flavor and follow below instruction to setup.
 In `app/src/main/jni/make` you will find JNI Makefile config `config.mk` and copy it to `app/src/main/jni` and modify it.
 
 ```bash
@@ -64,9 +64,6 @@ Here's a piece of example for `config.mk`.
 APP_ABI = arm64-v8a
 
 APP_PLATFORM = android-17
-
-# whether enable OpenCL during compile
-USE_OPENCL = 0
 ```
 
 Now use Gradle to compile JNI, resolve Java dependencies and build the Android application together with tvm4j. Run following script to generate the apk file.
@@ -82,28 +79,11 @@ Upload `tvmdemo-release.apk` to your Android device and install it.
 
 ### Build with OpenCL
 
-Application does not link with OpenCL library unless you configure it to. Modify JNI Makefile config `app/src/main/jni` with proper target OpenCL configuration.
-
-Here's a piece of example for `config.mk`.
-
-```makefile
-APP_ABI = arm64-v8a
-
-APP_PLATFORM = android-17
-
-# whether enable OpenCL during compile
-USE_OPENCL = 1
-
-# the additional include headers you want to add, e.g., SDK_PATH/adrenosdk/Development/Inc
-ADD_C_INCLUDES = /opt/adrenosdk-osx/Development/Inc
-
-# the additional link libs you want to add, e.g., ANDROID_LIB_PATH/libOpenCL.so
-ADD_LDLIBS = libOpenCL.so
-```
-
-Note that you should specify the correct GPU development headers for your android device. Run `adb shell dumpsys | grep GLES` to find out what GPU your android device uses. It is very likely the library (libOpenCL.so) is already present on the mobile device. For instance, I found it under `/system/vendor/lib64`. You can do `adb pull /system/vendor/lib64/libOpenCL.so ./` to get the file to your desktop.
-
-After you setup the `config.mk`, follow the instructions in [Build APK](#buildapk) to build the Android package with OpenCL flavor.
+Application is building with OpenCL support by default.
+[OpenCL-wrapper](../../src/runtime/opencl/opencl_wrapper) is used and will dynamically load OpenCL library on the device.
+If the device doesn't have OpenCL library on it, then you'll see in the runtime that OpenCL library cannot be opened.
+If you want to build this application without OpenCL then set `USE_OPENCL = 0`
+in [config.mk](./app/src/main/jni/make/config.mk)
 
 ## Cross Compile and Run on Android Devices
 

diff --git a/apps/android_deploy/app/src/main/jni/Android.mk b/apps/android_deploy/app/src/main/jni/Android.mk
@@ -38,7 +38,8 @@ LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog
 
 LOCAL_C_INCLUDES := $(ROOT_PATH)/include \
                     $(ROOT_PATH)/3rdparty/dlpack/include \
-                    $(ROOT_PATH)/3rdparty/dmlc-core/include
+                    $(ROOT_PATH)/3rdparty/dmlc-core/include \
+                    $(ROOT_PATH)/3rdparty/OpenCL-Headers
 
 LOCAL_MODULE = tvm4j_runtime_packed
 

diff --git a/apps/android_deploy/app/src/main/jni/make/config.mk b/apps/android_deploy/app/src/main/jni/make/config.mk
@@ -34,7 +34,7 @@ APP_ABI = all
 APP_PLATFORM = android-17
 
 # whether enable OpenCL during compile
-USE_OPENCL = 0
+USE_OPENCL = 1
 
 # the additional include headers you want to add, e.g., SDK_PATH/adrenosdk/Development/Inc
 ADD_C_INCLUDES =

diff --git a/apps/android_deploy/app/src/main/jni/tvm_runtime.h b/apps/android_deploy/app/src/main/jni/tvm_runtime.h
@@ -47,4 +47,7 @@
 #ifdef TVM_OPENCL_RUNTIME
 #include "../src/runtime/opencl/opencl_device_api.cc"
 #include "../src/runtime/opencl/opencl_module.cc"
+#include "../src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc"
+#include "../src/runtime/opencl/texture_pool.cc"
+#include "../src/runtime/source_utils.cc"
 #endif
diff --git a/apps/android_rpc/README.md b/apps/android_rpc/README.md
@@ -74,33 +74,11 @@ $ANDROID_HOME/platform-tools/adb uninstall org.apache.tvm.tvmrpc
 
 ### Build with OpenCL
 
-This application does not link any OpenCL library unless you configure it to. In `app/src/main/jni/make` you will find JNI Makefile config `config.mk`. Copy it to `app/src/main/jni` and modify it.
-
-```bash
-cd apps/android_rpc/app/src/main/jni
-cp make/config.mk .
-```
-
-Here's a piece of example for `config.mk`.
-
-```makefile
-APP_ABI = arm64-v8a
-
-APP_PLATFORM = android-17
-
-# whether enable OpenCL during compile
-USE_OPENCL = 1
-
-# the additional include headers you want to add, e.g., SDK_PATH/adrenosdk/Development/Inc
-ADD_C_INCLUDES = /opt/adrenosdk-osx/Development/Inc
-
-# the additional link libs you want to add, e.g., ANDROID_LIB_PATH/libOpenCL.so
-ADD_LDLIBS = libOpenCL.so
-```
-
-Note that you should specify the correct GPU development headers for your android device. Run `adb shell dumpsys | grep GLES` to find out what GPU your android device uses. It is very likely the library (libOpenCL.so) is already present on the mobile device. For instance, I found it under `/system/vendor/lib64`. You can do `adb pull /system/vendor/lib64/libOpenCL.so ./` to get the file to your desktop.
-
-After you setup the `config.mk`, follow the instructions in [Build APK](#buildapk) to build the Android package.
+Application is building with OpenCL support by default.
+[OpenCL-wrapper](../../src/runtime/opencl/opencl_wrapper) is used and will dynamically load OpenCL library on the device.
+If the device doesn't have OpenCL library on it, then you'll see in the runtime that OpenCL library cannot be opened.
+If you want to build this application without OpenCL then set `USE_OPENCL = 0`
+in [config.mk](./app/src/main/jni/make/config.mk)
 
 ## Cross Compile and Run on Android Devices
 

diff --git a/apps/android_rpc/app/src/main/jni/Android.mk b/apps/android_rpc/app/src/main/jni/Android.mk
@@ -38,7 +38,8 @@ LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog
 
 LOCAL_C_INCLUDES := $(ROOT_PATH)/include \
                     $(ROOT_PATH)/3rdparty/dlpack/include \
-                    $(ROOT_PATH)/3rdparty/dmlc-core/include
+                    $(ROOT_PATH)/3rdparty/dmlc-core/include \
+                    $(ROOT_PATH)/3rdparty/OpenCL-Headers
 
 LOCAL_MODULE = tvm4j_runtime_packed
 

diff --git a/apps/android_rpc/app/src/main/jni/make/config.mk b/apps/android_rpc/app/src/main/jni/make/config.mk
@@ -34,7 +34,7 @@ APP_ABI = all
 APP_PLATFORM = android-24
 
 # whether enable OpenCL during compile
-USE_OPENCL = 0
+USE_OPENCL = 1
 
 # whether to enable Vulkan during compile
 USE_VULKAN = 0

diff --git a/apps/android_rpc/app/src/main/jni/tvm_runtime.h b/apps/android_rpc/app/src/main/jni/tvm_runtime.h
@@ -64,6 +64,7 @@
 #ifdef TVM_OPENCL_RUNTIME
 #include "../src/runtime/opencl/opencl_device_api.cc"
 #include "../src/runtime/opencl/opencl_module.cc"
+#include "../src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc"
 #include "../src/runtime/opencl/texture_pool.cc"
 #include "../src/runtime/source_utils.cc"
 #endif

diff --git a/apps/cpp_rpc/README.md b/apps/cpp_rpc/README.md
@@ -37,7 +37,15 @@ This folder contains a simple recipe to make RPC server in c++.
   # Path to the desired C++ cross compiler
   set(CMAKE_CXX_COMPILER /path/to/cross/compiler/executable)
 ```
-- If linking against a custom device OpenCL library is needed, in the config specify the path to the OpenCL SDK containing the include/CL headers and lib/ or lib64/libOpenCL.so:
+- If you need to build cpp_rpc with OpenCL support, specify variable `USE_OPENCL` in the config:
+  ```
+  set(USE_OPENCL ON)
+  ```
+  In this case [OpenCL-wrapper](../../src/runtime/opencl/opencl_wrapper) or OpenCL installed to your system will be used.
+  When OpenCL-wrapper is used, it will dynamically load OpenCL library on the device.
+  If the device doesn't have OpenCL library on it, then you'll see in the runtime that OpenCL library cannot be opened.
+
+  If linking against a custom device OpenCL library is needed, in the config specify the path to the OpenCL SDK containing the include/CL headers and lib/ or lib64/libOpenCL.so:
 ```
   set(USE_OPENCL /path/to/opencl-sdk)
 ```

diff --git a/cmake/config.cmake b/cmake/config.cmake
@@ -65,7 +65,8 @@ set(USE_AOCL OFF)
 # Whether enable OpenCL runtime
 #
 # Possible values:
-# - ON: enable OpenCL with cmake's auto search
+# - ON: enable OpenCL with OpenCL wrapper to remove dependency during build
+#       time and trigger dynamic search and loading of OpenCL in runtime
 # - OFF: disable OpenCL
 # - /path/to/opencl-sdk: use specific path to opencl-sdk
 set(USE_OPENCL OFF)

diff --git a/cmake/modules/OpenCL.cmake b/cmake/modules/OpenCL.cmake
@@ -15,15 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# OPENCL Module
-find_opencl(${USE_OPENCL})
-
-if(OpenCL_FOUND)
-  # always set the includedir when cuda is available
-  # avoid global retrigger of cmake
-  include_directories(SYSTEM ${OpenCL_INCLUDE_DIRS})
-endif(OpenCL_FOUND)
-
 if(USE_SDACCEL)
   message(STATUS "Build with SDAccel support")
   tvm_file_glob(GLOB RUNTIME_SDACCEL_SRCS src/runtime/opencl/sdaccel/*.cc)
@@ -49,12 +40,23 @@ else()
 endif(USE_AOCL)
 
 if(USE_OPENCL)
-  if (NOT OpenCL_FOUND)
-    find_package(OpenCL REQUIRED)
-  endif()
-  message(STATUS "Build with OpenCL support")
   tvm_file_glob(GLOB RUNTIME_OPENCL_SRCS src/runtime/opencl/*.cc)
-  list(APPEND TVM_RUNTIME_LINKER_LIBS ${OpenCL_LIBRARIES})
+
+  if(${USE_OPENCL} MATCHES ${IS_TRUE_PATTERN})
+    message(WARNING "Build with OpenCL wrapper")
+    file_glob_append(RUNTIME_OPENCL_SRCS
+      "src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc"
+    )
+    include_directories(SYSTEM "3rdparty/OpenCL-Headers")
+  else()
+    find_opencl(${USE_OPENCL})
+    if(NOT OpenCL_FOUND)
+        message(FATAL_ERROR "Error! Cannot find specified OpenCL library")
+    endif()
+    message(STATUS "Build with OpenCL support")
+    include_directories(SYSTEM ${OpenCL_INCLUDE_DIRS})
+    list(APPEND TVM_RUNTIME_LINKER_LIBS ${OpenCL_LIBRARIES})
+  endif()
 
   if(DEFINED USE_OPENCL_GTEST AND EXISTS ${USE_OPENCL_GTEST})
     file_glob_append(RUNTIME_OPENCL_SRCS

diff --git a/cmake/utils/FindOpenCL.cmake b/cmake/utils/FindOpenCL.cmake
@@ -21,7 +21,7 @@
 # Usage:
 #   find_opencl(${USE_OPENCL})
 #
-# - When USE_OPENCL=ON, use auto search
+# - When USE_OPENCL=ON, use OpenCL wrapper for dynamic linking
 # - When USE_OPENCL=/path/to/opencl-sdk-path, use the sdk.
 #   Can be useful when cross compiling and cannot rely on
 #   CMake to provide the correct library as part of the

diff --git a/gallery/how_to/deploy_models/deploy_model_on_android.py b/gallery/how_to/deploy_models/deploy_model_on_android.py
@@ -137,11 +137,10 @@
 #
 #   # the additional include headers you want to add, e.g., SDK_PATH/adrenosdk/Development/Inc
 #   ADD_C_INCLUDES += /work/adrenosdk-linux-5_0/Development/Inc
-#   # downloaded from https://github.com/KhronosGroup/OpenCL-Headers
-#   ADD_C_INCLUDES += /usr/local/OpenCL-Headers/
+#   ADD_C_INCLUDES =
 #
 #   # the additional link libs you want to add, e.g., ANDROID_LIB_PATH/libOpenCL.so
-#   ADD_LDLIBS = /workspace/pull-from-android-device/libOpenCL.so
+#   ADD_LDLIBS =
 #
 # .. note::
 #

diff --git a/src/runtime/opencl/opencl_module.cc b/src/runtime/opencl/opencl_module.cc
@@ -232,7 +232,7 @@ cl_kernel OpenCLModuleNode::InstallKernel(cl::OpenCLWorkspace* w, cl::OpenCLThre
       cl_int err;
       cl_device_id dev = w->devices[device_id];
       programs_[func_name][device_id] =
-          clCreateProgramWithBinary(w->context, 1, &dev, &len, &s, NULL, &err);
+          clCreateProgramWithBinary(w->context, 1, &dev, &len, &s, nullptr, &err);
       OPENCL_CHECK_ERROR(err);
     } else {
       LOG(FATAL) << "Unknown OpenCL format " << fmt_;

diff --git a/src/runtime/opencl/opencl_wrapper/README.md b/src/runtime/opencl/opencl_wrapper/README.md
@@ -0,0 +1,25 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# OpenCL Wrapper
+
+This wrapper helps dynamically loading OpenCL library. It allows us to avoid of
+looking for and copying library from phone to host, looking for OpenCL SDK.
+
+This can be done because OpenCL is a standard and number of functions are
+limited. We can safely wrap all required functions and their number will not
+grow.