flexflow · wmdi · Jul 19, 2023 · Jul 19, 2023 · Jul 21, 2023 · Jul 20, 2023
diff --git a/.gitignore b/.gitignore
@@ -1,11 +1,14 @@
+# custom tooling
+/.tools/
+/.state/
+
 # ctest-created files
 Testing/
 
 # clang compilation db
 compile_commands.json
 
 # Compiled files
-/.tools/
 /python/flexflow_python
 /python/flexflow/core/legion_cffi.py
 python/flexflow/core/flexflow_cffi_header.py

diff --git a/.gitmodules b/.gitmodules
@@ -43,3 +43,9 @@
 [submodule "deps/any"]
 	path = deps/any
 	url = https://github.com/thelink2012/any.git
+[submodule "deps/nameof"]
+	path = deps/nameof
+	url = git@github.com:Neargye/nameof.git
+[submodule "deps/boost_preprocessor"]
+	path = deps/boost_preprocessor
+	url = https://github.com/boostorg/preprocessor.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,84 +3,11 @@ project(FlexFlow)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_LIST_DIR}/cmake)
 
-set(FF_MAX_DIM "5" CACHE STRING "Maximum tensor order")
-set(FF_MAX_OPNAME "128" CACHE STRING "Maximum op name length")
-set(FF_MAX_NUM_OUTPUTS "256" CACHE STRING "Maximum number of outputs (per operator)")
-set(FF_MAX_NUM_INPUTS "256" CACHE STRING "Maximum number of inputs (per operator)")
-set(FF_MAX_NUM_WEIGHTS "64" CACHE STRING "Maximum number of weights (per operator)")
-set(FF_MAX_NUM_FUSED_OPERATORS "64" CACHE STRING "Maximum number of fused tensors")
-set(FF_MAX_NUM_FUSED_TENSORS "64" CACHE STRING "Maximum number of input and output tensors per fused op")
-set(FF_MAX_NUM_WORKERS "1024" CACHE STRING "Maximum number of GPUs")
-set(FF_MAX_NUM_TASK_REGIONS "20" CACHE STRING 
-  "Maximum number of regions that can be passed to a task through the TaskSpec interface")
-set(FF_MAX_NUM_TASK_ARGUMENTS "5" CACHE STRING
-  "Maximum number of arguments that can be declared in a TaskSignature")
-option(FF_USE_NCCL "Run FlexFlow with NCCL" OFF)
-option(FF_USE_PREBUILT_NCCL "Enable use of NCCL pre-compiled library, if available" ON)
-option(FF_USE_PREBUILT_LEGION "Enable use of Legion pre-compiled library, if available" ON)
-option(FF_USE_ALL_PREBUILT_LIBRARIES "Enable use of all pre-compiled libraries, if available" OFF)
-option(FF_USE_PYTHON "Enable Python" ON)
-option(FF_BUILD_FROM_PYPI "Build from pypi" OFF)
-
-set(FF_GASNET_CONDUITS aries udp mpi ibv ucx)
-set(FF_GASNET_CONDUIT "mpi" CACHE STRING "Select GASNet conduit ${FF_GASNET_CONDUITS}")
-set_property(CACHE FF_GASNET_CONDUIT PROPERTY STRINGS ${FF_GASNET_CONDUITS})
-set(FF_LEGION_NETWORKS "" CACHE STRING "Network backend(s) to use")
-
-set(FF_GPU_BACKENDS cuda hip_cuda hip_rocm intel)
-set(FF_GPU_BACKEND "cuda" CACHE STRING "Select GPU Backend ${FF_GPU_BACKENDS}")
-set_property(CACHE FF_GPU_BACKEND PROPERTY STRINGS ${FF_GPU_BACKENDS})
-
-option(FF_USE_EXTERNAL_LEGION "Use pre-installed Legion" OFF)
-option(FF_BUILD_RESNET "build resnet example" OFF)
-option(FF_BUILD_RESNEXT "build resnext example" OFF)
-option(FF_BUILD_ALEXNET "build alexnet example" OFF)
-option(FF_BUILD_DLRM "build DLRM example" OFF)
-option(FF_BUILD_XDL "build XDL example" OFF)
-option(FF_BUILD_INCEPTION "build inception example" OFF)
-option(FF_BUILD_CANDLE_UNO "build candle uno example" OFF)
-option(FF_BUILD_TRANSFORMER "build transformer example" OFF)
-option(FF_BUILD_MOE "build mixture of experts example" OFF)
-option(FF_BUILD_MLP_UNIFY "build mlp unify example" OFF)
-option(FF_BUILD_SPLIT_TEST "build split test example" OFF)
-option(FF_BUILD_SPLIT_TEST_2 "build split test 2 example" OFF)
-option(FF_BUILD_ALL_EXAMPLES "build all examples. Overrides others" OFF)
-option(FF_BUILD_UNIT_TESTS "build non-operator unit tests" OFF)
-option(FF_BUILD_SUBSTITUTION_TOOL "build substitution conversion tool" OFF)
-option(FF_BUILD_VISUALIZATION_TOOL "build substitution visualization tool" OFF)
-option(FF_BUILD_ARG_PARSER "build command line argument parser" OFF)
-
-set(FF_CUDA_ARCH "autodetect" CACHE STRING "Target CUDA Arch")
-if (FF_CUDA_ARCH STREQUAL "")
-  message(FATAL_ERROR "FF_CUDA_ARCH cannot be an empty string. Set it to `autodetect`, `all`, or pass one or multiple valid CUDA archs.")
-endif()
-
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
-  set(LIBEXT ".so")
-endif()
-
-include(cuda)
-include(cudnn)
-include(nccl)
 # set_property(CACHE FF_GPU_BACKEND PROPERTY STRINGS ${FF_GPU_BACKENDS})
 
-include(json)
-include(optional)
-include(expected)
-include(spdlog)
-include(variant)
-include(doctest)
-include(visit_struct)
 include(CTest)
-include(fmt)
-include(legion)
-include(rapidcheck)
-include(invoke)
-include(any)
-#include(gtest)
-#include(fmt)
-
-include(flexflow-utils)
+include(utils)
+include(deps)
 
 # TODO @lockshaw remove me
 # https://discourse.nixos.org/t/get-clangd-to-find-standard-headers-in-nix-shell/11268/6

diff --git a/cmake/any.cmake b/cmake/any.cmake
diff --git a/cmake/deps.cmake b/cmake/deps.cmake
@@ -0,0 +1,14 @@
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_LIST_DIR}/deps)
+
+include(boost_preprocessor)
+include(cuda)
+include(cudnn)
+include(doctest)
+include(fmt)
+include(json)
+include(legion)
+include(nameof)
+include(nccl)
+include(rapidcheck)
+include(spdlog)
+include(visit_struct)
diff --git a/cmake/deps/boost_preprocessor.cmake b/cmake/deps/boost_preprocessor.cmake
@@ -0,0 +1 @@
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/deps/boost_preprocessor)
diff --git a/cmake/cuda.cmake → cmake/deps/cuda.cmake b/cmake/cuda.cmake → cmake/deps/cuda.cmake
diff --git a/cmake/cudnn.cmake → cmake/deps/cudnn.cmake b/cmake/cudnn.cmake → cmake/deps/cudnn.cmake
diff --git a/cmake/doctest.cmake → cmake/deps/doctest.cmake b/cmake/doctest.cmake → cmake/deps/doctest.cmake
diff --git a/cmake/fmt.cmake → cmake/deps/fmt.cmake b/cmake/fmt.cmake → cmake/deps/fmt.cmake
diff --git a/cmake/json.cmake → cmake/deps/json.cmake b/cmake/json.cmake → cmake/deps/json.cmake
diff --git a/cmake/legion.cmake → cmake/deps/legion.cmake b/cmake/legion.cmake → cmake/deps/legion.cmake
diff --git a/cmake/deps/nameof.cmake b/cmake/deps/nameof.cmake
@@ -0,0 +1 @@
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/deps/nameof)
diff --git a/cmake/nccl.cmake → cmake/deps/nccl.cmake b/cmake/nccl.cmake → cmake/deps/nccl.cmake
diff --git a/cmake/rapidcheck.cmake → cmake/deps/rapidcheck.cmake b/cmake/rapidcheck.cmake → cmake/deps/rapidcheck.cmake
diff --git a/cmake/spdlog.cmake → cmake/deps/spdlog.cmake b/cmake/spdlog.cmake → cmake/deps/spdlog.cmake
diff --git a/cmake/ucx.cmake → cmake/deps/ucx.cmake b/cmake/ucx.cmake → cmake/deps/ucx.cmake
diff --git a/cmake/visit_struct.cmake → cmake/deps/visit_struct.cmake b/cmake/visit_struct.cmake → cmake/deps/visit_struct.cmake
diff --git a/cmake/zlib.cmake → cmake/deps/zlib.cmake b/cmake/zlib.cmake → cmake/deps/zlib.cmake
diff --git a/cmake/expected.cmake b/cmake/expected.cmake
diff --git a/cmake/invoke.cmake b/cmake/invoke.cmake
diff --git a/cmake/optional.cmake b/cmake/optional.cmake
diff --git a/cmake/utils.cmake b/cmake/utils.cmake
@@ -1,56 +1,6 @@
-set(known_gpu_archs "")
-function(remove_duplicate_args __string)
-  if(${__string})
-    set(__list ${${__string}})
-    separate_arguments(__list)
-    list(REMOVE_DUPLICATES __list)
-    foreach(__e ${__list})
-      set(__str "${__str} ${__e}")
-    endforeach()
-    set(${__string} ${__str} PARENT_SCOPE)
-  endif()
-endfunction()
-function(detect_installed_gpus out_variable)
-  if(NOT CUDA_gpu_detect_output)
-    set(__cufile ${PROJECT_BINARY_DIR}/detect_cuda_archs.cu)
-    file(WRITE ${__cufile} ""
-      "#include <cstdio>\n"
-      "int main()\n"
-      "{\n"
-      "  int count = 0;\n"
-      "  if (cudaSuccess != cudaGetDeviceCount(&count)) return -1;\n"
-      "  if (count == 0) return -1;\n"
-      "  for (int device = 0; device < count; ++device)\n"
-      "  {\n"
-      "    cudaDeviceProp prop;\n"
-      "    if (cudaSuccess == cudaGetDeviceProperties(&prop, device))\n"
-      "      std::printf(\"%d.%d \", prop.major, prop.minor);\n"
-      "  }\n"
-      "  return 0;\n"
-      "}\n")
-    execute_process(COMMAND "${CUDA_NVCC_EXECUTABLE}" "--run" "${__cufile}"
-                    WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/"
-                    RESULT_VARIABLE __nvcc_res OUTPUT_VARIABLE __nvcc_out
-                    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if(__nvcc_res EQUAL 0)
-      message(STATUS "No result from nvcc so building for 2.0")
-      string(REPLACE "2.1" "2.1(2.0)" __nvcc_out "${__nvcc_out}")
-      set(CUDA_gpu_detect_output ${__nvcc_out} CACHE INTERNAL "Returned GPU architetures from detect_gpus tool" FORCE)
-    endif()
-  endif()
-  if(NOT CUDA_gpu_detect_output)
-    message(STATUS "Automatic GPU detection failed, Architecture is not set: ${known_gpu_archs}.")
-    set(${out_variable} ${known_gpu_archs} PARENT_SCOPE)
-  else()
-    remove_duplicate_args(CUDA_gpu_detect_output)
-    #Strip leading and trailing whitespaces
-    string(STRIP "${CUDA_gpu_detect_output}" CUDA_gpu_detect_output)
-    #Replace spaces in between with commas so you go from "5.2 6.1" to "5.2,6.1"
-    string(REGEX REPLACE " " "," CUDA_gpu_detect_output "${CUDA_gpu_detect_output}")
-    # message(${CUDA_gpu_detect_output})
-    string(REPLACE "." "" CUDA_gpu_detect_output "${CUDA_gpu_detect_output}")
-    # message(${CUDA_gpu_detect_output})
-    set(${out_variable} ${CUDA_gpu_detect_output} PARENT_SCOPE)
-    # message(STATUS "Automatic GPU ARCH detection: ${CUDA_gpu_detect_output}")
-  endif()
-endfunction()
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_LIST_DIR}/utils)
+
+include(flexflow-utils)
+include(aliasing)
+include(build-options)
+include(libext)
diff --git a/cmake/aliasing.cmake → cmake/utils/aliasing.cmake b/cmake/aliasing.cmake → cmake/utils/aliasing.cmake
diff --git a/cmake/utils/build-options.cmake b/cmake/utils/build-options.cmake
@@ -0,0 +1,51 @@
+set(FF_MAX_DIM "5" CACHE STRING "Maximum tensor order")
+set(FF_MAX_OPNAME "128" CACHE STRING "Maximum op name length")
+set(FF_MAX_NUM_OUTPUTS "256" CACHE STRING "Maximum number of outputs (per operator)")
+set(FF_MAX_NUM_INPUTS "256" CACHE STRING "Maximum number of inputs (per operator)")
+set(FF_MAX_NUM_WEIGHTS "64" CACHE STRING "Maximum number of weights (per operator)")
+set(FF_MAX_NUM_FUSED_OPERATORS "64" CACHE STRING "Maximum number of fused tensors")
+set(FF_MAX_NUM_FUSED_TENSORS "64" CACHE STRING "Maximum number of input and output tensors per fused op")
+set(FF_MAX_NUM_WORKERS "1024" CACHE STRING "Maximum number of GPUs")
+set(FF_MAX_NUM_TASK_REGIONS "20" CACHE STRING 
+  "Maximum number of regions that can be passed to a task through the TaskSpec interface")
+set(FF_MAX_NUM_TASK_ARGUMENTS "5" CACHE STRING
+  "Maximum number of arguments that can be declared in a TaskSignature")
+option(FF_USE_NCCL "Run FlexFlow with NCCL" OFF)
+option(FF_USE_PREBUILT_NCCL "Enable use of NCCL pre-compiled library, if available" ON)
+option(FF_USE_PREBUILT_LEGION "Enable use of Legion pre-compiled library, if available" ON)
+option(FF_USE_ALL_PREBUILT_LIBRARIES "Enable use of all pre-compiled libraries, if available" OFF)
+option(FF_USE_PYTHON "Enable Python" ON)
+option(FF_BUILD_FROM_PYPI "Build from pypi" OFF)
+
+set(FF_GASNET_CONDUITS aries udp mpi ibv ucx)
+set(FF_GASNET_CONDUIT "mpi" CACHE STRING "Select GASNet conduit ${FF_GASNET_CONDUITS}")
+set_property(CACHE FF_GASNET_CONDUIT PROPERTY STRINGS ${FF_GASNET_CONDUITS})
+set(FF_LEGION_NETWORKS "" CACHE STRING "Network backend(s) to use")
+
+set(FF_GPU_BACKENDS cuda hip_cuda hip_rocm intel)
+set(FF_GPU_BACKEND "cuda" CACHE STRING "Select GPU Backend ${FF_GPU_BACKENDS}")
+set_property(CACHE FF_GPU_BACKEND PROPERTY STRINGS ${FF_GPU_BACKENDS})
+
+option(FF_USE_EXTERNAL_LEGION "Use pre-installed Legion" OFF)
+option(FF_BUILD_RESNET "build resnet example" OFF)
+option(FF_BUILD_RESNEXT "build resnext example" OFF)
+option(FF_BUILD_ALEXNET "build alexnet example" OFF)
+option(FF_BUILD_DLRM "build DLRM example" OFF)
+option(FF_BUILD_XDL "build XDL example" OFF)
+option(FF_BUILD_INCEPTION "build inception example" OFF)
+option(FF_BUILD_CANDLE_UNO "build candle uno example" OFF)
+option(FF_BUILD_TRANSFORMER "build transformer example" OFF)
+option(FF_BUILD_MOE "build mixture of experts example" OFF)
+option(FF_BUILD_MLP_UNIFY "build mlp unify example" OFF)
+option(FF_BUILD_SPLIT_TEST "build split test example" OFF)
+option(FF_BUILD_SPLIT_TEST_2 "build split test 2 example" OFF)
+option(FF_BUILD_ALL_EXAMPLES "build all examples. Overrides others" OFF)
+option(FF_BUILD_UNIT_TESTS "build non-operator unit tests" OFF)
+option(FF_BUILD_SUBSTITUTION_TOOL "build substitution conversion tool" OFF)
+option(FF_BUILD_VISUALIZATION_TOOL "build substitution visualization tool" OFF)
+option(FF_BUILD_ARG_PARSER "build command line argument parser" OFF)
+
+set(FF_CUDA_ARCH "autodetect" CACHE STRING "Target CUDA Arch")
+if (FF_CUDA_ARCH STREQUAL "")
+  message(FATAL_ERROR "FF_CUDA_ARCH cannot be an empty string. Set it to `autodetect`, `all`, or pass one or multiple valid CUDA archs.")
+endif()
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/deps/boost_preprocessor)