add local backend support for executing task_impl locally

.clang-format-for-format-sh

@@ -0,0 +1,182 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignArrayOfStructures: None
+AlignConsecutiveMacros: None
+AlignConsecutiveAssignments: None
+AlignConsecutiveBitFields: None
+AlignConsecutiveDeclarations: None
+AlignEscapedNewlines: Right
+AlignOperands:   Align
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: Yes
+AttributeMacros:
+  - __capability
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeConceptDeclarations: true
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: false
+DisableFormat:   false
+EmptyLineAfterAccessModifier: Never
+EmptyLineBeforeAccessModifier: LogicalBlock
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IfMacros:
+  - KJ_IF_MAYBE
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+    CaseSensitive:   false
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentAccessModifiers: false
+IndentCaseLabels: true
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentExternBlock: AfterExternBlock
+IndentRequires:  false
+IndentWidth:     2
+IndentWrappedFunctionNames: true
+InsertBraces: true
+InsertTrailingCommas: None
+InsertNewlineAtEOF: true
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+LambdaBodyIndentation: Signature
+LineEnding: LF
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PenaltyIndentedWhitespace: 0
+PointerAlignment: Right
+PPIndentWidth:   -1
+QualifierAlignment: Right
+ReferenceAlignment: Pointer
+ReflowComments:  true
+ShortNamespaceLines: 1
+SortIncludes:    CaseSensitive
+SortJavaStaticImport: Before
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceAroundPointerQualifiers: Default
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  Never
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInLineCommentPrefix:
+  Minimum:         1
+  Maximum:         -1
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+BitFieldColonSpacing: Both
+Standard:        Latest
+StatementAttributeLikeMacros:
+  - Q_EMIT
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseCRLF:         false
+UseTab:          Never
+WhitespaceSensitiveMacros:
+  - STRINGIZE
+  - PP_STRINGIZE
+  - BOOST_PP_STRINGIZE
+  - NS_SWIFT_NAME
+  - CF_SWIFT_NAME
+...
+

.github/workflows/clang-format-check.yml

@@ -5,21 +5,25 @@ jobs:
     name: Formatting Check
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         path:
-          - check: "src"
-            exclude: '\.proto$'
-          - check: "include"
-          - check: "nmt"
-          - check: "python"
-          - check: "scripts"
+          - check: "lib/compiler"
+          - check: "lib/ffi"
+          - check: "lib/kernels"
+          - check: "lib/op-attrs"
+          - check: "lib/pcg"
+          - check: "lib/runtime"
+          - check: "lib/substitutions"
+          - check: "lib/utils"
           - check: "tests"
           - check: "examples"
+          - check: "bindings"
     steps:
       - uses: actions/checkout@v2
       - name: Run clang-format style check for C/C++/Protobuf programs.
-        uses: jidicula/clang-format-action@v4.8.0
+        uses: lockshaw/clang-format-action@v4.11.0-flexflow-3
         with:
-          clang-format-version: "15"
+          clang-format-version: "16"
           check-path: ${{ matrix.path['check'] }}
           exclude-regex: ${{ matrix.path['exclude'] }}

examples/cpp/DLRM/strategies/dlrm_strategy.cc

@@ -356,4 +356,4 @@ int main(int argc, char **argv) {
   std::to_string(gpu) + "gpus.pb"; std::fstream outputFile(output.c_str(),
   std::ios::out | std::ios::trunc); strategy.SerializeToOstream(&outputFile);
   */
-}
+}

lib/CMakeLists.txt

@@ -1,8 +1,8 @@
 #add_subdirectory(pcg)
 #add_subdirectory(compiler)
-# add_subdirectory(runtime)
-#add_subdirectory(op-attrs)
-#add_subdirectory(kernels)
+add_subdirectory(runtime)
+add_subdirectory(op-attrs)
+add_subdirectory(kernels)
 add_subdirectory(utils)
 # add_subdirectory(ffi)
 #add_subdirectory(substitutions)

lib/compiler/include/compiler/machine_mapping.h

@@ -39,4 +39,4 @@ MachineMapping optimal_cost(
 
 MAKE_VISIT_HASHABLE(::FlexFlow::MachineMapping);
 
-#endif
+#endif

lib/compiler/test/test_dp.cc

@@ -51,4 +51,4 @@ TEST_CASE("optimal_cost") {
       optimal_cost(pcg, allowed_machine_views, TestCostEstimator{}, resource);
 
   // TODO: check result
-}
+}

lib/compiler/test/test_open_graph.cc

@@ -102,4 +102,4 @@ TEST_CASE("get_cut") {
 
   GraphSplit gs1{{ns[0], ns[1], ns[2], ns[3]}, {ns[4]}};
   CHECK(get_cut(g, gs1) == std::unordered_set<MultiDiEdge>{e3, e4});
-}
+}

lib/kernels/include/kernels/reshape_kernels.h

@@ -30,4 +30,4 @@ void backward_kernel(ffStream_t stream,
 } // namespace Kernels
 } // namespace FlexFlow
 
-#endif // _FLEXFLOW_OPS_KERNELS_RESHAPE_KERNELS_H
+#endif // _FLEXFLOW_OPS_KERNELS_RESHAPE_KERNELS_H

lib/kernels/src/allocation.cc

@@ -0,0 +1,13 @@
+#include "kernels/allocation.h"
+
+namespace FlexFlow {
+
+void *Allocator::allocate(size_t size) {
+  return i_allocator->allocate(size);
+}
+
+void Allocator::deallocate(void *ptr) {
+  i_allocator->deallocate(ptr);
+}
+
+} // namespace FlexFlow

lib/kernels/src/array_shape.cc

@@ -10,4 +10,6 @@ std::size_t ArrayShape::get_volume() const {
   return product(this->dims);
 }
 
+ArrayShape::ArrayShape(std::vector<std::size_t> const &_dims) : dims(_dims) {}
+
 } // namespace FlexFlow

lib/kernels/src/hip/aggregate_kernels.cpp

@@ -296,4 +296,4 @@ AggregatePerDeviceState::~AggregatePerDeviceState(void) {
 
 } // namespace Aggregate
 } // namespace Kernels
-} // namespace FlexFlow
+} // namespace FlexFlow

lib/kernels/src/hip/optimizer_kernel.cpp

@@ -245,4 +245,4 @@ __host__ void AdamOptimizer::nccl_update_task_gpu(AdamOptimizer const *op,
 }
 #endif
 
-}; // namespace FlexFlow
+}; // namespace FlexFlow

lib/op-attrs/include/op-attrs/parallel_tensor_dims.h

@@ -11,6 +11,7 @@ struct ParallelTensorDims : public use_visitable_cmp<ParallelTensorDims> {
 
   size_t get_volume() const;
   size_t num_dims() const;
+  std::vector<size_t> get_dims() const;
 
   using iterator = typename FFOrdered<ParallelDim>::iterator;
   using const_iterator = typename FFOrdered<ParallelDim>::const_iterator;

lib/op-attrs/include/op-attrs/parallel_tensor_shape.h

@@ -26,7 +26,6 @@ struct ParallelTensorShape : public use_visitable_cmp<ParallelTensorShape> {
   ParallelTensorShape(TensorShape const &);
 
   int num_dims() const;
-
   ParallelDim const &at(ff_dim_t const &) const;
   ParallelDim &at(ff_dim_t const &);
   ParallelDim const &operator[](ff_dim_t const &) const;

lib/op-attrs/include/op-attrs/tensor_shape.h

@@ -16,7 +16,7 @@ struct TensorShape : public use_visitable_cmp<TensorShape> {
 
   template <typename Dims>
   TensorShape(Dims const &dims, DataType data_type)
-      : dims(dims), data_type(data_type) {}
+      : dims(this->dims), data_type(this->data_type) {}
 
   size_t at(ff_dim_t) const;
   size_t operator[](ff_dim_t) const;

lib/op-attrs/src/datatype.cc

@@ -0,0 +1,24 @@
+#include "op-attrs/datatype.h"
+
+namespace FlexFlow {
+
+size_t size_of(DataType data_type) {
+  switch (data_type) {
+    case DataType::BOOL:
+      return sizeof(bool);
+    case DataType::INT32:
+      return sizeof(int32_t);
+    case DataType::INT64:
+      return sizeof(int64_t);
+    case DataType::HALF:
+      return sizeof(float) / 2;
+    case DataType::FLOAT:
+      return sizeof(float);
+    case DataType::DOUBLE:
+      return sizeof(double);
+    default:
+      throw mk_runtime_error("Unknown data type");
+  }
+}
+
+} // namespace FlexFlow

lib/op-attrs/src/parallel_tensor_shape.cc

@@ -16,13 +16,35 @@ static std::vector<ParallelDim> lift_dims(TensorDims const &dims) {
 ParallelTensorDims::ParallelTensorDims(TensorDims const &dims)
     : data(lift_dims(dims)) {}
 
+std::vector<size_t> ParallelTensorDims::get_dims() const {
+  std::vector<size_t> dims;
+  for (ParallelDim const &d : this->data) {
+    dims.push_back(d.size);
+  }
+  return dims;
+}
+
+size_t ParallelTensorDims::get_volume() const {
+
+  // this function can use contains.h to optimize the code
+  size_t volume = 1;
+  for (ParallelDim const &d : this->data) {
+    volume *= d.size;
+  }
+  return volume;
+}
+
 ParallelTensorShape::ParallelTensorShape(TensorShape const &tensor_shape)
     : dims(tensor_shape.dims), data_type(tensor_shape.data_type) {}
 
 int get_num_replica_dims(ParallelTensorShape const &shape) {
   return count(shape.dims, is_replica_dim);
 }
 
+TensorShape get_piece_shape(ParallelTensorShape const &parall_tensor_shape) {
+  return TensorShape(parall_tensor_shape.dims, parall_tensor_shape.data_type);
+}
+
 int get_num_replicas(ParallelTensorShape const &shape) {
   return product(
       transform(filter(as_vector(shape.dims), is_replica_dim),