From 314e06f229235f9e148f98c9a95eddadc67801ab Mon Sep 17 00:00:00 2001 From: huajsj Date: Mon, 9 Aug 2021 22:13:16 -0700 Subject: [PATCH 01/34] [Runtime] Pipeline Executor Initial patch. This patch is one of serial patch for PR 7892 splitting.this is the initial part of the pipeline executor, this patch include the cmake change and python and C++ interface for pipeline executor. --- CMakeLists.txt | 6 + cmake/config.cmake | 3 + python/tvm/contrib/pipeline_executor.py | 176 +++++++++++++++++++ src/runtime/pipeline/pipeline_executor.cc | 49 ++++++ src/runtime/pipeline/pipeline_executor.h | 72 ++++++++ tests/python/relay/test_pipeline_executor.py | 136 ++++++++++++++ 6 files changed, 442 insertions(+) create mode 100644 python/tvm/contrib/pipeline_executor.py create mode 100644 src/runtime/pipeline/pipeline_executor.cc create mode 100644 src/runtime/pipeline/pipeline_executor.h create mode 100644 tests/python/relay/test_pipeline_executor.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 62598cbdf4a7..ab10575b7f4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -389,6 +389,12 @@ if(GTEST_INCLUDE_DIR AND GTEST_LIB) include(GoogleTest) endif() +if(USE_PIPELINE_EXECUTOR) + message(STATUS "Build with Pipeline Executor support...") + file(GLOB RUNTIME_PIPELINE_SRCS src/runtime/pipeline/*.cc) + list(APPEND RUNTIME_SRCS ${RUNTIME_PIPELINE_SRCS}) +endif(USE_PIPELINE_EXECUTOR) + # Module rules include(cmake/modules/VTA.cmake) include(cmake/modules/StandaloneCrt.cmake) diff --git a/cmake/config.cmake b/cmake/config.cmake index 8d8186c1b4f0..1b51e78f1023 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -105,6 +105,9 @@ set(USE_GRAPH_EXECUTOR ON) # Whether enable tiny graph executor with CUDA Graph set(USE_GRAPH_EXECUTOR_CUDA_GRAPH OFF) +# Whether enable subgraph runtime. +set(USE_PIPELINE_EXECUTOR OFF) + # Whether to enable the profiler for the graph executor and vm set(USE_PROFILER ON) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py new file mode 100644 index 000000000000..671d14c19f55 --- /dev/null +++ b/python/tvm/contrib/pipeline_executor.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Pipeline executor that executes pipeline containing TVM PackedFunc.""" +import json +import tvm._ffi +from tvm import relay +from tvm.contrib import graph_executor + + +def pipeline_executor_enabled(): + """check if pipeline executor enabled. + Return + ------ + enable: bool + return pipeline executor get enabled or not + """ + pipeline_enabled = False + try: + pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create") + assert pipelinecreate + pipeline_enabled = True + except ValueError: + print("pipeline executor not enabled!") + + return pipeline_enabled + + +def build_pipeline(mod_n_configs): + """build module list that can use for pipeline execution. + + Parameters + ---------- + mod_n_configs: Dict[IRModule, Dict[str, Any]] + build configuration informaton, structure like following. + {IRModule: {"target":target, + "target_host":target_host, + "params":params, + "mod_name"mod_name, + "build":build}} + + Returns + ------- + ret: List[IRModule] + list of IRModule + string_config: Dict[int, Dict[str, any]] + pipeline configuration + """ + mods = {} + config_len = len(mod_n_configs) + string_config = [{} for _ in range(config_len)] + for _, (ir_mod, mod_config) in enumerate(mod_n_configs.items()): + # init lib_name and json_name params with empty + lib_name = "" + json_name = "" + params_name = "" + # Get module configuration + assert "pipeline" in mod_config and "mod_indx" in mod_config["pipeline"] + # Get module index in pipeline configuration + mconf = mod_config["pipeline"].copy() + # Get mod device config + dev = mod_config["dev"] + mod_indx = mconf["mod_indx"] - 1 + target = mod_config["target"] + assert mod_indx < config_len + build_func = relay.build + # if there is a self defined build function then use it. + if "build" in mod_config and mod_config["build"]: + build_func = mod_config["build"] + + # build IRModule + mod = build_func( + ir_mod, + target, + params=mod_config["params"], + target_host=mod_config["target_host"], + mod_name=mod_config["mod_name"], + ) + + mconf["lib_name"] = lib_name + mconf["json_name"] = json_name + mconf["params_name"] = params_name + mconf["dev"] = "{},{}".format(dev.device_type, dev.device_id) + # Create pipeline configuration + string_config[mod_indx] = mconf + # associate mod with device + mods[mod] = {"dev": dev} + + # return IRModule list and pipeline configuration + return mods, string_config + + +def create(pipeline_mods, mod_config): + """Create a pipeline runtime executor. + + Parameters + ---------- + pipeline_mods : List[IRModule] + list of IRModule + + mod_config : Dict[int, Dict[str, Any]] + modules and modules dependency configuration informaiton. + + Returns + ------- + submodule : PipelineModule + Runtime pipeline module. + """ + + submodule = PipelineModule(pipeline_mods, mod_config) + return submodule + + +class PipelineModule(object): + """Wrapper runtime module. This is a thin wrapper of the underlying TVM module. + you can also directly call set_input, run, and get_output of underlying module functions. + + Parameters + ---------- + graph_module : List[GraphModule] + The internal tvm module that holds the actual graph functions. + + pipeline_config : Dict[IRModule, Dict[str, Any]] + modules and modules dependency configuration informaiton. + + """ + + def graph_executor_create(self, pipeline_mods, mod_config): + """Create a pipeline runtime executor. + + Parameters + ---------- + pipeline_mods : List[IRModule] + list of IRModule + + mod_config : Dict[int, Dict[str, Any]] + modules and modules dependency configuration informaiton. + + Returns + ------- + mods : GreaphModule + Runtime graph module. + """ + + mods = [] + for pipeline_mod in pipeline_mods: + mod = graph_executor.GraphModule( + pipeline_mod["default"](pipeline_mods[pipeline_mod]["dev"]) + ) + mods.append(mod.module) + + return mods, json.dumps(mod_config) + + def __init__(self, pipeline_mods, mod_config): + self.pipeline_mods = pipeline_mods + self.mod_config = mod_config + mods, config = self.graph_executor_create(pipeline_mods, mod_config) + + pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create") + assert pipelinecreate + module = pipelinecreate(mods, config) + + self.module_ = module diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc new file mode 100644 index 000000000000..44440f341805 --- /dev/null +++ b/src/runtime/pipeline/pipeline_executor.cc @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file pipeline_executor.cc + */ +#include "pipeline_executor.h" + +namespace tvm { +namespace runtime { + +void SubGraphRuntime::Init(const Array& modules, + const std::string& pipeline_json) { + return; +} + +PackedFunc SubGraphRuntime::GetFunction(const std::string& name, + const ObjectPtr& sptr_to_self) { + return PackedFunc(); +} + +Module PipelineRuntimeCreate(const Array& m, + const std::string& pipeline_json) { + auto exec = make_object(); + exec->Init(m, pipeline_json); + return Module(exec); +} + +TVM_REGISTER_GLOBAL("tvm.pipeline_executor.create").set_body([](TVMArgs args, TVMRetValue* rv) { + *rv = PipelineRuntimeCreate(args[0], args[1]); +}); +} // namespace runtime +} // namespace tvm diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h new file mode 100644 index 000000000000..fe92874975e9 --- /dev/null +++ b/src/runtime/pipeline/pipeline_executor.h @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \brief pipeline executor + * \file pipeline_executor.h + */ +#ifndef TVM_RUNTIME_PIPELINE_PIPELINE_EXECUTOR_H_ +#define TVM_RUNTIME_PIPELINE_PIPELINE_EXECUTOR_H_ +#include + +#include +#include +#include +#include + +#include "../file_utils.h" +using namespace std; +namespace tvm { +namespace runtime { + +/*! + * \brief pipeline runtime. + * + * This runtime can be acccesibly in various language via + * TVM runtime PackedFunc API. + */ +class TVM_DLL SubGraphRuntime : public ModuleNode { + public: + /*! + * \return The type key of the executor. + */ + const char* type_key() const final { return "SubGraphRuntime"; } + /*! + * \brief Initialize the graph executor with graph and context. + * \param graph_json The execution graph. + * \param module The module containing the compiled functions for the host + * processor. + * \param ctxs The context of the host and devices where graph nodes will be + * executed on. + * \param lookup_linked_param_func If given, a PackedFunc invoked to lookup linked parameters + * by storage_id. If not given, linked parameters are looked-up using an internal implementation, + * which is not compatible with RPCModules. + */ + void Init(const Array& modules, const std::string& pipeline_json); + /*! + * \brief Get member function to front-end + * \param name The name of the function. + * \param sptr_to_self The pointer to the module node. + * \return The corresponding member function. + */ + virtual PackedFunc GetFunction(const std::string& name, const ObjectPtr& sptr_to_self); +}; +} // namespace runtime +} // namespace tvm +#endif // TVM_RUNTIME_PIPELINE_PIPELINE_EXECUTOR_H_ diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py new file mode 100644 index 000000000000..2a043918542f --- /dev/null +++ b/tests/python/relay/test_pipeline_executor.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +import tvm +import tvm.testing +from tvm import relay +from tvm.relay import transform +from tvm.contrib import graph_executor, pipeline_executor + + +def get_mannual_mod(): + mods = [] + dshape = (3, 3) + data = relay.var("data_0", relay.TensorType(dshape, "float32")) + data21 = relay.var("data_1", relay.TensorType(dshape, "float32")) + data_net1_output_1 = relay.var("data_0", relay.TensorType(dshape, "float32")) + data_net1_output_2 = relay.var("data_1", relay.TensorType(dshape, "float32")) + data_net2_output_1 = relay.var("data_0", relay.TensorType(dshape, "float32")) + mvalue1 = np.full((1), 1).astype("float32") + mvalue2 = np.full((1), 2).astype("float32") + mvalue3 = np.full((1), 3).astype("float32") + mv1 = relay.Constant(tvm.nd.array(mvalue1)) + mv2 = relay.Constant(tvm.nd.array(mvalue2)) + mv3 = relay.Constant(tvm.nd.array(mvalue3)) + + # net1 have three output, output3 is final output + net_output1 = relay.add(data, mv1) + net_output2 = relay.subtract(data, mv2) + net_output3 = relay.multiply(data, mv3) + + # net2 use net1 output1 as input + net2 = relay.add(data_net1_output_1, mv2) + net2 = relay.add(net2, data21) + net2 = relay.add(net2, mv3) + + # net3 use net2 output1 and net1 outpu2 as input + net3 = relay.multiply(data_net2_output_1, mv3) + net3 = relay.add(net3, data_net1_output_2) + + mods.append( + tvm.IRModule.from_expr( + relay.Function([data], relay.Tuple([net_output1, net_output2, net_output3])) + ) + ) + mods.append(tvm.IRModule.from_expr(relay.Function([data_net1_output_1, data21], net2))) + mods.append( + tvm.IRModule.from_expr(relay.Function([data_net1_output_2, data_net2_output_1], net3)) + ) + + return mods, dshape + + +def pipeline(target): + """ + #Get 4 pipeline module. + """ + mods, dshape = get_mannual_mod() + """ + #Prepare batch data for pipeline feeding + """ + datas = [] + for i in range(len(mods) + 1): + datas.append(np.full(dshape, 3 + i).astype("float32")) + + # set configure + indx = 0 + mod_config = {} + mconfig = {"target_host": None, "mod_name": "default", "build": None, "params": None} + mconfig1 = mconfig.copy() + mconfig1["target"] = target[0] + mconfig1["dev"] = target[1] + # third output is final output, second output for mod3, first for mod2 + # input + mconfig1["pipeline"] = { + "mod_indx": 1, + "output": [ + {"output_indx": 1, "dependent": [{"mod_indx": 2, "input_name": "data_0"}]}, + {"output_indx": 2, "dependent": [{"mod_indx": 3, "input_name": "data_0"}]}, + {"output_indx": 3, "dependent": [{"mod_indx": 0, "input_name": "1"}]}, + ], + } + mod_config[mods[0]] = mconfig1 + + mconfig2 = mconfig.copy() + mconfig2["target"] = "llvm" + mconfig2["dev"] = tvm.cpu(0) + mconfig2["pipeline"] = { + "mod_indx": 2, + "output": [ + {"output_indx": 1, "dependent": [{"mod_indx": 3, "input_name": "data_1"}]}, + ], + } + mod_config[mods[1]] = mconfig2 + + mconfig3 = mconfig.copy() + mconfig3["target"] = "llvm" + mconfig3["dev"] = tvm.cpu(0) + + mconfig3["pipeline"] = { + "mod_indx": 3, + "output": [{"output_indx": 1, "dependent": [{"mod_indx": 0, "input_name": "2"}]}], + } + mod_config[mods[2]] = mconfig3 + """ + #build and create pipeline module + """ + with relay.build_config(opt_level=3): + pipeline_mods, string_config = pipeline_executor.build_pipeline(mod_config) + + pipeline_module = pipeline_executor.create(pipeline_mods, string_config) + + +def test_pipeline(): + if pipeline_executor.pipeline_executor_enabled(): + target_list = tvm.testing.enabled_targets() + for target in target_list: + pipeline(target) + + +if __name__ == "__main__": + test_pipeline() From a1e03d3ffc183f9a0229456169d80dca9295b933 Mon Sep 17 00:00:00 2001 From: huajsj Date: Fri, 20 Aug 2021 00:05:43 -0700 Subject: [PATCH 02/34] add pipeline config --- tests/python/relay/test_pipeline_executor.py | 90 +++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 2a043918542f..0d6dcf40c327 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -22,6 +22,68 @@ from tvm.relay import transform from tvm.contrib import graph_executor, pipeline_executor +class PipelineModuleConfig: + class interface: + def __init__(self, owner, itype, name): + self.owner_ = owner + self.itype_ = itype + self.name_ = name + self.dependent_ = [] + return + + def get_dependent_str(self): + name = "" + for dependent in self.dependent_: + name = name + dependent.name_ + return name + + def addDependent(self, dependent): + self.dependent_.append(dependent) + + class instance: + def __init__(self): + self.interfaces_ = {1:{}, 2:{}} + return + + def get_interface(self, itype, name): + if name not in self.interfaces_[itype]: + self.interfaces_[itype][name] = PipelineModuleConfig.interface(0, itype, name) + + return self.interfaces_[itype][name] + + def input(self, name): + return self.get_interface(1, name) + + def output(self, index): + return self.get_interface(2, index) + + + def __init__(self, mods): + self.pipe_instance = self.instance() + self.mod_instance = {m:self.instance() for m in mods} + return + + def __str__(self): + dump = "Inputs\n" + for input_name in self.pipe_instance.interfaces_[1]: + inf = self.pipe_instance.interfaces_[1][input_name] + dump = dump + " |" +input_name + ": " + inf.get_dependent_str() + "\n" + return dump + + def __getitem__(self, key): + return self.mod_instance[key] + + def pipe_input(self, name): + return self.pipe_instance.input(name) + + def pipe_output(self, index): + return self.pipe_instance.output(index) + + def connect(self, left:interface, right:interface): + left.addDependent(right) + return + + def get_mannual_mod(): mods = [] @@ -131,6 +193,32 @@ def test_pipeline(): for target in target_list: pipeline(target) +def test_config(): + (mod1, mod2, mod3), dshape = get_mannual_mod() + pipe_config = PipelineModuleConfig([mod1, mod2, mod3]) + pipe_config.connect(pipe_config.pipe_input("data_0"), + pipe_config[mod1].input("data_0")) + + pipe_config.connect(pipe_config.pipe_input("data_1"), + pipe_config[mod2].input("data_1")) + + pipe_config.connect(pipe_config[mod1].output(0), + pipe_config[mod2].input("data_0")) + + pipe_config.connect(pipe_config[mod1].output(1), + pipe_config[mod3].input("data_0")) + + pipe_config.connect(pipe_config[mod2].output(0), + pipe_config[mod3].input("data_1")) + + pipe_config.connect(pipe_config[mod1].output(2), + pipe_config.pipe_output("0")) + + pipe_config.connect(pipe_config[mod3].output(0), + pipe_config.pipe_output("1")) + + print(pipe_config) if __name__ == "__main__": - test_pipeline() + #test_pipeline() + test_config() From 6c1485a007595301704564141873178f9832a4ca Mon Sep 17 00:00:00 2001 From: huajsj Date: Tue, 24 Aug 2021 23:00:04 -0700 Subject: [PATCH 03/34] add config connect logic. --- python/tvm/contrib/pipeline_executor.py | 155 ++++++++++++++ tests/python/relay/test_pipeline_executor.py | 213 ++++++++++--------- 2 files changed, 262 insertions(+), 106 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 671d14c19f55..fbd6fb3b6fa9 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -174,3 +174,158 @@ def __init__(self, pipeline_mods, mod_config): module = pipelinecreate(mods, config) self.module_ = module + + +class PipelineModuleConfig: + class interface: + def __init__(self, owner, itype, name): + self.owner_ = owner + self.itype_ = itype + self.name_ = str(name) + self.dependent_ = [] + return + + def get_name(self): + mname = "" + mindx = 0 + if self.owner_: + mname = self.owner_.name_ + + return mname, self.name_ + + def get_owner_indx(self): + return self.owner_.indx_ + + def get_dependent_str(self): + name = "" + for dependent in self.dependent_: + mname, dname = dependent.get_name() + name = name + (mname + ":output(" + dname if self.itype_ == 2 else "") + name = name + (")" if self.itype_ == 2 else mname + ":" + dname) + return name + + def addDependent(self, dependent): + """ + # check if the dependency setting correct. + # correct connection are following + # 1. global input to module input + # 2. module output to next module input + # 3. module output to global output + """ + owner_indx = self.get_owner_indx() + dep_owner_indx = dependent.get_owner_indx() + assert owner_indx != dep_owner_indx, f"can not set self as dependent." + assert not (owner_indx > dep_owner_indx and \ + not (dependent.itype_ == 2 and dep_owner_indx == 0)), \ + f"dependent only can be next module interface or global output." + assert not (owner_indx == 0 and dependent.itype_ != 1), \ + f"global input only can set dependent with module input." + + self.dependent_.append(dependent) + + class instance: + def __init__(self, indx = 0): + self.indx_ = indx + self.name_ = "mod" + str(indx) if indx else "" + self.interfaces_ = {1:{}, 2:{}} + return + + def get_interface(self, itype, name): + if name not in self.interfaces_[itype]: + self.interfaces_[itype][name] = PipelineModuleConfig.interface(self, itype, name) + + return self.interfaces_[itype][name] + + def input(self, name): + return self.get_interface(1, name) + + def output(self, index): + return self.get_interface(2, index) + + + def __init__(self, mods): + """ + # input + """ + self.pipe_instance = self.instance(0) + self.mod_instance = { + m:self.instance(i + 1) for m, i in zip(mods, range(len(mods)))} + return + + def __str__(self): + # get input + input_dump = "Inputs\n" + for input_name in self.pipe_instance.interfaces_[1]: + inf = self.pipe_instance.interfaces_[1][input_name] + input_dump += " |" +input_name + ": " + inf.get_dependent_str() + "\n" + + # connections + output = {} + connections_dump = "\nconnections\n" + for mod in self.mod_instance: + for _, interface in self.mod_instance[mod].interfaces_[2].items(): + if len(interface.dependent_): + mname, dname = interface.get_name() + iname = mname + ".output(" + dname + ")->" + for dep in interface.dependent_: + dep_mname, dep_dname = dep.get_name() + if dep.owner_.indx_ > 0: + iname += " " + dep_mname + "." + dep_dname + connections_dump += " |" + iname +"\n" + else: + output[dep_dname] = mname + ".output(" + dname + ")" + + # get output + output_dump = "\noutput\n" + for name in sorted(output.keys()): + output_dump += " |output(" + name + ") : " + output[name] + "\n" + + + return input_dump + output_dump + connections_dump + + def get_config(self): + mconfig = {} + for mod in self.mod_instance: + mconf = {} + output_conf = [] + instance = self.mod_instance[mod] + for _, interface in instance.interfaces_[2].items(): + dep_conf = [] + output = {} + if len(interface.dependent_): + for dep in interface.dependent_: + dep_item = {} + _, dname = dep.get_name() + dep_item["mod_indx"] = dep.get_owner_indx() + dep_item["input_name"] = dname + dep_conf.append(dep_item) + + """ + # in configuration the ouput_indx start from 1 + """ + output["output_indx"] = int(interface.name_) + 1 + output["dependent"] = dep_conf + output_conf.append(output) + mconf["mod_indx"] = interface.get_owner_indx() + mconf["output"] = output_conf + mconfig[mod] = {"pipeline" : mconf} + + return mconfig + + + def __getitem__(self, key): + return self.mod_instance[key] + + def get_mod_indx(self, mod): + indx = self.mod_instance[mod].indx_ + return indx + + def pipe_input(self, name): + return self.pipe_instance.input(name) + + def pipe_output(self, index): + return self.pipe_instance.output(index) + + def connect(self, left:interface, right:interface): + left.addDependent(right) + return diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 0d6dcf40c327..62b0ea92c8cb 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -22,69 +22,6 @@ from tvm.relay import transform from tvm.contrib import graph_executor, pipeline_executor -class PipelineModuleConfig: - class interface: - def __init__(self, owner, itype, name): - self.owner_ = owner - self.itype_ = itype - self.name_ = name - self.dependent_ = [] - return - - def get_dependent_str(self): - name = "" - for dependent in self.dependent_: - name = name + dependent.name_ - return name - - def addDependent(self, dependent): - self.dependent_.append(dependent) - - class instance: - def __init__(self): - self.interfaces_ = {1:{}, 2:{}} - return - - def get_interface(self, itype, name): - if name not in self.interfaces_[itype]: - self.interfaces_[itype][name] = PipelineModuleConfig.interface(0, itype, name) - - return self.interfaces_[itype][name] - - def input(self, name): - return self.get_interface(1, name) - - def output(self, index): - return self.get_interface(2, index) - - - def __init__(self, mods): - self.pipe_instance = self.instance() - self.mod_instance = {m:self.instance() for m in mods} - return - - def __str__(self): - dump = "Inputs\n" - for input_name in self.pipe_instance.interfaces_[1]: - inf = self.pipe_instance.interfaces_[1][input_name] - dump = dump + " |" +input_name + ": " + inf.get_dependent_str() + "\n" - return dump - - def __getitem__(self, key): - return self.mod_instance[key] - - def pipe_input(self, name): - return self.pipe_instance.input(name) - - def pipe_output(self, index): - return self.pipe_instance.output(index) - - def connect(self, left:interface, right:interface): - left.addDependent(right) - return - - - def get_mannual_mod(): mods = [] dshape = (3, 3) @@ -127,25 +64,10 @@ def get_mannual_mod(): return mods, dshape -def pipeline(target): - """ - #Get 4 pipeline module. - """ - mods, dshape = get_mannual_mod() - """ - #Prepare batch data for pipeline feeding - """ - datas = [] - for i in range(len(mods) + 1): - datas.append(np.full(dshape, 3 + i).astype("float32")) - - # set configure - indx = 0 +def manual_conf(mods): mod_config = {} - mconfig = {"target_host": None, "mod_name": "default", "build": None, "params": None} - mconfig1 = mconfig.copy() - mconfig1["target"] = target[0] - mconfig1["dev"] = target[1] + # set configure + mconfig1 = {} # third output is final output, second output for mod3, first for mod2 # input mconfig1["pipeline"] = { @@ -158,9 +80,7 @@ def pipeline(target): } mod_config[mods[0]] = mconfig1 - mconfig2 = mconfig.copy() - mconfig2["target"] = "llvm" - mconfig2["dev"] = tvm.cpu(0) + mconfig2 = {} mconfig2["pipeline"] = { "mod_indx": 2, "output": [ @@ -169,33 +89,28 @@ def pipeline(target): } mod_config[mods[1]] = mconfig2 - mconfig3 = mconfig.copy() - mconfig3["target"] = "llvm" - mconfig3["dev"] = tvm.cpu(0) + mconfig3 = {} mconfig3["pipeline"] = { "mod_indx": 3, "output": [{"output_indx": 1, "dependent": [{"mod_indx": 0, "input_name": "2"}]}], } mod_config[mods[2]] = mconfig3 + return mod_config + +def pipeline(target): """ - #build and create pipeline module + #Get 4 pipeline module. """ - with relay.build_config(opt_level=3): - pipeline_mods, string_config = pipeline_executor.build_pipeline(mod_config) - - pipeline_module = pipeline_executor.create(pipeline_mods, string_config) - - -def test_pipeline(): - if pipeline_executor.pipeline_executor_enabled(): - target_list = tvm.testing.enabled_targets() - for target in target_list: - pipeline(target) - -def test_config(): (mod1, mod2, mod3), dshape = get_mannual_mod() - pipe_config = PipelineModuleConfig([mod1, mod2, mod3]) + """ + #Prepare batch data for pipeline feeding + """ + datas = [] + for i in range(5): + datas.append(np.full(dshape, 3 + i).astype("float32")) + + pipe_config = pipeline_executor.PipelineModuleConfig([mod1, mod2, mod3]) pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].input("data_0")) @@ -212,13 +127,99 @@ def test_config(): pipe_config[mod3].input("data_1")) pipe_config.connect(pipe_config[mod1].output(2), - pipe_config.pipe_output("0")) + pipe_config.pipe_output("1")) pipe_config.connect(pipe_config[mod3].output(0), - pipe_config.pipe_output("1")) + pipe_config.pipe_output("2")) + """ + # print configueration, the expect result like following. + # + #Inputs + # |data_0: mod1:data_0 + # |data_1: mod2:data_1 + # + #output + # |output(1) : mod1.output(2) + # |output(2) : mod3.output(0) + # + #connections + # |mod1.output(0)-> mod2.data_0 + # |mod1.output(1)-> mod3.data_0 + # |mod2.output(0)-> mod3.data_1 + """ print(pipe_config) + """ + # connection correctness veify + """ + try: + pipe_config.connect(pipe_config[mod2].output(0), + pipe_config[mod1].input("data_0")) + assert 0, f"wrong module connect order check not pass!" + pipe_config.connect(pipe_config.pipe_input("data_0"), + pipe_config[mod1].output(0)) + assert 0, f"wrong global input connect check not pass!" + except: + print("wrong connect check pass") + + """ + # get text format configuration. + """ + + pconfig = pipe_config.get_config() + + """ + # check if the configuration match expectation. + """ + assert pconfig == manual_conf([mod1, mod2, mod3]) + + """ + # generate configure for build process + """ + + mod_config = {} + mconfig1 = pconfig[mod1] + mconfig1["target_host"] = None + mconfig1["mod_name"] = "default" + mconfig1["build"] = None + mconfig1["params"] = None + mconfig1["target"] = target[0] + mconfig1["dev"] = target[1] + mod_config[mod1] = mconfig1 + + mconfig2 = pconfig[mod2] + mconfig2["target_host"] = None + mconfig2["mod_name"] = "default" + mconfig2["build"] = None + mconfig2["params"] = None + mconfig2["target"] = "llvm" + mconfig2["dev"] = tvm.cpu(0) + mod_config[mod2] = mconfig2 + + mconfig3 = pconfig[mod3] + mconfig3["target_host"] = None + mconfig3["mod_name"] = "default" + mconfig3["build"] = None + mconfig3["params"] = None + mconfig3["target"] = "llvm" + mconfig3["dev"] = tvm.cpu(0) + mod_config[mod3] = mconfig3 + + """ + # build and create pipeline module + """ + with relay.build_config(opt_level=3): + pipeline_mods, string_config = pipeline_executor.build_pipeline(mod_config) + + pipeline_module = pipeline_executor.create(pipeline_mods, string_config) + + +def test_pipeline(): + if pipeline_executor.pipeline_executor_enabled(): + target_list = tvm.testing.enabled_targets() + for target in target_list: + pipeline(target) + if __name__ == "__main__": - #test_pipeline() - test_config() + test_pipeline() From d9ba4ca8effeede54b099aca522302862e1f1c09 Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 26 Aug 2021 12:32:33 -0700 Subject: [PATCH 04/34] fix build issue. --- CMakeLists.txt | 9 + python/tvm/contrib/pipeline_executor.py | 197 ++++++++++--------- tests/python/relay/test_pipeline_executor.py | 91 ++++++--- 3 files changed, 179 insertions(+), 118 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ab10575b7f4c..f671ce7f790b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -395,6 +395,15 @@ if(USE_PIPELINE_EXECUTOR) list(APPEND RUNTIME_SRCS ${RUNTIME_PIPELINE_SRCS}) endif(USE_PIPELINE_EXECUTOR) +# Enable ctest if gtest is available +find_path(GTEST_INCLUDE_DIR gtest/gtest.h) +find_library(GTEST_LIB gtest "$ENV{GTEST_LIB}") +if(GTEST_INCLUDE_DIR AND GTEST_LIB) + enable_testing() + include(CTest) + include(GoogleTest) +endif() + # Module rules include(cmake/modules/VTA.cmake) include(cmake/modules/StandaloneCrt.cmake) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index fbd6fb3b6fa9..5ab19cef7e21 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -126,11 +126,9 @@ def create(pipeline_mods, mod_config): class PipelineModule(object): """Wrapper runtime module. This is a thin wrapper of the underlying TVM module. - you can also directly call set_input, run, and get_output of underlying module functions. - Parameters ---------- - graph_module : List[GraphModule] + pipeline_mods : List[GraphModule] The internal tvm module that holds the actual graph functions. pipeline_config : Dict[IRModule, Dict[str, Any]] @@ -138,6 +136,17 @@ class PipelineModule(object): """ + def __init__(self, pipeline_mods, pipeline_config): + self.pipeline_mods = pipeline_mods + self.mod_config = pipeline_config + mods, config = self.graph_executor_create(pipeline_mods, pipeline_config) + + pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create") + assert pipelinecreate + module = pipelinecreate(mods, config) + + self.module_ = module + def graph_executor_create(self, pipeline_mods, mod_config): """Create a pipeline runtime executor. @@ -164,75 +173,95 @@ def graph_executor_create(self, pipeline_mods, mod_config): return mods, json.dumps(mod_config) - def __init__(self, pipeline_mods, mod_config): - self.pipeline_mods = pipeline_mods - self.mod_config = mod_config - mods, config = self.graph_executor_create(pipeline_mods, mod_config) - pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create") - assert pipelinecreate - module = pipelinecreate(mods, config) +class PipelineModuleConfig: + """Pipeline Configuration Class, in this class there are 2 internal class, + first is Instance which use to represent Module, second is Interface which use + to represent Module input/output and Pipeline Module input/output, by setting + dependency relation between Interfaces this class can build the module + connection relation. + + The class Hierarchical as following. + PipelineModuleConfig ---> Pipe Instance ---> Interface(input/output) + ---> Module Instance ---> Interface(input/output) + """ - self.module_ = module + class Instance: + """The class use use to represent Module and storage module index and + Interface information. + """ + class Interface: + """The class that use to storage module connection information. + There are 2 types Interface Input:1 Output:2 + Parameters + ---------- -class PipelineModuleConfig: - class interface: - def __init__(self, owner, itype, name): - self.owner_ = owner - self.itype_ = itype - self.name_ = str(name) - self.dependent_ = [] - return - - def get_name(self): - mname = "" - mindx = 0 - if self.owner_: - mname = self.owner_.name_ - - return mname, self.name_ - - def get_owner_indx(self): - return self.owner_.indx_ - - def get_dependent_str(self): - name = "" - for dependent in self.dependent_: - mname, dname = dependent.get_name() - name = name + (mname + ":output(" + dname if self.itype_ == 2 else "") - name = name + (")" if self.itype_ == 2 else mname + ":" + dname) - return name - - def addDependent(self, dependent): - """ - # check if the dependency setting correct. - # correct connection are following - # 1. global input to module input - # 2. module output to next module input - # 3. module output to global output + owner : Instance + The class that own this interface, in such class there are + Module information like index, module name + + itype : integer + Interface type, 1 is input interface, 2 is output interface + + name : str/integer + Interface name, for input that is string for example "data0" + for output that is integer for example 0. """ - owner_indx = self.get_owner_indx() - dep_owner_indx = dependent.get_owner_indx() - assert owner_indx != dep_owner_indx, f"can not set self as dependent." - assert not (owner_indx > dep_owner_indx and \ - not (dependent.itype_ == 2 and dep_owner_indx == 0)), \ - f"dependent only can be next module interface or global output." - assert not (owner_indx == 0 and dependent.itype_ != 1), \ - f"global input only can set dependent with module input." - - self.dependent_.append(dependent) - - class instance: - def __init__(self, indx = 0): + + def __init__(self, owner, itype, name): + self.owner_ = owner + self.itype_ = itype + self.name_ = str(name) + self.dependent_ = [] + + def get_name(self): + mname = "" + if self.owner_: + mname = self.owner_.name_ + + return mname, self.name_ + + def get_owner_indx(self): + return self.owner_.indx_ + + def get_dependent_str(self): + name = "" + for dependent in self.dependent_: + mname, dname = dependent.get_name() + name = name + (mname + ":output(" + dname if self.itype_ == 2 else "") + name = name + (")" if self.itype_ == 2 else mname + ":" + dname) + return name + + def add_dependent(self, dependent): + """ + # check if the dependency setting correct. + # correct connection are following + # 1. global input to module input + # 2. module output to next module input + # 3. module output to global output + """ + owner_indx = self.get_owner_indx() + dep_owner_indx = dependent.get_owner_indx() + assert owner_indx != dep_owner_indx, f"can not set self as dependent." + assert not ( + owner_indx > dep_owner_indx + and not (dependent.itype_ == 2 and dep_owner_indx == 0) + ), f"dependent only can be next module interface or global output." + assert not ( + owner_indx == 0 and dependent.itype_ != 1 + ), f"global input only can set dependent with module input." + + self.dependent_.append(dependent) + + def __init__(self, indx=0): self.indx_ = indx self.name_ = "mod" + str(indx) if indx else "" - self.interfaces_ = {1:{}, 2:{}} - return + self.interfaces_ = {1: {}, 2: {}} - def get_interface(self, itype, name): + def get_interface(self, itype, name): if name not in self.interfaces_[itype]: - self.interfaces_[itype][name] = PipelineModuleConfig.interface(self, itype, name) + self.interfaces_[itype][name] = self.Interface(self, itype, name) return self.interfaces_[itype][name] @@ -242,48 +271,43 @@ def input(self, name): def output(self, index): return self.get_interface(2, index) - def __init__(self, mods): - """ - # input - """ - self.pipe_instance = self.instance(0) - self.mod_instance = { - m:self.instance(i + 1) for m, i in zip(mods, range(len(mods)))} - return + self.pipe_instance = self.Instance(0) + self.mod_instance = {m: self.Instance(i + 1) for m, i in zip(mods, range(len(mods)))} def __str__(self): + """ Get configuration in string type""" # get input input_dump = "Inputs\n" for input_name in self.pipe_instance.interfaces_[1]: inf = self.pipe_instance.interfaces_[1][input_name] - input_dump += " |" +input_name + ": " + inf.get_dependent_str() + "\n" + input_dump += " |" + input_name + ": " + inf.get_dependent_str() + "\n" - # connections + # get connections output = {} connections_dump = "\nconnections\n" for mod in self.mod_instance: for _, interface in self.mod_instance[mod].interfaces_[2].items(): - if len(interface.dependent_): + if interface.dependent_: mname, dname = interface.get_name() iname = mname + ".output(" + dname + ")->" for dep in interface.dependent_: dep_mname, dep_dname = dep.get_name() if dep.owner_.indx_ > 0: iname += " " + dep_mname + "." + dep_dname - connections_dump += " |" + iname +"\n" + connections_dump += " |" + iname + "\n" else: - output[dep_dname] = mname + ".output(" + dname + ")" + output[dep_dname] = mname + ".output(" + dname + ")" # get output output_dump = "\noutput\n" for name in sorted(output.keys()): output_dump += " |output(" + name + ") : " + output[name] + "\n" - return input_dump + output_dump + connections_dump def get_config(self): + """ Get configuration in dictionary format.""" mconfig = {} for mod in self.mod_instance: mconf = {} @@ -292,7 +316,7 @@ def get_config(self): for _, interface in instance.interfaces_[2].items(): dep_conf = [] output = {} - if len(interface.dependent_): + if interface.dependent_: for dep in interface.dependent_: dep_item = {} _, dname = dep.get_name() @@ -300,24 +324,22 @@ def get_config(self): dep_item["input_name"] = dname dep_conf.append(dep_item) - """ - # in configuration the ouput_indx start from 1 - """ + # in configuration the ouput_indx start from 1. + output["output_indx"] = int(interface.name_) + 1 output["dependent"] = dep_conf output_conf.append(output) - mconf["mod_indx"] = interface.get_owner_indx() + mconf["mod_indx"] = instance.indx_ mconf["output"] = output_conf - mconfig[mod] = {"pipeline" : mconf} + mconfig[mod] = {"pipeline": mconf} return mconfig - def __getitem__(self, key): return self.mod_instance[key] def get_mod_indx(self, mod): - indx = self.mod_instance[mod].indx_ + indx = self.mod_instance[mod].indx_ return indx def pipe_input(self, name): @@ -326,6 +348,5 @@ def pipe_input(self, name): def pipe_output(self, index): return self.pipe_instance.output(index) - def connect(self, left:interface, right:interface): - left.addDependent(right) - return + def connect(self, left: Instance.Interface, right: Instance.Interface): + left.add_dependent(right) diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 62b0ea92c8cb..67fd7ca91340 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -22,7 +22,11 @@ from tvm.relay import transform from tvm.contrib import graph_executor, pipeline_executor + def get_mannual_mod(): + """ + # get list of module that represent a subgraph + """ mods = [] dshape = (3, 3) data = relay.var("data_0", relay.TensorType(dshape, "float32")) @@ -37,17 +41,24 @@ def get_mannual_mod(): mv2 = relay.Constant(tvm.nd.array(mvalue2)) mv3 = relay.Constant(tvm.nd.array(mvalue3)) - # net1 have three output, output3 is final output + """ + # net1 have three output, output3 is final output. + """ + net_output1 = relay.add(data, mv1) net_output2 = relay.subtract(data, mv2) net_output3 = relay.multiply(data, mv3) - # net2 use net1 output1 as input + """ + # net2 use net1 output1 as input. + """ net2 = relay.add(data_net1_output_1, mv2) net2 = relay.add(net2, data21) net2 = relay.add(net2, mv3) - # net3 use net2 output1 and net1 outpu2 as input + """ + # net3 use net2 output1 and net1 outpu2 as input. + """ net3 = relay.multiply(data_net2_output_1, mv3) net3 = relay.add(net3, data_net1_output_2) @@ -64,12 +75,21 @@ def get_mannual_mod(): return mods, dshape -def manual_conf(mods): +def get_manual_conf(mods): + """ + # This function use to generate manual pipe line configueration, + # the result use to verify if the pipe configuration can generate + # correct result. + """ mod_config = {} + """ # set configure + """ mconfig1 = {} + """ # third output is final output, second output for mod3, first for mod2 # input + """ mconfig1["pipeline"] = { "mod_indx": 1, "output": [ @@ -98,39 +118,42 @@ def manual_conf(mods): mod_config[mods[2]] = mconfig3 return mod_config -def pipeline(target): + +def pipeline_module_create(target): """ - #Get 4 pipeline module. + #Get 3 pipeline module. """ (mod1, mod2, mod3), dshape = get_mannual_mod() - """ - #Prepare batch data for pipeline feeding - """ + + # Prepare batch data for pipeline feeding datas = [] for i in range(5): datas.append(np.full(dshape, 3 + i).astype("float32")) pipe_config = pipeline_executor.PipelineModuleConfig([mod1, mod2, mod3]) - pipe_config.connect(pipe_config.pipe_input("data_0"), - pipe_config[mod1].input("data_0")) - pipe_config.connect(pipe_config.pipe_input("data_1"), - pipe_config[mod2].input("data_1")) + # Create pipeline compute input/output and subgraph dependent relation. + + # pipeline compute input "data_0" would get forward to mod1 as input "data_0" + pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].input("data_0")) - pipe_config.connect(pipe_config[mod1].output(0), - pipe_config[mod2].input("data_0")) + # pipeline compute input "data_1" would get forward to mod2 as input "data_1" + pipe_config.connect(pipe_config.pipe_input("data_1"), pipe_config[mod2].input("data_1")) - pipe_config.connect(pipe_config[mod1].output(1), - pipe_config[mod3].input("data_0")) + # mod1 output(0) would get forward to mod2 as input "data_0" + pipe_config.connect(pipe_config[mod1].output(0), pipe_config[mod2].input("data_0")) - pipe_config.connect(pipe_config[mod2].output(0), - pipe_config[mod3].input("data_1")) + # mod1 output(1) would get forward to mod3 as input "data_0" + pipe_config.connect(pipe_config[mod1].output(1), pipe_config[mod3].input("data_0")) - pipe_config.connect(pipe_config[mod1].output(2), - pipe_config.pipe_output("1")) + # mod2 output(0) would get forward to mod3 as input "data_1" + pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod3].input("data_1")) - pipe_config.connect(pipe_config[mod3].output(0), - pipe_config.pipe_output("2")) + # mod1 output(2) would get forward as final pipeline compute output(1) + pipe_config.connect(pipe_config[mod1].output(2), pipe_config.pipe_output("1")) + + # mod3 output(0) would get forward as final pipeline compute output(2) + pipe_config.connect(pipe_config[mod3].output(0), pipe_config.pipe_output("2")) """ # print configueration, the expect result like following. # @@ -154,14 +177,12 @@ def pipeline(target): # connection correctness veify """ try: - pipe_config.connect(pipe_config[mod2].output(0), - pipe_config[mod1].input("data_0")) + pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod1].input("data_0")) assert 0, f"wrong module connect order check not pass!" - pipe_config.connect(pipe_config.pipe_input("data_0"), - pipe_config[mod1].output(0)) + pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].output(0)) assert 0, f"wrong global input connect check not pass!" except: - print("wrong connect check pass") + print("connection correctness check pass") """ # get text format configuration. @@ -172,7 +193,7 @@ def pipeline(target): """ # check if the configuration match expectation. """ - assert pconfig == manual_conf([mod1, mod2, mod3]) + assert pconfig == get_manual_conf([mod1, mod2, mod3]) """ # generate configure for build process @@ -207,12 +228,21 @@ def pipeline(target): mod_config[mod3] = mconfig3 """ - # build and create pipeline module + # Test build and create pipeline module """ with relay.build_config(opt_level=3): pipeline_mods, string_config = pipeline_executor.build_pipeline(mod_config) pipeline_module = pipeline_executor.create(pipeline_mods, string_config) + return pipeline_module + + +def pipeline(target): + module = pipeline_module_create(target) + """ + # Check if pipeline executor create value is valid. + """ + assert module def test_pipeline(): @@ -221,5 +251,6 @@ def test_pipeline(): for target in target_list: pipeline(target) + if __name__ == "__main__": test_pipeline() From 157e9c01261ede28d13b096896c13d9a35c8ebed Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 26 Aug 2021 20:34:27 -0700 Subject: [PATCH 05/34] set output index start from 0 --- python/tvm/contrib/pipeline_executor.py | 4 ++-- tests/python/relay/test_pipeline_executor.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 5ab19cef7e21..aabad78d6335 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -324,9 +324,9 @@ def get_config(self): dep_item["input_name"] = dname dep_conf.append(dep_item) - # in configuration the ouput_indx start from 1. + # in configuration the ouput_indx start from 0. - output["output_indx"] = int(interface.name_) + 1 + output["output_indx"] = int(interface.name_) output["dependent"] = dep_conf output_conf.append(output) mconf["mod_indx"] = instance.indx_ diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 67fd7ca91340..7f03183d02c9 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -93,9 +93,9 @@ def get_manual_conf(mods): mconfig1["pipeline"] = { "mod_indx": 1, "output": [ - {"output_indx": 1, "dependent": [{"mod_indx": 2, "input_name": "data_0"}]}, - {"output_indx": 2, "dependent": [{"mod_indx": 3, "input_name": "data_0"}]}, - {"output_indx": 3, "dependent": [{"mod_indx": 0, "input_name": "1"}]}, + {"output_indx": 0, "dependent": [{"mod_indx": 2, "input_name": "data_0"}]}, + {"output_indx": 1, "dependent": [{"mod_indx": 3, "input_name": "data_0"}]}, + {"output_indx": 2, "dependent": [{"mod_indx": 0, "input_name": "0"}]}, ], } mod_config[mods[0]] = mconfig1 @@ -104,7 +104,7 @@ def get_manual_conf(mods): mconfig2["pipeline"] = { "mod_indx": 2, "output": [ - {"output_indx": 1, "dependent": [{"mod_indx": 3, "input_name": "data_1"}]}, + {"output_indx": 0, "dependent": [{"mod_indx": 3, "input_name": "data_1"}]}, ], } mod_config[mods[1]] = mconfig2 @@ -113,7 +113,7 @@ def get_manual_conf(mods): mconfig3["pipeline"] = { "mod_indx": 3, - "output": [{"output_indx": 1, "dependent": [{"mod_indx": 0, "input_name": "2"}]}], + "output": [{"output_indx": 0, "dependent": [{"mod_indx": 0, "input_name": "1"}]}], } mod_config[mods[2]] = mconfig3 return mod_config @@ -150,10 +150,10 @@ def pipeline_module_create(target): pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod3].input("data_1")) # mod1 output(2) would get forward as final pipeline compute output(1) - pipe_config.connect(pipe_config[mod1].output(2), pipe_config.pipe_output("1")) + pipe_config.connect(pipe_config[mod1].output(2), pipe_config.pipe_output("0")) # mod3 output(0) would get forward as final pipeline compute output(2) - pipe_config.connect(pipe_config[mod3].output(0), pipe_config.pipe_output("2")) + pipe_config.connect(pipe_config[mod3].output(0), pipe_config.pipe_output("1")) """ # print configueration, the expect result like following. # From 6783d90a53626af5b23c0e53ea906ca89dba0ac0 Mon Sep 17 00:00:00 2001 From: huajsj Date: Fri, 27 Aug 2021 22:57:24 -0700 Subject: [PATCH 06/34] address review comments Co-authored-by: Cody Yu --- cmake/config.cmake | 2 +- python/tvm/contrib/pipeline_executor.py | 19 ++++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/cmake/config.cmake b/cmake/config.cmake index 1b51e78f1023..0ab498695fbf 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -105,7 +105,7 @@ set(USE_GRAPH_EXECUTOR ON) # Whether enable tiny graph executor with CUDA Graph set(USE_GRAPH_EXECUTOR_CUDA_GRAPH OFF) -# Whether enable subgraph runtime. +# Whether enable pipeline executor. set(USE_PIPELINE_EXECUTOR OFF) # Whether to enable the profiler for the graph executor and vm diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index aabad78d6335..94ae810e4ecb 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Pipeline executor that executes pipeline containing TVM PackedFunc.""" +"""Pipeline executor that executes a series of modules in a pipeline fashion.""" import json import tvm._ffi from tvm import relay @@ -22,21 +22,14 @@ def pipeline_executor_enabled(): - """check if pipeline executor enabled. + """check if pipeline executor is enabled. + Return - ------ + ------- enable: bool - return pipeline executor get enabled or not + Return pipeline executor is enabled or not. """ - pipeline_enabled = False - try: - pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create") - assert pipelinecreate - pipeline_enabled = True - except ValueError: - print("pipeline executor not enabled!") - - return pipeline_enabled + return tvm._ffi.get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None def build_pipeline(mod_n_configs): From a1bbb857d0b3d1e94970aaa60f969708e1ebc4e5 Mon Sep 17 00:00:00 2001 From: huajsj Date: Sat, 28 Aug 2021 23:37:39 -0700 Subject: [PATCH 07/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 101 ++++++++++----- src/runtime/pipeline/pipeline_executor.cc | 5 - src/runtime/pipeline/pipeline_executor.h | 7 -- tests/python/relay/test_pipeline_executor.py | 125 +++++++++---------- 4 files changed, 131 insertions(+), 107 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 94ae810e4ecb..6b46a81d62db 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -32,7 +32,7 @@ def pipeline_executor_enabled(): return tvm._ffi.get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None -def build_pipeline(mod_n_configs): +def build(pipe_configs): """build module list that can use for pipeline execution. Parameters @@ -53,9 +53,11 @@ def build_pipeline(mod_n_configs): pipeline configuration """ mods = {} + mod_n_configs = pipe_configs.get_config() config_len = len(mod_n_configs) string_config = [{} for _ in range(config_len)] - for _, (ir_mod, mod_config) in enumerate(mod_n_configs.items()): + #for _, (ir_mod, mod_config) in enumerate(mod_n_configs.items()): + for ir_mod, mod_config in mod_n_configs.items(): # init lib_name and json_name params with empty lib_name = "" json_name = "" @@ -133,15 +135,18 @@ def __init__(self, pipeline_mods, pipeline_config): self.pipeline_mods = pipeline_mods self.mod_config = pipeline_config mods, config = self.graph_executor_create(pipeline_mods, pipeline_config) - - pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create") + assert pipeline_executor_enabled(), \ + "Pipeline executor is not enabled. Please \ + re-build TVM with USE_PIPELINE_EXECUTOR=ON" + pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create", + allow_missing=False) assert pipelinecreate module = pipelinecreate(mods, config) self.module_ = module def graph_executor_create(self, pipeline_mods, mod_config): - """Create a pipeline runtime executor. + """Create graph_executor list and return string format config. Parameters ---------- @@ -167,19 +172,19 @@ def graph_executor_create(self, pipeline_mods, mod_config): return mods, json.dumps(mod_config) -class PipelineModuleConfig: +class PipelineConfig(object): """Pipeline Configuration Class, in this class there are 2 internal class, - first is Instance which use to represent Module, second is Interface which use + first is Module which use to represent Module, second is Interface which use to represent Module input/output and Pipeline Module input/output, by setting dependency relation between Interfaces this class can build the module connection relation. The class Hierarchical as following. - PipelineModuleConfig ---> Pipe Instance ---> Interface(input/output) - ---> Module Instance ---> Interface(input/output) + PipelineConfig ---> Pipeline Module ---> Interface(input/output) + ---> Subgraph Module ---> Interface(input/output) """ - class Instance: + class Module: """The class use use to represent Module and storage module index and Interface information. """ @@ -190,7 +195,7 @@ class Interface: Parameters ---------- - owner : Instance + owner : Module The class that own this interface, in such class there are Module information like index, module name @@ -251,6 +256,13 @@ def __init__(self, indx=0): self.indx_ = indx self.name_ = "mod" + str(indx) if indx else "" self.interfaces_ = {1: {}, 2: {}} + self.target_host_ = None + self.mod_name_ = "default" + self.build_func_ = None + self.params_ = None + self.target_ = None + self.dev_ = None + def get_interface(self, itype, name): if name not in self.interfaces_[itype]: @@ -264,23 +276,41 @@ def input(self, name): def output(self, index): return self.get_interface(2, index) + def set_target_host(self, host): + self.target_host_ = host + + def set_mod_name(self, name): + self.mod_name_ = name + + def set_build_func(self, build_func): + self.build_func_ = build_func + + def set_params(self, params): + self.params_ = params + + def set_target(self, target): + self.target_ = target + + def set_dev(self, dev): + self.dev_ = dev + def __init__(self, mods): - self.pipe_instance = self.Instance(0) - self.mod_instance = {m: self.Instance(i + 1) for m, i in zip(mods, range(len(mods)))} + self.pipe_module = self.Module(0) + self.mod_module = {m: self.Module(i + 1) for m, i in zip(mods, range(len(mods)))} def __str__(self): """ Get configuration in string type""" # get input input_dump = "Inputs\n" - for input_name in self.pipe_instance.interfaces_[1]: - inf = self.pipe_instance.interfaces_[1][input_name] + for input_name in self.pipe_module.interfaces_[1]: + inf = self.pipe_module.interfaces_[1][input_name] input_dump += " |" + input_name + ": " + inf.get_dependent_str() + "\n" # get connections output = {} connections_dump = "\nconnections\n" - for mod in self.mod_instance: - for _, interface in self.mod_instance[mod].interfaces_[2].items(): + for mod in self.mod_module: + for _, interface in self.mod_module[mod].interfaces_[2].items(): if interface.dependent_: mname, dname = interface.get_name() iname = mname + ".output(" + dname + ")->" @@ -302,11 +332,12 @@ def __str__(self): def get_config(self): """ Get configuration in dictionary format.""" mconfig = {} - for mod in self.mod_instance: + for mod in self.mod_module: + # get pipeline configure mconf = {} output_conf = [] - instance = self.mod_instance[mod] - for _, interface in instance.interfaces_[2].items(): + module = self.mod_module[mod] + for _, interface in module.interfaces_[2].items(): dep_conf = [] output = {} if interface.dependent_: @@ -317,29 +348,43 @@ def get_config(self): dep_item["input_name"] = dname dep_conf.append(dep_item) - # in configuration the ouput_indx start from 0. + # ouput_indx start from 0. output["output_indx"] = int(interface.name_) output["dependent"] = dep_conf output_conf.append(output) - mconf["mod_indx"] = instance.indx_ + mconf["mod_indx"] = module.indx_ mconf["output"] = output_conf - mconfig[mod] = {"pipeline": mconf} + + # build module configuration with pipeline and other parameters. + mconfig[mod] = {"pipeline": mconf, + "target_host": module.target_host_, + "mod_name": module.mod_name_, + "build": module.build_func_, + "params": module.params_, + "target": module.target_, + "dev": module.dev_, + } return mconfig def __getitem__(self, key): - return self.mod_instance[key] + return self.mod_module[key] def get_mod_indx(self, mod): - indx = self.mod_instance[mod].indx_ + indx = self.mod_module[mod].indx_ return indx def pipe_input(self, name): - return self.pipe_instance.input(name) + return self.pipe_module.input(name) def pipe_output(self, index): - return self.pipe_instance.output(index) + return self.pipe_module.output(index) - def connect(self, left: Instance.Interface, right: Instance.Interface): + def connect(self, left: Module.Interface, right: Module.Interface): left.add_dependent(right) + + +def PipeModuleConfig(object): + def __init__(self): + return diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc index 44440f341805..fc6406425e22 100644 --- a/src/runtime/pipeline/pipeline_executor.cc +++ b/src/runtime/pipeline/pipeline_executor.cc @@ -30,11 +30,6 @@ void SubGraphRuntime::Init(const Array& modules, return; } -PackedFunc SubGraphRuntime::GetFunction(const std::string& name, - const ObjectPtr& sptr_to_self) { - return PackedFunc(); -} - Module PipelineRuntimeCreate(const Array& m, const std::string& pipeline_json) { auto exec = make_object(); diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index fe92874975e9..ed1fc05548b0 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -59,13 +59,6 @@ class TVM_DLL SubGraphRuntime : public ModuleNode { * which is not compatible with RPCModules. */ void Init(const Array& modules, const std::string& pipeline_json); - /*! - * \brief Get member function to front-end - * \param name The name of the function. - * \param sptr_to_self The pointer to the module node. - * \return The corresponding member function. - */ - virtual PackedFunc GetFunction(const std::string& name, const ObjectPtr& sptr_to_self); }; } // namespace runtime } // namespace tvm diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 7f03183d02c9..4558a28e3045 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import pytest import numpy as np import tvm import tvm.testing @@ -75,7 +76,7 @@ def get_mannual_mod(): return mods, dshape -def get_manual_conf(mods): +def get_manual_conf(mods, target): """ # This function use to generate manual pipe line configueration, # the result use to verify if the pipe configuration can generate @@ -85,12 +86,12 @@ def get_manual_conf(mods): """ # set configure """ - mconfig1 = {} + """ # third output is final output, second output for mod3, first for mod2 # input """ - mconfig1["pipeline"] = { + pipe_config1 = { "mod_indx": 1, "output": [ {"output_indx": 0, "dependent": [{"mod_indx": 2, "input_name": "data_0"}]}, @@ -98,28 +99,46 @@ def get_manual_conf(mods): {"output_indx": 2, "dependent": [{"mod_indx": 0, "input_name": "0"}]}, ], } - mod_config[mods[0]] = mconfig1 - - mconfig2 = {} - mconfig2["pipeline"] = { + mod_config[mods[0]] = {"pipeline": pipe_config1, + "target_host": None, + "mod_name": "default", + "build": None, + "params": None, + "target": target[0], + "dev": target[1], + } + + pipe_config2 = { "mod_indx": 2, "output": [ {"output_indx": 0, "dependent": [{"mod_indx": 3, "input_name": "data_1"}]}, ], } - mod_config[mods[1]] = mconfig2 - - mconfig3 = {} - - mconfig3["pipeline"] = { + mod_config[mods[1]] = {"pipeline": pipe_config2, + "target_host": None, + "mod_name": "default", + "build": None, + "params": None, + "target": "llvm", + "dev": tvm.cpu(0), + } + + pipe_config3 = { "mod_indx": 3, "output": [{"output_indx": 0, "dependent": [{"mod_indx": 0, "input_name": "1"}]}], } - mod_config[mods[2]] = mconfig3 + mod_config[mods[2]] = {"pipeline": pipe_config3, + "target_host": None, + "mod_name": "default", + "build": None, + "params": None, + "target": "llvm", + "dev": tvm.cpu(0), + } return mod_config -def pipeline_module_create(target): +def pipeline(target): """ #Get 3 pipeline module. """ @@ -130,7 +149,7 @@ def pipeline_module_create(target): for i in range(5): datas.append(np.full(dshape, 3 + i).astype("float32")) - pipe_config = pipeline_executor.PipelineModuleConfig([mod1, mod2, mod3]) + pipe_config = pipeline_executor.PipelineConfig([mod1, mod2, mod3]) # Create pipeline compute input/output and subgraph dependent relation. @@ -155,7 +174,7 @@ def pipeline_module_create(target): # mod3 output(0) would get forward as final pipeline compute output(2) pipe_config.connect(pipe_config[mod3].output(0), pipe_config.pipe_output("1")) """ - # print configueration, the expect result like following. + # print configueration (print(pipe_config)), the expect result like following. # #Inputs # |data_0: mod1:data_0 @@ -171,78 +190,50 @@ def pipeline_module_create(target): # |mod2.output(0)-> mod3.data_1 """ - print(pipe_config) - """ # connection correctness veify """ - try: - pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod1].input("data_0")) - assert 0, f"wrong module connect order check not pass!" - pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].output(0)) - assert 0, f"wrong global input connect check not pass!" - except: - print("connection correctness check pass") """ - # get text format configuration. + # try wrong module order connection check, expect assert. """ - pconfig = pipe_config.get_config() + with pytest.raises(AssertionError): + pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod1].input("data_0")) """ - # check if the configuration match expectation. + # try pipeline module input with module output connection check, expect assert. """ - assert pconfig == get_manual_conf([mod1, mod2, mod3]) + + with pytest.raises(AssertionError): + pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].output(0)) + assert 0, f"wrong global input connect check not pass!" """ - # generate configure for build process + # set other parameter. """ + pipe_config[mod1].set_target(target[0]) + pipe_config[mod1].set_dev(target[1]) - mod_config = {} - mconfig1 = pconfig[mod1] - mconfig1["target_host"] = None - mconfig1["mod_name"] = "default" - mconfig1["build"] = None - mconfig1["params"] = None - mconfig1["target"] = target[0] - mconfig1["dev"] = target[1] - mod_config[mod1] = mconfig1 - - mconfig2 = pconfig[mod2] - mconfig2["target_host"] = None - mconfig2["mod_name"] = "default" - mconfig2["build"] = None - mconfig2["params"] = None - mconfig2["target"] = "llvm" - mconfig2["dev"] = tvm.cpu(0) - mod_config[mod2] = mconfig2 - - mconfig3 = pconfig[mod3] - mconfig3["target_host"] = None - mconfig3["mod_name"] = "default" - mconfig3["build"] = None - mconfig3["params"] = None - mconfig3["target"] = "llvm" - mconfig3["dev"] = tvm.cpu(0) - mod_config[mod3] = mconfig3 + pipe_config[mod2].set_target("llvm") + pipe_config[mod2].set_dev(tvm.cpu(0)) + + pipe_config[mod3].set_target("llvm") + pipe_config[mod3].set_dev(tvm.cpu(0)) + + """ + # check if the configuration match expectation. + """ + assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) """ # Test build and create pipeline module """ with relay.build_config(opt_level=3): - pipeline_mods, string_config = pipeline_executor.build_pipeline(mod_config) + pipeline_mods, string_config = pipeline_executor.build(pipe_config) pipeline_module = pipeline_executor.create(pipeline_mods, string_config) - return pipeline_module - - -def pipeline(target): - module = pipeline_module_create(target) - """ - # Check if pipeline executor create value is valid. - """ - assert module + assert pipeline_module def test_pipeline(): From 872f075aa39f8ad2c5722c9f24906a4d5f9adc07 Mon Sep 17 00:00:00 2001 From: huajsj Date: Sun, 29 Aug 2021 12:07:02 -0700 Subject: [PATCH 08/34] address review comments. --- CMakeLists.txt | 9 -- python/tvm/contrib/pipeline_executor.py | 105 ++++++++++--------- src/runtime/pipeline/pipeline_executor.cc | 4 +- src/runtime/pipeline/pipeline_executor.h | 4 +- tests/python/relay/test_pipeline_executor.py | 4 +- 5 files changed, 61 insertions(+), 65 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f671ce7f790b..ab10575b7f4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -395,15 +395,6 @@ if(USE_PIPELINE_EXECUTOR) list(APPEND RUNTIME_SRCS ${RUNTIME_PIPELINE_SRCS}) endif(USE_PIPELINE_EXECUTOR) -# Enable ctest if gtest is available -find_path(GTEST_INCLUDE_DIR gtest/gtest.h) -find_library(GTEST_LIB gtest "$ENV{GTEST_LIB}") -if(GTEST_INCLUDE_DIR AND GTEST_LIB) - enable_testing() - include(CTest) - include(GoogleTest) -endif() - # Module rules include(cmake/modules/VTA.cmake) include(cmake/modules/StandaloneCrt.cmake) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 6b46a81d62db..ad24d2174bb0 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -29,7 +29,8 @@ def pipeline_executor_enabled(): enable: bool Return pipeline executor is enabled or not. """ - return tvm._ffi.get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None + return tvm._ffi.get_global_func("tvm.pipeline_executor.create", + allow_missing=True) is not None def build(pipe_configs): @@ -58,19 +59,11 @@ def build(pipe_configs): string_config = [{} for _ in range(config_len)] #for _, (ir_mod, mod_config) in enumerate(mod_n_configs.items()): for ir_mod, mod_config in mod_n_configs.items(): - # init lib_name and json_name params with empty - lib_name = "" - json_name = "" - params_name = "" - # Get module configuration - assert "pipeline" in mod_config and "mod_indx" in mod_config["pipeline"] - # Get module index in pipeline configuration mconf = mod_config["pipeline"].copy() + mod_indx = mconf["mod_indx"] - 1 # Get mod device config dev = mod_config["dev"] - mod_indx = mconf["mod_indx"] - 1 target = mod_config["target"] - assert mod_indx < config_len build_func = relay.build # if there is a self defined build function then use it. if "build" in mod_config and mod_config["build"]: @@ -85,29 +78,24 @@ def build(pipe_configs): mod_name=mod_config["mod_name"], ) - mconf["lib_name"] = lib_name - mconf["json_name"] = json_name - mconf["params_name"] = params_name mconf["dev"] = "{},{}".format(dev.device_type, dev.device_id) # Create pipeline configuration string_config[mod_indx] = mconf # associate mod with device mods[mod] = {"dev": dev} - # return IRModule list and pipeline configuration - return mods, string_config + # return PipeModuleConfig + return PipeModuleConfig(mods, string_config) -def create(pipeline_mods, mod_config): +def create(pipe_mod_config): """Create a pipeline runtime executor. Parameters ---------- - pipeline_mods : List[IRModule] - list of IRModule - mod_config : Dict[int, Dict[str, Any]] - modules and modules dependency configuration informaiton. + pipe_mod_config : PipeModuleConfig + class to storage IRModule list and pipeline configuration. Returns ------- @@ -115,26 +103,23 @@ def create(pipeline_mods, mod_config): Runtime pipeline module. """ - submodule = PipelineModule(pipeline_mods, mod_config) - return submodule - + return PipelineModule(pipe_mod_config) class PipelineModule(object): """Wrapper runtime module. This is a thin wrapper of the underlying TVM module. + Parameters ---------- pipeline_mods : List[GraphModule] The internal tvm module that holds the actual graph functions. - pipeline_config : Dict[IRModule, Dict[str, Any]] modules and modules dependency configuration informaiton. - """ - def __init__(self, pipeline_mods, pipeline_config): - self.pipeline_mods = pipeline_mods - self.mod_config = pipeline_config - mods, config = self.graph_executor_create(pipeline_mods, pipeline_config) + def __init__(self, pipe_mod_config): + self.pipeline_mods_ = pipe_mod_config.pipeline_mods_ + self.mod_config_ = pipe_mod_config.mods_config_ + mods, config = self.graph_executor_create(self.pipeline_mods_, self.mod_config_) assert pipeline_executor_enabled(), \ "Pipeline executor is not enabled. Please \ re-build TVM with USE_PIPELINE_EXECUTOR=ON" @@ -150,6 +135,7 @@ def graph_executor_create(self, pipeline_mods, mod_config): Parameters ---------- + pipeline_mods : List[IRModule] list of IRModule @@ -158,8 +144,11 @@ def graph_executor_create(self, pipeline_mods, mod_config): Returns ------- - mods : GreaphModule + mods : List[GraphModule] Runtime graph module. + + mod_config : str + mods configuration """ mods = [] @@ -180,11 +169,10 @@ class PipelineConfig(object): connection relation. The class Hierarchical as following. - PipelineConfig ---> Pipeline Module ---> Interface(input/output) - ---> Subgraph Module ---> Interface(input/output) + PipelineConfig ---> ModuleWrapper ---> Interface(input/output) """ - class Module: + class ModuleWrapper: """The class use use to represent Module and storage module index and Interface information. """ @@ -195,7 +183,7 @@ class Interface: Parameters ---------- - owner : Module + owner : ModuleWrapper The class that own this interface, in such class there are Module information like index, module name @@ -295,22 +283,23 @@ def set_dev(self, dev): self.dev_ = dev def __init__(self, mods): - self.pipe_module = self.Module(0) - self.mod_module = {m: self.Module(i + 1) for m, i in zip(mods, range(len(mods)))} + self.pipe_module_name_ = "pipeline_module" + self.mod_wrapper = {m: self.ModuleWrapper(i + 1) for m, i in zip(mods, range(len(mods)))} + self.mod_wrapper[self.pipe_module_name_] = self.ModuleWrapper(0) def __str__(self): """ Get configuration in string type""" # get input input_dump = "Inputs\n" - for input_name in self.pipe_module.interfaces_[1]: - inf = self.pipe_module.interfaces_[1][input_name] + for input_name in self.mod_wrapper["pipeline_module"].interfaces_[1]: + inf = self.mod_wrapper["pipeline_module"].interfaces_[1][input_name] input_dump += " |" + input_name + ": " + inf.get_dependent_str() + "\n" # get connections output = {} connections_dump = "\nconnections\n" - for mod in self.mod_module: - for _, interface in self.mod_module[mod].interfaces_[2].items(): + for mod in self.mod_wrapper: + for _, interface in self.mod_wrapper[mod].interfaces_[2].items(): if interface.dependent_: mname, dname = interface.get_name() iname = mname + ".output(" + dname + ")->" @@ -332,11 +321,13 @@ def __str__(self): def get_config(self): """ Get configuration in dictionary format.""" mconfig = {} - for mod in self.mod_module: + for mod in self.mod_wrapper: + if mod == self.pipe_module_name_: + continue # get pipeline configure mconf = {} output_conf = [] - module = self.mod_module[mod] + module = self.mod_wrapper[mod] for _, interface in module.interfaces_[2].items(): dep_conf = [] output = {} @@ -369,22 +360,36 @@ def get_config(self): return mconfig def __getitem__(self, key): - return self.mod_module[key] + return self.mod_wrapper[key] def get_mod_indx(self, mod): - indx = self.mod_module[mod].indx_ + indx = self.mod_wrapper[mod].indx_ return indx def pipe_input(self, name): - return self.pipe_module.input(name) + return self.mod_wrapper[self.pipe_module_name_].input(name) def pipe_output(self, index): - return self.pipe_module.output(index) + return self.mod_wrapper[self.pipe_module_name_].output(index) - def connect(self, left: Module.Interface, right: Module.Interface): + def connect(self, left: ModuleWrapper.Interface, right: ModuleWrapper.Interface): left.add_dependent(right) -def PipeModuleConfig(object): - def __init__(self): - return +class PipeModuleConfig(object): + """This class use to storage pipeline IRModule and configurations. + + Parameters + ---------- + + pipeline_mods : List[IRModule] + list of IRModule + + mod_config : Dict[int, Dict[str, Any]] + modules and modules dependency configuration informaiton. + + """ + + def __init__(self, pipeline_mods, mods_config): + self.pipeline_mods_ = pipeline_mods + self.mods_config_ = mods_config diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc index fc6406425e22..7edb742d50d8 100644 --- a/src/runtime/pipeline/pipeline_executor.cc +++ b/src/runtime/pipeline/pipeline_executor.cc @@ -25,14 +25,14 @@ namespace tvm { namespace runtime { -void SubGraphRuntime::Init(const Array& modules, +void PipelineRuntime::Init(const Array& modules, const std::string& pipeline_json) { return; } Module PipelineRuntimeCreate(const Array& m, const std::string& pipeline_json) { - auto exec = make_object(); + auto exec = make_object(); exec->Init(m, pipeline_json); return Module(exec); } diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index ed1fc05548b0..49ce62ac01c3 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -41,12 +41,12 @@ namespace runtime { * This runtime can be acccesibly in various language via * TVM runtime PackedFunc API. */ -class TVM_DLL SubGraphRuntime : public ModuleNode { +class TVM_DLL PipelineRuntime : public ModuleNode { public: /*! * \return The type key of the executor. */ - const char* type_key() const final { return "SubGraphRuntime"; } + const char* type_key() const final { return "PipelineRuntime"; } /*! * \brief Initialize the graph executor with graph and context. * \param graph_json The execution graph. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 4558a28e3045..046175810b86 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -230,9 +230,9 @@ def pipeline(target): # Test build and create pipeline module """ with relay.build_config(opt_level=3): - pipeline_mods, string_config = pipeline_executor.build(pipe_config) + pipeline_mod_config = pipeline_executor.build(pipe_config) - pipeline_module = pipeline_executor.create(pipeline_mods, string_config) + pipeline_module = pipeline_executor.create(pipeline_mod_config) assert pipeline_module From 67242e371733cbd717d6f6c1ca3fd3101038767c Mon Sep 17 00:00:00 2001 From: huajsj Date: Mon, 30 Aug 2021 23:09:28 -0700 Subject: [PATCH 09/34] Update python/tvm/contrib/pipeline_executor.py Co-authored-by: Cody Yu Update python/tvm/contrib/pipeline_executor.py Co-authored-by: Cody Yu --- python/tvm/contrib/pipeline_executor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index ad24d2174bb0..44b377fc49fe 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -242,8 +242,9 @@ def add_dependent(self, dependent): def __init__(self, indx=0): self.indx_ = indx - self.name_ = "mod" + str(indx) if indx else "" - self.interfaces_ = {1: {}, 2: {}} + self.name = "mod{}".format(str(index) if index else "") + self.input_bindings = {} + self.output_bindings = {} self.target_host_ = None self.mod_name_ = "default" self.build_func_ = None From 607b758beb083c095fd6d52975065a7ccf6615a8 Mon Sep 17 00:00:00 2001 From: huajsj Date: Mon, 30 Aug 2021 23:52:43 -0700 Subject: [PATCH 10/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 98 +++++++++++-------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 44b377fc49fe..fde497d4c898 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -162,24 +162,19 @@ def graph_executor_create(self, pipeline_mods, mod_config): class PipelineConfig(object): - """Pipeline Configuration Class, in this class there are 2 internal class, - first is Module which use to represent Module, second is Interface which use - to represent Module input/output and Pipeline Module input/output, by setting - dependency relation between Interfaces this class can build the module - connection relation. - - The class Hierarchical as following. - PipelineConfig ---> ModuleWrapper ---> Interface(input/output) + """The wrapper of each module to be pipelined. The wrapper mainly includes the + module itself as well as the binding that represents the connections of this + module's inputs and outputs to other modules. """ class ModuleWrapper: """The class use use to represent Module and storage module index and - Interface information. + Binding information. """ - class Interface: + class Binding: """The class that use to storage module connection information. - There are 2 types Interface Input:1 Output:2 + There are 2 types Binding Input:1 Output:2 Parameters ---------- @@ -187,36 +182,33 @@ class Interface: The class that own this interface, in such class there are Module information like index, module name - itype : integer - Interface type, 1 is input interface, 2 is output interface + io_type : str + The type of this binding. It can be either "input" or "output". name : str/integer - Interface name, for input that is string for example "data0" + Binding name, for input that is string for example "data0" for output that is integer for example 0. """ - def __init__(self, owner, itype, name): - self.owner_ = owner - self.itype_ = itype - self.name_ = str(name) - self.dependent_ = [] + def __init__(self, owner, stype, name): + self.io_owner = owner + self.io_type = stype + self.name = str(name) + self.bindings = [] def get_name(self): - mname = "" - if self.owner_: - mname = self.owner_.name_ - - return mname, self.name_ + return self.io_owner.name, self.name def get_owner_indx(self): - return self.owner_.indx_ + return self.io_owner.indx_ - def get_dependent_str(self): + def get_bindings_str(self): name = "" - for dependent in self.dependent_: + for dependent in self.bindings: mname, dname = dependent.get_name() - name = name + (mname + ":output(" + dname if self.itype_ == 2 else "") - name = name + (")" if self.itype_ == 2 else mname + ":" + dname) + name += (mname + ":output(" + dname \ + if self.io_type == "output" else "") + name += (")" if self.io_type == "output" else mname + ":" + dname) return name def add_dependent(self, dependent): @@ -232,45 +224,41 @@ def add_dependent(self, dependent): assert owner_indx != dep_owner_indx, f"can not set self as dependent." assert not ( owner_indx > dep_owner_indx - and not (dependent.itype_ == 2 and dep_owner_indx == 0) + and not (dependent.io_type == "output" and dep_owner_indx == 0) ), f"dependent only can be next module interface or global output." assert not ( - owner_indx == 0 and dependent.itype_ != 1 + owner_indx == 0 and dependent.io_type != "input" ), f"global input only can set dependent with module input." - self.dependent_.append(dependent) + self.bindings.append(dependent) - def __init__(self, indx=0): - self.indx_ = indx + def __init__(self, index=0): + self.indx_ = index self.name = "mod{}".format(str(index) if index else "") self.input_bindings = {} self.output_bindings = {} self.target_host_ = None - self.mod_name_ = "default" self.build_func_ = None self.params_ = None self.target_ = None self.dev_ = None - def get_interface(self, itype, name): - if name not in self.interfaces_[itype]: - self.interfaces_[itype][name] = self.Interface(self, itype, name) - - return self.interfaces_[itype][name] - def input(self, name): - return self.get_interface(1, name) + if name not in self.input_bindings: + self.input_bindings[name] = self.Binding(self, "input", name) + + return self.input_bindings[name] def output(self, index): - return self.get_interface(2, index) + if index not in self.output_bindings: + self.output_bindings[index] = self.Binding(self, "output", index) + + return self.output_bindings[index] def set_target_host(self, host): self.target_host_ = host - def set_mod_name(self, name): - self.mod_name_ = name - def set_build_func(self, build_func): self.build_func_ = build_func @@ -294,17 +282,17 @@ def __str__(self): input_dump = "Inputs\n" for input_name in self.mod_wrapper["pipeline_module"].interfaces_[1]: inf = self.mod_wrapper["pipeline_module"].interfaces_[1][input_name] - input_dump += " |" + input_name + ": " + inf.get_dependent_str() + "\n" + input_dump += " |" + input_name + ": " + inf.get_bindings_str() + "\n" # get connections output = {} connections_dump = "\nconnections\n" for mod in self.mod_wrapper: for _, interface in self.mod_wrapper[mod].interfaces_[2].items(): - if interface.dependent_: + if interface.bindings: mname, dname = interface.get_name() iname = mname + ".output(" + dname + ")->" - for dep in interface.dependent_: + for dep in interface.bindings: dep_mname, dep_dname = dep.get_name() if dep.owner_.indx_ > 0: iname += " " + dep_mname + "." + dep_dname @@ -329,11 +317,11 @@ def get_config(self): mconf = {} output_conf = [] module = self.mod_wrapper[mod] - for _, interface in module.interfaces_[2].items(): + for _, binding in module.output_bindings.items(): dep_conf = [] output = {} - if interface.dependent_: - for dep in interface.dependent_: + if binding.bindings: + for dep in binding.bindings: dep_item = {} _, dname = dep.get_name() dep_item["mod_indx"] = dep.get_owner_indx() @@ -342,7 +330,7 @@ def get_config(self): # ouput_indx start from 0. - output["output_indx"] = int(interface.name_) + output["output_indx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) mconf["mod_indx"] = module.indx_ @@ -351,7 +339,7 @@ def get_config(self): # build module configuration with pipeline and other parameters. mconfig[mod] = {"pipeline": mconf, "target_host": module.target_host_, - "mod_name": module.mod_name_, + "mod_name": "default", "build": module.build_func_, "params": module.params_, "target": module.target_, @@ -373,7 +361,7 @@ def pipe_input(self, name): def pipe_output(self, index): return self.mod_wrapper[self.pipe_module_name_].output(index) - def connect(self, left: ModuleWrapper.Interface, right: ModuleWrapper.Interface): + def connect(self, left: ModuleWrapper.Binding, right: ModuleWrapper.Binding): left.add_dependent(right) From 8d476aee3e038fbad2414238f57bf9a73ced1863 Mon Sep 17 00:00:00 2001 From: huajsj Date: Tue, 31 Aug 2021 13:34:44 -0700 Subject: [PATCH 11/34] address review comments --- python/tvm/contrib/pipeline_executor.py | 74 +++++++++++++++----- tests/python/relay/test_pipeline_executor.py | 26 +++---- 2 files changed, 68 insertions(+), 32 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index fde497d4c898..a32aac4bb861 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -160,7 +160,6 @@ def graph_executor_create(self, pipeline_mods, mod_config): return mods, json.dumps(mod_config) - class PipelineConfig(object): """The wrapper of each module to be pipelined. The wrapper mainly includes the module itself as well as the binding that represents the connections of this @@ -197,10 +196,17 @@ def __init__(self, owner, stype, name): self.bindings = [] def get_name(self): - return self.io_owner.name, self.name + owner_name = "" + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + owner_name = self.io_owner.name + + return owner_name, self.name def get_owner_indx(self): - return self.io_owner.indx_ + owner_indx = 0 + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + return self.io_owner.indx_ + return 0 def get_bindings_str(self): name = "" @@ -211,7 +217,7 @@ def get_bindings_str(self): name += (")" if self.io_type == "output" else mname + ":" + dname) return name - def add_dependent(self, dependent): + def connect(self, dependent): """ # check if the dependency setting correct. # correct connection are following @@ -219,6 +225,7 @@ def add_dependent(self, dependent): # 2. module output to next module input # 3. module output to global output """ + ''' owner_indx = self.get_owner_indx() dep_owner_indx = dependent.get_owner_indx() assert owner_indx != dep_owner_indx, f"can not set self as dependent." @@ -229,20 +236,28 @@ def add_dependent(self, dependent): assert not ( owner_indx == 0 and dependent.io_type != "input" ), f"global input only can set dependent with module input." - + ''' self.bindings.append(dependent) def __init__(self, index=0): self.indx_ = index self.name = "mod{}".format(str(index) if index else "") - self.input_bindings = {} - self.output_bindings = {} + self.input_bindings = PipelineConfig.BindingList(self, "input") + self.output_bindings = PipelineConfig.BindingList(self, "output") self.target_host_ = None self.build_func_ = None self.params_ = None self.target_ = None self.dev_ = None + def __getitem__(self, key): + if (isinstance(key, str)): + if (key == "input"): + return self.input_bindings + + if (key == "output"): + return self.output_bindings + assert 0, "key not found" def input(self, name): if name not in self.input_bindings: @@ -271,30 +286,44 @@ def set_target(self, target): def set_dev(self, dev): self.dev_ = dev + class BindingList: + def __init__(self, owner, type_name): + self.bindings = {} + self.io_owner = owner + self.binding_type = type_name + + def __getitem__(self, key): + if key not in self.bindings: + self.bindings[key] = \ + PipelineConfig.ModuleWrapper.Binding(self.io_owner, + self.binding_type, key) + + return self.bindings[key] + def __init__(self, mods): - self.pipe_module_name_ = "pipeline_module" self.mod_wrapper = {m: self.ModuleWrapper(i + 1) for m, i in zip(mods, range(len(mods)))} - self.mod_wrapper[self.pipe_module_name_] = self.ModuleWrapper(0) + self.input_bindings = self.BindingList(self, "input") + self.output_bindings = self.BindingList(self, "output") def __str__(self): """ Get configuration in string type""" # get input input_dump = "Inputs\n" - for input_name in self.mod_wrapper["pipeline_module"].interfaces_[1]: - inf = self.mod_wrapper["pipeline_module"].interfaces_[1][input_name] + for input_name in self.input_bindings.bindings: + inf = self.input_bindings.bindings[input_name] input_dump += " |" + input_name + ": " + inf.get_bindings_str() + "\n" # get connections output = {} connections_dump = "\nconnections\n" for mod in self.mod_wrapper: - for _, interface in self.mod_wrapper[mod].interfaces_[2].items(): + for _, interface in self.mod_wrapper[mod].output_bindings.bindings.items(): if interface.bindings: mname, dname = interface.get_name() iname = mname + ".output(" + dname + ")->" for dep in interface.bindings: dep_mname, dep_dname = dep.get_name() - if dep.owner_.indx_ > 0: + if isinstance(dep.io_owner, PipelineConfig.ModuleWrapper): iname += " " + dep_mname + "." + dep_dname connections_dump += " |" + iname + "\n" else: @@ -311,13 +340,11 @@ def get_config(self): """ Get configuration in dictionary format.""" mconfig = {} for mod in self.mod_wrapper: - if mod == self.pipe_module_name_: - continue # get pipeline configure mconf = {} output_conf = [] module = self.mod_wrapper[mod] - for _, binding in module.output_bindings.items(): + for _, binding in module.output_bindings.bindings.items(): dep_conf = [] output = {} if binding.bindings: @@ -349,17 +376,26 @@ def get_config(self): return mconfig def __getitem__(self, key): - return self.mod_wrapper[key] + if (isinstance(key, tvm.ir.module.IRModule)): + return self.mod_wrapper[key] + + if (isinstance(key, str)): + if (key == "input"): + return self.input_bindings + if (key == "output"): + return self.output_bindings + + assert 0, "key not find!" def get_mod_indx(self, mod): indx = self.mod_wrapper[mod].indx_ return indx def pipe_input(self, name): - return self.mod_wrapper[self.pipe_module_name_].input(name) + return self.input_bindings[name] def pipe_output(self, index): - return self.mod_wrapper[self.pipe_module_name_].output(index) + return self.output_bindings[index] def connect(self, left: ModuleWrapper.Binding, right: ModuleWrapper.Binding): left.add_dependent(right) diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 046175810b86..08cbddf4cc8a 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -154,25 +154,25 @@ def pipeline(target): # Create pipeline compute input/output and subgraph dependent relation. # pipeline compute input "data_0" would get forward to mod1 as input "data_0" - pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].input("data_0")) + pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) # pipeline compute input "data_1" would get forward to mod2 as input "data_1" - pipe_config.connect(pipe_config.pipe_input("data_1"), pipe_config[mod2].input("data_1")) + pipe_config["input"]["data_1"].connect(pipe_config[mod2]["input"]["data_1"]) # mod1 output(0) would get forward to mod2 as input "data_0" - pipe_config.connect(pipe_config[mod1].output(0), pipe_config[mod2].input("data_0")) + pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) # mod1 output(1) would get forward to mod3 as input "data_0" - pipe_config.connect(pipe_config[mod1].output(1), pipe_config[mod3].input("data_0")) + pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) # mod2 output(0) would get forward to mod3 as input "data_1" - pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod3].input("data_1")) + pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) # mod1 output(2) would get forward as final pipeline compute output(1) - pipe_config.connect(pipe_config[mod1].output(2), pipe_config.pipe_output("0")) + pipe_config[mod1].output_bindings[2].connect(pipe_config.output_bindings["0"]) # mod3 output(0) would get forward as final pipeline compute output(2) - pipe_config.connect(pipe_config[mod3].output(0), pipe_config.pipe_output("1")) + pipe_config[mod3].output_bindings[0].connect(pipe_config.output_bindings["1"]) """ # print configueration (print(pipe_config)), the expect result like following. # @@ -189,7 +189,7 @@ def pipeline(target): # |mod1.output(1)-> mod3.data_0 # |mod2.output(0)-> mod3.data_1 """ - + print(pipe_config) """ # connection correctness veify """ @@ -198,16 +198,16 @@ def pipeline(target): # try wrong module order connection check, expect assert. """ - with pytest.raises(AssertionError): - pipe_config.connect(pipe_config[mod2].output(0), pipe_config[mod1].input("data_0")) + #with pytest.raises(AssertionError): + # pipe_config[mod2].output(0).connect(pipe_config[mod1].input("data_0")) """ # try pipeline module input with module output connection check, expect assert. """ - with pytest.raises(AssertionError): - pipe_config.connect(pipe_config.pipe_input("data_0"), pipe_config[mod1].output(0)) - assert 0, f"wrong global input connect check not pass!" + #with pytest.raises(AssertionError): + # pipe_config.pipe_input("data_0").connect(pipe_config[mod1].output(0)) + # assert 0, f"wrong global input connect check not pass!" """ # set other parameter. From e39b7479c547a60a5e791d60b6e558f01f2b7ec7 Mon Sep 17 00:00:00 2001 From: huajsj Date: Tue, 31 Aug 2021 21:15:57 -0700 Subject: [PATCH 12/34] add topology sort --- python/tvm/contrib/pipeline_executor.py | 90 +++++++++++++++----- tests/python/relay/test_pipeline_executor.py | 5 +- 2 files changed, 72 insertions(+), 23 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index a32aac4bb861..840843398dd1 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -194,6 +194,7 @@ def __init__(self, owner, stype, name): self.io_type = stype self.name = str(name) self.bindings = [] + self.parents = [] def get_name(self): owner_name = "" @@ -203,9 +204,11 @@ def get_name(self): return owner_name, self.name def get_owner_indx(self): - owner_indx = 0 if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - return self.io_owner.indx_ + return self.io_owner.index + + # if not ModuleWrapper then owner is PipelineConfig, return 0 + # to identify this is global interface return 0 def get_bindings_str(self): @@ -238,10 +241,11 @@ def connect(self, dependent): ), f"global input only can set dependent with module input." ''' self.bindings.append(dependent) + if isinstance(self, PipelineConfig.ModuleWrapper): + dependent.parents.append(self) def __init__(self, index=0): - self.indx_ = index - self.name = "mod{}".format(str(index) if index else "") + self.set_index(index) self.input_bindings = PipelineConfig.BindingList(self, "input") self.output_bindings = PipelineConfig.BindingList(self, "output") self.target_host_ = None @@ -257,7 +261,24 @@ def __getitem__(self, key): if (key == "output"): return self.output_bindings - assert 0, "key not found" + + assert 0, "{} not found!".format(key) + + def set_index(self, index): + self.index = index + self.name = "mod{}".format(str(index)) + + def is_root_mod(self): + for name, binding in self.input_bindings.bindings.items(): + if binding.parents: + return False + + return True + + def remove_self_from_bindings(self): + for _, binding in self.input_bindings.bindings.items(): + if binding in binding.bindings: + binding.bindings.remove(binding) def input(self, name): if name not in self.input_bindings: @@ -300,13 +321,16 @@ def __getitem__(self, key): return self.bindings[key] - def __init__(self, mods): - self.mod_wrapper = {m: self.ModuleWrapper(i + 1) for m, i in zip(mods, range(len(mods)))} + def __init__(self): + self.mod_wrapper = {} self.input_bindings = self.BindingList(self, "input") self.output_bindings = self.BindingList(self, "output") def __str__(self): """ Get configuration in string type""" + # Sort moudles + self.sub_module_sort() + # get input input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: @@ -336,8 +360,27 @@ def __str__(self): return input_dump + output_dump + connections_dump + def __getitem__(self, key): + if (isinstance(key, tvm.ir.module.IRModule)): + if key not in self.mod_wrapper: + self.mod_wrapper[key] = self.ModuleWrapper() + + return self.mod_wrapper[key] + + if (isinstance(key, str)): + if (key == "input"): + return self.input_bindings + if (key == "output"): + return self.output_bindings + + assert 0, "key not find!" + def get_config(self): """ Get configuration in dictionary format.""" + + # Sort moudles + self.sub_module_sort() + mconfig = {} for mod in self.mod_wrapper: # get pipeline configure @@ -360,7 +403,7 @@ def get_config(self): output["output_indx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) - mconf["mod_indx"] = module.indx_ + mconf["mod_indx"] = module.index mconf["output"] = output_conf # build module configuration with pipeline and other parameters. @@ -375,20 +418,28 @@ def get_config(self): return mconfig - def __getitem__(self, key): - if (isinstance(key, tvm.ir.module.IRModule)): - return self.mod_wrapper[key] + def sub_module_sort(self): + mlist = [] + mod_wrapper = self.mod_wrapper.copy() + while mod_wrapper: + temp_list = [] + for mod, wrapper in mod_wrapper.items(): + if wrapper.is_root_mod(): + temp_list.append(mod) + wrapper.remove_self_from_bindings() - if (isinstance(key, str)): - if (key == "input"): - return self.input_bindings - if (key == "output"): - return self.output_bindings + for mod in temp_list: + mod_wrapper.pop(mod, None) - assert 0, "key not find!" + mlist += temp_list + + for mod, i in zip(mlist, range(len(mlist))): + self.mod_wrapper[mod].set_index(i + 1) + + return def get_mod_indx(self, mod): - indx = self.mod_wrapper[mod].indx_ + indx = self.mod_wrapper[mod].index return indx def pipe_input(self, name): @@ -397,9 +448,6 @@ def pipe_input(self, name): def pipe_output(self, index): return self.output_bindings[index] - def connect(self, left: ModuleWrapper.Binding, right: ModuleWrapper.Binding): - left.add_dependent(right) - class PipeModuleConfig(object): """This class use to storage pipeline IRModule and configurations. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 08cbddf4cc8a..c82465202b48 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -149,10 +149,11 @@ def pipeline(target): for i in range(5): datas.append(np.full(dshape, 3 + i).astype("float32")) - pipe_config = pipeline_executor.PipelineConfig([mod1, mod2, mod3]) + pipe_config = pipeline_executor.PipelineConfig() # Create pipeline compute input/output and subgraph dependent relation. + # Test in key mode for binding find. # pipeline compute input "data_0" would get forward to mod1 as input "data_0" pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) @@ -168,6 +169,7 @@ def pipeline(target): # mod2 output(0) would get forward to mod3 as input "data_1" pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) + # Testing in API mode for binding find. # mod1 output(2) would get forward as final pipeline compute output(1) pipe_config[mod1].output_bindings[2].connect(pipe_config.output_bindings["0"]) @@ -189,7 +191,6 @@ def pipeline(target): # |mod1.output(1)-> mod3.data_0 # |mod2.output(0)-> mod3.data_1 """ - print(pipe_config) """ # connection correctness veify """ From 31ee5a60916f17371c1d3fcf9a4396abff36fad7 Mon Sep 17 00:00:00 2001 From: huajsj Date: Wed, 1 Sep 2021 19:34:13 -0700 Subject: [PATCH 13/34] add binding check logic. --- python/tvm/contrib/pipeline_executor.py | 149 ++++++++++++++----- tests/python/relay/test_pipeline_executor.py | 60 ++++++-- 2 files changed, 154 insertions(+), 55 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 840843398dd1..21379268e7ec 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -18,6 +18,7 @@ import json import tvm._ffi from tvm import relay +from tvm.relay.transform import InferType from tvm.contrib import graph_executor @@ -96,9 +97,9 @@ def create(pipe_mod_config): pipe_mod_config : PipeModuleConfig class to storage IRModule list and pipeline configuration. + ------- Returns - ------- submodule : PipelineModule Runtime pipeline module. """ @@ -189,13 +190,17 @@ class Binding: for output that is integer for example 0. """ - def __init__(self, owner, stype, name): + def __init__(self, owner, stype, name, data_type): self.io_owner = owner self.io_type = stype self.name = str(name) + # These item that have dependency relation with self self.bindings = [] + # The item that self depend self.parents = [] + self.data_type = data_type + def get_name(self): owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): @@ -211,41 +216,69 @@ def get_owner_indx(self): # to identify this is global interface return 0 - def get_bindings_str(self): - name = "" - for dependent in self.bindings: - mname, dname = dependent.get_name() - name += (mname + ":output(" + dname \ - if self.io_type == "output" else "") - name += (")" if self.io_type == "output" else mname + ":" + dname) - return name - - def connect(self, dependent): + def __repr__(self): + # Get all binding(input data), exepect like |data_0: mod1:data_0 + ret = " |{}: ".format(self.name) + for binding in self.bindings: + mname, dname = binding.get_name() + ret += "{0}:{1} ".format(mname, dname) + return ret + + def dag_acircle_check(self, start, inputs): + for name, binding in inputs.items(): + if start == binding.io_owner: + return False + for p in binding.parents: + if not self.dag_acircle_check(start, p.io_owner.input_bindings.bindings): + return False + + return True + + def connect(self, binding): """ - # check if the dependency setting correct. + # check if the bindendency setting correct. # correct connection are following # 1. global input to module input - # 2. module output to next module input - # 3. module output to global output + # 2. module output to global output """ - ''' owner_indx = self.get_owner_indx() - dep_owner_indx = dependent.get_owner_indx() - assert owner_indx != dep_owner_indx, f"can not set self as dependent." - assert not ( - owner_indx > dep_owner_indx - and not (dependent.io_type == "output" and dep_owner_indx == 0) - ), f"dependent only can be next module interface or global output." - assert not ( - owner_indx == 0 and dependent.io_type != "input" - ), f"global input only can set dependent with module input." - ''' - self.bindings.append(dependent) - if isinstance(self, PipelineConfig.ModuleWrapper): - dependent.parents.append(self) - - def __init__(self, index=0): + bind_owner_indx = binding.get_owner_indx() + if owner_indx == bind_owner_indx: + raise RuntimeError(f"can not set self as binding.") + + if owner_indx != 0 and self.io_type == "input": + raise RuntimeError(f"Module only can start binding from output!") + + if owner_indx != 0 and bind_owner_indx != 0 and binding.io_type == "output": + raise RuntimeError(f"Module output can not binding with module output!") + + if owner_indx != 0 and bind_owner_indx == 0 and binding.io_type == "input": + raise RuntimeError(f"Module output can not binding with global input!") + + if owner_indx == 0 and self.io_type != "input": + raise RuntimeError(f"Global only can start binding from input!") + + if owner_indx == 0 and binding.io_type != "input": + raise RuntimeError(f"Global input only can set binding with module input.") + + self.bindings.append(binding) + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + # check if the source and target data_type same + if (isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) + and self.data_type != binding.data_type): + raise RuntimeError(f"Illegal type:binding type is not same!") + + binding.parents.append(self) + # after + if not self.dag_acircle_check(binding.io_owner, + self.io_owner.input_bindings.bindings): + raise RuntimeError(f"Illegal connection: cause a circle!") + + def __init__(self, mod = None, index = 0): + self.input_params = InferType()(mod)["main"].params + self.output_values = InferType()(mod)["main"].checked_type.ret_type self.set_index(index) + self.mod = mod self.input_bindings = PipelineConfig.BindingList(self, "input") self.output_bindings = PipelineConfig.BindingList(self, "output") self.target_host_ = None @@ -264,6 +297,22 @@ def __getitem__(self, key): assert 0, "{} not found!".format(key) + def get_type(self, key, stype): + if stype == "input": + for param in self.input_params: + if param.name_hint == key: + return param._checked_type_ + + if stype == "output": + if isinstance(self.output_values, tvm.ir.type.TupleType): + if int(key) < len(self.output_values.fields): + return self.output_values.fields[int(key)] + elif int(key) == 0: + return self.output_values + + return None + + def set_index(self, index): self.index = index self.name = "mod{}".format(str(index)) @@ -276,9 +325,10 @@ def is_root_mod(self): return True def remove_self_from_bindings(self): - for _, binding in self.input_bindings.bindings.items(): - if binding in binding.bindings: - binding.bindings.remove(binding) + for _, binding in self.output_bindings.bindings.items(): + for child in binding.bindings: + if binding in child.parents: + child.parents.remove(binding) def input(self, name): if name not in self.input_bindings: @@ -313,15 +363,28 @@ def __init__(self, owner, type_name): self.io_owner = owner self.binding_type = type_name + def get_binding_type(self, key): + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + return self.io_owner.get_type(key, self.binding_type) + + return None + def __getitem__(self, key): if key not in self.bindings: + data_type = self.get_binding_type(key) + if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + raise RuntimeError(f"Illegal name: {self.binding_type}:{key} cannot find") + self.bindings[key] = \ PipelineConfig.ModuleWrapper.Binding(self.io_owner, - self.binding_type, key) + self.binding_type, + key, + data_type) return self.bindings[key] def __init__(self): + self.last_mod_indx = 0 self.mod_wrapper = {} self.input_bindings = self.BindingList(self, "input") self.output_bindings = self.BindingList(self, "output") @@ -329,13 +392,13 @@ def __init__(self): def __str__(self): """ Get configuration in string type""" # Sort moudles - self.sub_module_sort() + self.dag_topology_sort() # get input input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: inf = self.input_bindings.bindings[input_name] - input_dump += " |" + input_name + ": " + inf.get_bindings_str() + "\n" + input_dump += inf.__repr__() + "\n" # get connections output = {} @@ -363,7 +426,11 @@ def __str__(self): def __getitem__(self, key): if (isinstance(key, tvm.ir.module.IRModule)): if key not in self.mod_wrapper: - self.mod_wrapper[key] = self.ModuleWrapper() + # self.last_mod_indx start from 1 and be initialize value, + # the final value for mod index would get generate by function + # dag_topology_sort + self.last_mod_indx += 1 + self.mod_wrapper[key] = self.ModuleWrapper(key, self.last_mod_indx) return self.mod_wrapper[key] @@ -379,7 +446,7 @@ def get_config(self): """ Get configuration in dictionary format.""" # Sort moudles - self.sub_module_sort() + self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: @@ -418,7 +485,9 @@ def get_config(self): return mconfig - def sub_module_sort(self): + def dag_topology_sort(self): + """ Do topology sort to get pipeline module order.""" + mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index c82465202b48..46c2afc6ea1c 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -149,6 +149,9 @@ def pipeline(target): for i in range(5): datas.append(np.full(dshape, 3 + i).astype("float32")) + # Runtime error check + pipe_config_check(mod1, mod2, mod3) + pipe_config = pipeline_executor.PipelineConfig() # Create pipeline compute input/output and subgraph dependent relation. @@ -194,21 +197,7 @@ def pipeline(target): """ # connection correctness veify """ - - """ - # try wrong module order connection check, expect assert. - """ - - #with pytest.raises(AssertionError): - # pipe_config[mod2].output(0).connect(pipe_config[mod1].input("data_0")) - - """ - # try pipeline module input with module output connection check, expect assert. - """ - - #with pytest.raises(AssertionError): - # pipe_config.pipe_input("data_0").connect(pipe_config[mod1].output(0)) - # assert 0, f"wrong global input connect check not pass!" + print(pipe_config) """ # set other parameter. @@ -237,6 +226,47 @@ def pipeline(target): assert pipeline_module +def pipe_config_check(mod1, mod2, mod3): + """ + # try invalid input/output name exepect runtime error + """ + pipe_error = pipeline_executor.PipelineConfig() + with pytest.raises(RuntimeError): + pipe_error[mod1]["output"][9] + + with pytest.raises(RuntimeError): + pipe_error[mod1]["input"]["data_9"] + + """ + # try cirle connection , expect runtime error + """ + with pytest.raises(RuntimeError): + pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) + pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) + + """ + # try wrong module order connection check, expect runtime error. + """ + + with pytest.raises(RuntimeError): + pipe_error[mod1]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) + + with pytest.raises(RuntimeError): + pipe_error[mod1]["input"]["data_0"].connect(pipe_error[mod1]["input"]["data_0"]) + + with pytest.raises(RuntimeError): + pipe_error[mod1]["input"]["data_0"].connect(pipe_error[mod2]["input"]["data_0"]) + + with pytest.raises(RuntimeError): + pipe_error[mod1]["output"][0].connect(pipe_error["input"]["data_0"]) + + with pytest.raises(RuntimeError): + pipe_error["input"]["data_0"].connect(pipe_error[mod1]["output"][0]) + + with pytest.raises(RuntimeError): + pipe_error["output"]["0"].connect(pipe_error[mod1]["output"][0]) + + def test_pipeline(): if pipeline_executor.pipeline_executor_enabled(): target_list = tvm.testing.enabled_targets() From 5c50ba44e80609003706cdecb7caa4c7e065dd03 Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 2 Sep 2021 15:24:12 -0700 Subject: [PATCH 14/34] fix plint error. --- python/tvm/contrib/pipeline_executor.py | 156 +++++++++++-------- tests/python/relay/test_pipeline_executor.py | 64 ++++---- 2 files changed, 120 insertions(+), 100 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 21379268e7ec..4c5f8bf5dd39 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -30,8 +30,7 @@ def pipeline_executor_enabled(): enable: bool Return pipeline executor is enabled or not. """ - return tvm._ffi.get_global_func("tvm.pipeline_executor.create", - allow_missing=True) is not None + return tvm._ffi.get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None def build(pipe_configs): @@ -58,7 +57,6 @@ def build(pipe_configs): mod_n_configs = pipe_configs.get_config() config_len = len(mod_n_configs) string_config = [{} for _ in range(config_len)] - #for _, (ir_mod, mod_config) in enumerate(mod_n_configs.items()): for ir_mod, mod_config in mod_n_configs.items(): mconf = mod_config["pipeline"].copy() mod_indx = mconf["mod_indx"] - 1 @@ -106,6 +104,7 @@ class to storage IRModule list and pipeline configuration. return PipelineModule(pipe_mod_config) + class PipelineModule(object): """Wrapper runtime module. This is a thin wrapper of the underlying TVM module. @@ -121,11 +120,13 @@ def __init__(self, pipe_mod_config): self.pipeline_mods_ = pipe_mod_config.pipeline_mods_ self.mod_config_ = pipe_mod_config.mods_config_ mods, config = self.graph_executor_create(self.pipeline_mods_, self.mod_config_) - assert pipeline_executor_enabled(), \ - "Pipeline executor is not enabled. Please \ + assert ( + pipeline_executor_enabled() + ), "Pipeline executor is not enabled. Please \ re-build TVM with USE_PIPELINE_EXECUTOR=ON" - pipelinecreate = tvm._ffi.get_global_func("tvm.pipeline_executor.create", - allow_missing=False) + pipelinecreate = tvm._ffi.get_global_func( + "tvm.pipeline_executor.create", allow_missing=False + ) assert pipelinecreate module = pipelinecreate(mods, config) @@ -148,8 +149,8 @@ def graph_executor_create(self, pipeline_mods, mod_config): mods : List[GraphModule] Runtime graph module. - mod_config : str - mods configuration + mod_config : str + mods configuration """ mods = [] @@ -161,6 +162,7 @@ def graph_executor_create(self, pipeline_mods, mod_config): return mods, json.dumps(mod_config) + class PipelineConfig(object): """The wrapper of each module to be pipelined. The wrapper mainly includes the module itself as well as the binding that represents the connections of this @@ -190,7 +192,7 @@ class Binding: for output that is integer for example 0. """ - def __init__(self, owner, stype, name, data_type): + def __init__(self, owner, stype, name, data_type=None): self.io_owner = owner self.io_type = stype self.name = str(name) @@ -202,6 +204,7 @@ def __init__(self, owner, stype, name, data_type): self.data_type = data_type def get_name(self): + """get owner name and self name""" owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -209,6 +212,7 @@ def get_name(self): return owner_name, self.name def get_owner_indx(self): + """return index if owner is ModuleWrapper, if not return 0""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.index @@ -217,7 +221,7 @@ def get_owner_indx(self): return 0 def __repr__(self): - # Get all binding(input data), exepect like |data_0: mod1:data_0 + """Get all binding(input data), exepect like |data_0: mod1:data_0""" ret = " |{}: ".format(self.name) for binding in self.bindings: mname, dname = binding.get_name() @@ -225,7 +229,8 @@ def __repr__(self): return ret def dag_acircle_check(self, start, inputs): - for name, binding in inputs.items(): + """check if the DAG that current binding stay is acircle""" + for _, binding in inputs.items(): if start == binding.io_owner: return False for p in binding.parents: @@ -240,6 +245,7 @@ def connect(self, binding): # correct connection are following # 1. global input to module input # 2. module output to global output + # 3. module output to moudle input """ owner_indx = self.get_owner_indx() bind_owner_indx = binding.get_owner_indx() @@ -250,11 +256,11 @@ def connect(self, binding): raise RuntimeError(f"Module only can start binding from output!") if owner_indx != 0 and bind_owner_indx != 0 and binding.io_type == "output": - raise RuntimeError(f"Module output can not binding with module output!") + raise RuntimeError(f"Module output can not binding with module output!") if owner_indx != 0 and bind_owner_indx == 0 and binding.io_type == "input": - raise RuntimeError(f"Module output can not binding with global input!") - + raise RuntimeError(f"Module output can not binding with global input!") + if owner_indx == 0 and self.io_type != "input": raise RuntimeError(f"Global only can start binding from input!") @@ -264,20 +270,24 @@ def connect(self, binding): self.bindings.append(binding) if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): # check if the source and target data_type same - if (isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) - and self.data_type != binding.data_type): + if ( + isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) + and self.data_type != binding.data_type + ): raise RuntimeError(f"Illegal type:binding type is not same!") binding.parents.append(self) - # after - if not self.dag_acircle_check(binding.io_owner, - self.io_owner.input_bindings.bindings): + # Do acircle check after add the in-degree. + if not self.dag_acircle_check( + binding.io_owner, self.io_owner.input_bindings.bindings + ): raise RuntimeError(f"Illegal connection: cause a circle!") - def __init__(self, mod = None, index = 0): + def __init__(self, mod=None, index=0): + """init class""" self.input_params = InferType()(mod)["main"].params self.output_values = InferType()(mod)["main"].checked_type.ret_type - self.set_index(index) + self.set_index_name(index) self.mod = mod self.input_bindings = PipelineConfig.BindingList(self, "input") self.output_bindings = PipelineConfig.BindingList(self, "output") @@ -288,16 +298,19 @@ def __init__(self, mod = None, index = 0): self.dev_ = None def __getitem__(self, key): - if (isinstance(key, str)): - if (key == "input"): + """get item by key""" + if isinstance(key, str): + if key == "input": return self.input_bindings - if (key == "output"): + if key == "output": return self.output_bindings - assert 0, "{} not found!".format(key) + raise RuntimeError(f"{key} not found!") + + def get_data_type(self, key, stype): + """get module input/output data type.""" - def get_type(self, key, stype): if stype == "input": for param in self.input_params: if param.name_hint == key: @@ -308,78 +321,84 @@ def get_type(self, key, stype): if int(key) < len(self.output_values.fields): return self.output_values.fields[int(key)] elif int(key) == 0: - return self.output_values + return self.output_values return None - - def set_index(self, index): + def set_index_name(self, index): + """generate name by index and storage index value""" self.index = index self.name = "mod{}".format(str(index)) def is_root_mod(self): - for name, binding in self.input_bindings.bindings.items(): + """use by DAG topology sort, identify if this item is root item and in-degree is 0""" + for _, binding in self.input_bindings.bindings.items(): if binding.parents: return False return True def remove_self_from_bindings(self): + """use by DAG topology sort, by remove self from binding to reduce child in-degree""" for _, binding in self.output_bindings.bindings.items(): for child in binding.bindings: if binding in child.parents: child.parents.remove(binding) - def input(self, name): - if name not in self.input_bindings: - self.input_bindings[name] = self.Binding(self, "input", name) - - return self.input_bindings[name] - - def output(self, index): - if index not in self.output_bindings: - self.output_bindings[index] = self.Binding(self, "output", index) - - return self.output_bindings[index] - def set_target_host(self, host): + """set target host that use by build function""" self.target_host_ = host def set_build_func(self, build_func): + """set build funciton that use by build function""" self.build_func_ = build_func def set_params(self, params): + """set params that use by build function""" self.params_ = params def set_target(self, target): + """set target that use by build function""" self.target_ = target def set_dev(self, dev): + """set dev that use by build function""" self.dev_ = dev class BindingList: + """Use to storage Binding list. + Parameters + ---------- + + owner : ModuleWrapper/PipelineConfig + who own this list, it can be ModuleWrapper or PipelineConfig + + type_name : str + The type of this binding list. It can be either "input" or "output". + """ + def __init__(self, owner, type_name): self.bindings = {} self.io_owner = owner self.binding_type = type_name - def get_binding_type(self, key): + def get_binding_data_type(self, key): + """return binding data type""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - return self.io_owner.get_type(key, self.binding_type) + return self.io_owner.get_data_type(key, self.binding_type) return None def __getitem__(self, key): + """return item by key""" if key not in self.bindings: - data_type = self.get_binding_type(key) + data_type = self.get_binding_data_type(key) if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): raise RuntimeError(f"Illegal name: {self.binding_type}:{key} cannot find") - self.bindings[key] = \ - PipelineConfig.ModuleWrapper.Binding(self.io_owner, - self.binding_type, - key, - data_type) + self.bindings[key] = PipelineConfig.ModuleWrapper.Binding( + self.io_owner, self.binding_type, key, data_type + ) return self.bindings[key] @@ -424,7 +443,8 @@ def __str__(self): return input_dump + output_dump + connections_dump def __getitem__(self, key): - if (isinstance(key, tvm.ir.module.IRModule)): + """return item by key""" + if isinstance(key, tvm.ir.module.IRModule): if key not in self.mod_wrapper: # self.last_mod_indx start from 1 and be initialize value, # the final value for mod index would get generate by function @@ -434,13 +454,13 @@ def __getitem__(self, key): return self.mod_wrapper[key] - if (isinstance(key, str)): - if (key == "input"): + if isinstance(key, str): + if key == "input": return self.input_bindings - if (key == "output"): + if key == "output": return self.output_bindings - assert 0, "key not find!" + raise RuntimeError(f"{key} not found.") def get_config(self): """ Get configuration in dictionary format.""" @@ -474,14 +494,15 @@ def get_config(self): mconf["output"] = output_conf # build module configuration with pipeline and other parameters. - mconfig[mod] = {"pipeline": mconf, - "target_host": module.target_host_, - "mod_name": "default", - "build": module.build_func_, - "params": module.params_, - "target": module.target_, - "dev": module.dev_, - } + mconfig[mod] = { + "pipeline": mconf, + "target_host": module.target_host_, + "mod_name": "default", + "build": module.build_func_, + "params": module.params_, + "target": module.target_, + "dev": module.dev_, + } return mconfig @@ -503,18 +524,19 @@ def dag_topology_sort(self): mlist += temp_list for mod, i in zip(mlist, range(len(mlist))): - self.mod_wrapper[mod].set_index(i + 1) - - return + self.mod_wrapper[mod].set_index_name(i + 1) def get_mod_indx(self, mod): + """return index for specify mod""" indx = self.mod_wrapper[mod].index return indx def pipe_input(self, name): + """return input binding by name""" return self.input_bindings[name] def pipe_output(self, index): + """return output binding by index""" return self.output_bindings[index] diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 46c2afc6ea1c..4b86d4452c9e 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -99,14 +99,15 @@ def get_manual_conf(mods, target): {"output_indx": 2, "dependent": [{"mod_indx": 0, "input_name": "0"}]}, ], } - mod_config[mods[0]] = {"pipeline": pipe_config1, - "target_host": None, - "mod_name": "default", - "build": None, - "params": None, - "target": target[0], - "dev": target[1], - } + mod_config[mods[0]] = { + "pipeline": pipe_config1, + "target_host": None, + "mod_name": "default", + "build": None, + "params": None, + "target": target[0], + "dev": target[1], + } pipe_config2 = { "mod_indx": 2, @@ -114,27 +115,29 @@ def get_manual_conf(mods, target): {"output_indx": 0, "dependent": [{"mod_indx": 3, "input_name": "data_1"}]}, ], } - mod_config[mods[1]] = {"pipeline": pipe_config2, - "target_host": None, - "mod_name": "default", - "build": None, - "params": None, - "target": "llvm", - "dev": tvm.cpu(0), - } + mod_config[mods[1]] = { + "pipeline": pipe_config2, + "target_host": None, + "mod_name": "default", + "build": None, + "params": None, + "target": "llvm", + "dev": tvm.cpu(0), + } pipe_config3 = { "mod_indx": 3, "output": [{"output_indx": 0, "dependent": [{"mod_indx": 0, "input_name": "1"}]}], } - mod_config[mods[2]] = {"pipeline": pipe_config3, - "target_host": None, - "mod_name": "default", - "build": None, - "params": None, - "target": "llvm", - "dev": tvm.cpu(0), - } + mod_config[mods[2]] = { + "pipeline": pipe_config3, + "target_host": None, + "mod_name": "default", + "build": None, + "params": None, + "target": "llvm", + "dev": tvm.cpu(0), + } return mod_config @@ -172,12 +175,11 @@ def pipeline(target): # mod2 output(0) would get forward to mod3 as input "data_1" pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) - # Testing in API mode for binding find. # mod1 output(2) would get forward as final pipeline compute output(1) - pipe_config[mod1].output_bindings[2].connect(pipe_config.output_bindings["0"]) + pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) # mod3 output(0) would get forward as final pipeline compute output(2) - pipe_config[mod3].output_bindings[0].connect(pipe_config.output_bindings["1"]) + pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) """ # print configueration (print(pipe_config)), the expect result like following. # @@ -194,10 +196,6 @@ def pipeline(target): # |mod1.output(1)-> mod3.data_0 # |mod2.output(0)-> mod3.data_1 """ - """ - # connection correctness veify - """ - print(pipe_config) """ # set other parameter. @@ -241,8 +239,8 @@ def pipe_config_check(mod1, mod2, mod3): # try cirle connection , expect runtime error """ with pytest.raises(RuntimeError): - pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) - pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) + pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) + pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) """ # try wrong module order connection check, expect runtime error. From 21a3064027a1526b7dce39bbf5814b9b58168bc4 Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 2 Sep 2021 17:29:39 -0700 Subject: [PATCH 15/34] Update python/tvm/contrib/pipeline_executor.py Co-authored-by: Cody Yu --- python/tvm/contrib/pipeline_executor.py | 12 ++++++------ tests/python/relay/test_pipeline_executor.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 4c5f8bf5dd39..a16ed61108e4 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -176,10 +176,10 @@ class ModuleWrapper: class Binding: """The class that use to storage module connection information. - There are 2 types Binding Input:1 Output:2 + The binding can be either "input" or "output". + Parameters ---------- - owner : ModuleWrapper The class that own this interface, in such class there are Module information like index, module name @@ -188,8 +188,8 @@ class Binding: The type of this binding. It can be either "input" or "output". name : str/integer - Binding name, for input that is string for example "data0" - for output that is integer for example 0. + Binding name, for input it is string such as "data0"; + for output it is the index integer such as 0. """ def __init__(self, owner, stype, name, data_type=None): @@ -230,7 +230,7 @@ def __repr__(self): def dag_acircle_check(self, start, inputs): """check if the DAG that current binding stay is acircle""" - for _, binding in inputs.items(): + for binding in inputs.values(): if start == binding.io_owner: return False for p in binding.parents: @@ -274,7 +274,7 @@ def connect(self, binding): isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) and self.data_type != binding.data_type ): - raise RuntimeError(f"Illegal type:binding type is not same!") + raise RuntimeError(f"Illegal type (%s vs. %s): binding type is not same!" % (self.data_type, binding.data_type)) binding.parents.append(self) # Do acircle check after add the in-degree. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 4b86d4452c9e..503084784eb7 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -273,4 +273,4 @@ def test_pipeline(): if __name__ == "__main__": - test_pipeline() + pytest.main([__file__]) From 04985614848fc1507930308c67969f88d37c917f Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 2 Sep 2021 18:46:36 -0700 Subject: [PATCH 16/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 158 ++++++++-------- tests/python/relay/test_pipeline_executor.py | 179 +++++++++---------- 2 files changed, 159 insertions(+), 178 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index a16ed61108e4..5447d9c444a2 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -38,20 +38,13 @@ def build(pipe_configs): Parameters ---------- - mod_n_configs: Dict[IRModule, Dict[str, Any]] - build configuration informaton, structure like following. - {IRModule: {"target":target, - "target_host":target_host, - "params":params, - "mod_name"mod_name, - "build":build}} + pipe_configs: PipelineConfig + build configuration informaton. Returns ------- - ret: List[IRModule] - list of IRModule - string_config: Dict[int, Dict[str, any]] - pipeline configuration + ret: PipeModuleConfig + the class that wrap module list and configuration. """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -59,7 +52,7 @@ def build(pipe_configs): string_config = [{} for _ in range(config_len)] for ir_mod, mod_config in mod_n_configs.items(): mconf = mod_config["pipeline"].copy() - mod_indx = mconf["mod_indx"] - 1 + mod_idx = mconf["mod_idx"] - 1 # Get mod device config dev = mod_config["dev"] target = mod_config["target"] @@ -79,12 +72,12 @@ def build(pipe_configs): mconf["dev"] = "{},{}".format(dev.device_type, dev.device_id) # Create pipeline configuration - string_config[mod_indx] = mconf + string_config[mod_idx] = mconf # associate mod with device mods[mod] = {"dev": dev} - # return PipeModuleConfig - return PipeModuleConfig(mods, string_config) + # return PipelineExecutorFactoryModule + return PipelineExecutorFactoryModule(mods, string_config) def create(pipe_mod_config): @@ -170,7 +163,7 @@ class PipelineConfig(object): """ class ModuleWrapper: - """The class use use to represent Module and storage module index and + """The class use use to represent Module and storage module idx and Binding information. """ @@ -182,14 +175,14 @@ class Binding: ---------- owner : ModuleWrapper The class that own this interface, in such class there are - Module information like index, module name + Module information like idx, module name io_type : str The type of this binding. It can be either "input" or "output". name : str/integer Binding name, for input it is string such as "data0"; - for output it is the index integer such as 0. + for output it is the idx integer such as 0. """ def __init__(self, owner, stype, name, data_type=None): @@ -211,15 +204,19 @@ def get_name(self): return owner_name, self.name - def get_owner_indx(self): - """return index if owner is ModuleWrapper, if not return 0""" + def get_owner_idx(self): + """return idx if owner is ModuleWrapper, if not return 0""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - return self.io_owner.index + return self.io_owner.idx # if not ModuleWrapper then owner is PipelineConfig, return 0 # to identify this is global interface return 0 + def is_global_interface(self): + """check if this interface is global""" + return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) + def __repr__(self): """Get all binding(input data), exepect like |data_0: mod1:data_0""" ret = " |{}: ".format(self.name) @@ -228,13 +225,13 @@ def __repr__(self): ret += "{0}:{1} ".format(mname, dname) return ret - def dag_acircle_check(self, start, inputs): + def check_dag_acyclic(self, start, inputs): """check if the DAG that current binding stay is acircle""" for binding in inputs.values(): if start == binding.io_owner: return False for p in binding.parents: - if not self.dag_acircle_check(start, p.io_owner.input_bindings.bindings): + if not self.check_dag_acyclic(start, p.io_owner.input_bindings.bindings): return False return True @@ -247,28 +244,30 @@ def connect(self, binding): # 2. module output to global output # 3. module output to moudle input """ - owner_indx = self.get_owner_indx() - bind_owner_indx = binding.get_owner_indx() - if owner_indx == bind_owner_indx: + if self.io_owner == binding.io_owner: raise RuntimeError(f"can not set self as binding.") - if owner_indx != 0 and self.io_type == "input": + if not self.is_global_interface() and self.io_type == "input": raise RuntimeError(f"Module only can start binding from output!") - if owner_indx != 0 and bind_owner_indx != 0 and binding.io_type == "output": + if (not self.is_global_interface() and + not binding.is_global_interface() and binding.io_type == "output" + ): raise RuntimeError(f"Module output can not binding with module output!") - if owner_indx != 0 and bind_owner_indx == 0 and binding.io_type == "input": + if (not self.is_global_interface() and + binding.is_global_interface() and binding.io_type == "input" + ): raise RuntimeError(f"Module output can not binding with global input!") - if owner_indx == 0 and self.io_type != "input": + if self.is_global_interface() and self.io_type != "input": raise RuntimeError(f"Global only can start binding from input!") - if owner_indx == 0 and binding.io_type != "input": + if self.is_global_interface() and binding.io_type != "input": raise RuntimeError(f"Global input only can set binding with module input.") self.bindings.append(binding) - if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + if not self.is_global_interface(): # check if the source and target data_type same if ( isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) @@ -278,24 +277,32 @@ def connect(self, binding): binding.parents.append(self) # Do acircle check after add the in-degree. - if not self.dag_acircle_check( + if not self.check_dag_acyclic( binding.io_owner, self.io_owner.input_bindings.bindings ): raise RuntimeError(f"Illegal connection: cause a circle!") - def __init__(self, mod=None, index=0): + def __init__(self, mod=None): """init class""" self.input_params = InferType()(mod)["main"].params self.output_values = InferType()(mod)["main"].checked_type.ret_type - self.set_index_name(index) + self.idx = None + self.name = None self.mod = mod self.input_bindings = PipelineConfig.BindingList(self, "input") self.output_bindings = PipelineConfig.BindingList(self, "output") - self.target_host_ = None - self.build_func_ = None - self.params_ = None - self.target_ = None - self.dev_ = None + self.target_host = None + self.build_func = None + self.params = None + self.target = None + self.dev = None + + def __eq__(self, other): + """check if self equl other""" + if isinstance(other, PipelineConfig.ModuleWrapper): + return self.mod == other.mod + + return False def __getitem__(self, key): """get item by key""" @@ -325,10 +332,10 @@ def get_data_type(self, key, stype): return None - def set_index_name(self, index): - """generate name by index and storage index value""" - self.index = index - self.name = "mod{}".format(str(index)) + def set_idx_name(self, idx): + """generate name by idx and storage idx value""" + self.idx = idx + self.name = "mod{}".format(str(idx)) def is_root_mod(self): """use by DAG topology sort, identify if this item is root item and in-degree is 0""" @@ -345,26 +352,6 @@ def remove_self_from_bindings(self): if binding in child.parents: child.parents.remove(binding) - def set_target_host(self, host): - """set target host that use by build function""" - self.target_host_ = host - - def set_build_func(self, build_func): - """set build funciton that use by build function""" - self.build_func_ = build_func - - def set_params(self, params): - """set params that use by build function""" - self.params_ = params - - def set_target(self, target): - """set target that use by build function""" - self.target_ = target - - def set_dev(self, dev): - """set dev that use by build function""" - self.dev_ = dev - class BindingList: """Use to storage Binding list. Parameters @@ -403,7 +390,6 @@ def __getitem__(self, key): return self.bindings[key] def __init__(self): - self.last_mod_indx = 0 self.mod_wrapper = {} self.input_bindings = self.BindingList(self, "input") self.output_bindings = self.BindingList(self, "output") @@ -446,11 +432,7 @@ def __getitem__(self, key): """return item by key""" if isinstance(key, tvm.ir.module.IRModule): if key not in self.mod_wrapper: - # self.last_mod_indx start from 1 and be initialize value, - # the final value for mod index would get generate by function - # dag_topology_sort - self.last_mod_indx += 1 - self.mod_wrapper[key] = self.ModuleWrapper(key, self.last_mod_indx) + self.mod_wrapper[key] = self.ModuleWrapper(key) return self.mod_wrapper[key] @@ -481,27 +463,27 @@ def get_config(self): for dep in binding.bindings: dep_item = {} _, dname = dep.get_name() - dep_item["mod_indx"] = dep.get_owner_indx() + dep_item["mod_idx"] = dep.get_owner_idx() dep_item["input_name"] = dname dep_conf.append(dep_item) - # ouput_indx start from 0. + # ouput_idx start from 0. - output["output_indx"] = int(binding.name) + output["output_idx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) - mconf["mod_indx"] = module.index + mconf["mod_idx"] = module.idx mconf["output"] = output_conf # build module configuration with pipeline and other parameters. mconfig[mod] = { "pipeline": mconf, - "target_host": module.target_host_, + "target_host": module.target_host, "mod_name": "default", - "build": module.build_func_, - "params": module.params_, - "target": module.target_, - "dev": module.dev_, + "build": module.build_func, + "params": module.params, + "target": module.target, + "dev": module.dev, } return mconfig @@ -524,23 +506,23 @@ def dag_topology_sort(self): mlist += temp_list for mod, i in zip(mlist, range(len(mlist))): - self.mod_wrapper[mod].set_index_name(i + 1) + self.mod_wrapper[mod].set_idx_name(i + 1) - def get_mod_indx(self, mod): - """return index for specify mod""" - indx = self.mod_wrapper[mod].index - return indx + def get_mod_idx(self, mod): + """return idx for specify mod""" + idx = self.mod_wrapper[mod].idx + return idx def pipe_input(self, name): """return input binding by name""" return self.input_bindings[name] - def pipe_output(self, index): - """return output binding by index""" - return self.output_bindings[index] + def pipe_output(self, idx): + """return output binding by idx""" + return self.output_bindings[idx] -class PipeModuleConfig(object): +class PipelineExecutorFactoryModule(object): """This class use to storage pipeline IRModule and configurations. Parameters diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 503084784eb7..1cf3d90b475f 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -92,11 +92,11 @@ def get_manual_conf(mods, target): # input """ pipe_config1 = { - "mod_indx": 1, + "mod_idx": 1, "output": [ - {"output_indx": 0, "dependent": [{"mod_indx": 2, "input_name": "data_0"}]}, - {"output_indx": 1, "dependent": [{"mod_indx": 3, "input_name": "data_0"}]}, - {"output_indx": 2, "dependent": [{"mod_indx": 0, "input_name": "0"}]}, + {"output_idx": 0, "dependent": [{"mod_idx": 2, "input_name": "data_0"}]}, + {"output_idx": 1, "dependent": [{"mod_idx": 3, "input_name": "data_0"}]}, + {"output_idx": 2, "dependent": [{"mod_idx": 0, "input_name": "0"}]}, ], } mod_config[mods[0]] = { @@ -110,9 +110,9 @@ def get_manual_conf(mods, target): } pipe_config2 = { - "mod_indx": 2, + "mod_idx": 2, "output": [ - {"output_indx": 0, "dependent": [{"mod_indx": 3, "input_name": "data_1"}]}, + {"output_idx": 0, "dependent": [{"mod_idx": 3, "input_name": "data_1"}]}, ], } mod_config[mods[1]] = { @@ -126,8 +126,8 @@ def get_manual_conf(mods, target): } pipe_config3 = { - "mod_indx": 3, - "output": [{"output_indx": 0, "dependent": [{"mod_indx": 0, "input_name": "1"}]}], + "mod_idx": 3, + "output": [{"output_idx": 0, "dependent": [{"mod_idx": 0, "input_name": "1"}]}], } mod_config[mods[2]] = { "pipeline": pipe_config3, @@ -141,90 +141,11 @@ def get_manual_conf(mods, target): return mod_config -def pipeline(target): +def test_pipe_config_check(): """ #Get 3 pipeline module. """ (mod1, mod2, mod3), dshape = get_mannual_mod() - - # Prepare batch data for pipeline feeding - datas = [] - for i in range(5): - datas.append(np.full(dshape, 3 + i).astype("float32")) - - # Runtime error check - pipe_config_check(mod1, mod2, mod3) - - pipe_config = pipeline_executor.PipelineConfig() - - # Create pipeline compute input/output and subgraph dependent relation. - - # Test in key mode for binding find. - # pipeline compute input "data_0" would get forward to mod1 as input "data_0" - pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) - - # pipeline compute input "data_1" would get forward to mod2 as input "data_1" - pipe_config["input"]["data_1"].connect(pipe_config[mod2]["input"]["data_1"]) - - # mod1 output(0) would get forward to mod2 as input "data_0" - pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) - - # mod1 output(1) would get forward to mod3 as input "data_0" - pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) - - # mod2 output(0) would get forward to mod3 as input "data_1" - pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) - - # mod1 output(2) would get forward as final pipeline compute output(1) - pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) - - # mod3 output(0) would get forward as final pipeline compute output(2) - pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) - """ - # print configueration (print(pipe_config)), the expect result like following. - # - #Inputs - # |data_0: mod1:data_0 - # |data_1: mod2:data_1 - # - #output - # |output(1) : mod1.output(2) - # |output(2) : mod3.output(0) - # - #connections - # |mod1.output(0)-> mod2.data_0 - # |mod1.output(1)-> mod3.data_0 - # |mod2.output(0)-> mod3.data_1 - """ - - """ - # set other parameter. - """ - pipe_config[mod1].set_target(target[0]) - pipe_config[mod1].set_dev(target[1]) - - pipe_config[mod2].set_target("llvm") - pipe_config[mod2].set_dev(tvm.cpu(0)) - - pipe_config[mod3].set_target("llvm") - pipe_config[mod3].set_dev(tvm.cpu(0)) - - """ - # check if the configuration match expectation. - """ - assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) - - """ - # Test build and create pipeline module - """ - with relay.build_config(opt_level=3): - pipeline_mod_config = pipeline_executor.build(pipe_config) - - pipeline_module = pipeline_executor.create(pipeline_mod_config) - assert pipeline_module - - -def pipe_config_check(mod1, mod2, mod3): """ # try invalid input/output name exepect runtime error """ @@ -269,8 +190,86 @@ def test_pipeline(): if pipeline_executor.pipeline_executor_enabled(): target_list = tvm.testing.enabled_targets() for target in target_list: - pipeline(target) + """ + #Get 3 pipeline module. + """ + (mod1, mod2, mod3), dshape = get_mannual_mod() + + # Prepare batch data for pipeline feeding + datas = [] + for i in range(5): + datas.append(np.full(dshape, 3 + i).astype("float32")) + + pipe_config = pipeline_executor.PipelineConfig() + + # Create pipeline compute input/output and subgraph dependent relation. + + # Test in key mode for binding find. + # pipeline compute input "data_0" would get forward to mod1 as input "data_0" + pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) + + # pipeline compute input "data_1" would get forward to mod2 as input "data_1" + pipe_config["input"]["data_1"].connect(pipe_config[mod2]["input"]["data_1"]) + + # mod1 output(0) would get forward to mod2 as input "data_0" + pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) + + # mod1 output(1) would get forward to mod3 as input "data_0" + pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) + + # mod2 output(0) would get forward to mod3 as input "data_1" + pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) + + # mod1 output(2) would get forward as final pipeline compute output(1) + pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) + + # mod3 output(0) would get forward as final pipeline compute output(2) + pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) + """ + # print configueration (print(pipe_config)), the expect result like following. + # + #Inputs + # |data_0: mod1:data_0 + # |data_1: mod2:data_1 + # + #output + # |output(1) : mod1.output(2) + # |output(2) : mod3.output(0) + # + #connections + # |mod1.output(0)-> mod2.data_0 + # |mod1.output(1)-> mod3.data_0 + # |mod2.output(0)-> mod3.data_1 + """ + + """ + # set other parameter. + """ + pipe_config[mod1].target = target[0] + pipe_config[mod1].dev = target[1] + + pipe_config[mod2].target = "llvm" + pipe_config[mod2].dev = tvm.cpu(0) + + pipe_config[mod3].target = "llvm" + pipe_config[mod3].dev = tvm.cpu(0) + + """ + # check if the configuration match expectation. + """ + assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) + + """ + # Test build and create pipeline module + """ + with tvm.transform.PassContext(opt_level=3): + pipeline_mod_config = pipeline_executor.build(pipe_config) + + pipeline_module = pipeline_executor.create(pipeline_mod_config) + assert pipeline_module if __name__ == "__main__": - pytest.main([__file__]) + #pytest.main([__file__]) + test_pipeline() + test_pipe_config_check() From 814651387ea1e67ce8066821151e343d2b60a670 Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 2 Sep 2021 21:33:03 -0700 Subject: [PATCH 17/34] fix plint issue. --- python/tvm/contrib/pipeline_executor.py | 53 +++++++++----------- tests/python/relay/test_pipeline_executor.py | 4 +- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 5447d9c444a2..6fb4a4e3eeef 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -85,10 +85,8 @@ def create(pipe_mod_config): Parameters ---------- - pipe_mod_config : PipeModuleConfig class to storage IRModule list and pipeline configuration. - ------- Returns submodule : PipelineModule @@ -130,7 +128,6 @@ def graph_executor_create(self, pipeline_mods, mod_config): Parameters ---------- - pipeline_mods : List[IRModule] list of IRModule @@ -250,13 +247,17 @@ def connect(self, binding): if not self.is_global_interface() and self.io_type == "input": raise RuntimeError(f"Module only can start binding from output!") - if (not self.is_global_interface() and - not binding.is_global_interface() and binding.io_type == "output" + if ( + not self.is_global_interface() + and not binding.is_global_interface() + and binding.io_type == "output" ): raise RuntimeError(f"Module output can not binding with module output!") - if (not self.is_global_interface() and - binding.is_global_interface() and binding.io_type == "input" + if ( + not self.is_global_interface() + and binding.is_global_interface() + and binding.io_type == "input" ): raise RuntimeError(f"Module output can not binding with global input!") @@ -273,10 +274,13 @@ def connect(self, binding): isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) and self.data_type != binding.data_type ): - raise RuntimeError(f"Illegal type (%s vs. %s): binding type is not same!" % (self.data_type, binding.data_type)) + raise RuntimeError( + f"Illegal type (%s vs. %s): binding type is not same!" + % (self.data_type, binding.data_type) + ) binding.parents.append(self) - # Do acircle check after add the in-degree. + # Do acyclic check after increase the in-degree. if not self.check_dag_acyclic( binding.io_owner, self.io_owner.input_bindings.bindings ): @@ -284,18 +288,18 @@ def connect(self, binding): def __init__(self, mod=None): """init class""" - self.input_params = InferType()(mod)["main"].params - self.output_values = InferType()(mod)["main"].checked_type.ret_type - self.idx = None - self.name = None - self.mod = mod - self.input_bindings = PipelineConfig.BindingList(self, "input") - self.output_bindings = PipelineConfig.BindingList(self, "output") self.target_host = None self.build_func = None self.params = None self.target = None + self.name = None self.dev = None + self.idx = None + self.mod = mod + self.input_params = InferType()(mod)["main"].params + self.output_values = InferType()(mod)["main"].checked_type.ret_type + self.input_bindings = PipelineConfig.BindingList(self, "input") + self.output_bindings = PipelineConfig.BindingList(self, "output") def __eq__(self, other): """check if self equl other""" @@ -317,7 +321,6 @@ def __getitem__(self, key): def get_data_type(self, key, stype): """get module input/output data type.""" - if stype == "input": for param in self.input_params: if param.name_hint == key: @@ -338,7 +341,7 @@ def set_idx_name(self, idx): self.name = "mod{}".format(str(idx)) def is_root_mod(self): - """use by DAG topology sort, identify if this item is root item and in-degree is 0""" + """use by dag topology sort, identify if this item is root item""" for _, binding in self.input_bindings.bindings.items(): if binding.parents: return False @@ -354,9 +357,9 @@ def remove_self_from_bindings(self): class BindingList: """Use to storage Binding list. + Parameters ---------- - owner : ModuleWrapper/PipelineConfig who own this list, it can be ModuleWrapper or PipelineConfig @@ -373,7 +376,6 @@ def get_binding_data_type(self, key): """return binding data type""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.get_data_type(key, self.binding_type) - return None def __getitem__(self, key): @@ -396,9 +398,8 @@ def __init__(self): def __str__(self): """ Get configuration in string type""" - # Sort moudles + # topology sort to get correct module order in list. self.dag_topology_sort() - # get input input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: @@ -433,7 +434,6 @@ def __getitem__(self, key): if isinstance(key, tvm.ir.module.IRModule): if key not in self.mod_wrapper: self.mod_wrapper[key] = self.ModuleWrapper(key) - return self.mod_wrapper[key] if isinstance(key, str): @@ -447,9 +447,8 @@ def __getitem__(self, key): def get_config(self): """ Get configuration in dictionary format.""" - # Sort moudles + # topology sort to get correct module order in list. self.dag_topology_sort() - mconfig = {} for mod in self.mod_wrapper: # get pipeline configure @@ -468,10 +467,10 @@ def get_config(self): dep_conf.append(dep_item) # ouput_idx start from 0. - output["output_idx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) + mconf["mod_idx"] = module.idx mconf["output"] = output_conf @@ -490,7 +489,6 @@ def get_config(self): def dag_topology_sort(self): """ Do topology sort to get pipeline module order.""" - mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: @@ -527,7 +525,6 @@ class PipelineExecutorFactoryModule(object): Parameters ---------- - pipeline_mods : List[IRModule] list of IRModule diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 1cf3d90b475f..8d4136bd5413 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -270,6 +270,4 @@ def test_pipeline(): if __name__ == "__main__": - #pytest.main([__file__]) - test_pipeline() - test_pipe_config_check() + pytest.main([__file__]) From a5c5215a1d1175b42d847c4a36a84295a46525a8 Mon Sep 17 00:00:00 2001 From: huajsj Date: Fri, 3 Sep 2021 12:34:15 -0700 Subject: [PATCH 18/34] address review comments. --- src/runtime/pipeline/pipeline_executor.cc | 7 +++++++ src/runtime/pipeline/pipeline_executor.h | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc index 7edb742d50d8..80e9286b5bfd 100644 --- a/src/runtime/pipeline/pipeline_executor.cc +++ b/src/runtime/pipeline/pipeline_executor.cc @@ -30,6 +30,13 @@ void PipelineRuntime::Init(const Array& modules, return; } +/* GetFunction can not be pure abstract function, implement a empty function for initial. + */ +PackedFunc PipelineRuntime::GetFunction(const std::string& name, + const ObjectPtr& sptr_to_self) { + return nullptr; +} + Module PipelineRuntimeCreate(const Array& m, const std::string& pipeline_json) { auto exec = make_object(); diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index 49ce62ac01c3..80e4aa564c81 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -25,16 +25,9 @@ #define TVM_RUNTIME_PIPELINE_PIPELINE_EXECUTOR_H_ #include -#include #include -#include -#include - -#include "../file_utils.h" -using namespace std; namespace tvm { namespace runtime { - /*! * \brief pipeline runtime. * @@ -59,6 +52,13 @@ class TVM_DLL PipelineRuntime : public ModuleNode { * which is not compatible with RPCModules. */ void Init(const Array& modules, const std::string& pipeline_json); + /*! + * \brief Get member function to front-end + * \param name The name of the function. + * \param sptr_to_self The pointer to the module node. + * \return The corresponding member function. + */ + virtual PackedFunc GetFunction(const std::string& name, const ObjectPtr& sptr_to_self); }; } // namespace runtime } // namespace tvm From f75ea731ab21ad5cf877886d6aef38c33fe849bb Mon Sep 17 00:00:00 2001 From: huajsj Date: Tue, 7 Sep 2021 12:03:45 -0700 Subject: [PATCH 19/34] trigger build. --- src/runtime/pipeline/pipeline_executor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index 80e4aa564c81..7868d8d4ce90 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -53,7 +53,7 @@ class TVM_DLL PipelineRuntime : public ModuleNode { */ void Init(const Array& modules, const std::string& pipeline_json); /*! - * \brief Get member function to front-end + * \brief Get member function to front-end. * \param name The name of the function. * \param sptr_to_self The pointer to the module node. * \return The corresponding member function. From abdded1f4df5615ac80bbd97a91d074ab8afd2d7 Mon Sep 17 00:00:00 2001 From: Hua Jiang Date: Wed, 8 Sep 2021 11:22:54 -0700 Subject: [PATCH 20/34] Update python/tvm/contrib/pipeline_executor.py Co-authored-by: Cody Yu address review comments --- python/tvm/contrib/pipeline_executor.py | 30 +++++++++++-------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 6fb4a4e3eeef..0a737923b3c1 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -43,7 +43,7 @@ def build(pipe_configs): Returns ------- - ret: PipeModuleConfig + ret: PipelineExecutorFactoryModule the class that wrap module list and configuration. """ mods = {} @@ -80,20 +80,20 @@ def build(pipe_configs): return PipelineExecutorFactoryModule(mods, string_config) -def create(pipe_mod_config): +def create(pipe_executor_factory_module): """Create a pipeline runtime executor. Parameters ---------- - pipe_mod_config : PipeModuleConfig - class to storage IRModule list and pipeline configuration. + pipe_executor_factory_module : PipelineExecutorFactoryModule + Executor factory to storage IRModule list and pipeline configuration. Returns submodule : PipelineModule Runtime pipeline module. """ - return PipelineModule(pipe_mod_config) + return PipelineModule(pipe_executor_factory_module) class PipelineModule(object): @@ -342,15 +342,11 @@ def set_idx_name(self, idx): def is_root_mod(self): """use by dag topology sort, identify if this item is root item""" - for _, binding in self.input_bindings.bindings.items(): - if binding.parents: - return False - - return True + return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): """use by DAG topology sort, by remove self from binding to reduce child in-degree""" - for _, binding in self.output_bindings.bindings.items(): + for binding in self.output_bindings.bindings.values(): for child in binding.bindings: if binding in child.parents: child.parents.remove(binding) @@ -383,7 +379,7 @@ def __getitem__(self, key): if key not in self.bindings: data_type = self.get_binding_data_type(key) if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - raise RuntimeError(f"Illegal name: {self.binding_type}:{key} cannot find") + raise RuntimeError(f"Cannot find {key} in binding list {self.binding_type}") self.bindings[key] = PipelineConfig.ModuleWrapper.Binding( self.io_owner, self.binding_type, key, data_type @@ -410,22 +406,22 @@ def __str__(self): output = {} connections_dump = "\nconnections\n" for mod in self.mod_wrapper: - for _, interface in self.mod_wrapper[mod].output_bindings.bindings.items(): + for interface in self.mod_wrapper[mod].output_bindings.bindings.values(): if interface.bindings: mname, dname = interface.get_name() iname = mname + ".output(" + dname + ")->" for dep in interface.bindings: dep_mname, dep_dname = dep.get_name() if isinstance(dep.io_owner, PipelineConfig.ModuleWrapper): - iname += " " + dep_mname + "." + dep_dname - connections_dump += " |" + iname + "\n" + iname += f" {dep_mname}.{dep_dname}" + connections_dump += f" |{iname}\n" else: - output[dep_dname] = mname + ".output(" + dname + ")" + output[dep_dname] = f"{mname}.output({dname})" # get output output_dump = "\noutput\n" for name in sorted(output.keys()): - output_dump += " |output(" + name + ") : " + output[name] + "\n" + output_dump += f" |output({name}) : {output[name]}\n" return input_dump + output_dump + connections_dump From f12b4ef2023fe71ac3dda5df6ec67649bbf2c115 Mon Sep 17 00:00:00 2001 From: huajsj Date: Wed, 8 Sep 2021 16:05:14 -0700 Subject: [PATCH 21/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 343 +++++++++++----------- src/runtime/pipeline/pipeline_executor.cc | 2 +- src/runtime/pipeline/pipeline_executor.h | 23 +- 3 files changed, 182 insertions(+), 186 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 0a737923b3c1..4c26fda3367e 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -76,7 +76,6 @@ def build(pipe_configs): # associate mod with device mods[mod] = {"dev": dev} - # return PipelineExecutorFactoryModule return PipelineExecutorFactoryModule(mods, string_config) @@ -108,18 +107,18 @@ class PipelineModule(object): """ def __init__(self, pipe_mod_config): - self.pipeline_mods_ = pipe_mod_config.pipeline_mods_ - self.mod_config_ = pipe_mod_config.mods_config_ - mods, config = self.graph_executor_create(self.pipeline_mods_, self.mod_config_) + self.pipeline_mods = pipe_mod_config.pipeline_mods + self.mod_config = pipe_mod_config.mods_config + mods, config = self.graph_executor_create(self.pipeline_mods, self.mod_config) assert ( pipeline_executor_enabled() ), "Pipeline executor is not enabled. Please \ re-build TVM with USE_PIPELINE_EXECUTOR=ON" - pipelinecreate = tvm._ffi.get_global_func( + pipeline_create = tvm._ffi.get_global_func( "tvm.pipeline_executor.create", allow_missing=False ) - assert pipelinecreate - module = pipelinecreate(mods, config) + assert pipeline_create + module = pipeline_create(mods, config) self.module_ = module @@ -128,16 +127,16 @@ def graph_executor_create(self, pipeline_mods, mod_config): Parameters ---------- - pipeline_mods : List[IRModule] - list of IRModule + pipeline_mods : List[GraphExecutorFactoryModule] + list of GraphExecutorFactoryModule - mod_config : Dict[int, Dict[str, Any]] - modules and modules dependency configuration informaiton. + mod_config : Dict[str, Any] + modules dependency configuration informaiton. Returns ------- - mods : List[GraphModule] - Runtime graph module. + mods : List[Module] + Module list. mod_config : str mods configuration @@ -159,133 +158,168 @@ class PipelineConfig(object): module's inputs and outputs to other modules. """ - class ModuleWrapper: - """The class use use to represent Module and storage module idx and - Binding information. - """ + class Binding: + """The class that use to storage module connection information. + The binding can be either "input" or "output". - class Binding: - """The class that use to storage module connection information. - The binding can be either "input" or "output". + Parameters + ---------- + owner : ModuleWrapper + The class that own this interface, in such class there are + Module information like idx, module name - Parameters - ---------- - owner : ModuleWrapper - The class that own this interface, in such class there are - Module information like idx, module name + io_type : str + The type of this binding. It can be either "input" or "output". - io_type : str - The type of this binding. It can be either "input" or "output". + name : str/integer + Binding name, for input it is string such as "data0"; + for output it is the idx integer such as 0. + """ - name : str/integer - Binding name, for input it is string such as "data0"; - for output it is the idx integer such as 0. - """ + def __init__(self, owner, stype, name, data_type=None): + self.io_owner = owner + self.io_type = stype + self.name = str(name) + # These item that have dependency relation with self + self.bindings = [] + # The item that self depend + self.parents = [] + + self.data_type = data_type + + def get_name(self): + """get owner name and self name""" + owner_name = "" + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + owner_name = self.io_owner.name + + return owner_name, self.name - def __init__(self, owner, stype, name, data_type=None): - self.io_owner = owner - self.io_type = stype - self.name = str(name) - # These item that have dependency relation with self - self.bindings = [] - # The item that self depend - self.parents = [] - - self.data_type = data_type - - def get_name(self): - """get owner name and self name""" - owner_name = "" - if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - owner_name = self.io_owner.name - - return owner_name, self.name - - def get_owner_idx(self): - """return idx if owner is ModuleWrapper, if not return 0""" - if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - return self.io_owner.idx - - # if not ModuleWrapper then owner is PipelineConfig, return 0 - # to identify this is global interface - return 0 - - def is_global_interface(self): - """check if this interface is global""" - return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) - - def __repr__(self): - """Get all binding(input data), exepect like |data_0: mod1:data_0""" - ret = " |{}: ".format(self.name) - for binding in self.bindings: - mname, dname = binding.get_name() - ret += "{0}:{1} ".format(mname, dname) - return ret - - def check_dag_acyclic(self, start, inputs): - """check if the DAG that current binding stay is acircle""" - for binding in inputs.values(): - if start == binding.io_owner: + def get_owner_idx(self): + """return idx if owner is ModuleWrapper, if not return 0""" + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + return self.io_owner.idx + + # if not ModuleWrapper then owner is PipelineConfig, return 0 + # to identify this is global interface + return 0 + + def is_global_interface(self): + """check if this interface is global""" + return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) + + def __repr__(self): + """Get all binding(input data), exepect like |data_0: mod1:data_0""" + ret = " |{}: ".format(self.name) + for binding in self.bindings: + mname, dname = binding.get_name() + ret += "{0}:{1} ".format(mname, dname) + return ret + + def check_dag_acyclic(self, start, inputs): + """check if the DAG that current binding stay is acircle""" + for binding in inputs.values(): + if start == binding.io_owner: + return False + for p in binding.parents: + if not self.check_dag_acyclic(start, p.io_owner.input_bindings.bindings): return False - for p in binding.parents: - if not self.check_dag_acyclic(start, p.io_owner.input_bindings.bindings): - return False - - return True - - def connect(self, binding): - """ - # check if the bindendency setting correct. - # correct connection are following - # 1. global input to module input - # 2. module output to global output - # 3. module output to moudle input - """ - if self.io_owner == binding.io_owner: - raise RuntimeError(f"can not set self as binding.") - - if not self.is_global_interface() and self.io_type == "input": - raise RuntimeError(f"Module only can start binding from output!") - if ( - not self.is_global_interface() - and not binding.is_global_interface() - and binding.io_type == "output" - ): - raise RuntimeError(f"Module output can not binding with module output!") + return True + def connect(self, binding): + """ + # check if the bindendency setting correct. + # correct connection are following + # 1. global input to module input + # 2. module output to global output + # 3. module output to moudle input + """ + if self.io_owner == binding.io_owner: + raise RuntimeError(f"can not set self as binding.") + + if not self.is_global_interface() and self.io_type == "input": + raise RuntimeError(f"Module only can start binding from output!") + + if ( + not self.is_global_interface() + and not binding.is_global_interface() + and binding.io_type == "output" + ): + raise RuntimeError(f"Module output can not binding with module output!") + + if ( + not self.is_global_interface() + and binding.is_global_interface() + and binding.io_type == "input" + ): + raise RuntimeError(f"Module output can not binding with global input!") + + if self.is_global_interface() and self.io_type != "input": + raise RuntimeError(f"Global only can start binding from input!") + + if self.is_global_interface() and binding.io_type != "input": + raise RuntimeError(f"Global input only can set binding with module input.") + + self.bindings.append(binding) + if not self.is_global_interface(): + # check if the source and target data_type same if ( - not self.is_global_interface() - and binding.is_global_interface() - and binding.io_type == "input" + isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) + and self.data_type != binding.data_type ): - raise RuntimeError(f"Module output can not binding with global input!") - - if self.is_global_interface() and self.io_type != "input": - raise RuntimeError(f"Global only can start binding from input!") - - if self.is_global_interface() and binding.io_type != "input": - raise RuntimeError(f"Global input only can set binding with module input.") - - self.bindings.append(binding) - if not self.is_global_interface(): - # check if the source and target data_type same - if ( - isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) - and self.data_type != binding.data_type - ): - raise RuntimeError( - f"Illegal type (%s vs. %s): binding type is not same!" - % (self.data_type, binding.data_type) - ) - - binding.parents.append(self) - # Do acyclic check after increase the in-degree. - if not self.check_dag_acyclic( - binding.io_owner, self.io_owner.input_bindings.bindings - ): - raise RuntimeError(f"Illegal connection: cause a circle!") + raise RuntimeError( + f"Illegal type (%s vs. %s): binding type is not same!" + % (self.data_type, binding.data_type) + ) + + binding.parents.append(self) + # Do acyclic check after increase the in-degree. + if not self.check_dag_acyclic( + binding.io_owner, self.io_owner.input_bindings.bindings + ): + raise RuntimeError(f"Illegal connection: cause a circle!") + + class BindingList: + """Container for bindings(input or output interface). + + Parameters + ---------- + owner : ModuleWrapper/PipelineConfig + The owner of this list, it can be ModuleWrapper or PipelineConfig + + type_name : str + The type of this binding list. It can be either "input" or "output". + """ + + def __init__(self, owner, type_name): + self.bindings = {} + self.io_owner = owner + self.binding_type = type_name + + def get_binding_data_type(self, key): + """return binding data type""" + if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + return self.io_owner.get_data_type(key, self.binding_type) + return None + + def __getitem__(self, key): + """return item by key""" + if key not in self.bindings: + data_type = self.get_binding_data_type(key) + if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): + raise RuntimeError(f"Cannot find {key} in binding list {self.binding_type}") + + self.bindings[key] = PipelineConfig.Binding( + self.io_owner, self.binding_type, key, data_type + ) + return self.bindings[key] + + class ModuleWrapper: + """Module Wrapper with information like module index, binding information + and building informations. + """ def __init__(self, mod=None): """init class""" self.target_host = None @@ -336,56 +370,21 @@ def get_data_type(self, key, stype): return None def set_idx_name(self, idx): - """generate name by idx and storage idx value""" + """Set index and generating name by index""" self.idx = idx self.name = "mod{}".format(str(idx)) def is_root_mod(self): - """use by dag topology sort, identify if this item is root item""" + """Identify if this item is root item, used by DAG topological sort.""" return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): - """use by DAG topology sort, by remove self from binding to reduce child in-degree""" + """Remove self from binding to reduce child in-degree, used by DAG topological sort.""" for binding in self.output_bindings.bindings.values(): for child in binding.bindings: if binding in child.parents: child.parents.remove(binding) - class BindingList: - """Use to storage Binding list. - - Parameters - ---------- - owner : ModuleWrapper/PipelineConfig - who own this list, it can be ModuleWrapper or PipelineConfig - - type_name : str - The type of this binding list. It can be either "input" or "output". - """ - - def __init__(self, owner, type_name): - self.bindings = {} - self.io_owner = owner - self.binding_type = type_name - - def get_binding_data_type(self, key): - """return binding data type""" - if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - return self.io_owner.get_data_type(key, self.binding_type) - return None - - def __getitem__(self, key): - """return item by key""" - if key not in self.bindings: - data_type = self.get_binding_data_type(key) - if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - raise RuntimeError(f"Cannot find {key} in binding list {self.binding_type}") - - self.bindings[key] = PipelineConfig.ModuleWrapper.Binding( - self.io_owner, self.binding_type, key, data_type - ) - - return self.bindings[key] def __init__(self): self.mod_wrapper = {} @@ -394,7 +393,7 @@ def __init__(self): def __str__(self): """ Get configuration in string type""" - # topology sort to get correct module order in list. + # topological sort to get correct module order in list. self.dag_topology_sort() # get input input_dump = "Inputs\n" @@ -443,7 +442,7 @@ def __getitem__(self, key): def get_config(self): """ Get configuration in dictionary format.""" - # topology sort to get correct module order in list. + # topological sort to get correct module order in list. self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: @@ -484,7 +483,7 @@ def get_config(self): return mconfig def dag_topology_sort(self): - """ Do topology sort to get pipeline module order.""" + """ Do topological sort to get pipeline module order.""" mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: @@ -530,5 +529,5 @@ class PipelineExecutorFactoryModule(object): """ def __init__(self, pipeline_mods, mods_config): - self.pipeline_mods_ = pipeline_mods - self.mods_config_ = mods_config + self.pipeline_mods = pipeline_mods + self.mods_config = mods_config diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc index 80e9286b5bfd..adb8c1c2d21e 100644 --- a/src/runtime/pipeline/pipeline_executor.cc +++ b/src/runtime/pipeline/pipeline_executor.cc @@ -30,7 +30,7 @@ void PipelineRuntime::Init(const Array& modules, return; } -/* GetFunction can not be pure abstract function, implement a empty function for initial. +/* GetFunction can not be pure abstract function, implement an empty function for initial. */ PackedFunc PipelineRuntime::GetFunction(const std::string& name, const ObjectPtr& sptr_to_self) { diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index 7868d8d4ce90..919a477f0f10 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -29,9 +29,12 @@ namespace tvm { namespace runtime { /*! - * \brief pipeline runtime. + * \brief pipeline executor. + * This executor take Module list and Dependency relations of Modules as + * inputs, then execute modules on heterogeneous hardware target in + * pipeline parallism mode to improve processing throughput. * - * This runtime can be acccesibly in various language via + * This executor can be acccesibly in various language via * TVM runtime PackedFunc API. */ class TVM_DLL PipelineRuntime : public ModuleNode { @@ -41,22 +44,16 @@ class TVM_DLL PipelineRuntime : public ModuleNode { */ const char* type_key() const final { return "PipelineRuntime"; } /*! - * \brief Initialize the graph executor with graph and context. - * \param graph_json The execution graph. - * \param module The module containing the compiled functions for the host - * processor. - * \param ctxs The context of the host and devices where graph nodes will be - * executed on. - * \param lookup_linked_param_func If given, a PackedFunc invoked to lookup linked parameters - * by storage_id. If not given, linked parameters are looked-up using an internal implementation, - * which is not compatible with RPCModules. + * \brief Initialize the pipeline executor with module Array and json text. + * \param modules The module list that use for pipeline build. + * \param pipeline_json The configuration for module dependent in pipeline. */ void Init(const Array& modules, const std::string& pipeline_json); /*! - * \brief Get member function to front-end. + * \brief Give frontends an access to packed functions. * \param name The name of the function. * \param sptr_to_self The pointer to the module node. - * \return The corresponding member function. + * \return The corresponding packed functions. */ virtual PackedFunc GetFunction(const std::string& name, const ObjectPtr& sptr_to_self); }; From ce25b283a37557b4757dcf186cb347ef8f4321e2 Mon Sep 17 00:00:00 2001 From: huajsj Date: Wed, 8 Sep 2021 17:35:19 -0700 Subject: [PATCH 22/34] polish doc and comments. --- python/tvm/contrib/pipeline_executor.py | 62 ++++++++++---------- src/runtime/pipeline/pipeline_executor.h | 2 +- tests/python/relay/test_pipeline_executor.py | 38 ++++++------ 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 4c26fda3367e..aec3961e26d6 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -23,7 +23,7 @@ def pipeline_executor_enabled(): - """check if pipeline executor is enabled. + """Check if pipeline executor get enabled. Return ------- @@ -34,7 +34,8 @@ def pipeline_executor_enabled(): def build(pipe_configs): - """build module list that can use for pipeline execution. + """Build IRModule with configurations of pipe_configs then + return Module list and Module dependency configuration. Parameters ---------- @@ -44,7 +45,7 @@ def build(pipe_configs): Returns ------- ret: PipelineExecutorFactoryModule - the class that wrap module list and configuration. + the class that wrap module list and module dependency configuration. """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -85,9 +86,10 @@ def create(pipe_executor_factory_module): Parameters ---------- pipe_executor_factory_module : PipelineExecutorFactoryModule - Executor factory to storage IRModule list and pipeline configuration. + Executor factory with IRModule list and pipeline configuration. Returns + ------- submodule : PipelineModule Runtime pipeline module. """ @@ -96,13 +98,11 @@ def create(pipe_executor_factory_module): class PipelineModule(object): - """Wrapper runtime module. This is a thin wrapper of the underlying TVM module. + """Wrapper of runtime module. Parameters ---------- - pipeline_mods : List[GraphModule] - The internal tvm module that holds the actual graph functions. - pipeline_config : Dict[IRModule, Dict[str, Any]] + pipeline_config : Dict[GraphExecutorFactoryModule, Dict[str, Any]] modules and modules dependency configuration informaiton. """ @@ -123,7 +123,7 @@ def __init__(self, pipe_mod_config): self.module_ = module def graph_executor_create(self, pipeline_mods, mod_config): - """Create graph_executor list and return string format config. + """Create graph_executor list and return text format configuration. Parameters ---------- @@ -188,7 +188,7 @@ def __init__(self, owner, stype, name, data_type=None): self.data_type = data_type def get_name(self): - """get owner name and self name""" + """Get owner name and self name""" owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -196,16 +196,16 @@ def get_name(self): return owner_name, self.name def get_owner_idx(self): - """return idx if owner is ModuleWrapper, if not return 0""" + """Return owner idex if owner is ModuleWrapper, if not return 0""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.idx - # if not ModuleWrapper then owner is PipelineConfig, return 0 + # If not ModuleWrapper then owner is PipelineConfig, return 0 # to identify this is global interface return 0 def is_global_interface(self): - """check if this interface is global""" + """Check if this interface is global interface.""" return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) def __repr__(self): @@ -217,7 +217,7 @@ def __repr__(self): return ret def check_dag_acyclic(self, start, inputs): - """check if the DAG that current binding stay is acircle""" + """Check if the DAG that current binding stay is acircle""" for binding in inputs.values(): if start == binding.io_owner: return False @@ -229,7 +229,7 @@ def check_dag_acyclic(self, start, inputs): def connect(self, binding): """ - # check if the bindendency setting correct. + # Check if the bindendency setting correct. # correct connection are following # 1. global input to module input # 2. module output to global output @@ -320,8 +320,9 @@ class ModuleWrapper: """Module Wrapper with information like module index, binding information and building informations. """ + def __init__(self, mod=None): - """init class""" + """Init class""" self.target_host = None self.build_func = None self.params = None @@ -336,14 +337,14 @@ def __init__(self, mod=None): self.output_bindings = PipelineConfig.BindingList(self, "output") def __eq__(self, other): - """check if self equl other""" + """Check if self equl other""" if isinstance(other, PipelineConfig.ModuleWrapper): return self.mod == other.mod return False def __getitem__(self, key): - """get item by key""" + """Get item by key""" if isinstance(key, str): if key == "input": return self.input_bindings @@ -354,7 +355,7 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found!") def get_data_type(self, key, stype): - """get module input/output data type.""" + """Get module input/output data type.""" if stype == "input": for param in self.input_params: if param.name_hint == key: @@ -385,7 +386,6 @@ def remove_self_from_bindings(self): if binding in child.parents: child.parents.remove(binding) - def __init__(self): self.mod_wrapper = {} self.input_bindings = self.BindingList(self, "input") @@ -425,7 +425,7 @@ def __str__(self): return input_dump + output_dump + connections_dump def __getitem__(self, key): - """return item by key""" + """Return item by key""" if isinstance(key, tvm.ir.module.IRModule): if key not in self.mod_wrapper: self.mod_wrapper[key] = self.ModuleWrapper(key) @@ -442,11 +442,11 @@ def __getitem__(self, key): def get_config(self): """ Get configuration in dictionary format.""" - # topological sort to get correct module order in list. + # Tpological sort to get correct module order in list. self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: - # get pipeline configure + # Get pipeline configure. mconf = {} output_conf = [] module = self.mod_wrapper[mod] @@ -461,7 +461,7 @@ def get_config(self): dep_item["input_name"] = dname dep_conf.append(dep_item) - # ouput_idx start from 0. + # Ouput_idx start from 0. output["output_idx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) @@ -469,7 +469,7 @@ def get_config(self): mconf["mod_idx"] = module.idx mconf["output"] = output_conf - # build module configuration with pipeline and other parameters. + # Build module configuration with pipeline and other parameters. mconfig[mod] = { "pipeline": mconf, "target_host": module.target_host, @@ -502,26 +502,26 @@ def dag_topology_sort(self): self.mod_wrapper[mod].set_idx_name(i + 1) def get_mod_idx(self, mod): - """return idx for specify mod""" + """Return idx for specify mod""" idx = self.mod_wrapper[mod].idx return idx def pipe_input(self, name): - """return input binding by name""" + """Return input binding by name""" return self.input_bindings[name] def pipe_output(self, idx): - """return output binding by idx""" + """Return output binding by idx""" return self.output_bindings[idx] class PipelineExecutorFactoryModule(object): - """This class use to storage pipeline IRModule and configurations. + """This is a factory with list of GraphExecutorFactoryModule and Module configurations. Parameters ---------- - pipeline_mods : List[IRModule] - list of IRModule + pipeline_mods : List[GraphExecutorFactoryModule] + list of GraphExecutorFactoryModule mod_config : Dict[int, Dict[str, Any]] modules and modules dependency configuration informaiton. diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index 919a477f0f10..464157c84b92 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -32,7 +32,7 @@ namespace runtime { * \brief pipeline executor. * This executor take Module list and Dependency relations of Modules as * inputs, then execute modules on heterogeneous hardware target in - * pipeline parallism mode to improve processing throughput. + * pipeline parallism mode to improve processing throughput. * * This executor can be acccesibly in various language via * TVM runtime PackedFunc API. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 8d4136bd5413..6d9fa0467528 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -26,7 +26,7 @@ def get_mannual_mod(): """ - # get list of module that represent a subgraph + # Get list of module that represent a subgraph. """ mods = [] dshape = (3, 3) @@ -43,7 +43,7 @@ def get_mannual_mod(): mv3 = relay.Constant(tvm.nd.array(mvalue3)) """ - # net1 have three output, output3 is final output. + # Net1 have three output, output3 is final output. """ net_output1 = relay.add(data, mv1) @@ -51,14 +51,14 @@ def get_mannual_mod(): net_output3 = relay.multiply(data, mv3) """ - # net2 use net1 output1 as input. + # Net2 use net1 output1 as input. """ net2 = relay.add(data_net1_output_1, mv2) net2 = relay.add(net2, data21) net2 = relay.add(net2, mv3) """ - # net3 use net2 output1 and net1 outpu2 as input. + # Net3 use net2 output1 and net1 outpu2 as input. """ net3 = relay.multiply(data_net2_output_1, mv3) net3 = relay.add(net3, data_net1_output_2) @@ -84,11 +84,11 @@ def get_manual_conf(mods, target): """ mod_config = {} """ - # set configure + # Set configure """ """ - # third output is final output, second output for mod3, first for mod2 + # Third output is final output, second output for mod3, first for mod2 # input """ pipe_config1 = { @@ -147,7 +147,7 @@ def test_pipe_config_check(): """ (mod1, mod2, mod3), dshape = get_mannual_mod() """ - # try invalid input/output name exepect runtime error + # Try invalid input/output name exepect runtime error """ pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): @@ -157,14 +157,14 @@ def test_pipe_config_check(): pipe_error[mod1]["input"]["data_9"] """ - # try cirle connection , expect runtime error + # Try cirle connection , expect runtime error """ with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) """ - # try wrong module order connection check, expect runtime error. + # Try wrong module order connection check, expect runtime error. """ with pytest.raises(RuntimeError): @@ -205,28 +205,28 @@ def test_pipeline(): # Create pipeline compute input/output and subgraph dependent relation. # Test in key mode for binding find. - # pipeline compute input "data_0" would get forward to mod1 as input "data_0" + # Pipeline compute input "data_0" would get forward to mod1 as input "data_0" pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) - # pipeline compute input "data_1" would get forward to mod2 as input "data_1" + # Pipeline compute input "data_1" would get forward to mod2 as input "data_1" pipe_config["input"]["data_1"].connect(pipe_config[mod2]["input"]["data_1"]) - # mod1 output(0) would get forward to mod2 as input "data_0" + # Mod1 output(0) would get forward to mod2 as input "data_0" pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) - # mod1 output(1) would get forward to mod3 as input "data_0" + # Mod1 output(1) would get forward to mod3 as input "data_0" pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) - # mod2 output(0) would get forward to mod3 as input "data_1" + # Mod2 output(0) would get forward to mod3 as input "data_1" pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) - # mod1 output(2) would get forward as final pipeline compute output(1) + # Mod1 output(2) would get forward as final pipeline compute output(1) pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) - # mod3 output(0) would get forward as final pipeline compute output(2) + # Mod3 output(0) would get forward as final pipeline compute output(2) pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) """ - # print configueration (print(pipe_config)), the expect result like following. + # Print configueration (print(pipe_config)), the expect result like following. # #Inputs # |data_0: mod1:data_0 @@ -243,7 +243,7 @@ def test_pipeline(): """ """ - # set other parameter. + # Set other parameter. """ pipe_config[mod1].target = target[0] pipe_config[mod1].dev = target[1] @@ -255,7 +255,7 @@ def test_pipeline(): pipe_config[mod3].dev = tvm.cpu(0) """ - # check if the configuration match expectation. + # Check if the configuration match expectation. """ assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) From 2ae18f1b18ffff3e8ee3f668f21ad59fc32161d9 Mon Sep 17 00:00:00 2001 From: huajsj Date: Wed, 8 Sep 2021 20:06:50 -0700 Subject: [PATCH 23/34] polish doc and address review comments. --- python/tvm/contrib/pipeline_executor.py | 42 +++++------ src/runtime/pipeline/pipeline_executor.h | 4 +- tests/python/relay/test_pipeline_executor.py | 77 ++++++++++---------- 3 files changed, 60 insertions(+), 63 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index aec3961e26d6..a5b2e5bce74c 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -23,7 +23,7 @@ def pipeline_executor_enabled(): - """Check if pipeline executor get enabled. + """Check if pipeline executor is enabled. Return ------- @@ -45,7 +45,7 @@ def build(pipe_configs): Returns ------- ret: PipelineExecutorFactoryModule - the class that wrap module list and module dependency configuration. + a class that wraps module list and module dependency configuration. """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -123,7 +123,7 @@ def __init__(self, pipe_mod_config): self.module_ = module def graph_executor_create(self, pipeline_mods, mod_config): - """Create graph_executor list and return text format configuration. + """Create graph_executor list and return configuration as a json string. Parameters ---------- @@ -159,13 +159,13 @@ class PipelineConfig(object): """ class Binding: - """The class that use to storage module connection information. + """The class to storage module connection information. The binding can be either "input" or "output". Parameters ---------- owner : ModuleWrapper - The class that own this interface, in such class there are + The class that owns this interface, in such class there are Module information like idx, module name io_type : str @@ -176,19 +176,19 @@ class Binding: for output it is the idx integer such as 0. """ - def __init__(self, owner, stype, name, data_type=None): + def __init__(self, owner, io_type, name, data_type=None): self.io_owner = owner - self.io_type = stype + self.io_type = io_type self.name = str(name) # These item that have dependency relation with self self.bindings = [] - # The item that self depend + # The items that this binding depend on. self.parents = [] self.data_type = data_type def get_name(self): - """Get owner name and self name""" + """Return this interface name and name of owner who own this interface.""" owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -217,7 +217,7 @@ def __repr__(self): return ret def check_dag_acyclic(self, start, inputs): - """Check if the DAG that current binding stay is acircle""" + """Here to check if the DAG that build by bindings connections is acyclic.""" for binding in inputs.values(): if start == binding.io_owner: return False @@ -229,11 +229,11 @@ def check_dag_acyclic(self, start, inputs): def connect(self, binding): """ - # Check if the bindendency setting correct. + # Check if the binding settings is correct. # correct connection are following # 1. global input to module input # 2. module output to global output - # 3. module output to moudle input + # 3. module output to module input """ if self.io_owner == binding.io_owner: raise RuntimeError(f"can not set self as binding.") @@ -298,13 +298,11 @@ def __init__(self, owner, type_name): self.binding_type = type_name def get_binding_data_type(self, key): - """return binding data type""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.get_data_type(key, self.binding_type) return None def __getitem__(self, key): - """return item by key""" if key not in self.bindings: data_type = self.get_binding_data_type(key) if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): @@ -337,14 +335,12 @@ def __init__(self, mod=None): self.output_bindings = PipelineConfig.BindingList(self, "output") def __eq__(self, other): - """Check if self equl other""" if isinstance(other, PipelineConfig.ModuleWrapper): return self.mod == other.mod return False def __getitem__(self, key): - """Get item by key""" if isinstance(key, str): if key == "input": return self.input_bindings @@ -371,7 +367,7 @@ def get_data_type(self, key, stype): return None def set_idx_name(self, idx): - """Set index and generating name by index""" + """Set index and generate name by index""" self.idx = idx self.name = "mod{}".format(str(idx)) @@ -392,14 +388,15 @@ def __init__(self): self.output_bindings = self.BindingList(self, "output") def __str__(self): - """ Get configuration in string type""" + """ Get configuration as string""" # topological sort to get correct module order in list. self.dag_topology_sort() # get input input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: inf = self.input_bindings.bindings[input_name] - input_dump += inf.__repr__() + "\n" + #input_dump += inf.__repr__() + "\n" + input_dump += str(inf) + "\n" # get connections output = {} @@ -425,7 +422,6 @@ def __str__(self): return input_dump + output_dump + connections_dump def __getitem__(self, key): - """Return item by key""" if isinstance(key, tvm.ir.module.IRModule): if key not in self.mod_wrapper: self.mod_wrapper[key] = self.ModuleWrapper(key) @@ -442,7 +438,7 @@ def __getitem__(self, key): def get_config(self): """ Get configuration in dictionary format.""" - # Tpological sort to get correct module order in list. + # Topological sort to get correct module order in list. self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: @@ -502,7 +498,7 @@ def dag_topology_sort(self): self.mod_wrapper[mod].set_idx_name(i + 1) def get_mod_idx(self, mod): - """Return idx for specify mod""" + """Return idx for specified mod""" idx = self.mod_wrapper[mod].idx return idx @@ -516,7 +512,7 @@ def pipe_output(self, idx): class PipelineExecutorFactoryModule(object): - """This is a factory with list of GraphExecutorFactoryModule and Module configurations. + """This class is a wrapper of GraphExecutorFactoryModule list and Module configurations. Parameters ---------- diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index 464157c84b92..9b16a5dd12d3 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -45,8 +45,8 @@ class TVM_DLL PipelineRuntime : public ModuleNode { const char* type_key() const final { return "PipelineRuntime"; } /*! * \brief Initialize the pipeline executor with module Array and json text. - * \param modules The module list that use for pipeline build. - * \param pipeline_json The configuration for module dependent in pipeline. + * \param modules The module list that used for building pipeline. + * \param pipeline_json The configuration of modules dependencies. */ void Init(const Array& modules, const std::string& pipeline_json); /*! diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 6d9fa0467528..a3ffc0834b13 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -43,32 +43,35 @@ def get_mannual_mod(): mv3 = relay.Constant(tvm.nd.array(mvalue3)) """ - # Net1 have three output, output3 is final output. + # The first model has three output. """ - net_output1 = relay.add(data, mv1) - net_output2 = relay.subtract(data, mv2) - net_output3 = relay.multiply(data, mv3) + net1_output1 = relay.add(data, mv1) + net1_output2 = relay.subtract(data, mv2) + net1_output3 = relay.multiply(data, mv3) """ - # Net2 use net1 output1 as input. + # The second model use first model net1_output1 as first input, + # here data_net1_output_1 represent net_output1, the second input + # of this model is data21. """ net2 = relay.add(data_net1_output_1, mv2) net2 = relay.add(net2, data21) - net2 = relay.add(net2, mv3) + net2_output = relay.add(net2, mv3) """ - # Net3 use net2 output1 and net1 outpu2 as input. + # The third model use the second model net2_output as first input + # and use the first model net1_output2 as second input. """ net3 = relay.multiply(data_net2_output_1, mv3) net3 = relay.add(net3, data_net1_output_2) mods.append( tvm.IRModule.from_expr( - relay.Function([data], relay.Tuple([net_output1, net_output2, net_output3])) + relay.Function([data], relay.Tuple([net1_output1, net1_output2, net1_output3])) ) ) - mods.append(tvm.IRModule.from_expr(relay.Function([data_net1_output_1, data21], net2))) + mods.append(tvm.IRModule.from_expr(relay.Function([data_net1_output_1, data21], net2_output))) mods.append( tvm.IRModule.from_expr(relay.Function([data_net1_output_2, data_net2_output_1], net3)) ) @@ -78,18 +81,12 @@ def get_mannual_mod(): def get_manual_conf(mods, target): """ - # This function use to generate manual pipe line configueration, - # the result use to verify if the pipe configuration can generate - # correct result. + # This function is used to generate manual pipeline configuration. """ mod_config = {} """ - # Set configure - """ - - """ - # Third output is final output, second output for mod3, first for mod2 - # input + # The third output is the final output, the second output is for mod3, the first is for mod2 + # input. """ pipe_config1 = { "mod_idx": 1, @@ -143,11 +140,15 @@ def get_manual_conf(mods, target): def test_pipe_config_check(): """ - #Get 3 pipeline module. + # This function is used to trigger runtime error by appling wrong logic connection. + """ + + """ + #Here get three pipeline modules. """ (mod1, mod2, mod3), dshape = get_mannual_mod() """ - # Try invalid input/output name exepect runtime error + # Runing error will be expected by trying invalid input/output name. """ pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): @@ -157,14 +158,14 @@ def test_pipe_config_check(): pipe_error[mod1]["input"]["data_9"] """ - # Try cirle connection , expect runtime error + # Runtime error will be expected by trying circle connection. """ with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) """ - # Try wrong module order connection check, expect runtime error. + # Runtime error will be expected by trying wrong module order connection check. """ with pytest.raises(RuntimeError): @@ -190,40 +191,40 @@ def test_pipeline(): if pipeline_executor.pipeline_executor_enabled(): target_list = tvm.testing.enabled_targets() for target in target_list: - """ - #Get 3 pipeline module. - """ + # Here get three pipeline modules. (mod1, mod2, mod3), dshape = get_mannual_mod() - # Prepare batch data for pipeline feeding + # Batch data is prepared for pipeline feeding. datas = [] for i in range(5): datas.append(np.full(dshape, 3 + i).astype("float32")) pipe_config = pipeline_executor.PipelineConfig() - # Create pipeline compute input/output and subgraph dependent relation. - - # Test in key mode for binding find. - # Pipeline compute input "data_0" would get forward to mod1 as input "data_0" + # Input named "data_0" of global pipeline will be transfered to input + # named "data_0" of mod1 when executor is running after the 'connect' + # function calling. pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) - # Pipeline compute input "data_1" would get forward to mod2 as input "data_1" + # Input named "data_1" of global pipeline will be transfered to input + # named "data_1" of mod2 when executor is running. pipe_config["input"]["data_1"].connect(pipe_config[mod2]["input"]["data_1"]) - # Mod1 output(0) would get forward to mod2 as input "data_0" + # Output[0] of mod1 will be transfered to input named "data_0" of mod2 when + # executor is running. pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) - # Mod1 output(1) would get forward to mod3 as input "data_0" + # Output[1] of mod1 will be transfered to input named "data_0" of mod3 when + # executor is running. pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) - # Mod2 output(0) would get forward to mod3 as input "data_1" + # Output[2] of mod2 will be transfered to input named "data_1" of mod3" pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) - # Mod1 output(2) would get forward as final pipeline compute output(1) + # Output[2] of mod1 will be transfered to global pipeline output[1] pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) - # Mod3 output(0) would get forward as final pipeline compute output(2) + # Output[0] of mod3 will be transfered to global pipeline output[2] pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) """ # Print configueration (print(pipe_config)), the expect result like following. @@ -255,12 +256,12 @@ def test_pipeline(): pipe_config[mod3].dev = tvm.cpu(0) """ - # Check if the configuration match expectation. + # Here is to check correctness for configuration generated by API. """ assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) """ - # Test build and create pipeline module + # build then create pipeline module. """ with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) From 8fa82c1f0e3412cf6ad3fb696ae631fa946d27ed Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 9 Sep 2021 13:14:41 -0700 Subject: [PATCH 24/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 114 +++++++++++------------ src/runtime/pipeline/pipeline_executor.h | 10 +- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index a5b2e5bce74c..759191563637 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -23,29 +23,28 @@ def pipeline_executor_enabled(): - """Check if pipeline executor is enabled. + """Check if the pipeline executor is enabled. Return ------- enable: bool - Return pipeline executor is enabled or not. + Return whether pipeline executor is enabled. """ return tvm._ffi.get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None def build(pipe_configs): - """Build IRModule with configurations of pipe_configs then - return Module list and Module dependency configuration. + """Use pipe_config to build and return Module list and Module dependency configuration. Parameters ---------- pipe_configs: PipelineConfig - build configuration informaton. + Configuration information for build. Returns ------- ret: PipelineExecutorFactoryModule - a class that wraps module list and module dependency configuration. + A class that wraps module list and module dependency configuration. """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -54,15 +53,15 @@ def build(pipe_configs): for ir_mod, mod_config in mod_n_configs.items(): mconf = mod_config["pipeline"].copy() mod_idx = mconf["mod_idx"] - 1 - # Get mod device config + # Get mod device configuration. dev = mod_config["dev"] target = mod_config["target"] build_func = relay.build - # if there is a self defined build function then use it. + # Check whether there is a customized build function. if "build" in mod_config and mod_config["build"]: build_func = mod_config["build"] - # build IRModule + # Build. mod = build_func( ir_mod, target, @@ -72,9 +71,9 @@ def build(pipe_configs): ) mconf["dev"] = "{},{}".format(dev.device_type, dev.device_id) - # Create pipeline configuration + # Create pipeline configuration. string_config[mod_idx] = mconf - # associate mod with device + # Set device. mods[mod] = {"dev": dev} return PipelineExecutorFactoryModule(mods, string_config) @@ -86,7 +85,7 @@ def create(pipe_executor_factory_module): Parameters ---------- pipe_executor_factory_module : PipelineExecutorFactoryModule - Executor factory with IRModule list and pipeline configuration. + It is wrapper class which include IRModule list and pipeline configuration. Returns ------- @@ -103,7 +102,7 @@ class PipelineModule(object): Parameters ---------- pipeline_config : Dict[GraphExecutorFactoryModule, Dict[str, Any]] - modules and modules dependency configuration informaiton. + Modules and modules dependency configuration informaitons. """ def __init__(self, pipe_mod_config): @@ -128,10 +127,10 @@ def graph_executor_create(self, pipeline_mods, mod_config): Parameters ---------- pipeline_mods : List[GraphExecutorFactoryModule] - list of GraphExecutorFactoryModule + List of GraphExecutorFactoryModule mod_config : Dict[str, Any] - modules dependency configuration informaiton. + Modules dependency configuration information. Returns ------- @@ -139,7 +138,7 @@ def graph_executor_create(self, pipeline_mods, mod_config): Module list. mod_config : str - mods configuration + Mods configuration. """ mods = [] @@ -159,36 +158,35 @@ class PipelineConfig(object): """ class Binding: - """The class to storage module connection information. - The binding can be either "input" or "output". + """This class define the module connection information. + The type can only be either "input" or "output". Parameters ---------- owner : ModuleWrapper - The class that owns this interface, in such class there are - Module information like idx, module name + The class who owns this interface. io_type : str - The type of this binding. It can be either "input" or "output". + The type of this interface. It can only be either "input" or "output". name : str/integer - Binding name, for input it is string such as "data0"; - for output it is the idx integer such as 0. + Name, for input it is string such as "data0", for output it is the + idx integer such as 0. """ def __init__(self, owner, io_type, name, data_type=None): self.io_owner = owner self.io_type = io_type self.name = str(name) - # These item that have dependency relation with self + # Child nodes. self.bindings = [] - # The items that this binding depend on. + # Parents nodes. self.parents = [] self.data_type = data_type def get_name(self): - """Return this interface name and name of owner who own this interface.""" + """Return the interface name and name of owner who own this interface.""" owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -205,11 +203,11 @@ def get_owner_idx(self): return 0 def is_global_interface(self): - """Check if this interface is global interface.""" + """It is to check whether this interface is global interface.""" return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) def __repr__(self): - """Get all binding(input data), exepect like |data_0: mod1:data_0""" + """Get all binding(input data) informations that looks like '|data_0: mod1:data_0'""" ret = " |{}: ".format(self.name) for binding in self.bindings: mname, dname = binding.get_name() @@ -217,7 +215,7 @@ def __repr__(self): return ret def check_dag_acyclic(self, start, inputs): - """Here to check if the DAG that build by bindings connections is acyclic.""" + """It is to check whether the DAG that contain the inputs interfaces is acyclic.""" for binding in inputs.values(): if start == binding.io_owner: return False @@ -229,37 +227,37 @@ def check_dag_acyclic(self, start, inputs): def connect(self, binding): """ - # Check if the binding settings is correct. + # Check whether the binding settings is correct or not. # correct connection are following # 1. global input to module input # 2. module output to global output # 3. module output to module input """ if self.io_owner == binding.io_owner: - raise RuntimeError(f"can not set self as binding.") + raise RuntimeError(f"Can not bind itself.") if not self.is_global_interface() and self.io_type == "input": - raise RuntimeError(f"Module only can start binding from output!") + raise RuntimeError(f"Module can only bind from output interface!") if ( not self.is_global_interface() and not binding.is_global_interface() and binding.io_type == "output" ): - raise RuntimeError(f"Module output can not binding with module output!") + raise RuntimeError(f"Can not bind module output with another module output!") if ( not self.is_global_interface() and binding.is_global_interface() and binding.io_type == "input" ): - raise RuntimeError(f"Module output can not binding with global input!") + raise RuntimeError(f"Can not bind module output with global input!") - if self.is_global_interface() and self.io_type != "input": - raise RuntimeError(f"Global only can start binding from input!") + if self.is_global_interface() and self.io_type == "output": + raise RuntimeError(f"Global output can not be used as binding start point.") if self.is_global_interface() and binding.io_type != "input": - raise RuntimeError(f"Global input only can set binding with module input.") + raise RuntimeError(f"Global input can only bind with module input.") self.bindings.append(binding) if not self.is_global_interface(): @@ -278,7 +276,7 @@ def connect(self, binding): if not self.check_dag_acyclic( binding.io_owner, self.io_owner.input_bindings.bindings ): - raise RuntimeError(f"Illegal connection: cause a circle!") + raise RuntimeError(f"Illegal connection: Cause a circle!") class BindingList: """Container for bindings(input or output interface). @@ -286,10 +284,10 @@ class BindingList: Parameters ---------- owner : ModuleWrapper/PipelineConfig - The owner of this list, it can be ModuleWrapper or PipelineConfig + The owner of this list can be ModuleWrapper or PipelineConfig type_name : str - The type of this binding list. It can be either "input" or "output". + The type of this binding list can be either "input" or "output". """ def __init__(self, owner, type_name): @@ -306,7 +304,7 @@ def __getitem__(self, key): if key not in self.bindings: data_type = self.get_binding_data_type(key) if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - raise RuntimeError(f"Cannot find {key} in binding list {self.binding_type}") + raise RuntimeError(f"Can not find {key} in binding list {self.binding_type}") self.bindings[key] = PipelineConfig.Binding( self.io_owner, self.binding_type, key, data_type @@ -315,8 +313,8 @@ def __getitem__(self, key): return self.bindings[key] class ModuleWrapper: - """Module Wrapper with information like module index, binding information - and building informations. + """This class is a wrapper that represent the module, contains module informations, + binding informations and building informations. """ def __init__(self, mod=None): @@ -351,7 +349,7 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found!") def get_data_type(self, key, stype): - """Get module input/output data type.""" + """Get the data type of the input or output interface of the module.""" if stype == "input": for param in self.input_params: if param.name_hint == key: @@ -367,16 +365,18 @@ def get_data_type(self, key, stype): return None def set_idx_name(self, idx): - """Set index and generate name by index""" + """Sepecify the index value and generate the module name.""" self.idx = idx self.name = "mod{}".format(str(idx)) def is_root_mod(self): - """Identify if this item is root item, used by DAG topological sort.""" + """Check whether it is root node and use it in DAG topological sort.""" return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): - """Remove self from binding to reduce child in-degree, used by DAG topological sort.""" + """Remove itself from child reference to reduce child node in-degree + and use it in DAG topological sort. + """ for binding in self.output_bindings.bindings.values(): for child in binding.bindings: if binding in child.parents: @@ -395,7 +395,6 @@ def __str__(self): input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: inf = self.input_bindings.bindings[input_name] - #input_dump += inf.__repr__() + "\n" input_dump += str(inf) + "\n" # get connections @@ -436,13 +435,13 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found.") def get_config(self): - """ Get configuration in dictionary format.""" + """ Get configuration information in dictionary form.""" - # Topological sort to get correct module order in list. + # Use topological sort to get correct order of modules. self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: - # Get pipeline configure. + # Generate pipeline configure. mconf = {} output_conf = [] module = self.mod_wrapper[mod] @@ -465,7 +464,7 @@ def get_config(self): mconf["mod_idx"] = module.idx mconf["output"] = output_conf - # Build module configuration with pipeline and other parameters. + # Build configuration with parameters. mconfig[mod] = { "pipeline": mconf, "target_host": module.target_host, @@ -479,7 +478,7 @@ def get_config(self): return mconfig def dag_topology_sort(self): - """ Do topological sort to get pipeline module order.""" + """ Use topological sort to get order of pipeline modules.""" mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: @@ -498,21 +497,22 @@ def dag_topology_sort(self): self.mod_wrapper[mod].set_idx_name(i + 1) def get_mod_idx(self, mod): - """Return idx for specified mod""" + """Return module index.""" idx = self.mod_wrapper[mod].idx return idx def pipe_input(self, name): - """Return input binding by name""" + """Return the corresponding input binding interface accordding the name""" return self.input_bindings[name] def pipe_output(self, idx): - """Return output binding by idx""" + """Return the corresponding output binding interface accordding the name""" return self.output_bindings[idx] class PipelineExecutorFactoryModule(object): - """This class is a wrapper of GraphExecutorFactoryModule list and Module configurations. + """This is a wrapper class which contains GraphExecutorFactoryModule list + and Module configurations. Parameters ---------- diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index 9b16a5dd12d3..c99a7243472b 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -30,9 +30,9 @@ namespace tvm { namespace runtime { /*! * \brief pipeline executor. - * This executor take Module list and Dependency relations of Modules as - * inputs, then execute modules on heterogeneous hardware target in - * pipeline parallism mode to improve processing throughput. + * This executor class use module list and dependency relations of modules as + * the parameters and executes these modules on heterogeneous pipeline parallel + * to improve throughput. * * This executor can be acccesibly in various language via * TVM runtime PackedFunc API. @@ -40,11 +40,11 @@ namespace runtime { class TVM_DLL PipelineRuntime : public ModuleNode { public: /*! - * \return The type key of the executor. + * \Return the type key of the executor. */ const char* type_key() const final { return "PipelineRuntime"; } /*! - * \brief Initialize the pipeline executor with module Array and json text. + * \brief Initialize the pipeline executor with module array and json text. * \param modules The module list that used for building pipeline. * \param pipeline_json The configuration of modules dependencies. */ From 31e0819c594a46258623742294d7a3795773d4ee Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 9 Sep 2021 16:12:40 -0700 Subject: [PATCH 25/34] doc change. --- python/tvm/contrib/pipeline_executor.py | 18 +++++++++--------- tests/python/relay/test_pipeline_executor.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 759191563637..c56c2fa9e7ec 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -194,7 +194,7 @@ def get_name(self): return owner_name, self.name def get_owner_idx(self): - """Return owner idex if owner is ModuleWrapper, if not return 0""" + """Return owner idex if owner is ModuleWrapper, if not return 0.""" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.idx @@ -207,7 +207,7 @@ def is_global_interface(self): return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) def __repr__(self): - """Get all binding(input data) informations that looks like '|data_0: mod1:data_0'""" + """Get all binding(input data) informations that looks like '|data_0: mod1:data_0'.""" ret = " |{}: ".format(self.name) for binding in self.bindings: mname, dname = binding.get_name() @@ -284,7 +284,7 @@ class BindingList: Parameters ---------- owner : ModuleWrapper/PipelineConfig - The owner of this list can be ModuleWrapper or PipelineConfig + The owner of this list can be ModuleWrapper or PipelineConfig. type_name : str The type of this binding list can be either "input" or "output". @@ -304,7 +304,7 @@ def __getitem__(self, key): if key not in self.bindings: data_type = self.get_binding_data_type(key) if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper): - raise RuntimeError(f"Can not find {key} in binding list {self.binding_type}") + raise RuntimeError(f"Can not find {key} in binding list {self.binding_type}.") self.bindings[key] = PipelineConfig.Binding( self.io_owner, self.binding_type, key, data_type @@ -397,7 +397,7 @@ def __str__(self): inf = self.input_bindings.bindings[input_name] input_dump += str(inf) + "\n" - # get connections + # get connections. output = {} connections_dump = "\nconnections\n" for mod in self.mod_wrapper: @@ -502,22 +502,22 @@ def get_mod_idx(self, mod): return idx def pipe_input(self, name): - """Return the corresponding input binding interface accordding the name""" + """Return the corresponding input binding interface accordding the name.""" return self.input_bindings[name] def pipe_output(self, idx): - """Return the corresponding output binding interface accordding the name""" + """Return the corresponding output binding interface accordding the name.""" return self.output_bindings[idx] class PipelineExecutorFactoryModule(object): - """This is a wrapper class which contains GraphExecutorFactoryModule list + """This is a wrapper class which contains GraphExecutorFactoryModule list. and Module configurations. Parameters ---------- pipeline_mods : List[GraphExecutorFactoryModule] - list of GraphExecutorFactoryModule + list of GraphExecutorFactoryModule. mod_config : Dict[int, Dict[str, Any]] modules and modules dependency configuration informaiton. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index a3ffc0834b13..59e2f8f85b0b 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -261,7 +261,7 @@ def test_pipeline(): assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) """ - # build then create pipeline module. + # Build then create pipeline module. """ with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) From 1b167fd9a014b36428124d0a04e9ffe2316684dc Mon Sep 17 00:00:00 2001 From: huajsj Date: Thu, 9 Sep 2021 21:03:55 -0700 Subject: [PATCH 26/34] doc change. --- tests/python/relay/test_pipeline_executor.py | 34 +++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 59e2f8f85b0b..1466ddb492d7 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -144,11 +144,11 @@ def test_pipe_config_check(): """ """ - #Here get three pipeline modules. + # Get three pipeline modules here. """ (mod1, mod2, mod3), dshape = get_mannual_mod() """ - # Runing error will be expected by trying invalid input/output name. + # The input/output name is illegal and expects a runtime error. """ pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): @@ -158,14 +158,14 @@ def test_pipe_config_check(): pipe_error[mod1]["input"]["data_9"] """ - # Runtime error will be expected by trying circle connection. + # The connection will cause a circle in DAG and exepects runtime error. """ with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) """ - # Runtime error will be expected by trying wrong module order connection check. + # The module connection is illegal and expects runtime error. """ with pytest.raises(RuntimeError): @@ -191,7 +191,7 @@ def test_pipeline(): if pipeline_executor.pipeline_executor_enabled(): target_list = tvm.testing.enabled_targets() for target in target_list: - # Here get three pipeline modules. + # Get three pipeline modules here. (mod1, mod2, mod3), dshape = get_mannual_mod() # Batch data is prepared for pipeline feeding. @@ -201,33 +201,29 @@ def test_pipeline(): pipe_config = pipeline_executor.PipelineConfig() - # Input named "data_0" of global pipeline will be transfered to input - # named "data_0" of mod1 when executor is running after the 'connect' - # function calling. + # The global input named "data_0" will be connected to a input + # named "data_0" of mod1. pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) - # Input named "data_1" of global pipeline will be transfered to input - # named "data_1" of mod2 when executor is running. + # The global Input named "data_1" will be connected to a input named "data_1" of mod2. pipe_config["input"]["data_1"].connect(pipe_config[mod2]["input"]["data_1"]) - # Output[0] of mod1 will be transfered to input named "data_0" of mod2 when - # executor is running. + # The mod1 output[0] will be connected to a input named "data_0" of mod2. pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) - # Output[1] of mod1 will be transfered to input named "data_0" of mod3 when - # executor is running. + # Mod1 output[1] will be connected to a input named "data_0" of mod3. pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) - # Output[2] of mod2 will be transfered to input named "data_1" of mod3" + # Mod2 output[2] will be connected to a input named "data_1" of mod3. pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) - # Output[2] of mod1 will be transfered to global pipeline output[1] + # Mod1 output[2] will be connected to global output[1]. pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) - # Output[0] of mod3 will be transfered to global pipeline output[2] + # Mod3 output[0] will be connected to global output[2]. pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) """ - # Print configueration (print(pipe_config)), the expect result like following. + # Print configueration (print(pipe_config)), the result looks like following. # #Inputs # |data_0: mod1:data_0 @@ -261,7 +257,7 @@ def test_pipeline(): assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) """ - # Build then create pipeline module. + # Build and create pipeline module. """ with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) From 4d869f55c9d73d755339a1c91a2bbde1d5f81ff6 Mon Sep 17 00:00:00 2001 From: huajsj Date: Fri, 10 Sep 2021 12:05:35 -0700 Subject: [PATCH 27/34] Trigger build. --- tests/python/relay/test_pipeline_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 1466ddb492d7..e62448fc539b 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -148,7 +148,7 @@ def test_pipe_config_check(): """ (mod1, mod2, mod3), dshape = get_mannual_mod() """ - # The input/output name is illegal and expects a runtime error. + # The input or output name is illegal and expects a runtime error. """ pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): From 3cf01e99ddc3972623fc78f4c0425720cedf1fad Mon Sep 17 00:00:00 2001 From: huajsj Date: Fri, 10 Sep 2021 19:09:01 -0700 Subject: [PATCH 28/34] trigge build. --- tests/python/relay/test_pipeline_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index e62448fc539b..1466ddb492d7 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -148,7 +148,7 @@ def test_pipe_config_check(): """ (mod1, mod2, mod3), dshape = get_mannual_mod() """ - # The input or output name is illegal and expects a runtime error. + # The input/output name is illegal and expects a runtime error. """ pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): From 73b6c9809fe3ef0b683d8de697704deb99c909ff Mon Sep 17 00:00:00 2001 From: huajsj Date: Sun, 12 Sep 2021 12:33:34 -0700 Subject: [PATCH 29/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 144 +++++++++---------- src/runtime/pipeline/pipeline_executor.cc | 2 +- src/runtime/pipeline/pipeline_executor.h | 10 +- tests/python/relay/test_pipeline_executor.py | 83 +++++------ 4 files changed, 108 insertions(+), 131 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index c56c2fa9e7ec..9759d80c94f6 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -34,7 +34,7 @@ def pipeline_executor_enabled(): def build(pipe_configs): - """Use pipe_config to build and return Module list and Module dependency configuration. + """Use pipe_config to build and return module list and module dependency configuration. Parameters ---------- @@ -97,58 +97,16 @@ def create(pipe_executor_factory_module): class PipelineModule(object): - """Wrapper of runtime module. + """Wrapper of runtime module, caller can use this module to set parameters and get outputs. Parameters ---------- - pipeline_config : Dict[GraphExecutorFactoryModule, Dict[str, Any]] - Modules and modules dependency configuration informaitons. + module : PipelineExecutorFactoryModule + Common interface for pipeline executor factory modules. """ - def __init__(self, pipe_mod_config): - self.pipeline_mods = pipe_mod_config.pipeline_mods - self.mod_config = pipe_mod_config.mods_config - mods, config = self.graph_executor_create(self.pipeline_mods, self.mod_config) - assert ( - pipeline_executor_enabled() - ), "Pipeline executor is not enabled. Please \ - re-build TVM with USE_PIPELINE_EXECUTOR=ON" - pipeline_create = tvm._ffi.get_global_func( - "tvm.pipeline_executor.create", allow_missing=False - ) - assert pipeline_create - module = pipeline_create(mods, config) - - self.module_ = module - - def graph_executor_create(self, pipeline_mods, mod_config): - """Create graph_executor list and return configuration as a json string. - - Parameters - ---------- - pipeline_mods : List[GraphExecutorFactoryModule] - List of GraphExecutorFactoryModule - - mod_config : Dict[str, Any] - Modules dependency configuration information. - - Returns - ------- - mods : List[Module] - Module list. - - mod_config : str - Mods configuration. - """ - - mods = [] - for pipeline_mod in pipeline_mods: - mod = graph_executor.GraphModule( - pipeline_mod["default"](pipeline_mods[pipeline_mod]["dev"]) - ) - mods.append(mod.module) - - return mods, json.dumps(mod_config) + def __init__(self, module): + self.module = module.module class PipelineConfig(object): @@ -186,7 +144,7 @@ def __init__(self, owner, io_type, name, data_type=None): self.data_type = data_type def get_name(self): - """Return the interface name and name of owner who own this interface.""" + """Return the interface name and name of owner who owns this interface.""" owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -203,7 +161,11 @@ def get_owner_idx(self): return 0 def is_global_interface(self): - """It is to check whether this interface is global interface.""" + """The global interface is the interface visible to the caller which use pipeline + executor, the global input interface is responsible for passing parameters to the + internal module interface, and the global output interface is responsible for + outputting the pipeline executor compute results to caller. + """ return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) def __repr__(self): @@ -215,7 +177,7 @@ def __repr__(self): return ret def check_dag_acyclic(self, start, inputs): - """It is to check whether the DAG that contain the inputs interfaces is acyclic.""" + """It is to check whether the DAG that contains the inputs interfaces is acyclic.""" for binding in inputs.values(): if start == binding.io_owner: return False @@ -226,12 +188,11 @@ def check_dag_acyclic(self, start, inputs): return True def connect(self, binding): - """ - # Check whether the binding settings is correct or not. - # correct connection are following - # 1. global input to module input - # 2. module output to global output - # 3. module output to module input + """Check whether the binding settings is correct or not. + correct connection are following + 1. global input to module input + 2. module output to global output + 3. module output to module input """ if self.io_owner == binding.io_owner: raise RuntimeError(f"Can not bind itself.") @@ -276,7 +237,7 @@ def connect(self, binding): if not self.check_dag_acyclic( binding.io_owner, self.io_owner.input_bindings.bindings ): - raise RuntimeError(f"Illegal connection: Cause a circle!") + raise RuntimeError(f"Illegal connection: Cause a cycle!") class BindingList: """Container for bindings(input or output interface). @@ -314,11 +275,10 @@ def __getitem__(self, key): class ModuleWrapper: """This class is a wrapper that represent the module, contains module informations, - binding informations and building informations. + binding informations and building information. """ def __init__(self, mod=None): - """Init class""" self.target_host = None self.build_func = None self.params = None @@ -348,14 +308,14 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found!") - def get_data_type(self, key, stype): - """Get the data type of the input or output interface of the module.""" - if stype == "input": + def get_data_type(self, key, interface_type): + """Get module interface data type.""" + if interface_type == "input": for param in self.input_params: if param.name_hint == key: return param._checked_type_ - if stype == "output": + if interface_type == "output": if isinstance(self.output_values, tvm.ir.type.TupleType): if int(key) < len(self.output_values.fields): return self.output_values.fields[int(key)] @@ -370,12 +330,12 @@ def set_idx_name(self, idx): self.name = "mod{}".format(str(idx)) def is_root_mod(self): - """Check whether it is root node and use it in DAG topological sort.""" + """Check whether it is root node, this function used by DAG topological sort.""" return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): - """Remove itself from child reference to reduce child node in-degree - and use it in DAG topological sort. + """Remove itself from child reference to reduce child node in-degree. + This function used by DAG topological sort. """ for binding in self.output_bindings.bindings.values(): for child in binding.bindings: @@ -502,17 +462,16 @@ def get_mod_idx(self, mod): return idx def pipe_input(self, name): - """Return the corresponding input binding interface accordding the name.""" + """Return the corresponding input binding interface accordding to the name.""" return self.input_bindings[name] def pipe_output(self, idx): - """Return the corresponding output binding interface accordding the name.""" + """Return the corresponding output binding interface accordding to the name.""" return self.output_bindings[idx] class PipelineExecutorFactoryModule(object): - """This is a wrapper class which contains GraphExecutorFactoryModule list. - and Module configurations. + """Common interface for pipeline executor factory modules. Parameters ---------- @@ -520,10 +479,47 @@ class PipelineExecutorFactoryModule(object): list of GraphExecutorFactoryModule. mod_config : Dict[int, Dict[str, Any]] - modules and modules dependency configuration informaiton. + modules and modules dependency configuration information. """ def __init__(self, pipeline_mods, mods_config): - self.pipeline_mods = pipeline_mods - self.mods_config = mods_config + mods, config = self.graph_executor_create(pipeline_mods, mods_config) + assert ( + pipeline_executor_enabled() + ), "Pipeline executor is not enabled. Please \ + re-build TVM with USE_PIPELINE_EXECUTOR=ON" + pipeline_create = tvm._ffi.get_global_func( + "tvm.pipeline_executor.create", allow_missing=False + ) + assert pipeline_create + self.module = pipeline_create(mods, config) + + def graph_executor_create(self, pipeline_mods, mod_config): + """Create graph_executor list and return configuration as a json string. + + Parameters + ---------- + pipeline_mods : List[GraphExecutorFactoryModule] + List of GraphExecutorFactoryModule + + mod_config : Dict[str, Any] + Modules dependency configuration information. + + Returns + ------- + mods : List[Module] + Module list. + + mod_config : str + Mods configuration. + """ + + mods = [] + for pipeline_mod in pipeline_mods: + mod = graph_executor.GraphModule( + pipeline_mod["default"](pipeline_mods[pipeline_mod]["dev"]) + ) + mods.append(mod.module) + + return mods, json.dumps(mod_config) diff --git a/src/runtime/pipeline/pipeline_executor.cc b/src/runtime/pipeline/pipeline_executor.cc index adb8c1c2d21e..41f867057282 100644 --- a/src/runtime/pipeline/pipeline_executor.cc +++ b/src/runtime/pipeline/pipeline_executor.cc @@ -30,7 +30,7 @@ void PipelineRuntime::Init(const Array& modules, return; } -/* GetFunction can not be pure abstract function, implement an empty function for initial. +/* GetFunction can not be pure abstract function, implement an empty function for now. */ PackedFunc PipelineRuntime::GetFunction(const std::string& name, const ObjectPtr& sptr_to_self) { diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index c99a7243472b..d0d3fb87299a 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -31,10 +31,10 @@ namespace runtime { /*! * \brief pipeline executor. * This executor class use module list and dependency relations of modules as - * the parameters and executes these modules on heterogeneous pipeline parallel - * to improve throughput. + * the parameters and executes these modules on heterogeneous targets in pipeline + * parallel to improve throughput. * - * This executor can be acccesibly in various language via + * This executor can be accessed by various language via * TVM runtime PackedFunc API. */ class TVM_DLL PipelineRuntime : public ModuleNode { @@ -45,7 +45,7 @@ class TVM_DLL PipelineRuntime : public ModuleNode { const char* type_key() const final { return "PipelineRuntime"; } /*! * \brief Initialize the pipeline executor with module array and json text. - * \param modules The module list that used for building pipeline. + * \param modules The module list used for building pipeline. * \param pipeline_json The configuration of modules dependencies. */ void Init(const Array& modules, const std::string& pipeline_json); @@ -53,7 +53,7 @@ class TVM_DLL PipelineRuntime : public ModuleNode { * \brief Give frontends an access to packed functions. * \param name The name of the function. * \param sptr_to_self The pointer to the module node. - * \return The corresponding packed functions. + * \return The corresponding packed function. */ virtual PackedFunc GetFunction(const std::string& name, const ObjectPtr& sptr_to_self); }; diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 1466ddb492d7..4d2fefec009d 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -25,9 +25,7 @@ def get_mannual_mod(): - """ - # Get list of module that represent a subgraph. - """ + """Get list of module that represent a subgraph.""" mods = [] dshape = (3, 3) data = relay.var("data_0", relay.TensorType(dshape, "float32")) @@ -42,26 +40,22 @@ def get_mannual_mod(): mv2 = relay.Constant(tvm.nd.array(mvalue2)) mv3 = relay.Constant(tvm.nd.array(mvalue3)) - """ - # The first model has three output. - """ + """The first model has three output.""" net1_output1 = relay.add(data, mv1) net1_output2 = relay.subtract(data, mv2) net1_output3 = relay.multiply(data, mv3) - """ - # The second model use first model net1_output1 as first input, - # here data_net1_output_1 represent net_output1, the second input - # of this model is data21. + """The second model use first model net1_output1 as first input, + here data_net1_output_1 represent net_output1, the second input + of this model is data21. """ net2 = relay.add(data_net1_output_1, mv2) net2 = relay.add(net2, data21) net2_output = relay.add(net2, mv3) - """ - # The third model use the second model net2_output as first input - # and use the first model net1_output2 as second input. + """The third model use the second model net2_output as first input + and use the first model net1_output2 as second input. """ net3 = relay.multiply(data_net2_output_1, mv3) net3 = relay.add(net3, data_net1_output_2) @@ -80,13 +74,10 @@ def get_mannual_mod(): def get_manual_conf(mods, target): - """ - # This function is used to generate manual pipeline configuration. - """ + """This function is used to generate manual pipeline configuration.""" mod_config = {} - """ - # The third output is the final output, the second output is for mod3, the first is for mod2 - # input. + """The third output is the final output, the second output is for mod3, the first is for mod2 + input. """ pipe_config1 = { "mod_idx": 1, @@ -139,16 +130,12 @@ def get_manual_conf(mods, target): def test_pipe_config_check(): - """ - # This function is used to trigger runtime error by appling wrong logic connection. - """ + """This function is used to trigger runtime error by appling wrong logic connection.""" - """ - # Get three pipeline modules here. + """Get three pipeline modules here. """ (mod1, mod2, mod3), dshape = get_mannual_mod() - """ - # The input/output name is illegal and expects a runtime error. + """The input/output name is illegal and expects a runtime error. """ pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): @@ -157,15 +144,13 @@ def test_pipe_config_check(): with pytest.raises(RuntimeError): pipe_error[mod1]["input"]["data_9"] - """ - # The connection will cause a circle in DAG and exepects runtime error. + """The connection will cause a circle in DAG and exepects runtime error. """ with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) - """ - # The module connection is illegal and expects runtime error. + """The module connection is illegal and expects runtime error. """ with pytest.raises(RuntimeError): @@ -222,25 +207,23 @@ def test_pipeline(): # Mod3 output[0] will be connected to global output[2]. pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) - """ - # Print configueration (print(pipe_config)), the result looks like following. - # - #Inputs - # |data_0: mod1:data_0 - # |data_1: mod2:data_1 - # - #output - # |output(1) : mod1.output(2) - # |output(2) : mod3.output(0) - # - #connections - # |mod1.output(0)-> mod2.data_0 - # |mod1.output(1)-> mod3.data_0 - # |mod2.output(0)-> mod3.data_1 + """Print configueration (print(pipe_config)), the result looks like following. + + Inputs + |data_0: mod1:data_0 + |data_1: mod2:data_1 + + output + |output(1) : mod1.output(2) + |output(2) : mod3.output(0) + + connections + |mod1.output(0)-> mod2.data_0 + |mod1.output(1)-> mod3.data_0 + |mod2.output(0)-> mod3.data_1 """ - """ - # Set other parameter. + """Set other parameter. """ pipe_config[mod1].target = target[0] pipe_config[mod1].dev = target[1] @@ -251,13 +234,11 @@ def test_pipeline(): pipe_config[mod3].target = "llvm" pipe_config[mod3].dev = tvm.cpu(0) - """ - # Here is to check correctness for configuration generated by API. + """Here is to check correctness for configuration generated by API. """ assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) - """ - # Build and create pipeline module. + """Build and create pipeline module. """ with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) From beac1983f6a471bc1ac0c6a93ae5cde5aec62033 Mon Sep 17 00:00:00 2001 From: huajsj Date: Mon, 13 Sep 2021 17:43:54 -0700 Subject: [PATCH 30/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 113 ++++++++++++------- src/runtime/pipeline/pipeline_executor.h | 6 +- tests/python/relay/test_pipeline_executor.py | 43 ++++--- 3 files changed, 94 insertions(+), 68 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 9759d80c94f6..c2870ebd71b9 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -34,7 +34,7 @@ def pipeline_executor_enabled(): def build(pipe_configs): - """Use pipe_config to build and return module list and module dependency configuration. + """Use pipe_config to build and return PipelineExecutorFactoryModule. Parameters ---------- @@ -44,7 +44,8 @@ def build(pipe_configs): Returns ------- ret: PipelineExecutorFactoryModule - A class that wraps module list and module dependency configuration. + A factory class is responsible for receiving module and configuration + information and maintaining executor module """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -85,7 +86,8 @@ def create(pipe_executor_factory_module): Parameters ---------- pipe_executor_factory_module : PipelineExecutorFactoryModule - It is wrapper class which include IRModule list and pipeline configuration. + A factory class is responsible for receiving module and configuration + information and maintaining executor module. Returns ------- @@ -116,7 +118,7 @@ class PipelineConfig(object): """ class Binding: - """This class define the module connection information. + """This class defines the module connection information. The type can only be either "input" or "output". Parameters @@ -128,8 +130,7 @@ class Binding: The type of this interface. It can only be either "input" or "output". name : str/integer - Name, for input it is string such as "data0", for output it is the - idx integer such as 0. + Name, for input it is string such as "data0", for output it is an integer such as 0. """ def __init__(self, owner, io_type, name, data_type=None): @@ -144,7 +145,7 @@ def __init__(self, owner, io_type, name, data_type=None): self.data_type = data_type def get_name(self): - """Return the interface name and name of owner who owns this interface.""" + # Return the interface name and name of owner who owns this interface. owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -152,7 +153,7 @@ def get_name(self): return owner_name, self.name def get_owner_idx(self): - """Return owner idex if owner is ModuleWrapper, if not return 0.""" + # Return owner idex if owner is ModuleWrapper, if not return 0. if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.idx @@ -161,15 +162,15 @@ def get_owner_idx(self): return 0 def is_global_interface(self): - """The global interface is the interface visible to the caller which use pipeline + """The global interface is the interface visible to the caller which use a pipeline executor, the global input interface is responsible for passing parameters to the internal module interface, and the global output interface is responsible for - outputting the pipeline executor compute results to caller. + outputting the results computed by the pipeline executor to a caller. """ return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper) def __repr__(self): - """Get all binding(input data) informations that looks like '|data_0: mod1:data_0'.""" + # Get all binding information. ret = " |{}: ".format(self.name) for binding in self.bindings: mname, dname = binding.get_name() @@ -177,7 +178,19 @@ def __repr__(self): return ret def check_dag_acyclic(self, start, inputs): - """It is to check whether the DAG that contains the inputs interfaces is acyclic.""" + """This is to check whether the DAG containing these input interfaces is acyclic. + Parameters + ---------- + start: ModuleWrapper + The starting node of the cycle check algorithm. + + inputs: Binding + These interfaces are used to connect to each other to build DAG. + + Return + ------ + Return True if there is no cycle in DAG. + """ for binding in inputs.values(): if start == binding.io_owner: return False @@ -188,12 +201,17 @@ def check_dag_acyclic(self, start, inputs): return True def connect(self, binding): - """Check whether the binding settings is correct or not. - correct connection are following - 1. global input to module input - 2. module output to global output - 3. module output to module input + """Connect the current interface to the destination interface. + correct connections as following 1. global input to module input, + 2. module output to global output, 3. module output to module input + + Parameters + ---------- + binding: Binding + The destination of this connection. """ + + # Check whether the binding setting is correct or not. if self.io_owner == binding.io_owner: raise RuntimeError(f"Can not bind itself.") @@ -222,7 +240,7 @@ def connect(self, binding): self.bindings.append(binding) if not self.is_global_interface(): - # check if the source and target data_type same + # Check whether the data types of the source and destination are the same. if ( isinstance(binding.io_owner, PipelineConfig.ModuleWrapper) and self.data_type != binding.data_type @@ -274,8 +292,8 @@ def __getitem__(self, key): return self.bindings[key] class ModuleWrapper: - """This class is a wrapper that represent the module, contains module informations, - binding informations and building information. + """This class is a wrapper that represents the module, contains module information, + binding information and building information. """ def __init__(self, mod=None): @@ -288,7 +306,7 @@ def __init__(self, mod=None): self.idx = None self.mod = mod self.input_params = InferType()(mod)["main"].params - self.output_values = InferType()(mod)["main"].checked_type.ret_type + self.output_type = InferType()(mod)["main"].checked_type.ret_type self.input_bindings = PipelineConfig.BindingList(self, "input") self.output_bindings = PipelineConfig.BindingList(self, "output") @@ -309,33 +327,44 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found!") def get_data_type(self, key, interface_type): - """Get module interface data type.""" + """Get module interface data type. + Parameters + ---------- + key: str + The interface name. + interface_type: + The interface type. + + Return + ------- + Return data type. + """ if interface_type == "input": for param in self.input_params: if param.name_hint == key: return param._checked_type_ if interface_type == "output": - if isinstance(self.output_values, tvm.ir.type.TupleType): - if int(key) < len(self.output_values.fields): - return self.output_values.fields[int(key)] + if isinstance(self.output_type, tvm.ir.type.TupleType): + if int(key) < len(self.output_type.fields): + return self.output_type.fields[int(key)] elif int(key) == 0: - return self.output_values + return self.output_type return None def set_idx_name(self, idx): - """Sepecify the index value and generate the module name.""" + # Set the index value and generate the module name. self.idx = idx self.name = "mod{}".format(str(idx)) def is_root_mod(self): - """Check whether it is root node, this function used by DAG topological sort.""" + # Check whether it is root node and is used by DAG topological sort. return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): """Remove itself from child reference to reduce child node in-degree. - This function used by DAG topological sort. + This function is used by DAG topological sort. """ for binding in self.output_bindings.bindings.values(): for child in binding.bindings: @@ -348,16 +377,17 @@ def __init__(self): self.output_bindings = self.BindingList(self, "output") def __str__(self): - """ Get configuration as string""" - # topological sort to get correct module order in list. + # Get configuration as string. + + # Use topological sort to get correct module order. self.dag_topology_sort() - # get input + # Get input. input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: inf = self.input_bindings.bindings[input_name] input_dump += str(inf) + "\n" - # get connections. + # Get connections. output = {} connections_dump = "\nconnections\n" for mod in self.mod_wrapper: @@ -373,7 +403,7 @@ def __str__(self): else: output[dep_dname] = f"{mname}.output({dname})" - # get output + # Get output. output_dump = "\noutput\n" for name in sorted(output.keys()): output_dump += f" |output({name}) : {output[name]}\n" @@ -395,7 +425,7 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found.") def get_config(self): - """ Get configuration information in dictionary form.""" + """Get configuration information in dictionary form.""" # Use topological sort to get correct order of modules. self.dag_topology_sort() @@ -416,7 +446,7 @@ def get_config(self): dep_item["input_name"] = dname dep_conf.append(dep_item) - # Ouput_idx start from 0. + # The ouput_idx start from 0. output["output_idx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) @@ -438,7 +468,7 @@ def get_config(self): return mconfig def dag_topology_sort(self): - """ Use topological sort to get order of pipeline modules.""" + """Use topological sort to get order of pipeline modules.""" mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: @@ -457,21 +487,22 @@ def dag_topology_sort(self): self.mod_wrapper[mod].set_idx_name(i + 1) def get_mod_idx(self, mod): - """Return module index.""" + # Return module index. idx = self.mod_wrapper[mod].idx return idx def pipe_input(self, name): - """Return the corresponding input binding interface accordding to the name.""" + # Return the corresponding input binding interface according to the name. return self.input_bindings[name] def pipe_output(self, idx): - """Return the corresponding output binding interface accordding to the name.""" + # Return the corresponding output binding interface according to the name. return self.output_bindings[idx] class PipelineExecutorFactoryModule(object): - """Common interface for pipeline executor factory modules. + """A factory class is responsible for receiving module and configuration + information and maintaining executor module Parameters ---------- diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index d0d3fb87299a..f42f2e916c3c 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -30,9 +30,9 @@ namespace tvm { namespace runtime { /*! * \brief pipeline executor. - * This executor class use module list and dependency relations of modules as - * the parameters and executes these modules on heterogeneous targets in pipeline - * parallel to improve throughput. + * This executor class use a module list and dependency relations of modules as + * the parameters and executes these modules on heterogeneous targets in a pipeline + * parallel manner to improve throughput. * * This executor can be accessed by various language via * TVM runtime PackedFunc API. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 4d2fefec009d..18986fa1e6ea 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -25,7 +25,7 @@ def get_mannual_mod(): - """Get list of module that represent a subgraph.""" + # Get list of module that represent a subgraph. mods = [] dshape = (3, 3) data = relay.var("data_0", relay.TensorType(dshape, "float32")) @@ -40,7 +40,7 @@ def get_mannual_mod(): mv2 = relay.Constant(tvm.nd.array(mvalue2)) mv3 = relay.Constant(tvm.nd.array(mvalue3)) - """The first model has three output.""" + # The first model has three output. net1_output1 = relay.add(data, mv1) net1_output2 = relay.subtract(data, mv2) @@ -74,7 +74,7 @@ def get_mannual_mod(): def get_manual_conf(mods, target): - """This function is used to generate manual pipeline configuration.""" + # This function is used to generate manual pipeline configuration. mod_config = {} """The third output is the final output, the second output is for mod3, the first is for mod2 input. @@ -130,13 +130,12 @@ def get_manual_conf(mods, target): def test_pipe_config_check(): - """This function is used to trigger runtime error by appling wrong logic connection.""" + # This function is used to trigger runtime error by applying wrong logic connection. - """Get three pipeline modules here. - """ + # Get three pipeline modules here. (mod1, mod2, mod3), dshape = get_mannual_mod() - """The input/output name is illegal and expects a runtime error. - """ + + # The input/output name is illegal and expects a runtime error. pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): pipe_error[mod1]["output"][9] @@ -144,14 +143,12 @@ def test_pipe_config_check(): with pytest.raises(RuntimeError): pipe_error[mod1]["input"]["data_9"] - """The connection will cause a circle in DAG and exepects runtime error. - """ + # The connection will cause a cycle in DAG and expects runtime error. with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) - """The module connection is illegal and expects runtime error. - """ + # The module connection is illegal and expects runtime error. with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) @@ -186,8 +183,9 @@ def test_pipeline(): pipe_config = pipeline_executor.PipelineConfig() - # The global input named "data_0" will be connected to a input - # named "data_0" of mod1. + """ The global input named "data_0" will be connected to a input + named "data_0" of mod1. + """ pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) # The global Input named "data_1" will be connected to a input named "data_1" of mod2. @@ -196,16 +194,16 @@ def test_pipeline(): # The mod1 output[0] will be connected to a input named "data_0" of mod2. pipe_config[mod1]["output"][0].connect(pipe_config[mod2]["input"]["data_0"]) - # Mod1 output[1] will be connected to a input named "data_0" of mod3. + # The mod1 output[1] will be connected to a input named "data_0" of mod3. pipe_config[mod1]["output"][1].connect(pipe_config[mod3]["input"]["data_0"]) - # Mod2 output[2] will be connected to a input named "data_1" of mod3. + # The mod2 output[2] will be connected to a input named "data_1" of mod3. pipe_config[mod2]["output"][0].connect(pipe_config[mod3]["input"]["data_1"]) - # Mod1 output[2] will be connected to global output[1]. + # The mod1 output[2] will be connected to global output[1]. pipe_config[mod1]["output"][2].connect(pipe_config["output"]["0"]) - # Mod3 output[0] will be connected to global output[2]. + # The mod3 output[0] will be connected to global output[2]. pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) """Print configueration (print(pipe_config)), the result looks like following. @@ -223,8 +221,7 @@ def test_pipeline(): |mod2.output(0)-> mod3.data_1 """ - """Set other parameter. - """ + # Set other parameter. pipe_config[mod1].target = target[0] pipe_config[mod1].dev = target[1] @@ -234,12 +231,10 @@ def test_pipeline(): pipe_config[mod3].target = "llvm" pipe_config[mod3].dev = tvm.cpu(0) - """Here is to check correctness for configuration generated by API. - """ + # Here is to check correctness for configuration generated by API. assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) - """Build and create pipeline module. - """ + # Build and create pipeline module. with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) From 8a088bf238a4ee0dda124d7f58d669cc7d973b1c Mon Sep 17 00:00:00 2001 From: huajsj Date: Mon, 13 Sep 2021 21:08:21 -0700 Subject: [PATCH 31/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 29 +++----------------- tests/python/relay/test_pipeline_executor.py | 2 +- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index c2870ebd71b9..54e0f0cec5fe 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -34,7 +34,8 @@ def pipeline_executor_enabled(): def build(pipe_configs): - """Use pipe_config to build and return PipelineExecutorFactoryModule. + """Build these modules used in the pipeline executor, then use these modules and configuration + to create a pipeline executor by using the PipelineExecutorFactoryModule class. Parameters ---------- @@ -44,8 +45,7 @@ def build(pipe_configs): Returns ------- ret: PipelineExecutorFactoryModule - A factory class is responsible for receiving module and configuration - information and maintaining executor module + A factory class is responsible for maintaining executor module """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -54,7 +54,6 @@ def build(pipe_configs): for ir_mod, mod_config in mod_n_configs.items(): mconf = mod_config["pipeline"].copy() mod_idx = mconf["mod_idx"] - 1 - # Get mod device configuration. dev = mod_config["dev"] target = mod_config["target"] build_func = relay.build @@ -62,7 +61,6 @@ def build(pipe_configs): if "build" in mod_config and mod_config["build"]: build_func = mod_config["build"] - # Build. mod = build_func( ir_mod, target, @@ -72,32 +70,13 @@ def build(pipe_configs): ) mconf["dev"] = "{},{}".format(dev.device_type, dev.device_id) - # Create pipeline configuration. + # Create a pipeline configuration. string_config[mod_idx] = mconf - # Set device. mods[mod] = {"dev": dev} return PipelineExecutorFactoryModule(mods, string_config) -def create(pipe_executor_factory_module): - """Create a pipeline runtime executor. - - Parameters - ---------- - pipe_executor_factory_module : PipelineExecutorFactoryModule - A factory class is responsible for receiving module and configuration - information and maintaining executor module. - - Returns - ------- - submodule : PipelineModule - Runtime pipeline module. - """ - - return PipelineModule(pipe_executor_factory_module) - - class PipelineModule(object): """Wrapper of runtime module, caller can use this module to set parameters and get outputs. diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 18986fa1e6ea..32ea0c34a951 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -238,7 +238,7 @@ def test_pipeline(): with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) - pipeline_module = pipeline_executor.create(pipeline_mod_config) + pipeline_module = pipeline_executor.PipelineModule(pipeline_mod_config) assert pipeline_module From a88770526563ca3cdb1020497abc7c13c4c9f7d9 Mon Sep 17 00:00:00 2001 From: huajsj Date: Mon, 13 Sep 2021 22:10:14 -0700 Subject: [PATCH 32/34] polish documents. --- python/tvm/contrib/pipeline_executor.py | 90 ++++++++++---------- src/runtime/pipeline/pipeline_executor.h | 3 +- tests/python/relay/test_pipeline_executor.py | 27 +++--- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 54e0f0cec5fe..9821aa077e5d 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -28,7 +28,7 @@ def pipeline_executor_enabled(): Return ------- enable: bool - Return whether pipeline executor is enabled. + Return whether the pipeline executor is enabled. """ return tvm._ffi.get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None @@ -40,12 +40,12 @@ def build(pipe_configs): Parameters ---------- pipe_configs: PipelineConfig - Configuration information for build. + Build Configuration information. Returns ------- ret: PipelineExecutorFactoryModule - A factory class is responsible for maintaining executor module + Common interface for pipeline executor factory modules. """ mods = {} mod_n_configs = pipe_configs.get_config() @@ -91,13 +91,13 @@ def __init__(self, module): class PipelineConfig(object): - """The wrapper of each module to be pipelined. The wrapper mainly includes the - module itself as well as the binding that represents the connections of this - module's inputs and outputs to other modules. + """Pipeline configuration information, this class contains the DAG that expresses + the dependency of each module involved by pipeline and the specific parameters + of each module build. """ class Binding: - """This class defines the module connection information. + """This class defines the module connections information. The type can only be either "input" or "output". Parameters @@ -106,25 +106,28 @@ class Binding: The class who owns this interface. io_type : str - The type of this interface. It can only be either "input" or "output". + The I/O type of this interface. It can only be either "input" or "output". name : str/integer Name, for input it is string such as "data0", for output it is an integer such as 0. + + data_type: TensorType + The data type of this interface. """ def __init__(self, owner, io_type, name, data_type=None): self.io_owner = owner self.io_type = io_type self.name = str(name) - # Child nodes. + # Child nodes that depend on this interface. self.bindings = [] - # Parents nodes. + # Parents nodes that this interface depend on. self.parents = [] self.data_type = data_type def get_name(self): - # Return the interface name and name of owner who owns this interface. + # Return name of this interface and the name of owner who owns this interface. owner_name = "" if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): owner_name = self.io_owner.name @@ -132,12 +135,10 @@ def get_name(self): return owner_name, self.name def get_owner_idx(self): - # Return owner idex if owner is ModuleWrapper, if not return 0. + # If the owner is ModuleWrapper return the owner index, if not return 0. if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): return self.io_owner.idx - # If not ModuleWrapper then owner is PipelineConfig, return 0 - # to identify this is global interface return 0 def is_global_interface(self): @@ -168,7 +169,7 @@ def check_dag_acyclic(self, start, inputs): Return ------ - Return True if there is no cycle in DAG. + Return true if there is no cycle in the DAG. """ for binding in inputs.values(): if start == binding.io_owner: @@ -181,8 +182,8 @@ def check_dag_acyclic(self, start, inputs): def connect(self, binding): """Connect the current interface to the destination interface. - correct connections as following 1. global input to module input, - 2. module output to global output, 3. module output to module input + correct connections as following 1. global input connect to module input, + 2. module output connect to global output, 3. module output connect to module input Parameters ---------- @@ -244,14 +245,14 @@ class BindingList: owner : ModuleWrapper/PipelineConfig The owner of this list can be ModuleWrapper or PipelineConfig. - type_name : str + io_type : str The type of this binding list can be either "input" or "output". """ - def __init__(self, owner, type_name): + def __init__(self, owner, io_type): self.bindings = {} self.io_owner = owner - self.binding_type = type_name + self.binding_type = io_type def get_binding_data_type(self, key): if isinstance(self.io_owner, PipelineConfig.ModuleWrapper): @@ -271,8 +272,8 @@ def __getitem__(self, key): return self.bindings[key] class ModuleWrapper: - """This class is a wrapper that represents the module, contains module information, - binding information and building information. + """This class is a wrapper representing the module and contains information such as + module information, binding information and building information. """ def __init__(self, mod=None): @@ -306,11 +307,12 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found!") def get_data_type(self, key, interface_type): - """Get module interface data type. + """Get the module interface data type according to the key value and interface type. Parameters ---------- key: str The interface name. + interface_type: The interface type. @@ -338,12 +340,14 @@ def set_idx_name(self, idx): self.name = "mod{}".format(str(idx)) def is_root_mod(self): - # Check whether it is root node and is used by DAG topological sort. + """Check whether this node is the root node in DAG, this function is used + in topological sorting. + """ return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): - """Remove itself from child reference to reduce child node in-degree. - This function is used by DAG topological sort. + """Remove the current node from child dependencies to reduce the in-degree + of child node, this function is used in topological sorting. """ for binding in self.output_bindings.bindings.values(): for child in binding.bindings: @@ -356,17 +360,17 @@ def __init__(self): self.output_bindings = self.BindingList(self, "output") def __str__(self): - # Get configuration as string. + # Get configuration information as a string. - # Use topological sort to get correct module order. + # Use topological sorting to get correct module order. self.dag_topology_sort() - # Get input. + # Get the input dependencies. input_dump = "Inputs\n" for input_name in self.input_bindings.bindings: inf = self.input_bindings.bindings[input_name] input_dump += str(inf) + "\n" - # Get connections. + # Get the connections information of each module. output = {} connections_dump = "\nconnections\n" for mod in self.mod_wrapper: @@ -382,7 +386,7 @@ def __str__(self): else: output[dep_dname] = f"{mname}.output({dname})" - # Get output. + # Get the output dependencies. output_dump = "\noutput\n" for name in sorted(output.keys()): output_dump += f" |output({name}) : {output[name]}\n" @@ -404,9 +408,9 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found.") def get_config(self): - """Get configuration information in dictionary form.""" + """Get the configuration information in dictionary form.""" - # Use topological sort to get correct order of modules. + # Use topological sorting to get the correct order of modules. self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: @@ -433,7 +437,6 @@ def get_config(self): mconf["mod_idx"] = module.idx mconf["output"] = output_conf - # Build configuration with parameters. mconfig[mod] = { "pipeline": mconf, "target_host": module.target_host, @@ -447,7 +450,7 @@ def get_config(self): return mconfig def dag_topology_sort(self): - """Use topological sort to get order of pipeline modules.""" + """Use topological sorting to get order of pipeline modules.""" mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: @@ -466,30 +469,29 @@ def dag_topology_sort(self): self.mod_wrapper[mod].set_idx_name(i + 1) def get_mod_idx(self, mod): - # Return module index. + # Return the module index. idx = self.mod_wrapper[mod].idx return idx def pipe_input(self, name): - # Return the corresponding input binding interface according to the name. + # Return the input interface according to the name. return self.input_bindings[name] def pipe_output(self, idx): - # Return the corresponding output binding interface according to the name. + # Return the output interface according to the name. return self.output_bindings[idx] class PipelineExecutorFactoryModule(object): - """A factory class is responsible for receiving module and configuration - information and maintaining executor module + """Common interface for pipeline executor factory modules. Parameters ---------- pipeline_mods : List[GraphExecutorFactoryModule] - list of GraphExecutorFactoryModule. + List of GraphExecutorFactoryModule. mod_config : Dict[int, Dict[str, Any]] - modules and modules dependency configuration information. + Modules dependency configuration information. """ @@ -519,10 +521,10 @@ def graph_executor_create(self, pipeline_mods, mod_config): Returns ------- mods : List[Module] - Module list. + The Module list. mod_config : str - Mods configuration. + The Modudle configuration. """ mods = [] diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index f42f2e916c3c..b4ecc259cf77 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -34,8 +34,7 @@ namespace runtime { * the parameters and executes these modules on heterogeneous targets in a pipeline * parallel manner to improve throughput. * - * This executor can be accessed by various language via - * TVM runtime PackedFunc API. + * This executor can be accessed by various language via TVM runtime PackedFunc API. */ class TVM_DLL PipelineRuntime : public ModuleNode { public: diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 32ea0c34a951..fc285f36b16e 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -25,7 +25,7 @@ def get_mannual_mod(): - # Get list of module that represent a subgraph. + # Get a list of modules representing subgraphs. mods = [] dshape = (3, 3) data = relay.var("data_0", relay.TensorType(dshape, "float32")) @@ -40,22 +40,21 @@ def get_mannual_mod(): mv2 = relay.Constant(tvm.nd.array(mvalue2)) mv3 = relay.Constant(tvm.nd.array(mvalue3)) - # The first model has three output. + # There are three output in the first model. net1_output1 = relay.add(data, mv1) net1_output2 = relay.subtract(data, mv2) net1_output3 = relay.multiply(data, mv3) - """The second model use first model net1_output1 as first input, - here data_net1_output_1 represent net_output1, the second input - of this model is data21. + """The second model use output named net1_output1 of the first model as the first input, + the second input of the second model is data21. """ net2 = relay.add(data_net1_output_1, mv2) net2 = relay.add(net2, data21) net2_output = relay.add(net2, mv3) - """The third model use the second model net2_output as first input - and use the first model net1_output2 as second input. + """The third model use the output named net2_output of the second model as the first input + and use the output named net1_output2 of the first model as the second input. """ net3 = relay.multiply(data_net2_output_1, mv3) net3 = relay.add(net3, data_net1_output_2) @@ -76,8 +75,8 @@ def get_mannual_mod(): def get_manual_conf(mods, target): # This function is used to generate manual pipeline configuration. mod_config = {} - """The third output is the final output, the second output is for mod3, the first is for mod2 - input. + """The third output is the final output, the second output is for mod3, the first output + is for mod2 input. """ pipe_config1 = { "mod_idx": 1, @@ -173,17 +172,17 @@ def test_pipeline(): if pipeline_executor.pipeline_executor_enabled(): target_list = tvm.testing.enabled_targets() for target in target_list: - # Get three pipeline modules here. + # Get the three pipeline modules here. (mod1, mod2, mod3), dshape = get_mannual_mod() - # Batch data is prepared for pipeline feeding. + # Prepare batch data for pipeline computation. datas = [] for i in range(5): datas.append(np.full(dshape, 3 + i).astype("float32")) pipe_config = pipeline_executor.PipelineConfig() - """ The global input named "data_0" will be connected to a input + """The global input named "data_0" will be connected to a input named "data_0" of mod1. """ pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) @@ -221,7 +220,7 @@ def test_pipeline(): |mod2.output(0)-> mod3.data_1 """ - # Set other parameter. + # Set other parameters. pipe_config[mod1].target = target[0] pipe_config[mod1].dev = target[1] @@ -234,7 +233,7 @@ def test_pipeline(): # Here is to check correctness for configuration generated by API. assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) - # Build and create pipeline module. + # Build and create a pipeline module. with tvm.transform.PassContext(opt_level=3): pipeline_mod_config = pipeline_executor.build(pipe_config) From efd91e1de95049c6040f2f9a4853922c4f33ec89 Mon Sep 17 00:00:00 2001 From: huajsj Date: Tue, 14 Sep 2021 00:08:42 -0700 Subject: [PATCH 33/34] Polish the document. --- python/tvm/contrib/pipeline_executor.py | 32 +++++++++++--------- src/runtime/pipeline/pipeline_executor.h | 2 +- tests/python/relay/test_pipeline_executor.py | 18 +++++------ 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 9821aa077e5d..44296fa39a44 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -35,7 +35,7 @@ def pipeline_executor_enabled(): def build(pipe_configs): """Build these modules used in the pipeline executor, then use these modules and configuration - to create a pipeline executor by using the PipelineExecutorFactoryModule class. + to create a pipeline executor. Parameters ---------- @@ -98,7 +98,7 @@ class PipelineConfig(object): class Binding: """This class defines the module connections information. - The type can only be either "input" or "output". + The type can only be "input" or "output". Parameters ---------- @@ -106,7 +106,7 @@ class Binding: The class who owns this interface. io_type : str - The I/O type of this interface. It can only be either "input" or "output". + The I/O type of this interface. It can only be "input" or "output". name : str/integer Name, for input it is string such as "data0", for output it is an integer such as 0. @@ -119,9 +119,9 @@ def __init__(self, owner, io_type, name, data_type=None): self.io_owner = owner self.io_type = io_type self.name = str(name) - # Child nodes that depend on this interface. + # Child interfaces that depend on this interface. self.bindings = [] - # Parents nodes that this interface depend on. + # Parents interfaces that this interface depend on. self.parents = [] self.data_type = data_type @@ -243,10 +243,10 @@ class BindingList: Parameters ---------- owner : ModuleWrapper/PipelineConfig - The owner of this list can be ModuleWrapper or PipelineConfig. + The owner of this class can be ModuleWrapper or PipelineConfig. io_type : str - The type of this binding list can be either "input" or "output". + The type of this class can be "input" or "output". """ def __init__(self, owner, io_type): @@ -341,13 +341,13 @@ def set_idx_name(self, idx): def is_root_mod(self): """Check whether this node is the root node in DAG, this function is used - in topological sorting. + in topological sort. """ return all([not b.parents for b in self.input_bindings.bindings.values()]) def remove_self_from_bindings(self): """Remove the current node from child dependencies to reduce the in-degree - of child node, this function is used in topological sorting. + of child node, this function is used in topological sort. """ for binding in self.output_bindings.bindings.values(): for child in binding.bindings: @@ -362,7 +362,7 @@ def __init__(self): def __str__(self): # Get configuration information as a string. - # Use topological sorting to get correct module order. + # Use topological sort to get correct module order. self.dag_topology_sort() # Get the input dependencies. input_dump = "Inputs\n" @@ -408,13 +408,15 @@ def __getitem__(self, key): raise RuntimeError(f"{key} not found.") def get_config(self): - """Get the configuration information in dictionary form.""" + """Get the configuration information in dictionary form, this configuration + will be used to create pipeline executor. + """ - # Use topological sorting to get the correct order of modules. + # Use topological sort to get the correct order of modules. self.dag_topology_sort() mconfig = {} for mod in self.mod_wrapper: - # Generate pipeline configure. + # Generate pipeline configuration. mconf = {} output_conf = [] module = self.mod_wrapper[mod] @@ -429,7 +431,7 @@ def get_config(self): dep_item["input_name"] = dname dep_conf.append(dep_item) - # The ouput_idx start from 0. + # The value of ouput_idx start from 0. output["output_idx"] = int(binding.name) output["dependent"] = dep_conf output_conf.append(output) @@ -450,7 +452,7 @@ def get_config(self): return mconfig def dag_topology_sort(self): - """Use topological sorting to get order of pipeline modules.""" + """Use topological sort to get order of pipeline modules.""" mlist = [] mod_wrapper = self.mod_wrapper.copy() while mod_wrapper: diff --git a/src/runtime/pipeline/pipeline_executor.h b/src/runtime/pipeline/pipeline_executor.h index b4ecc259cf77..c7625c62b724 100644 --- a/src/runtime/pipeline/pipeline_executor.h +++ b/src/runtime/pipeline/pipeline_executor.h @@ -30,7 +30,7 @@ namespace tvm { namespace runtime { /*! * \brief pipeline executor. - * This executor class use a module list and dependency relations of modules as + * This executor class use the module list and dependency configuration of modules as * the parameters and executes these modules on heterogeneous targets in a pipeline * parallel manner to improve throughput. * diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index fc285f36b16e..7348900efead 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -40,7 +40,7 @@ def get_mannual_mod(): mv2 = relay.Constant(tvm.nd.array(mvalue2)) mv3 = relay.Constant(tvm.nd.array(mvalue3)) - # There are three output in the first model. + # There are three outputs in the first model. net1_output1 = relay.add(data, mv1) net1_output2 = relay.subtract(data, mv2) @@ -131,10 +131,10 @@ def get_manual_conf(mods, target): def test_pipe_config_check(): # This function is used to trigger runtime error by applying wrong logic connection. - # Get three pipeline modules here. + # Get the three pipeline modules here. (mod1, mod2, mod3), dshape = get_mannual_mod() - # The input/output name is illegal and expects a runtime error. + # The input or output name is illegal and expects a runtime error. pipe_error = pipeline_executor.PipelineConfig() with pytest.raises(RuntimeError): pipe_error[mod1]["output"][9] @@ -142,7 +142,7 @@ def test_pipe_config_check(): with pytest.raises(RuntimeError): pipe_error[mod1]["input"]["data_9"] - # The connection will cause a cycle in DAG and expects runtime error. + # The module connection will cause a cycle in DAG and expects runtime error. with pytest.raises(RuntimeError): pipe_error[mod1]["output"][0].connect(pipe_error[mod2]["input"]["data_0"]) pipe_error[mod2]["output"][0].connect(pipe_error[mod1]["input"]["data_0"]) @@ -182,9 +182,7 @@ def test_pipeline(): pipe_config = pipeline_executor.PipelineConfig() - """The global input named "data_0" will be connected to a input - named "data_0" of mod1. - """ + # The global input named "data_0" will be connected to a input named "data_0" of mod1. pipe_config["input"]["data_0"].connect(pipe_config[mod1]["input"]["data_0"]) # The global Input named "data_1" will be connected to a input named "data_1" of mod2. @@ -230,14 +228,14 @@ def test_pipeline(): pipe_config[mod3].target = "llvm" pipe_config[mod3].dev = tvm.cpu(0) - # Here is to check correctness for configuration generated by API. + # Here is to check the correctness of the configuration generated by API. assert pipe_config.get_config() == get_manual_conf([mod1, mod2, mod3], target) # Build and create a pipeline module. with tvm.transform.PassContext(opt_level=3): - pipeline_mod_config = pipeline_executor.build(pipe_config) + pipeline_mod_factory = pipeline_executor.build(pipe_config) - pipeline_module = pipeline_executor.PipelineModule(pipeline_mod_config) + pipeline_module = pipeline_executor.PipelineModule(pipeline_mod_factory) assert pipeline_module From db8422d99fe7f72b3ba408d2b3dd0589a34784c3 Mon Sep 17 00:00:00 2001 From: huajsj Date: Tue, 14 Sep 2021 23:02:57 -0700 Subject: [PATCH 34/34] address review comments. --- python/tvm/contrib/pipeline_executor.py | 16 ++++--- tests/python/relay/test_pipeline_executor.py | 44 +++++++++----------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py index 44296fa39a44..36c03891d210 100644 --- a/python/tvm/contrib/pipeline_executor.py +++ b/python/tvm/contrib/pipeline_executor.py @@ -34,7 +34,7 @@ def pipeline_executor_enabled(): def build(pipe_configs): - """Build these modules used in the pipeline executor, then use these modules and configuration + """Build modules used in the pipeline executor, then use these modules and configuration to create a pipeline executor. Parameters @@ -92,8 +92,8 @@ def __init__(self, module): class PipelineConfig(object): """Pipeline configuration information, this class contains the DAG that expresses - the dependency of each module involved by pipeline and the specific parameters - of each module build. + the dependency of each module involved in a pipeline and the parameters for building + each module. """ class Binding: @@ -182,8 +182,9 @@ def check_dag_acyclic(self, start, inputs): def connect(self, binding): """Connect the current interface to the destination interface. - correct connections as following 1. global input connect to module input, - 2. module output connect to global output, 3. module output connect to module input + Correct connections are as follows: 1. global input connected to module input, + 2. module output connected to global output, 3. module output connected to + module input. Parameters ---------- @@ -231,7 +232,10 @@ def connect(self, binding): ) binding.parents.append(self) - # Do acyclic check after increase the in-degree. + + # Do acyclic check after increasing the in-degree of child node by setting + # current interface as a parent of the child node. + if not self.check_dag_acyclic( binding.io_owner, self.io_owner.input_bindings.bindings ): diff --git a/tests/python/relay/test_pipeline_executor.py b/tests/python/relay/test_pipeline_executor.py index 7348900efead..d9411c92c375 100644 --- a/tests/python/relay/test_pipeline_executor.py +++ b/tests/python/relay/test_pipeline_executor.py @@ -46,16 +46,14 @@ def get_mannual_mod(): net1_output2 = relay.subtract(data, mv2) net1_output3 = relay.multiply(data, mv3) - """The second model use output named net1_output1 of the first model as the first input, - the second input of the second model is data21. - """ + # The second model use output named net1_output1 of the first model as the first input, + # the second input of the second model is data21. net2 = relay.add(data_net1_output_1, mv2) net2 = relay.add(net2, data21) net2_output = relay.add(net2, mv3) - """The third model use the output named net2_output of the second model as the first input - and use the output named net1_output2 of the first model as the second input. - """ + # The third model use the output named net2_output of the second model as the first input + # and use the output named net1_output2 of the first model as the second input. net3 = relay.multiply(data_net2_output_1, mv3) net3 = relay.add(net3, data_net1_output_2) @@ -75,9 +73,8 @@ def get_mannual_mod(): def get_manual_conf(mods, target): # This function is used to generate manual pipeline configuration. mod_config = {} - """The third output is the final output, the second output is for mod3, the first output - is for mod2 input. - """ + # The third output is the final output, the second output is for mod3, the first output + # is for mod2 input. pipe_config1 = { "mod_idx": 1, "output": [ @@ -202,21 +199,20 @@ def test_pipeline(): # The mod3 output[0] will be connected to global output[2]. pipe_config[mod3]["output"][0].connect(pipe_config["output"]["1"]) - """Print configueration (print(pipe_config)), the result looks like following. - - Inputs - |data_0: mod1:data_0 - |data_1: mod2:data_1 - - output - |output(1) : mod1.output(2) - |output(2) : mod3.output(0) - - connections - |mod1.output(0)-> mod2.data_0 - |mod1.output(1)-> mod3.data_0 - |mod2.output(0)-> mod3.data_1 - """ + # Print configueration (print(pipe_config)), the result looks like following. + # + # Inputs + # |data_0: mod1:data_0 + # |data_1: mod2:data_1 + # + # output + # |output(1) : mod1.output(2) + # |output(2) : mod3.output(0) + # + # connections + # |mod1.output(0)-> mod2.data_0 + # |mod1.output(1)-> mod3.data_0 + # |mod2.output(0)-> mod3.data_1 # Set other parameters. pipe_config[mod1].target = target[0]