From 15611c1c4fd1bf8d3ff4abc5382a14c4f5abfa63 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 5 Apr 2019 11:21:01 -0700 Subject: [PATCH 001/146] Fix build_ccache_wrappers: * Fix broken links * Make it idempotent fixes https://github.com/apache/incubator-mxnet/pull/13456 fixes https://github.com/apache/incubator-mxnet/issues/14117 fixes https://github.com/apache/incubator-mxnet/issues/11516 --- ci/docker/runtime_functions.sh | 49 ++++++++++++++-------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index c3610d2452e0..56e078f5f8ed 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -66,37 +66,29 @@ build_ccache_wrappers() { # But in the beginning, we'll make this opt-in. In future, loads of processes like # the scala make step or numpy compilation and other pip package generations # could be heavily sped up by using ccache as well. - mkdir /tmp/ccache-redirects + mkdir -p /tmp/ccache-redirects export PATH=/tmp/ccache-redirects:$PATH - ln -s ccache /tmp/ccache-redirects/gcc - ln -s ccache /tmp/ccache-redirects/gcc-8 - ln -s ccache /tmp/ccache-redirects/g++ - ln -s ccache /tmp/ccache-redirects/g++-8 - ln -s ccache /tmp/ccache-redirects/nvcc - ln -s ccache /tmp/ccache-redirects/clang++-3.9 - ln -s ccache /tmp/ccache-redirects/clang-3.9 - ln -s ccache /tmp/ccache-redirects/clang++-5.0 - ln -s ccache /tmp/ccache-redirects/clang-5.0 - ln -s ccache /tmp/ccache-redirects/clang++-6.0 - ln -s ccache /tmp/ccache-redirects/clang-6.0 - ln -s ccache /usr/local/bin/gcc - ln -s ccache /usr/local/bin/gcc-8 - ln -s ccache /usr/local/bin/g++ - ln -s ccache /usr/local/bin/g++-8 - ln -s ccache /usr/local/bin/nvcc - ln -s ccache /usr/local/bin/clang++-3.9 - ln -s ccache /usr/local/bin/clang-3.9 - ln -s ccache /usr/local/bin/clang++-5.0 - ln -s ccache /usr/local/bin/clang-5.0 - ln -s ccache /usr/local/bin/clang++-6.0 - ln -s ccache /usr/local/bin/clang-6.0 - - export NVCC=ccache + CCACHE=`which ccache` + ln -sf $CCACHE /tmp/ccache-redirects/gcc + ln -sf $CCACHE /tmp/ccache-redirects/gcc-8 + ln -sf $CCACHE /tmp/ccache-redirects/g++ + ln -sf $CCACHE /tmp/ccache-redirects/g++-8 + ln -sf $CCACHE /tmp/ccache-redirects/nvcc + ln -sf $CCACHE /tmp/ccache-redirects/clang++-3.9 + ln -sf $CCACHE /tmp/ccache-redirects/clang-3.9 + ln -sf $CCACHE /tmp/ccache-redirects/clang++-5.0 + ln -sf $CCACHE /tmp/ccache-redirects/clang-5.0 + ln -sf $CCACHE /tmp/ccache-redirects/clang++-6.0 + ln -sf $CCACHE /tmp/ccache-redirects/clang-6.0 + # Doesn't work: https://github.com/ccache/ccache/issues/373 + #ln -sf $CCACHE /tmp/ccache-redirects/nvcc + #export NVCC="/tmp/ccache-redirects/nvcc" # Uncomment if you would like to debug CCache hit rates. # You can monitor using tail -f ccache-log - # export CCACHE_LOGFILE=/work/mxnet/ccache-log - # export CCACHE_DEBUG=1 + #export CCACHE_LOGFILE=/work/mxnet/ccache-log + #export CCACHE_LOGFILE=/tmp/ccache-log + #export CCACHE_DEBUG=1 } build_wheel() { @@ -667,8 +659,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ubuntu_gpu_cuda91_cudnn7() { set -ex - # unfortunately this build has problems in 3rdparty dependencies with ccache and make - # build_ccache_wrappers + build_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ From 7711a0bc57ef4781af491557e2455d9e3c0c408e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 8 Feb 2019 13:55:02 +0100 Subject: [PATCH 002/146] Optimize move semantics of NodeEntry https://github.com/dmlc/tvm/pull/2576 Making copies of shared_ptr is very expensive, 100x more than moving. This PR reduces lock contention by using move semantics in NNVM nodes --- .gitmodules | 2 +- 3rdparty/tvm | 2 +- src/c_api/c_api_function.cc | 2 +- src/executor/graph_executor.cc | 8 ++++---- src/imperative/cached_op.cc | 4 ++-- src/imperative/imperative.cc | 2 +- src/nnvm/legacy_op_util.cc | 23 +++++++++++------------ src/operator/custom/custom.cc | 2 +- src/operator/nn/lrn.cc | 2 +- src/operator/operator_common.h | 2 +- src/operator/tensor/elemwise_sum.cc | 6 +++--- 11 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.gitmodules b/.gitmodules index e0ffec11bfd0..151b9fe39a86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/dmlc/tvm + url = https://github.com/larroy/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git diff --git a/3rdparty/tvm b/3rdparty/tvm index 0f053c82a747..e6df3d2fc099 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 +Subproject commit e6df3d2fc099291411308e983fa7edd2df8a669a diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc index 50f9b32d6e47..3adc581b50a5 100644 --- a/src/c_api/c_api_function.cc +++ b/src/c_api/c_api_function.cc @@ -56,7 +56,7 @@ std::vector Gradient( std::vector ret; for (uint32_t i = 0; i < g->num_outputs(); ++i) { - ret.emplace_back(nnvm::NodeEntry{g, i, 0}); + ret.emplace_back(std::move(g), i, 0); } return ret; diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 4a4505581920..21a617cb505f 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -147,7 +147,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { ng->attrs.op = Op::Get("_zeros_without_dtype"); ng->attrs.name = "zeros_without_dtype"; ng->attrs.op->attr_parser(&(ng->attrs)); - return nnvm::NodeEntry{ng, 0, 0}; + return nnvm::NodeEntry(std::move(ng), 0, 0); } // remove zero in the sum. at least keep 1. @@ -168,7 +168,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { sum_node->attrs.dict["num_args"] = std::to_string(v.size()); sum_node->attrs.op->attr_parser(&(sum_node->attrs)); sum_node->inputs = std::move(v); - return nnvm::NodeEntry{sum_node, 0, 0}; + return nnvm::NodeEntry(std::move(sum_node), 0, 0); } else { // use a stream line of plus instead nnvm::NodeEntry ret = v[0]; @@ -198,7 +198,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { x->attrs.op = ewise_plus_op; x->attrs.name = os.str(); x->inputs = {ret, v[i]}; - ret = nnvm::NodeEntry{x, 0, 0}; + ret = nnvm::NodeEntry(std::move(x), 0, 0); } // identity node is used to avoid exposure of dummy plus node // when its output get assigned to another space. @@ -247,7 +247,7 @@ nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, } if (!need_grad_) return g; for (size_t i = 0; i < g.outputs.size(); ++i) { - NodeEntry ngrad{nnvm::Node::Create(), 0, 0}; + NodeEntry ngrad; head_grad_entry_.emplace_back(AttrHint(ngrad, g.outputs[i])); head_grad_map_[ngrad.node.get()] = i; } diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 7a5ed21432d3..1601d0e0494f 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -160,7 +160,7 @@ CachedOp::CachedOp( { ograd_entries_.reserve(fwd_graph_.outputs.size()); for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { - ograd_entries_.emplace_back(NodeEntry{Node::Create(), 0, 0}); + ograd_entries_.emplace_back(); } std::vector xs; @@ -169,7 +169,7 @@ CachedOp::CachedOp( auto nid = idx.input_nodes()[i]; if (idx.mutable_input_nodes().count(nid)) continue; fwd_input_to_grad_output_[i] = xs.size(); - xs.emplace_back(NodeEntry{idx[nid].weak_ref.lock(), 0, 0}); + xs.emplace_back(idx[nid].weak_ref.lock(), 0, 0); } CHECK_GT(xs.size(), 0) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index b027de0a0f6f..1597eee9ca26 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -305,7 +305,7 @@ std::vector Imperative::Backward( std::vector ograd_entries; ograd_entries.reserve(ograds.size()); for (size_t i = 0; i < outputs.size(); ++i) { - ograd_entries.emplace_back(NodeEntry{Node::Create(), 0, 0}); + ograd_entries.emplace_back(); AGInfo& info = AGInfo::Create(ograd_entries.back().node); info.ctx = outputs[i]->ctx(); if (ograds[i] != nullptr) { diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 16ad0053e29a..e738a5a47695 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -321,9 +321,10 @@ inline std::vector OpPropGradient( const NodePtr& ptr, const std::vector& out_grads) { auto& prop = nnvm::get(ptr->attrs.parsed); - std::vector out_data(prop.outputs.size()); - for (uint32_t i = 0; i < out_data.size(); ++i) { - out_data[i] = NodeEntry{ptr, i, 0}; + std::vector out_data; + out_data.reserve(prop.outputs.size()); + for (size_t i = 0; i < out_data.size(); ++i) { + out_data.emplace_back(ptr, i, 0); } std::vector in_data( ptr->inputs.begin(), ptr->inputs.begin() + prop.arguments.size()); @@ -331,7 +332,7 @@ inline std::vector OpPropGradient( out_grads.begin(), out_grads.begin() + prop.ptr->NumVisibleOutputs()); auto inputs = prop.ptr->BackwardInputs(ograd, in_data, out_data); // add all the auxiliary data - for (uint32_t i = 0; i < prop.aux_states.size(); ++i) { + for (size_t i = 0; i < prop.aux_states.size(); ++i) { inputs.emplace_back(ptr->inputs[i + prop.arguments.size()]); } NodePtr gnode = Node::Create(); @@ -340,17 +341,15 @@ inline std::vector OpPropGradient( gnode->attrs = ptr->attrs; gnode->attrs.op = back_op; gnode->attrs.name = ptr->attrs.name + "_backward"; - std::vector in_grad(prop.arguments.size()); - for (uint32_t i = 0; i < prop.arguments.size(); ++i) { - in_grad[i] = NodeEntry{gnode, i, 0}; + std::vector in_grad; + in_grad.reserve(prop.arguments.size() + prop.aux_states.size()); + for (size_t i = 0; i < prop.arguments.size(); ++i) { + in_grad.emplace_back(gnode, i, 0); } // attach no gradient node to forbid gradient on aux_state if (prop.aux_states.size() != 0) { - NodePtr ng = Node::Create(); - ng->attrs.op = Op::Get("_NoGradient"); - ng->attrs.name = "NoGradient"; - for (uint32_t i = 0; i < prop.aux_states.size(); ++i) { - in_grad.emplace_back(NodeEntry{ng, 0, 0}); + for (size_t i = 0; i < prop.aux_states.size(); ++i) { + in_grad.emplace_back(Node::Create(Op::Get("_NoGradient"), "NoGradient"), 0, 0); } } return in_grad; diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 412bfa1bc3aa..544fdc3a8dce 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -224,7 +224,7 @@ std::vector Gradient( size_t i = static_cast(t); if (i >= params.num_outs + params.num_args) { uint32_t idx = static_cast(i-params.num_outs-params.num_args); - g->inputs.push_back(nnvm::NodeEntry{n, idx, 0}); + g->inputs.push_back(n, idx, 0); } else if (i >= params.num_outs) { g->inputs.push_back(n->inputs[i-params.num_outs]); } else { diff --git a/src/operator/nn/lrn.cc b/src/operator/nn/lrn.cc index b632e35b57fe..3a3ca59f2be1 100644 --- a/src/operator/nn/lrn.cc +++ b/src/operator/nn/lrn.cc @@ -77,7 +77,7 @@ struct LRNGrad { std::vector heads; heads.push_back(ograds[0]); // out_grad heads.push_back(n->inputs[lrn_enum::kData]); - heads.emplace_back(nnvm::NodeEntry{n, lrn_enum::kTmpNorm, 0}); + heads.emplace_back(n, lrn_enum::kTmpNorm, 0); return MakeGradNode(op_name, n, heads, n->attrs.dict); } }; diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 59f572211d0e..0328dfa7de26 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -447,7 +447,7 @@ inline std::vector MakeNonlossGradNode( p->inputs.insert(p->inputs.end(), inputs.begin(), inputs.end()); std::vector ret; for (uint32_t i = 0; i < p->num_outputs(); ++i) { - ret.emplace_back(nnvm::NodeEntry{p, i, 0}); + ret.emplace_back(std::move(p), i, 0); } return ret; } diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index f1ec8b5ad387..dec57633be22 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -49,12 +49,12 @@ std::vector ElementWiseSumGrad( nnvm::Op::Get("identity"); CHECK_EQ(ograds.size(), 1); std::vector ret; - nnvm::NodeEntry n_out{n, 0, 0}; - for (size_t i = 0; i < n->inputs.size(); i++) { + nnvm::NodeEntry n_out(n, 0, 0); + for (size_t i = 0; i < n->inputs.size(); ++i) { nnvm::NodePtr id_node = nnvm::Node::Create(); id_node->attrs.op = copy_op; id_node->inputs = {ograds[0]}; - ret.push_back(nnvm::NodeEntry{id_node, 0, 0}); + ret.emplace_back(id_node, 0, 0); } return ret; } From 3294d84397988645b9e6f1d2a65eaca966a05ee5 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 8 Feb 2019 15:35:20 +0100 Subject: [PATCH 003/146] Compile fix --- src/operator/custom/custom.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 544fdc3a8dce..63b42007317b 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -224,7 +224,7 @@ std::vector Gradient( size_t i = static_cast(t); if (i >= params.num_outs + params.num_args) { uint32_t idx = static_cast(i-params.num_outs-params.num_args); - g->inputs.push_back(n, idx, 0); + g->inputs.emplace_back(n, idx, 0); } else if (i >= params.num_outs) { g->inputs.push_back(n->inputs[i-params.num_outs]); } else { From 9e4a084b639a568202d62548417bd4a3ab107984 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sun, 10 Feb 2019 22:58:21 +0100 Subject: [PATCH 004/146] Fix --- src/operator/operator_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 0328dfa7de26..732203f0e601 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -447,7 +447,7 @@ inline std::vector MakeNonlossGradNode( p->inputs.insert(p->inputs.end(), inputs.begin(), inputs.end()); std::vector ret; for (uint32_t i = 0; i < p->num_outputs(); ++i) { - ret.emplace_back(std::move(p), i, 0); + ret.emplace_back(p, i, 0); } return ret; } From cc08215ac59e8f093bc059a3873cdd0d3ad789f1 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 11 Feb 2019 23:25:41 +0100 Subject: [PATCH 005/146] Fix crash, uninitialized Node ptr --- src/executor/graph_executor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 21a617cb505f..bc80be5f92a5 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -247,7 +247,7 @@ nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, } if (!need_grad_) return g; for (size_t i = 0; i < g.outputs.size(); ++i) { - NodeEntry ngrad; + NodeEntry ngrad(nnvm::Node::Create(), 0, 0); head_grad_entry_.emplace_back(AttrHint(ngrad, g.outputs[i])); head_grad_map_[ngrad.node.get()] = i; } From 092c952685892c898f01cbc419c045f4a62221f8 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 12 Feb 2019 00:09:53 +0100 Subject: [PATCH 006/146] Fix --- src/c_api/c_api_function.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc index 3adc581b50a5..3cd70379b68f 100644 --- a/src/c_api/c_api_function.cc +++ b/src/c_api/c_api_function.cc @@ -56,7 +56,7 @@ std::vector Gradient( std::vector ret; for (uint32_t i = 0; i < g->num_outputs(); ++i) { - ret.emplace_back(std::move(g), i, 0); + ret.emplace_back(g, i, 0); } return ret; From 6e1764dd6135c9f28ef96f35f30f8ea78bb2ac17 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 14 Mar 2019 18:43:53 -0700 Subject: [PATCH 007/146] Fix autograd crash --- src/imperative/imperative.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 1597eee9ca26..b027de0a0f6f 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -305,7 +305,7 @@ std::vector Imperative::Backward( std::vector ograd_entries; ograd_entries.reserve(ograds.size()); for (size_t i = 0; i < outputs.size(); ++i) { - ograd_entries.emplace_back(); + ograd_entries.emplace_back(NodeEntry{Node::Create(), 0, 0}); AGInfo& info = AGInfo::Create(ograd_entries.back().node); info.ctx = outputs[i]->ctx(); if (ograds[i] != nullptr) { From 59abd4dfca0dffcfc69c8461f76ee7cb6c7608cd Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 22:42:54 +0000 Subject: [PATCH 008/146] Fix clang tidy errors --- src/imperative/cached_op.cc | 2 +- src/imperative/imperative.cc | 2 +- src/operator/tensor/broadcast_reduce_op_value.cc | 4 ++-- src/operator/tensor/elemwise_unary_op_basic.cc | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 1601d0e0494f..90e8ee9d7c6b 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -118,7 +118,7 @@ CachedOp::CachedOp( if (_copy->attr_parser != nullptr) { _copy->attr_parser(&(copy_node->attrs)); } - fwd_graph_.outputs.push_back(NodeEntry{copy_node, 0, 0}); + fwd_graph_.outputs.emplace_back(copy_node, 0, 0); } else { dedup_out.insert({i, 0}); fwd_graph_.outputs.push_back(i); diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index b027de0a0f6f..a1c41ee0df6b 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -363,7 +363,7 @@ std::vector Imperative::Backward( auto node = Node::Create(); node->attrs.op = copy_op; node->inputs.push_back(e); - graph.outputs.push_back(NodeEntry{node, 0, 0}); + graph.outputs.emplace_back(node, 0, 0); } else { graph.outputs.push_back(e); } diff --git a/src/operator/tensor/broadcast_reduce_op_value.cc b/src/operator/tensor/broadcast_reduce_op_value.cc index f890963c2cf1..3a598dbcf374 100644 --- a/src/operator/tensor/broadcast_reduce_op_value.cc +++ b/src/operator/tensor/broadcast_reduce_op_value.cc @@ -287,11 +287,11 @@ NNVM_REGISTER_OP(broadcast_like) [](const nnvm::NodePtr& n, const std::vector& ograds) { if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); - auto lhs = MakeNonlossGradNode("_broadcast_backward", n, ograds, {}, + std::vector lhs = MakeNonlossGradNode("_broadcast_backward", n, ograds, {}, {{"keepdims", "true"}}); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); + lhs.emplace_back(ng, 0, 0); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index 5114a5d0dbe3..f9d79ebf893e 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -356,11 +356,11 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) "FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); - auto lhs = MakeGradNode("_backward_copy", n, ograds, + std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); + lhs.emplace_back(ng, 0, 0); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") @@ -495,11 +495,11 @@ Negative indices are supported, and `None` can be used for either `lhs_end` or ` "FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); - auto lhs = MakeGradNode("_backward_copy", n, ograds, + std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); + lhs.emplace_back(ng, 0, 0); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") From 0679ca250f9d7c61651388fbf46735fa513e94cb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 22:44:30 +0000 Subject: [PATCH 009/146] Restore subrepo --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 151b9fe39a86..e0ffec11bfd0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/larroy/tvm + url = https://github.com/dmlc/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git From 33e99db5ead68e37a039277afc376f160cd59398 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 23:48:30 +0000 Subject: [PATCH 010/146] restore tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index e6df3d2fc099..51785062553b 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit e6df3d2fc099291411308e983fa7edd2df8a669a +Subproject commit 51785062553b19a219b2639cd230b7da7455cd0d From 5f8c4b8293c0e068a8ae8c8c2782ecde98b17a8b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 23:49:59 +0000 Subject: [PATCH 011/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 51785062553b..0f053c82a747 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 51785062553b19a219b2639cd230b7da7455cd0d +Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 From 3d955eb25017a561674e2a160ce77ad4db8fe814 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 18:21:43 -0700 Subject: [PATCH 012/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 0f053c82a747..2da23bd86905 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 +Subproject commit 2da23bd86905c3abec857a28b6135d069ea9515d From b85b9a67fbc5c9b9604fcde54693c0aa9cdfae5b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 18:22:17 -0700 Subject: [PATCH 013/146] Update NDarray with NodeEntry constructors and refine initializer lists --- include/mxnet/ndarray.h | 52 +++++++++++++++++++------------ src/operator/elemwise_op_common.h | 4 +-- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 05d3fa45683e..c015044baa15 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -94,8 +94,11 @@ class NDArray { NDArray(const mxnet::TShape &shape, Context ctx, bool delay_alloc = false, int dtype = mshadow::default_type_flag) : ptr_(std::make_shared(shape, ctx, delay_alloc, dtype)), - shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_({nullptr, 0, 0}) { + shape_(shape), + dtype_(dtype), + storage_type_(kDefaultStorage), + entry_() + { } /*! \brief constructor for NDArray with storage type */ @@ -109,11 +112,13 @@ class NDArray { * \param ctx context of NDArray * \param dtype data type of this ndarray */ - explicit NDArray(Context ctx, int dtype = mshadow::default_type_flag) { - ptr_ = std::make_shared(mxnet::TShape(mshadow::Shape1(0)), ctx, true, dtype); - dtype_ = dtype; - storage_type_ = kDefaultStorage; - entry_ = {nullptr, 0, 0}; + explicit NDArray(Context ctx, int dtype = mshadow::default_type_flag) + : ptr_(std::make_shared(mxnet::TShape(mshadow::Shape1(0)), ctx, true, dtype)), + shape_(), + dtype_(dtype), + storage_type_(kDefaultStorage), + entry_() + { } /*! * \brief constructing a static NDArray that shares data with TBlob @@ -123,9 +128,11 @@ class NDArray { * \param dev_id the device id this tensor sits at */ NDArray(const TBlob &data, int dev_id) - : ptr_(std::make_shared(data, dev_id)), shape_(data.shape_), - dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_({nullptr, 0, 0}) { + : ptr_(std::make_shared(data, dev_id)), + shape_(data.shape_), + dtype_(data.type_flag_), + storage_type_(kDefaultStorage), + entry_() { } /*! @@ -137,20 +144,22 @@ class NDArray { * \param deleter the function pointer of custom deleter */ NDArray(const TBlob &data, int dev_id, const std::function& deleter) - : ptr_(new Chunk(data, dev_id), - [deleter](Chunk *p) { - deleter(); // call custom deleter - delete p; // delete Chunk object + : ptr_(new Chunk(data, dev_id), [deleter](Chunk *p) { + deleter(); // call custom deleter + delete p; // delete Chunk object }), shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_({nullptr, 0, 0}) { + entry_() { } /*! \brief create ndarray from shared memory */ NDArray(int shared_pid, int shared_id, const mxnet::TShape& shape, int dtype) - : ptr_(std::make_shared(shared_pid, shared_id, shape, dtype)), shape_(shape), - dtype_(dtype), storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { + : ptr_(std::make_shared(shared_pid, shared_id, shape, dtype)), + shape_(shape), + dtype_(dtype), + storage_type_(kDefaultStorage), + entry_() { } /*! @@ -165,8 +174,11 @@ class NDArray { */ NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, const TBlob &data, const std::vector &aux_data, int dev_id) - : ptr_(std::make_shared(stype, data, aux_data, dev_id)), shape_(shape), - dtype_(data.type_flag_), storage_type_(stype), entry_({nullptr, 0, 0}) { + : ptr_(std::make_shared(stype, data, aux_data, dev_id)), + shape_(shape), + dtype_(data.type_flag_), + storage_type_(stype), + entry_() { } /*! * \brief initialize the NDArray, assuming it is not assigned a meaningful shape before @@ -640,7 +652,7 @@ class NDArray { */ NDArray Detach() const { NDArray ret(*this); - ret.entry_ = nnvm::NodeEntry{nullptr, 0, 0}; + ret.entry_ = nnvm::NodeEntry(); return ret; } diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index 2edaa55540c1..6dae2dfa20c4 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -203,7 +203,7 @@ struct ElemwiseGradUseOut { std::vector heads; uint32_t n_out = n->num_outputs(); for (uint32_t i = 0; i < n_out; ++i) { - heads.emplace_back(nnvm::NodeEntry{n, i, 0}); + heads.emplace_back(n, i, 0); } return MakeNonlossGradNode(op_name, n, ograds, heads, n->attrs.dict); } @@ -220,7 +220,7 @@ struct ElemwiseGradUseInOut { } uint32_t n_out = n->num_outputs(); for (uint32_t i = 0; i < n_out; ++i) { - heads.emplace_back(nnvm::NodeEntry{n, i, 0}); + heads.emplace_back(n, i, 0); } return MakeGradNode(op_name, n, heads, n->attrs.dict); } From aeec3069f0f42712dc313bc805c933581df2fb52 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 11 Apr 2019 20:26:34 +0000 Subject: [PATCH 014/146] Fix lint --- include/mxnet/ndarray.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index c015044baa15..bb7f2b2f5b6d 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -97,8 +97,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() - { + entry_() { } /*! \brief constructor for NDArray with storage type */ @@ -117,8 +116,7 @@ class NDArray { shape_(), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() - { + entry_() { } /*! * \brief constructing a static NDArray that shares data with TBlob From 778902f51ed7991c46aea3456a4ac23a4cb2a862 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 12:54:22 -0700 Subject: [PATCH 015/146] fix --- src/imperative/cached_op.cc | 42 ++++++++++++++++++------------------- src/ndarray/ndarray.cc | 2 +- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 90e8ee9d7c6b..8f01fa134b6f 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -108,20 +108,20 @@ CachedOp::CachedOp( // construct forward graph { NodeEntryMap dedup_out; - for (const auto& i : sym.outputs) { - if (dedup_out.count(i)) { + for (const NodeEntry& nodeEntry : sym.outputs) { + if (dedup_out.count(nodeEntry)) { NodePtr copy_node = Node::Create(); copy_node->attrs.op = _copy; copy_node->attrs.name = - i.node->attrs.name + "_copy" + std::to_string(dedup_out[i]++); - copy_node->inputs.emplace_back(i); + nodeEntry.node->attrs.name + "_copy" + std::to_string(dedup_out[nodeEntry]++); + copy_node->inputs.emplace_back(nodeEntry); if (_copy->attr_parser != nullptr) { _copy->attr_parser(&(copy_node->attrs)); } fwd_graph_.outputs.emplace_back(copy_node, 0, 0); } else { - dedup_out.insert({i, 0}); - fwd_graph_.outputs.push_back(i); + dedup_out.emplace(nodeEntry); + fwd_graph_.outputs.push_back(nodeEntry); } } const auto& idx = fwd_graph_.indexed_graph(); @@ -143,14 +143,15 @@ CachedOp::CachedOp( // Set params { - const auto& idx = fwd_graph_.indexed_graph(); + const auto& indexed_graph = fwd_graph_.indexed_graph(); if (config_.data_indices.ndim() || config_.param_indices.ndim()) { CHECK_EQ(config_.data_indices.ndim() + config_.param_indices.ndim(), - idx.input_nodes().size()); + indexed_graph.input_nodes().size()); } else { std::vector tmp; - for (size_t i = 0; i < idx.input_nodes().size(); ++i) { - tmp.push_back(i); + tmp.reserve(indexed_graph.input_nodes().size()); + for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) { + tmp.emplace_back(i); } config_.data_indices.assign(tmp.begin(), tmp.end()); } @@ -158,21 +159,18 @@ CachedOp::CachedOp( // construct backward graph { - ograd_entries_.reserve(fwd_graph_.outputs.size()); - for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { - ograd_entries_.emplace_back(); - } - + ograd_entries_.resize(fwd_graph_.outputs.size()); std::vector xs; - const auto& idx = fwd_graph_.indexed_graph(); - for (size_t i = 0; i < idx.input_nodes().size(); ++i) { - auto nid = idx.input_nodes()[i]; - if (idx.mutable_input_nodes().count(nid)) continue; - fwd_input_to_grad_output_[i] = xs.size(); - xs.emplace_back(idx[nid].weak_ref.lock(), 0, 0); + const IndexedGraph& indexed_graph = fwd_graph_.indexed_graph(); + for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) { + const uint32_t node_id = indexed_graph.input_nodes()[i]; + if (indexed_graph.mutable_input_nodes().count(node_id)) + continue; + fwd_input_to_grad_output_.at(i) = xs.size(); + xs.emplace_back(std::move(indexed_graph[node_id].weak_ref.lock())); } - CHECK_GT(xs.size(), 0) + CHECK(!xs.empty()) << "There are no inputs in computation graph that require gradients."; grad_graph_ = pass::MXGradient( diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 0bfca8c10a1a..c143a5a947d7 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -53,7 +53,7 @@ namespace mxnet { NDArray::NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc, int dtype, std::vector aux_types, mxnet::ShapeVector aux_shapes, mxnet::TShape storage_shape) : shape_(shape), - dtype_(dtype), storage_type_(stype), entry_({nullptr, 0, 0}) { + dtype_(dtype), storage_type_(stype), entry_(nullptr, 0, 0) { // Assign default aux types if not given if (aux_types.size() == 0 && stype != kDefaultStorage) { From fed230b91c6ddd52602b1bd2e34146a2122b1f5a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 12:54:46 -0700 Subject: [PATCH 016/146] Revert "Restore subrepo" This reverts commit 3e6f56e5c055616b2983825d6254cac7d061167e. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e0ffec11bfd0..151b9fe39a86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/dmlc/tvm + url = https://github.com/larroy/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git From 71e730c3b390c7663e30ce9c79cbe2f74ee3bebb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 12:55:32 -0700 Subject: [PATCH 017/146] Update tvm to my repo --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 2da23bd86905..9a2cf6cd9ce2 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 2da23bd86905c3abec857a28b6135d069ea9515d +Subproject commit 9a2cf6cd9ce2b8f5cebd1273042aea1111a19578 From 5fe33987ee18368c9d205af3e8ea633db5a1dff6 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 14:55:00 -0700 Subject: [PATCH 018/146] Fix --- include/mxnet/ndarray.h | 12 ++++++------ src/executor/graph_executor.cc | 1 + src/executor/infer_graph_attr_pass.cc | 3 ++- src/imperative/cached_op.cc | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index bb7f2b2f5b6d..13c83f2282a3 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -97,7 +97,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! \brief constructor for NDArray with storage type */ @@ -116,7 +116,7 @@ class NDArray { shape_(), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! * \brief constructing a static NDArray that shares data with TBlob @@ -130,7 +130,7 @@ class NDArray { shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! @@ -148,7 +148,7 @@ class NDArray { }), shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! \brief create ndarray from shared memory */ @@ -157,7 +157,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! @@ -176,7 +176,7 @@ class NDArray { shape_(shape), dtype_(data.type_flag_), storage_type_(stype), - entry_() { + entry_(nullptr) { } /*! * \brief initialize the NDArray, assuming it is not assigned a meaningful shape before diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index bc80be5f92a5..5cc28b4248bd 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -152,6 +152,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { // remove zero in the sum. at least keep 1. auto begin = std::remove_if(v.begin(), v.end(), [](const nnvm::NodeEntry& nodeEntry) { + CHECK(nodeEntry.node); return nodeEntry.node->op() == zeros_op || nodeEntry.node->op() == zeros_like_op; }); if (begin == v.begin()) ++begin; diff --git a/src/executor/infer_graph_attr_pass.cc b/src/executor/infer_graph_attr_pass.cc index fa7aee518486..fa5fa04f918a 100644 --- a/src/executor/infer_graph_attr_pass.cc +++ b/src/executor/infer_graph_attr_pass.cc @@ -628,7 +628,8 @@ nnvm::Graph InferShapeAttr(nnvm::Graph &&ret, } if (dispatch_mode_name) { for (size_t i = node_start; i < node_end; i++) { - if (dispatch_modes[i] == DispatchMode::kUndefined) ++num_unknown; + if (dispatch_modes[i] == DispatchMode::kUndefined) + ++num_unknown; } } ++i; diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 8f01fa134b6f..0189328b0097 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -120,7 +120,7 @@ CachedOp::CachedOp( } fwd_graph_.outputs.emplace_back(copy_node, 0, 0); } else { - dedup_out.emplace(nodeEntry); + dedup_out.emplace(nodeEntry, 0); fwd_graph_.outputs.push_back(nodeEntry); } } @@ -166,7 +166,7 @@ CachedOp::CachedOp( const uint32_t node_id = indexed_graph.input_nodes()[i]; if (indexed_graph.mutable_input_nodes().count(node_id)) continue; - fwd_input_to_grad_output_.at(i) = xs.size(); + fwd_input_to_grad_output_[i] = xs.size(); xs.emplace_back(std::move(indexed_graph[node_id].weak_ref.lock())); } From 331a3240e30db443363f563698ffe535a77dceec Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 13:39:46 -0700 Subject: [PATCH 019/146] Fix --- src/imperative/cached_op.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 0189328b0097..22d19fbe9c55 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -98,7 +98,7 @@ CachedOp::CachedOp( using namespace nnvm; using namespace imperative; static const std::vector zero_ops{Op::Get("zeros_like"), Op::Get("_zeros")}; - static const auto _copy = Op::Get("_copy"); + static const auto _copy_op = Op::Get("_copy"); config_.Init(flags); if (config_.static_shape) { @@ -107,18 +107,18 @@ CachedOp::CachedOp( // construct forward graph { - NodeEntryMap dedup_out; + NodeEntryMap dedup_out; for (const NodeEntry& nodeEntry : sym.outputs) { - if (dedup_out.count(nodeEntry)) { + if (dedup_out.find(nodeEntry) != dedup_out.end()) { NodePtr copy_node = Node::Create(); - copy_node->attrs.op = _copy; + copy_node->attrs.op = _copy_op; copy_node->attrs.name = nodeEntry.node->attrs.name + "_copy" + std::to_string(dedup_out[nodeEntry]++); copy_node->inputs.emplace_back(nodeEntry); - if (_copy->attr_parser != nullptr) { - _copy->attr_parser(&(copy_node->attrs)); + if (_copy_op->attr_parser != nullptr) { + _copy_op->attr_parser(&(copy_node->attrs)); } - fwd_graph_.outputs.emplace_back(copy_node, 0, 0); + fwd_graph_.outputs.emplace_back(std::move(copy_node)); } else { dedup_out.emplace(nodeEntry, 0); fwd_graph_.outputs.push_back(nodeEntry); @@ -197,7 +197,7 @@ CachedOp::CachedOp( } auto full_ref_count = fwd_graph_.GetAttr >("forward_ref_count"); - for (size_t i = 0; i < num_forward_entries; ++i) full_ref_count[i] += ref_count[i]; + for (size_t i = 0; i < num_forward_entries; ++i) full_ref_count.at(i) += ref_count[i]; fwd_graph_.attrs["full_ref_count"] = std::make_shared(std::move(full_ref_count)); From e4f84473649d987aef792ce6c01085a7dfa6c74b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 13:45:26 -0700 Subject: [PATCH 020/146] Fix --- src/ndarray/ndarray.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index c143a5a947d7..79ac3e981ba0 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -171,7 +171,7 @@ nnvm::Symbol NDArray::get_autograd_symbol() const { #if MXNET_USE_MKLDNN == 1 NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) - : storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { + : storage_type_(kDefaultStorage), entry_(nullptr) { auto mem_desc = mem_pd.desc(); shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); dtype_ = get_mxnet_type(mem_desc.data.data_type); @@ -181,7 +181,7 @@ NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) } NDArray::NDArray(const std::shared_ptr &mkldnn_mem) - : storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { + : storage_type_(kDefaultStorage), entry_(nullptr) { auto mem_pd = mkldnn_mem->get_primitive_desc(); auto mem_desc = mem_pd.desc(); shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); From b5fbc3ff22ea70b3a8010c6587a2b8869580399d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 14:30:05 -0700 Subject: [PATCH 021/146] Fix --- dev_menu.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index d439d8194f2a..6f1701fac5f3 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -99,15 +99,15 @@ def create_virtualenv(venv_exe, pyexe, venv) -> None: logging.warn("Skipping creation of virtualenv") return check_call([venv_exe, '-p', pyexe, venv]) - activate_this_py = os.path.join(venv, 'bin', 'activate_this.py') - # Activate virtualenv in this interpreter - exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) - check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) - check_call(['pip', 'install', '-r', 'tests/requirements.txt']) + # TODO: Activate virtualenv in this interpreter, this is not working as it is. + #activate_this_py = os.path.join(venv, 'bin', 'activate_this.py') + #exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) + #check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) + #check_call(['pip', 'install', '-r', 'tests/requirements.txt']) def create_virtualenv_default(): create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV) - logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV) + logging.info("You can use the virtualenv by executing 'source %s/bin/activate && pip install -e python'", DEFAULT_PYENV) COMMANDS = OrderedDict([ ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV), From 343581405087d2f196902579ab8714e80cb23da1 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 17:28:24 -0700 Subject: [PATCH 022/146] minor --- src/ndarray/ndarray.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 79ac3e981ba0..a8a34eaccac0 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -53,7 +53,7 @@ namespace mxnet { NDArray::NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc, int dtype, std::vector aux_types, mxnet::ShapeVector aux_shapes, mxnet::TShape storage_shape) : shape_(shape), - dtype_(dtype), storage_type_(stype), entry_(nullptr, 0, 0) { + dtype_(dtype), storage_type_(stype), entry_(nullptr) { // Assign default aux types if not given if (aux_types.size() == 0 && stype != kDefaultStorage) { From 23bc3df092c94655bde6f2e304cccab5bf6f96ee Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 18:49:19 -0700 Subject: [PATCH 023/146] Fixes --- include/mxnet/ndarray.h | 6 ++++-- src/imperative/cached_op.cc | 7 ++++++- src/imperative/imperative.cc | 2 +- src/nnvm/legacy_op_util.cc | 2 +- src/operator/custom/custom.cc | 4 ++-- src/operator/tensor/elemwise_sum.cc | 9 ++++----- src/operator/tensor/elemwise_unary_op_basic.cc | 10 ++++------ 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 13c83f2282a3..fecc26851b8c 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -82,7 +82,9 @@ class MKLDNNMemory; class NDArray { public: /*! \brief default constructor */ - NDArray() { + NDArray() + : entry_(nullptr) + { } /*! * \brief constructs a new dynamic NDArray @@ -650,7 +652,7 @@ class NDArray { */ NDArray Detach() const { NDArray ret(*this); - ret.entry_ = nnvm::NodeEntry(); + ret.entry_ = nnvm::NodeEntry(nullptr); return ret; } diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 22d19fbe9c55..acdfdd8fb6df 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -159,7 +159,12 @@ CachedOp::CachedOp( // construct backward graph { - ograd_entries_.resize(fwd_graph_.outputs.size()); + //ograd_entries_.resize(fwd_graph_.outputs.size()); + ograd_entries_.reserve(fwd_graph_.outputs.size()); + for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { + ograd_entries_.emplace_back(Node::Create()); + } + std::vector xs; const IndexedGraph& indexed_graph = fwd_graph_.indexed_graph(); for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) { diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index a1c41ee0df6b..35c5172effc7 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -363,7 +363,7 @@ std::vector Imperative::Backward( auto node = Node::Create(); node->attrs.op = copy_op; node->inputs.push_back(e); - graph.outputs.emplace_back(node, 0, 0); + graph.outputs.emplace_back(std::move(node)); } else { graph.outputs.push_back(e); } diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index e738a5a47695..2c4bcec3d7ca 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -344,7 +344,7 @@ inline std::vector OpPropGradient( std::vector in_grad; in_grad.reserve(prop.arguments.size() + prop.aux_states.size()); for (size_t i = 0; i < prop.arguments.size(); ++i) { - in_grad.emplace_back(gnode, i, 0); + in_grad.emplace_back(std::move(gnode), i, 0); } // attach no gradient node to forbid gradient on aux_state if (prop.aux_states.size() != 0) { diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 63b42007317b..f6def8a08538 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -238,14 +238,14 @@ std::vector Gradient( std::vector ret; for (size_t i = 0; i < params.num_args; ++i) { - ret.emplace_back(nnvm::NodeEntry{g, static_cast(i), 0}); + ret.emplace_back(std::move(g), static_cast(i), 0); } if (params.num_auxs) { nnvm::NodePtr ng = nnvm::Node::Create(); ng->attrs.op = nnvm::Op::Get("_NoGradient"); ng->attrs.name = "NoGradient"; for (size_t i = 0; i < params.num_auxs; ++i) { - ret.emplace_back(nnvm::NodeEntry{ng, 0, 0}); + ret.emplace_back(std::move(ng), 0, 0); } } diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index dec57633be22..2ba4e309f3ac 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -49,12 +49,11 @@ std::vector ElementWiseSumGrad( nnvm::Op::Get("identity"); CHECK_EQ(ograds.size(), 1); std::vector ret; - nnvm::NodeEntry n_out(n, 0, 0); for (size_t i = 0; i < n->inputs.size(); ++i) { - nnvm::NodePtr id_node = nnvm::Node::Create(); - id_node->attrs.op = copy_op; - id_node->inputs = {ograds[0]}; - ret.emplace_back(id_node, 0, 0); + nnvm::NodePtr node = nnvm::Node::Create(); + node->attrs.op = copy_op; + node->inputs = {ograds[0]}; + ret.emplace_back(std::move(node), 0, 0); } return ret; } diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index f9d79ebf893e..58d441f18f83 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -358,9 +358,8 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); - auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", - {n->inputs[1]}, nullptr, &n); - lhs.emplace_back(ng, 0, 0); + lhs.emplace_back(MakeNode("zeros_like", n->attrs.name + "_rhs_backward", + {n->inputs[1]}, nullptr, &n)); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") @@ -497,9 +496,8 @@ Negative indices are supported, and `None` can be used for either `lhs_end` or ` if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); - auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", - {n->inputs[1]}, nullptr, &n); - lhs.emplace_back(ng, 0, 0); + lhs.emplace_back(MakeNode("zeros_like", n->attrs.name + "_rhs_backward", + {n->inputs[1]}, nullptr, &n)); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") From 8180054685c8e7b9e8d31fb49f95609165e1f360 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 13 Apr 2019 02:03:55 +0000 Subject: [PATCH 024/146] update tvm and dmlc-core --- 3rdparty/dmlc-core | 2 +- 3rdparty/tvm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 3ffea8694adf..82bf4c2e2af3 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 3ffea8694adf9c0363f9abbf162dc0e4a45b22c5 +Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 diff --git a/3rdparty/tvm b/3rdparty/tvm index 9a2cf6cd9ce2..3e244d496689 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 9a2cf6cd9ce2b8f5cebd1273042aea1111a19578 +Subproject commit 3e244d496689b8a96dfb7fea30b5242492aacd8f From 0e1c99c7b7b9b80a3442cee0d1d1f75f8cf29829 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 13 Apr 2019 02:10:38 +0000 Subject: [PATCH 025/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 3e244d496689..b947674c79cb 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 3e244d496689b8a96dfb7fea30b5242492aacd8f +Subproject commit b947674c79cbc8bbe56712d1b48205cb3e43a4a2 From 2c7d653e4e38b24356bb4e8956342fa4b94ebebe Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 11 Apr 2019 21:55:01 +0000 Subject: [PATCH 026/146] Revert "update dmlc-core" This reverts commit fc51adb75348ad33c2ac685985940b5bd3f1ee99. --- 3rdparty/dmlc-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 82bf4c2e2af3..3ffea8694adf 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 +Subproject commit 3ffea8694adf9c0363f9abbf162dc0e4a45b22c5 From d2b4eeb3f9318e10f06395f1f3726c90dca31992 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 15 Apr 2019 23:01:23 +0000 Subject: [PATCH 027/146] Update dmlc-core and tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index b947674c79cb..f4ca10c23195 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit b947674c79cbc8bbe56712d1b48205cb3e43a4a2 +Subproject commit f4ca10c231953f13a8cef82d9855e5d9bf043b9f From 6914fa539c90dc56929c7e5033a1e58b5fcea789 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 15:40:15 -0700 Subject: [PATCH 028/146] Improve batch_norm with NodeEntry refactorings --- src/operator/nn/batch_norm.cc | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 622952cc4bc5..08d8839d507c 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -484,19 +484,20 @@ static inline bool BatchNormStorageType(const nnvm::NodeAttrs &attrs, std::vector BatchNormGrad(const nnvm::NodePtr& n, const std::vector& ograds) { std::vector out_data(n->num_outputs()); - for (uint32_t i = 0; i < out_data.size(); ++i) { - out_data[i] = nnvm::NodeEntry{n, i, 0}; + out_data.reserve(n->num_outputs()); + for (size_t i = 0; i < out_data.size(); ++i) { + out_data.emplace_back(n, i, 0); } std::vector heads; heads.reserve(8); - heads.push_back(ograds[0]); - heads.push_back(out_data[batchnorm::kMean]); - heads.push_back(out_data[batchnorm::kVar]); - heads.push_back(n->inputs[batchnorm::kData]); - heads.push_back(n->inputs[batchnorm::kGamma]); - heads.push_back(n->inputs[batchnorm::kBeta]); - heads.push_back(n->inputs[batchnorm::kInMovingMean]); - heads.push_back(n->inputs[batchnorm::kInMovingVar]); + heads.emplace_back(ograds[0]); + heads.emplace_back(out_data[batchnorm::kMean]); + heads.emplace_back(out_data[batchnorm::kVar]); + heads.emplace_back(n->inputs[batchnorm::kData]); + heads.emplace_back(n->inputs[batchnorm::kGamma]); + heads.emplace_back(n->inputs[batchnorm::kBeta]); + heads.emplace_back(n->inputs[batchnorm::kInMovingMean]); + heads.emplace_back(n->inputs[batchnorm::kInMovingVar]); nnvm::NodePtr gnode = nnvm::Node::Create(); gnode->inputs = std::move(heads); @@ -505,19 +506,17 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, gnode->attrs.op = nnvm::Op::Get("_backward_BatchNorm"); gnode->attrs.name = n->attrs.name + "_backward"; // The input of batchnorm - std::vector in_grad(5); - for (uint32_t i = 0; i < 3; ++i) { - in_grad[i] = nnvm::NodeEntry{gnode, i, 0}; - } - + std::vector in_grad; + in_grad.reserve(5); + for (size_t i = 0; i < 3; ++i) + in_grad.emplace_back(gnode, i, 0); // attach no gradient node to forbid gradient on aux_state nnvm::NodePtr ng = nnvm::Node::Create(); ng->attrs.op = Op::Get("_NoGradient"); ng->attrs.name = "NoGradient"; // the aux state of batchnorm - for (uint32_t i = 0; i < 2; ++i) { - in_grad[i + 3] = nnvm::NodeEntry{ng, 0, 0}; - } + for (size_t i = 3; i < 5; ++i) + in_grad.emplace_back(ng, 0, 0); return in_grad; } From 173ac2ec2c9fc0cc1ef76e0b5f01389ed9dd8707 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 15:40:44 -0700 Subject: [PATCH 029/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index f4ca10c23195..1acf86bbd080 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit f4ca10c231953f13a8cef82d9855e5d9bf043b9f +Subproject commit 1acf86bbd0809ebfe736ed045345719c24b7acb0 From 4fc969cead7263ac025f005443c37f71a18957f3 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 16:18:37 -0700 Subject: [PATCH 030/146] Fix bug introduced in batch_norm --- src/operator/nn/batch_norm.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 08d8839d507c..e5a68ebe227b 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -485,9 +485,8 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, const std::vector& ograds) { std::vector out_data(n->num_outputs()); out_data.reserve(n->num_outputs()); - for (size_t i = 0; i < out_data.size(); ++i) { + for (size_t i = 0; i < n->num_outputs(); ++i) out_data.emplace_back(n, i, 0); - } std::vector heads; heads.reserve(8); heads.emplace_back(ograds[0]); From 675c4b76548ed83f2e361ab42648178e44daff43 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:08:51 +0000 Subject: [PATCH 031/146] lint --- include/mxnet/ndarray.h | 3 +-- src/imperative/cached_op.cc | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index fecc26851b8c..0a3339b4a8a9 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -83,8 +83,7 @@ class NDArray { public: /*! \brief default constructor */ NDArray() - : entry_(nullptr) - { + : entry_(nullptr) { } /*! * \brief constructs a new dynamic NDArray diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index acdfdd8fb6df..16ba20d77338 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -159,11 +159,9 @@ CachedOp::CachedOp( // construct backward graph { - //ograd_entries_.resize(fwd_graph_.outputs.size()); ograd_entries_.reserve(fwd_graph_.outputs.size()); - for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { + for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) ograd_entries_.emplace_back(Node::Create()); - } std::vector xs; const IndexedGraph& indexed_graph = fwd_graph_.indexed_graph(); From 575b62c6ae019a85cf3ea4a5a2ff206084bb9a2c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:25:22 +0000 Subject: [PATCH 032/146] Add std::move --- src/operator/tensor/broadcast_reduce_op_value.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/operator/tensor/broadcast_reduce_op_value.cc b/src/operator/tensor/broadcast_reduce_op_value.cc index 3a598dbcf374..612d1f926650 100644 --- a/src/operator/tensor/broadcast_reduce_op_value.cc +++ b/src/operator/tensor/broadcast_reduce_op_value.cc @@ -286,12 +286,13 @@ NNVM_REGISTER_OP(broadcast_like) .set_attr("FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { - if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); + if (CheckGradAllZero(ograds)) + return MakeZeroGradNodes(n, ograds); std::vector lhs = MakeNonlossGradNode("_broadcast_backward", n, ograds, {}, - {{"keepdims", "true"}}); + {{"keepdims", "true"}}); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.emplace_back(ng, 0, 0); + lhs.emplace_back(std::move(ng)); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") From 4d4c06200e156a9b1ef2ebb7bf4a81f730a3ef5c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 23:35:48 -0700 Subject: [PATCH 033/146] Fix bugs --- src/nnvm/legacy_op_util.cc | 4 ++-- src/operator/nn/batch_norm.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 2c4bcec3d7ca..25cc46ef7309 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -323,9 +323,9 @@ inline std::vector OpPropGradient( auto& prop = nnvm::get(ptr->attrs.parsed); std::vector out_data; out_data.reserve(prop.outputs.size()); - for (size_t i = 0; i < out_data.size(); ++i) { + for (size_t i = 0; i < prop.outputs.size(); ++i) out_data.emplace_back(ptr, i, 0); - } + std::vector in_data( ptr->inputs.begin(), ptr->inputs.begin() + prop.arguments.size()); std::vector ograd( diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index e5a68ebe227b..1e1fbb9e7abe 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -483,7 +483,7 @@ static inline bool BatchNormStorageType(const nnvm::NodeAttrs &attrs, std::vector BatchNormGrad(const nnvm::NodePtr& n, const std::vector& ograds) { - std::vector out_data(n->num_outputs()); + std::vector out_data; out_data.reserve(n->num_outputs()); for (size_t i = 0; i < n->num_outputs(); ++i) out_data.emplace_back(n, i, 0); From 176c9101d88f7e7244289f1fb633aab9cc16480d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:18:42 -0700 Subject: [PATCH 034/146] Fix bug introduced in batch_norm --- src/operator/nn/batch_norm.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 1e1fbb9e7abe..1ade322f732f 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -489,14 +489,14 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, out_data.emplace_back(n, i, 0); std::vector heads; heads.reserve(8); - heads.emplace_back(ograds[0]); - heads.emplace_back(out_data[batchnorm::kMean]); - heads.emplace_back(out_data[batchnorm::kVar]); - heads.emplace_back(n->inputs[batchnorm::kData]); - heads.emplace_back(n->inputs[batchnorm::kGamma]); - heads.emplace_back(n->inputs[batchnorm::kBeta]); - heads.emplace_back(n->inputs[batchnorm::kInMovingMean]); - heads.emplace_back(n->inputs[batchnorm::kInMovingVar]); + heads.push_back(ograds[0]); + heads.push_back(out_data[batchnorm::kMean]); + heads.push_back(out_data[batchnorm::kVar]); + heads.push_back(n->inputs[batchnorm::kData]); + heads.push_back(n->inputs[batchnorm::kGamma]); + heads.push_back(n->inputs[batchnorm::kBeta]); + heads.push_back(n->inputs[batchnorm::kInMovingMean]); + heads.push_back(n->inputs[batchnorm::kInMovingVar]); nnvm::NodePtr gnode = nnvm::Node::Create(); gnode->inputs = std::move(heads); From 423fc6421369489f3efa1493e71ee50976122f2a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:26:53 -0700 Subject: [PATCH 035/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 1acf86bbd080..3073acda0f85 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 1acf86bbd0809ebfe736ed045345719c24b7acb0 +Subproject commit 3073acda0f854f320362138fd9165133cce64733 From 7c6345665d4d8709ed32af4224cfd862470c96b4 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:28:23 -0700 Subject: [PATCH 036/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 3073acda0f85..2db7717a0515 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 3073acda0f854f320362138fd9165133cce64733 +Subproject commit 2db7717a0515ae63c75ecb8aa841410d38d1f1d4 From 3895aac4c8a9531386fc762563e4c7206b0be9ec Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:29:20 -0700 Subject: [PATCH 037/146] Sync gradient.cc with tvm --- src/nnvm/gradient.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index 4927191a5964..c51704f82077 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -212,6 +212,8 @@ Graph Gradient(Graph src) { LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " << "because it didn't register FGradient attribute."; } + for(const auto& nodeEntry: input_grads) + CHECK(nodeEntry.node); auto git = input_grads.begin(); for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { auto& ge = output_grads[it->node.get()][it->index]; From 81a714bce65f1674ba065902714da223fde256ff Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 07:49:09 +0000 Subject: [PATCH 038/146] Fix bug --- src/nnvm/legacy_op_util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 25cc46ef7309..698666f94d90 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -344,7 +344,7 @@ inline std::vector OpPropGradient( std::vector in_grad; in_grad.reserve(prop.arguments.size() + prop.aux_states.size()); for (size_t i = 0; i < prop.arguments.size(); ++i) { - in_grad.emplace_back(std::move(gnode), i, 0); + in_grad.emplace_back(gnode, i, 0); } // attach no gradient node to forbid gradient on aux_state if (prop.aux_states.size() != 0) { From 412787a7b6f54c3c0e493a5a510d5b6f7576f010 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 07:52:40 +0000 Subject: [PATCH 039/146] Use at --- dev_menu.py | 4 ++-- src/operator/nn/batch_norm.cc | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index 6f1701fac5f3..5a80ef0b59e4 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -113,7 +113,7 @@ def create_virtualenv_default(): ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV), [ CMake(), - create_virtualenv_default, + #create_virtualenv_default, ]), ('[Local] Python Unit tests', "./py3_venv/bin/nosetests -v tests/python/unittest/" @@ -209,7 +209,7 @@ def build(args) -> None: else: cmake = CMake() cmake() - create_virtualenv(venv_exe, pyexe, args.venv) + #create_virtualenv(venv_exe, pyexe, args.venv) def main(): logging.getLogger().setLevel(logging.INFO) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 1ade322f732f..dbb0f10561b9 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -489,14 +489,14 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, out_data.emplace_back(n, i, 0); std::vector heads; heads.reserve(8); - heads.push_back(ograds[0]); - heads.push_back(out_data[batchnorm::kMean]); - heads.push_back(out_data[batchnorm::kVar]); - heads.push_back(n->inputs[batchnorm::kData]); - heads.push_back(n->inputs[batchnorm::kGamma]); - heads.push_back(n->inputs[batchnorm::kBeta]); - heads.push_back(n->inputs[batchnorm::kInMovingMean]); - heads.push_back(n->inputs[batchnorm::kInMovingVar]); + heads.emplace_back(ograds.at(0)); + heads.emplace_back(out_data.at(batchnorm::kMean)); + heads.emplace_back(out_data.at(batchnorm::kVar)); + heads.emplace_back(n->inputs.at(batchnorm::kData)); + heads.emplace_back(n->inputs.at(batchnorm::kGamma)); + heads.emplace_back(n->inputs.at(batchnorm::kBeta)); + heads.emplace_back(n->inputs.at(batchnorm::kInMovingMean)); + heads.emplace_back(n->inputs[batchnorm::kInMovingVar]); nnvm::NodePtr gnode = nnvm::Node::Create(); gnode->inputs = std::move(heads); From c49821fd9ce8fca2bc87a3b2ba9313d9137a6012 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:03:27 -0700 Subject: [PATCH 040/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 2db7717a0515..5922ccb588e5 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 2db7717a0515ae63c75ecb8aa841410d38d1f1d4 +Subproject commit 5922ccb588e5a9c09e51833c960489dc30df1db6 From a0da596532f827a6c5553ef3e87dec907ac57eb1 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 08:29:04 +0000 Subject: [PATCH 041/146] lint --- src/nnvm/gradient.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index c51704f82077..badc3f9e2b85 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -212,7 +212,7 @@ Graph Gradient(Graph src) { LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " << "because it didn't register FGradient attribute."; } - for(const auto& nodeEntry: input_grads) + for (const auto& nodeEntry : input_grads) CHECK(nodeEntry.node); auto git = input_grads.begin(); for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { From 4e5233aad52c99dbf6ca2518854d907b4a896202 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 08:41:52 +0000 Subject: [PATCH 042/146] Update subrepos --- 3rdparty/dmlc-core | 2 +- 3rdparty/tvm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 3ffea8694adf..82bf4c2e2af3 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 3ffea8694adf9c0363f9abbf162dc0e4a45b22c5 +Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 diff --git a/3rdparty/tvm b/3rdparty/tvm index 5922ccb588e5..824f236773e5 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 5922ccb588e5a9c09e51833c960489dc30df1db6 +Subproject commit 824f236773e503e86bd4218bd28dccf1ee970f99 From ed7a00dbc5a2988919d31023270d82feff7fbc58 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:46:17 -0700 Subject: [PATCH 043/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 824f236773e5..568235b75ce3 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 824f236773e503e86bd4218bd28dccf1ee970f99 +Subproject commit 568235b75ce38acf6d6660e4f1a75692fb0258cb From c39afa4f133332aaf8d3cc27d6743457f3ed5572 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 18 Apr 2019 17:21:39 -0700 Subject: [PATCH 044/146] Fix moves leaving nodes uninitialized --- src/imperative/cached_op.cc | 2 +- src/operator/custom/custom.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 16ba20d77338..47d4ac42006b 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -170,7 +170,7 @@ CachedOp::CachedOp( if (indexed_graph.mutable_input_nodes().count(node_id)) continue; fwd_input_to_grad_output_[i] = xs.size(); - xs.emplace_back(std::move(indexed_graph[node_id].weak_ref.lock())); + xs.emplace_back(indexed_graph[node_id].weak_ref.lock()); } CHECK(!xs.empty()) diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index f6def8a08538..77fe2e6e4b1c 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -238,14 +238,14 @@ std::vector Gradient( std::vector ret; for (size_t i = 0; i < params.num_args; ++i) { - ret.emplace_back(std::move(g), static_cast(i), 0); + ret.emplace_back(g, static_cast(i), 0); } if (params.num_auxs) { nnvm::NodePtr ng = nnvm::Node::Create(); ng->attrs.op = nnvm::Op::Get("_NoGradient"); ng->attrs.name = "NoGradient"; for (size_t i = 0; i < params.num_auxs; ++i) { - ret.emplace_back(std::move(ng), 0, 0); + ret.emplace_back(ng, 0, 0); } } From 0f555765000bb1beb2c8315b8bc8201869bb67f8 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 18 Apr 2019 17:48:19 -0700 Subject: [PATCH 045/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 568235b75ce3..cb7e33ca44ce 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 568235b75ce38acf6d6660e4f1a75692fb0258cb +Subproject commit cb7e33ca44cec59c896358ee69f0e5c26e5709f5 From cfa3d8b76a15ebf8bdf12c04e742d1ebf949c833 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 19 Apr 2019 12:09:20 -0700 Subject: [PATCH 046/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index cb7e33ca44ce..e031b39c5e1d 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit cb7e33ca44cec59c896358ee69f0e5c26e5709f5 +Subproject commit e031b39c5e1d4468b7db96f1db39dbb97e4beef8 From 57788134492a65eab849034b49c22e8098045bd6 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 23:34:21 +0000 Subject: [PATCH 047/146] restore gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 151b9fe39a86..e0ffec11bfd0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/larroy/tvm + url = https://github.com/dmlc/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git From b8b47b403d8beb86211c2addffee7b70dfe5ef5d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 23:39:19 +0000 Subject: [PATCH 048/146] restore tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index e031b39c5e1d..eed28f581553 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit e031b39c5e1d4468b7db96f1db39dbb97e4beef8 +Subproject commit eed28f581553525646a4312d377edd87fe827f42 From a0c3b14e176c705e8c67cb1e93a16f38fa0ffe1a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 25 Apr 2019 00:03:37 +0000 Subject: [PATCH 049/146] Revert "Fix build_ccache_wrappers:" This reverts commit 381ff13753d12f71dc5f29355de9766bb9a6cc18. --- ci/docker/runtime_functions.sh | 49 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 56e078f5f8ed..c3610d2452e0 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -66,29 +66,37 @@ build_ccache_wrappers() { # But in the beginning, we'll make this opt-in. In future, loads of processes like # the scala make step or numpy compilation and other pip package generations # could be heavily sped up by using ccache as well. - mkdir -p /tmp/ccache-redirects + mkdir /tmp/ccache-redirects export PATH=/tmp/ccache-redirects:$PATH - CCACHE=`which ccache` - ln -sf $CCACHE /tmp/ccache-redirects/gcc - ln -sf $CCACHE /tmp/ccache-redirects/gcc-8 - ln -sf $CCACHE /tmp/ccache-redirects/g++ - ln -sf $CCACHE /tmp/ccache-redirects/g++-8 - ln -sf $CCACHE /tmp/ccache-redirects/nvcc - ln -sf $CCACHE /tmp/ccache-redirects/clang++-3.9 - ln -sf $CCACHE /tmp/ccache-redirects/clang-3.9 - ln -sf $CCACHE /tmp/ccache-redirects/clang++-5.0 - ln -sf $CCACHE /tmp/ccache-redirects/clang-5.0 - ln -sf $CCACHE /tmp/ccache-redirects/clang++-6.0 - ln -sf $CCACHE /tmp/ccache-redirects/clang-6.0 - # Doesn't work: https://github.com/ccache/ccache/issues/373 - #ln -sf $CCACHE /tmp/ccache-redirects/nvcc - #export NVCC="/tmp/ccache-redirects/nvcc" + ln -s ccache /tmp/ccache-redirects/gcc + ln -s ccache /tmp/ccache-redirects/gcc-8 + ln -s ccache /tmp/ccache-redirects/g++ + ln -s ccache /tmp/ccache-redirects/g++-8 + ln -s ccache /tmp/ccache-redirects/nvcc + ln -s ccache /tmp/ccache-redirects/clang++-3.9 + ln -s ccache /tmp/ccache-redirects/clang-3.9 + ln -s ccache /tmp/ccache-redirects/clang++-5.0 + ln -s ccache /tmp/ccache-redirects/clang-5.0 + ln -s ccache /tmp/ccache-redirects/clang++-6.0 + ln -s ccache /tmp/ccache-redirects/clang-6.0 + ln -s ccache /usr/local/bin/gcc + ln -s ccache /usr/local/bin/gcc-8 + ln -s ccache /usr/local/bin/g++ + ln -s ccache /usr/local/bin/g++-8 + ln -s ccache /usr/local/bin/nvcc + ln -s ccache /usr/local/bin/clang++-3.9 + ln -s ccache /usr/local/bin/clang-3.9 + ln -s ccache /usr/local/bin/clang++-5.0 + ln -s ccache /usr/local/bin/clang-5.0 + ln -s ccache /usr/local/bin/clang++-6.0 + ln -s ccache /usr/local/bin/clang-6.0 + + export NVCC=ccache # Uncomment if you would like to debug CCache hit rates. # You can monitor using tail -f ccache-log - #export CCACHE_LOGFILE=/work/mxnet/ccache-log - #export CCACHE_LOGFILE=/tmp/ccache-log - #export CCACHE_DEBUG=1 + # export CCACHE_LOGFILE=/work/mxnet/ccache-log + # export CCACHE_DEBUG=1 } build_wheel() { @@ -659,7 +667,8 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ubuntu_gpu_cuda91_cudnn7() { set -ex - build_ccache_wrappers + # unfortunately this build has problems in 3rdparty dependencies with ccache and make + # build_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ From 2bfc51c1c5d053178f32c2a9d1bb271cf3f95413 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 19 Apr 2019 13:52:20 -0700 Subject: [PATCH 050/146] readability --- src/nnvm/gradient.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index badc3f9e2b85..c4850d46de45 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -144,13 +144,13 @@ Graph Gradient(Graph src) { << "because it is unreachable from the outputs."; } - // construct mirror reduece memory strategy if needed + // construct mirror as memory reduction strategy if needed std::unordered_map mirror_map; if (mirror_fun != nullptr) { - for (const NodePtr& n : topo_order) { - if (mirror_fun(*n)) { + for (const NodePtr& node_ptr : topo_order) { + if (mirror_fun(*node_ptr)) { NodePtr new_node = Node::Create(); - *new_node = *n; + *new_node = *node_ptr; new_node->attrs.name += "_mirror"; for (auto& e : new_node->inputs) { e.node = mirror_map.at(e.node.get()); @@ -158,9 +158,9 @@ Graph Gradient(Graph src) { for (auto& n : new_node->control_deps) { n = mirror_map.at(n.get()); } - mirror_map[n.get()] = std::move(new_node); + mirror_map[node_ptr.get()] = std::move(new_node); } else { - mirror_map[n.get()] = n; + mirror_map[node_ptr.get()] = node_ptr; } } } From a173495c958800b0d9d69cb35af775c3bdb68195 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 11:01:07 -0700 Subject: [PATCH 051/146] Add checks to gradient.cc --- src/nnvm/gradient.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index c4850d46de45..586027129a0b 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -186,7 +186,8 @@ Graph Gradient(Graph src) { if ((*rit)->inputs.size() != 0) { NodePtr fwd_node = (mirror_map.size() == 0 ? ptr : mirror_map.at(ptr.get())); std::vector input_grads; - if (grad_fun_map.count(ptr->op())) { + // Check for FGradient + if (grad_fun_map.contains(ptr->op())) { input_grads = grad_fun_map[ptr->op()](fwd_node, out_agg_grads); CHECK_EQ((*rit)->inputs.size(), input_grads.size()) << "Gradient function not returning enough gradient"; @@ -206,7 +207,7 @@ Graph Gradient(Graph src) { if (p->op()->attr_parser != nullptr) { p->op()->attr_parser(&(p->attrs)); } - input_grads.emplace_back(nnvm::NodeEntry{p, 0, 0}); + input_grads.emplace_back(p, 0, 0); } } else { LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " @@ -215,13 +216,14 @@ Graph Gradient(Graph src) { for (const auto& nodeEntry : input_grads) CHECK(nodeEntry.node); auto git = input_grads.begin(); + CHECK((*rit)->inputs.size() <= input_grads.size()); for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { - auto& ge = output_grads[it->node.get()][it->index]; + auto& output_grad_entry = output_grads[it->node.get()][it->index]; // if any of the backward op can do shape inference, the hint is not necessary. - if (finfer_shape.count(git->node->op())) { - ge.need_attr_hint = false; + if (finfer_shape.contains(git->node->op())) { + output_grad_entry.need_attr_hint = false; } - ge.grads.emplace_back(std::move(*git)); + output_grad_entry.grads.emplace_back(std::move(*git)); } } } From f268a983b09980c144152893ef450f64171e5253 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 11:56:40 -0700 Subject: [PATCH 052/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index eed28f581553..a706ad16f83d 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit eed28f581553525646a4312d377edd87fe827f42 +Subproject commit a706ad16f83d810f4a6268cf604cece79c2c4791 From 47c9d6e0280476d111caf2292b6fe059d1a644f8 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 14:15:06 -0700 Subject: [PATCH 053/146] Add backward to fully connected. (_backward_FullyConnected) --- src/operator/nn/fully_connected-inl.h | 1 + src/operator/nn/fully_connected.cc | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/operator/nn/fully_connected-inl.h b/src/operator/nn/fully_connected-inl.h index e4bb11f6bc56..f9ef3c3af66f 100644 --- a/src/operator/nn/fully_connected-inl.h +++ b/src/operator/nn/fully_connected-inl.h @@ -237,6 +237,7 @@ void FullyConnectedGradCompute(const nnvm::NodeAttrs& attrs, } } + } // namespace op } // namespace mxnet namespace std { diff --git a/src/operator/nn/fully_connected.cc b/src/operator/nn/fully_connected.cc index a097357ef5a3..e2594d369290 100644 --- a/src/operator/nn/fully_connected.cc +++ b/src/operator/nn/fully_connected.cc @@ -1,4 +1,4 @@ -/* + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -165,6 +165,7 @@ static bool FullyConnectedType(const nnvm::NodeAttrs& attrs, attrs, in_type, out_type, -1); } + struct FullyConnectedGrad { const char *op_name; std::vector operator()(const nnvm::NodePtr& n, @@ -176,6 +177,16 @@ struct FullyConnectedGrad { } }; + +std::vector FullyConnectedBackwardGrad(const nnvm::NodePtr& n, + const std::vector& ograds) { + auto zero_node = MakeNode("zeros_like", n->attrs.name + "_backward", {n->inputs[0]}, nullptr, &n); + std::vector ret; + ret.emplace_back(nnvm::NodeEntry{zero_node, 0, 0}); + return ret; +} + + inline static bool FCStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, DispatchMode* dispatch_mode, @@ -310,6 +321,7 @@ If ``no_bias`` is set to be true, then the ``bias`` term is ignored. .add_argument("bias", "NDArray-or-Symbol", "Bias parameter.") .add_arguments(FullyConnectedParam::__FIELDS__()); + NNVM_REGISTER_OP(_backward_FullyConnected) .set_num_inputs(3) .set_num_outputs([](const NodeAttrs& attrs) { @@ -325,6 +337,7 @@ NNVM_REGISTER_OP(_backward_FullyConnected) .set_attr("FInplaceOption", [](const NodeAttrs& attrs){ return std::vector >{{1, 0}}; }) +.set_attr("FGradient", FullyConnectedBackwardGrad) .set_attr("FInferStorageType", BackwardFCStorageType) .set_attr_parser(ParamParser) #if MXNET_USE_MKLDNN == 1 @@ -333,5 +346,6 @@ NNVM_REGISTER_OP(_backward_FullyConnected) #endif .set_attr("FCompute", FullyConnectedGradCompute); + } // namespace op } // namespace mxnet From 05bb7ea4af09313415915139f5163a013861c1a9 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 17:16:00 -0700 Subject: [PATCH 054/146] Add failing test --- 3rdparty/googletest | 2 +- 3rdparty/mshadow | 2 +- src/operator/nn/fully_connected.cc | 9 +++++++-- tests/python/unittest/test_gluon.py | 17 +++++++++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/3rdparty/googletest b/3rdparty/googletest index eb9225ce361a..ec44c6c1675c 160000 --- a/3rdparty/googletest +++ b/3rdparty/googletest @@ -1 +1 @@ -Subproject commit eb9225ce361affe561592e0912320b9db84985d0 +Subproject commit ec44c6c1675c25b9827aacd08c02433cccde7780 diff --git a/3rdparty/mshadow b/3rdparty/mshadow index 6e94643bdf1d..95ebe0f109ae 160000 --- a/3rdparty/mshadow +++ b/3rdparty/mshadow @@ -1 +1 @@ -Subproject commit 6e94643bdf1d51a505b147f28c358fb71070b8fd +Subproject commit 95ebe0f109ae021d0d66e2a627ccfc55c3253b55 diff --git a/src/operator/nn/fully_connected.cc b/src/operator/nn/fully_connected.cc index e2594d369290..32e1722f251f 100644 --- a/src/operator/nn/fully_connected.cc +++ b/src/operator/nn/fully_connected.cc @@ -180,9 +180,14 @@ struct FullyConnectedGrad { std::vector FullyConnectedBackwardGrad(const nnvm::NodePtr& n, const std::vector& ograds) { - auto zero_node = MakeNode("zeros_like", n->attrs.name + "_backward", {n->inputs[0]}, nullptr, &n); std::vector ret; - ret.emplace_back(nnvm::NodeEntry{zero_node, 0, 0}); + size_t i = 0; + for (const auto& x: n->inputs) { + std::ostringstream os; + os << n->attrs.name << "_backward_" << i; + ret.emplace_back(nnvm::NodeEntry{MakeNode("zeros_like", os.str(), {x}, nullptr, &n), 0, 0}); + ++i; + } return ret; } diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index efa04f4fa47a..92a4556176d9 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -915,6 +915,23 @@ def test_sequential_warning(): assert len(w) == 1 +@with_seed() +def test_dense_backward(): + import mxnet.autograd as ag + import mxnet.ndarray as nd + x = nd.array([[1,2,3,4]]) + net = gluon.nn.Sequential() + with net.name_scope(): + net.add(gluon.nn.Dense(1, in_units=x.shape[1])) + net.initialize(mx.initializer.Constant(.5)) + params = [x.data() for x in net.collect_params().values()] + with ag.record(): + y = net.forward(x) + params_grad = ag.grad(y, params, create_graph=True, retain_graph=True) + params_grad_grad = ag.grad(params_grad, x) + print(params_grad_grad.grad()) + + @with_seed() def test_global_norm_clip(): stypes = ['default', 'row_sparse'] From c2b441227b5aed9ea45133b03e6f412f68388fae Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 17:19:28 -0700 Subject: [PATCH 055/146] CR --- src/operator/nn/fully_connected.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/operator/nn/fully_connected.cc b/src/operator/nn/fully_connected.cc index 32e1722f251f..cc87285c1d71 100644 --- a/src/operator/nn/fully_connected.cc +++ b/src/operator/nn/fully_connected.cc @@ -1,4 +1,4 @@ - /* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -179,7 +179,7 @@ struct FullyConnectedGrad { std::vector FullyConnectedBackwardGrad(const nnvm::NodePtr& n, - const std::vector& ograds) { + const std::vector& ograds) { std::vector ret; size_t i = 0; for (const auto& x: n->inputs) { @@ -326,7 +326,6 @@ If ``no_bias`` is set to be true, then the ``bias`` term is ignored. .add_argument("bias", "NDArray-or-Symbol", "Bias parameter.") .add_arguments(FullyConnectedParam::__FIELDS__()); - NNVM_REGISTER_OP(_backward_FullyConnected) .set_num_inputs(3) .set_num_outputs([](const NodeAttrs& attrs) { @@ -351,6 +350,5 @@ NNVM_REGISTER_OP(_backward_FullyConnected) #endif .set_attr("FCompute", FullyConnectedGradCompute); - } // namespace op } // namespace mxnet From b26ffe18c8dee302968620ecfb5adfda41620756 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 18:41:50 -0700 Subject: [PATCH 056/146] Add a test for FC grad --- tests/python/unittest/test_gluon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index 92a4556176d9..1f769b50d4bb 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -21,7 +21,7 @@ import mxnet as mx from mxnet import gluon from mxnet.gluon import nn -from mxnet.test_utils import assert_almost_equal +from mxnet.test_utils import assert_almost_equal, same from mxnet.ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID from common import (setup_module, with_seed, assertRaises, teardown, assert_raises_cudnn_not_satisfied) @@ -919,17 +919,17 @@ def test_sequential_warning(): def test_dense_backward(): import mxnet.autograd as ag import mxnet.ndarray as nd - x = nd.array([[1,2,3,4]]) + x = nd.array([[1,2,3,400]]) net = gluon.nn.Sequential() with net.name_scope(): net.add(gluon.nn.Dense(1, in_units=x.shape[1])) net.initialize(mx.initializer.Constant(.5)) params = [x.data() for x in net.collect_params().values()] + x.attach_grad() with ag.record(): y = net.forward(x) - params_grad = ag.grad(y, params, create_graph=True, retain_graph=True) - params_grad_grad = ag.grad(params_grad, x) - print(params_grad_grad.grad()) + params_grad = ag.grad(y, params[0], create_graph=True, retain_graph=True)[0] + same(params_grad, x) @with_seed() From 4ef42d2b35153bbb34463a54114478f0f6b9d25c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 19:13:05 -0700 Subject: [PATCH 057/146] Fix test --- tests/python/unittest/test_gluon.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index 1f769b50d4bb..4d2f831938cb 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -1,4 +1,4 @@ -# Licensed to the Apache Software Foundation (ASF) under one + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file @@ -928,8 +928,9 @@ def test_dense_backward(): x.attach_grad() with ag.record(): y = net.forward(x) - params_grad = ag.grad(y, params[0], create_graph=True, retain_graph=True)[0] - same(params_grad, x) + y_grad = ag.grad(y, x, create_graph=True, retain_graph=True)[0] + y_grad.backward() + same(x.grad, nd.zeros(4)) @with_seed() From 12779257bae88c0a7268496402df4f47ce9a22c4 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 11:12:28 -0700 Subject: [PATCH 058/146] sync subrepos --- 3rdparty/googletest | 2 +- 3rdparty/mshadow | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/googletest b/3rdparty/googletest index ec44c6c1675c..eb9225ce361a 160000 --- a/3rdparty/googletest +++ b/3rdparty/googletest @@ -1 +1 @@ -Subproject commit ec44c6c1675c25b9827aacd08c02433cccde7780 +Subproject commit eb9225ce361affe561592e0912320b9db84985d0 diff --git a/3rdparty/mshadow b/3rdparty/mshadow index 95ebe0f109ae..6e94643bdf1d 160000 --- a/3rdparty/mshadow +++ b/3rdparty/mshadow @@ -1 +1 @@ -Subproject commit 95ebe0f109ae021d0d66e2a627ccfc55c3253b55 +Subproject commit 6e94643bdf1d51a505b147f28c358fb71070b8fd From a5ad61d65b37f0ef902c53523f241fa41cfbbbc8 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 13:45:44 -0700 Subject: [PATCH 059/146] Fix lint --- src/operator/nn/fully_connected.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/operator/nn/fully_connected.cc b/src/operator/nn/fully_connected.cc index cc87285c1d71..8137af694445 100644 --- a/src/operator/nn/fully_connected.cc +++ b/src/operator/nn/fully_connected.cc @@ -178,11 +178,12 @@ struct FullyConnectedGrad { }; -std::vector FullyConnectedBackwardGrad(const nnvm::NodePtr& n, - const std::vector& ograds) { +std::vector FullyConnectedBackwardGrad( + const nnvm::NodePtr& n, + const std::vector& ograds) { std::vector ret; size_t i = 0; - for (const auto& x: n->inputs) { + for (const auto& x : n->inputs) { std::ostringstream os; os << n->attrs.name << "_backward_" << i; ret.emplace_back(nnvm::NodeEntry{MakeNode("zeros_like", os.str(), {x}, nullptr, &n), 0, 0}); From e118a8bec2496bfffa78f35537efda852f6fd383 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 19:28:41 -0700 Subject: [PATCH 060/146] Check for null ptr in NDarray members --- include/mxnet/ndarray.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 0a3339b4a8a9..7ef358bf76d9 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -389,6 +389,7 @@ class NDArray { } /*! \return the associated variable of the ndarray.*/ inline Engine::VarHandle var() const { + CHECK(ptr_); return ptr_->var; } /*! \return byte offset in chunk of the ndarray*/ @@ -397,6 +398,7 @@ class NDArray { } /*! \brief return var version of the NDArray*/ inline size_t version() const { + CHECK(var()); return var()->version(); } /*! From f3feab54a17e3abaf2293e3ccaa9ce6cf5dc3f4a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 19:29:10 -0700 Subject: [PATCH 061/146] uint32_t -> size_t --- include/mxnet/imperative.h | 2 +- src/imperative/imperative.cc | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index ad209913ac53..23090bc714a0 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -168,7 +168,7 @@ class Imperative { /*! \brief find the input/output ndarrays that are needed for backward */ void GetBackwardDependency( const nnvm::NodePtr& node, - uint32_t num_inputs, uint32_t num_outputs, + size_t num_inputs, size_t num_outputs, std::vector *p_save_inputs, std::vector *p_save_outputs); /*! \brief indicate whether is training. */ diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 35c5172effc7..531491ede2f6 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -124,7 +124,7 @@ void Imperative::MarkVariables( const std::vector& variables, const std::vector& grad_reqs, const std::vector& gradients) { - for (uint32_t i = 0; i < variables.size(); ++i) { + for (size_t i = 0; i < variables.size(); ++i) { std::string str_c(std::to_string(variable_count_++)); variables[i]->entry_ = nnvm::NodeEntry{ @@ -146,7 +146,7 @@ void Imperative::MarkVariables( void Imperative::GetBackwardDependency( const nnvm::NodePtr& node, - uint32_t num_inputs, uint32_t num_outputs, + size_t num_inputs, size_t num_outputs, std::vector *p_save_inputs, std::vector *p_save_outputs) { static auto& fgradient = nnvm::Op::GetAttr("FGradient"); @@ -159,14 +159,14 @@ void Imperative::GetBackwardDependency( node->inputs.clear(); node->inputs.reserve(num_inputs); - for (uint32_t i = 0; i < num_inputs; ++i) { + for (size_t i = 0; i < num_inputs; ++i) { node->inputs.emplace_back(nnvm::NodeEntry{nullptr, i, 0}); } if (fgradient.count(node->op())) { std::vector ograd_entries; ograd_entries.reserve(num_outputs); - for (uint32_t i = 0; i < num_outputs; ++i) { + for (size_t i = 0; i < num_outputs; ++i) { ograd_entries.emplace_back(nnvm::NodeEntry{nullptr, i, 1}); } auto igrad_entries = fgradient[node->op()](node, ograd_entries); @@ -263,7 +263,7 @@ void Imperative::RecordOp( << "recording with autograd."; } - for (uint32_t i = 0; i < outputs.size(); ++i) { + for (size_t i = 0; i < outputs.size(); ++i) { if (save_outputs[i]) { info.outputs.emplace_back(outputs[i]->Detach()); } else { From 3f749060c2b13f86bef75f275cbbe574ebb70526 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 19:31:06 -0700 Subject: [PATCH 062/146] Fix warning in fully connected backward^2 --- src/operator/nn/fully_connected.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/nn/fully_connected.cc b/src/operator/nn/fully_connected.cc index 8137af694445..5c6994c747fe 100644 --- a/src/operator/nn/fully_connected.cc +++ b/src/operator/nn/fully_connected.cc @@ -182,7 +182,7 @@ std::vector FullyConnectedBackwardGrad( const nnvm::NodePtr& n, const std::vector& ograds) { std::vector ret; - size_t i = 0; + decltype(nnvm::NodeEntry::index) i = 0; for (const auto& x : n->inputs) { std::ostringstream os; os << n->attrs.name << "_backward_" << i; From 6eeedde253bfa87c52a1a9b292d272ad6409f750 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 26 Apr 2019 17:24:28 -0700 Subject: [PATCH 063/146] Refactor Backward --- include/mxnet/imperative.h | 92 ++++++++++++++++++------------- include/mxnet/ndarray.h | 3 +- src/c_api/c_api_ndarray.cc | 2 +- src/imperative/imperative.cc | 85 ++++++++++++++++------------ src/imperative/imperative_utils.h | 2 +- src/ndarray/ndarray.cc | 14 ++--- 6 files changed, 116 insertions(+), 82 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index 23090bc714a0..6e117ac81b3d 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -35,48 +35,53 @@ #include "./ndarray.h" namespace mxnet { -/*! \brief runtime functions for NDArray */ -class Imperative { +/*! + * Autograd Info used in class: nnvm::Node::info + */ +class AGInfo { public: - /*! \brief */ - class AGInfo { - public: - Context ctx; - OpReqType grad_req; - OpStatePtr state; - std::vector outputs; - std::vector out_grads; - bool fresh_out_grad; + Context ctx; + OpReqType grad_req; + OpStatePtr state; + std::vector outputs; + std::vector out_grads; + bool fresh_out_grad; - AGInfo() : - grad_req(kNullOp), fresh_out_grad(false) {} + AGInfo() : + grad_req(kNullOp), fresh_out_grad(false) {} + + static void Clear(const nnvm::NodePtr& node) { + if (node == nullptr || node->info.empty()) return; + AGInfo& info = Get(node); + if (info.grad_req != kNullOp) return; + node->info.clear(); + } + + static AGInfo& Get(const nnvm::NodePtr& node) { + return dmlc::get(node->info); + } - static void Clear(const nnvm::NodePtr& node) { - if (node == nullptr || node->info.empty()) return; - AGInfo& info = Get(node); - if (info.grad_req != kNullOp) return; - node->info.clear(); - } + static AGInfo& Create(const nnvm::NodePtr& node) { + node->info.construct(); + return Get(node); + } - static AGInfo& Get(const nnvm::NodePtr& node) { - return dmlc::get(node->info); - } + static bool IsNone(const NDArray& arr) { + return arr.entry_.node == nullptr || arr.entry_.node->info.empty(); + } - static AGInfo& Create(const nnvm::NodePtr& node) { - node->info.construct(); - return Get(node); - } + static bool IsVariable(const nnvm::NodePtr& node) { + AGInfo& info = Get(node); + return info.grad_req != kNullOp && info.outputs.size() == 1 + && info.out_grads.size() == 1; + } +}; - static bool IsNone(const NDArray& arr) { - return arr.entry_.node == nullptr || arr.entry_.node->info.empty(); - } +/*! \brief runtime functions for NDArray */ +class Imperative { + public: + /*! \brief */ - static bool IsVariable(const nnvm::NodePtr& node) { - AGInfo& info = Get(node); - return info.grad_req != kNullOp && info.outputs.size() == 1 - && info.out_grads.size() == 1; - } - }; /*! \brief whether operator recording is on. */ bool is_training() const { return is_train_; @@ -97,11 +102,11 @@ class Imperative { is_recording_ = is_recording; return old; } - /*! brief whether numpy compatibility is on. */ + /*! \brief whether numpy compatibility is on. */ bool is_np_comp() const { return is_np_comp_; } - /*! brief turn on or turn off numpy compatibility switch. */ + /*! \brief turn on or turn off numpy compatibility switch. */ bool set_is_np_comp(bool is_np_comp) { bool old = is_np_comp_; is_np_comp_ = is_np_comp; @@ -160,7 +165,18 @@ class Imperative { private: friend class NDArray; - /*! \brief make constructor protected. */ + /*! Create a forward graph + * @param output_nodes graph node vector to add nodes to + * @param outputs source ndarrays + * @return vector of nodes + */ + static std::vector CreateForwardGraph(const std::vector& outputs); + /*! Create gradient nodes using output shapes and ctx. + * Gradient heads are set to 1 if they are not present (nullptr) + * @return vector of nodes + */ + static std::vector CreateHeadGradients(const std::vector& outputs, + const std::vector& ograds); Imperative() { if (PreferBulkExecTrain()) backward_bulk_size_ = BulkExecMaxNodeTrainBwd(); diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 7ef358bf76d9..04ad1d0d51ef 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -80,6 +80,8 @@ class MKLDNNMemory; * \brief ndarray interface */ class NDArray { + friend class AGInfo; + friend class Imperative; public: /*! \brief default constructor */ NDArray() @@ -816,7 +818,6 @@ class NDArray { std::vector* keys); private: - friend class Imperative; /*! \brief the real data chunk that backs NDArray */ // shandle is used to store the actual values in the NDArray // aux_handles store the aux data(such as indices) if it's needed by non-default storage. diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 0e136b03ecd7..dcad058b6cba 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -355,7 +355,7 @@ int MXAutogradBackwardEx(mx_uint num_output, } auto grads = Imperative::Get()->Backward(outputs, ograds, variables, is_train, - retain_graph, create_graph); + retain_graph, create_graph); if (num_variables != 0) { ret->ret_handles.clear(); ret->out_types.clear(); diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 531491ede2f6..70ac116e8cd3 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -126,20 +126,22 @@ void Imperative::MarkVariables( const std::vector& gradients) { for (size_t i = 0; i < variables.size(); ++i) { std::string str_c(std::to_string(variable_count_++)); - - variables[i]->entry_ = nnvm::NodeEntry{ - nnvm::Symbol::CreateVariable("var" + str_c).outputs[0].node, 0, 0}; - AGInfo& info = AGInfo::Create(variables[i]->entry_.node); - info.outputs.emplace_back(variables[i]->Detach()); - info.out_grads.emplace_back(gradients[i]->Detach()); - info.grad_req = static_cast(grad_reqs[i]); - info.ctx = variables[i]->ctx(); - - gradients[i]->entry_ = nnvm::NodeEntry{ - nnvm::Symbol::CreateVariable("grad" + str_c).outputs[0].node, 0, 0}; - AGInfo& grad_info = AGInfo::Create(gradients[i]->entry_.node); - grad_info.outputs.emplace_back(gradients[i]->Detach()); - grad_info.ctx = gradients[i]->ctx(); + { + variables[i]->entry_ = nnvm::NodeEntry{ + nnvm::Symbol::CreateVariable("var" + str_c).outputs[0].node, 0, 0}; + AGInfo &info = AGInfo::Create(variables[i]->entry_.node); + info.outputs.emplace_back(variables[i]->Detach()); + info.out_grads.emplace_back(gradients[i]->Detach()); + info.grad_req = static_cast(grad_reqs[i]); + info.ctx = variables[i]->ctx(); + } + { + gradients[i]->entry_ = nnvm::NodeEntry{ + nnvm::Symbol::CreateVariable("grad" + str_c).outputs[0].node, 0, 0}; + AGInfo &grad_info = AGInfo::Create(gradients[i]->entry_.node); + grad_info.outputs.emplace_back(gradients[i]->Detach()); + grad_info.ctx = gradients[i]->ctx(); + } } } @@ -277,36 +279,30 @@ void Imperative::RecordOp( } } -std::vector Imperative::Backward( - const std::vector& outputs, - const std::vector& ograds, - const std::vector& variables, - bool is_train, bool retain_graph, - bool create_graph) { - using namespace nnvm; - using namespace imperative; - static const std::vector zero_ops{Op::Get("zeros_like"), Op::Get("_zeros")}; - static const Op* copy_op = Op::Get("_copy"); - - // Construct forward graph - Graph graph; - graph.outputs.reserve(outputs.size()); - for (const auto& i : outputs) { +std::vector Imperative::CreateForwardGraph(const std::vector& outputs) { + std::vector output_nodes; + output_nodes.reserve(outputs.size()); + for (const auto &i : outputs) { CHECK(!AGInfo::IsNone(*i)) << "Cannot differentiate node because it is not in a computational graph. " << "You need to set is_recording to true or use autograd.record() to save " << "computational graphs for backward. If you want to differentiate the same " << "graph twice, you need to pass retain_graph=True to backward."; - graph.outputs.emplace_back(i->entry_); + output_nodes.emplace_back(i->entry_); } - size_t num_forward_outputs = graph.outputs.size(); + return output_nodes; +} - // Prepare head gradients +std::vector Imperative::CreateHeadGradients( + const std::vector& outputs, + const std::vector& ograds) { + using nnvm::NodeEntry; + using nnvm::Node; std::vector ograd_entries; ograd_entries.reserve(ograds.size()); for (size_t i = 0; i < outputs.size(); ++i) { ograd_entries.emplace_back(NodeEntry{Node::Create(), 0, 0}); - AGInfo& info = AGInfo::Create(ograd_entries.back().node); + AGInfo &info = AGInfo::Create(ograd_entries.back().node); info.ctx = outputs[i]->ctx(); if (ograds[i] != nullptr) { info.outputs.emplace_back(*ograds[i]); @@ -316,6 +312,27 @@ std::vector Imperative::Backward( info.outputs.back() = static_cast(1.0); } } + return ograd_entries; +} + +std::vector Imperative::Backward( + const std::vector& outputs, + const std::vector& ograds, + const std::vector& variables, + bool is_train, bool retain_graph, + bool create_graph) { + using namespace nnvm; + using namespace imperative; + static const std::vector zero_ops{Op::Get("zeros_like"), Op::Get("_zeros")}; + static const Op* copy_op = Op::Get("_copy"); + + // Construct forward graph + Graph graph; + graph.outputs = CreateForwardGraph(outputs); + const size_t num_forward_outputs = graph.outputs.size(); + + // Prepare head gradients + std::vector ograd_entries = CreateHeadGradients(outputs, ograds); // Get gradient graph Symbol sym; @@ -323,7 +340,7 @@ std::vector Imperative::Backward( std::vector xs; std::vector x_grads; std::vector x_reqs; - if (variables.size()) { + if (!variables.empty()) { xs.reserve(variables.size()); x_grads.reserve(variables.size()); x_reqs.reserve(variables.size()); diff --git a/src/imperative/imperative_utils.h b/src/imperative/imperative_utils.h index 5c9706834b2d..f0f60ae9c5a2 100644 --- a/src/imperative/imperative_utils.h +++ b/src/imperative/imperative_utils.h @@ -732,7 +732,7 @@ inline std::vector PlaceDevice(const nnvm::IndexedGraph& idx) { // forward pass for (size_t i = 0; i < idx.num_nodes(); ++i) { if (!idx[i].source->info.empty()) { - vctx[i] = dmlc::get(idx[i].source->info).ctx; + vctx[i] = dmlc::get(idx[i].source->info).ctx; } else if (idx[i].source->op() == _copyto) { CHECK_GT(idx[i].source->control_deps.size(), 0); auto fwd_nid = idx.node_id(idx[i].source->control_deps[0].get()); diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index a8a34eaccac0..a41817064607 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -151,8 +151,8 @@ void NDArray::Chunk::CheckAndAllocData(const mxnet::TShape &shape, int dtype) { } NDArray NDArray::grad() const { - if (Imperative::AGInfo::IsNone(*this)) return NDArray(); - Imperative::AGInfo& info = Imperative::AGInfo::Get(entry_.node); + if (AGInfo::IsNone(*this)) return NDArray(); + AGInfo& info = AGInfo::Get(entry_.node); if (info.out_grads.size()) { CHECK_EQ(info.out_grads.size(), 1); return info.out_grads[0]; @@ -161,7 +161,7 @@ NDArray NDArray::grad() const { } nnvm::Symbol NDArray::get_autograd_symbol() const { - CHECK(!Imperative::AGInfo::IsNone(*this)) + CHECK(!AGInfo::IsNone(*this)) << "NDArray is not part of a computation graph. Did you forget to turn on recording?"; nnvm::Symbol ret; ret.outputs.emplace_back(entry_); @@ -366,16 +366,16 @@ NDArray NDArray::FromDLPack(const DLManagedTensor* tensor) { } bool NDArray::fresh_out_grad() const { - if (Imperative::AGInfo::IsNone(*this)) return false; - Imperative::AGInfo& info = Imperative::AGInfo::Get(entry_.node); + if (AGInfo::IsNone(*this)) return false; + AGInfo& info = AGInfo::Get(entry_.node); return info.fresh_out_grad; } void NDArray::set_fresh_out_grad(bool state) const { - CHECK(!Imperative::AGInfo::IsNone(*this)) + CHECK(!AGInfo::IsNone(*this)) << "NDArray has not been marked as a variable and does not have gradient state"; - Imperative::AGInfo& info = Imperative::AGInfo::Get(entry_.node); + AGInfo& info = AGInfo::Get(entry_.node); info.fresh_out_grad = state; } From 6d2e18d4c6152a46049de76e87bf1658adc8c67d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 26 Apr 2019 18:31:10 -0700 Subject: [PATCH 064/146] Refactor Imperative::Backward --- include/mxnet/c_api.h | 2 +- include/mxnet/imperative.h | 4 ++ src/imperative/imperative.cc | 100 ++++++++++++++++++++--------------- 3 files changed, 62 insertions(+), 44 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index d3c679455a57..8d434b1558bd 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -1109,7 +1109,7 @@ MXNET_DLL int MXAutogradBackward(mx_uint num_output, * \param output_handles output NDArrays * \param ograd_handles head gradient for NDArrays * \param num_variables number of variables - * \param + * \param var_handles variables to compute gradient with respect to (d / d var) * \param retain_graph whether to keep the graph after backward * \param is_train whether to do backward for training or inference * \return 0 when success, -1 when failure happens diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index 6e117ac81b3d..5ed896b22715 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -177,6 +177,10 @@ class Imperative { */ static std::vector CreateHeadGradients(const std::vector& outputs, const std::vector& ograds); + + class GradientGraph; + GradientGraph CreateGradientGraph(const std::vector& variables, + const std::vector& outputs); Imperative() { if (PreferBulkExecTrain()) backward_bulk_size_ = BulkExecMaxNodeTrainBwd(); diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 70ac116e8cd3..c02915e60d73 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -146,6 +146,9 @@ void Imperative::MarkVariables( } + + + void Imperative::GetBackwardDependency( const nnvm::NodePtr& node, size_t num_inputs, size_t num_outputs, @@ -294,8 +297,8 @@ std::vector Imperative::CreateForwardGraph(const std::vector Imperative::CreateHeadGradients( - const std::vector& outputs, - const std::vector& ograds) { + const std::vector& outputs, + const std::vector& ograds) { using nnvm::NodeEntry; using nnvm::Node; std::vector ograd_entries; @@ -315,6 +318,49 @@ std::vector Imperative::CreateHeadGradients( return ograd_entries; } +struct Imperative::GradientGraph { + std::vector variable_nodes; + std::vector gradients; + std::vector op_req_types; +}; + +Imperative::GradientGraph Imperative::CreateGradientGraph(const std::vector& variables, + const std::vector& outputs) { + GradientGraph gg; + if (!variables.empty()) { + gg.variable_nodes.reserve(variables.size()); + gg.gradients.reserve(variables.size()); + gg.op_req_types.reserve(variables.size()); + for (size_t i = 0; i < variables.size(); ++i) { + CHECK(!AGInfo::IsNone(*variables[i]) && + AGInfo::IsVariable(variables[i]->entry_.node)) + << "Cannot differentiate with respect to the " << i+1 << "-th variable" + << " because it does not require gradient."; + gg.variable_nodes.emplace_back(variables[i]->entry_); + gg.gradients.push_back(new NDArray()); + gg.op_req_types.push_back(kWriteTo); + } + } else { + std::vector ro_nodes = nnvm::Symbol::ListInputs(Symbol::kReadOnlyArgs, outputs); + gg.variable_nodes.reserve(ro_nodes.size()); + gg.gradients.reserve(ro_nodes.size()); + gg.op_req_types.reserve(ro_nodes.size()); + for (const auto& i : ro_nodes) { + AGInfo& info = AGInfo::Get(i); + if (info.grad_req != kNullOp) { + gg.variable_nodes.emplace_back(nnvm::NodeEntry{i, 0, 0}); + gg.gradients.push_back(&info.out_grads[0]); + gg.op_req_types.push_back(info.grad_req); + info.fresh_out_grad = true; + } + } + CHECK_GT(gg.variable_nodes.size(), 0) + << "There are no inputs in computation graph that require gradients."; + } +} + + + std::vector Imperative::Backward( const std::vector& outputs, const std::vector& ograds, @@ -335,46 +381,14 @@ std::vector Imperative::Backward( std::vector ograd_entries = CreateHeadGradients(outputs, ograds); // Get gradient graph - Symbol sym; - sym.outputs = graph.outputs; - std::vector xs; - std::vector x_grads; - std::vector x_reqs; - if (!variables.empty()) { - xs.reserve(variables.size()); - x_grads.reserve(variables.size()); - x_reqs.reserve(variables.size()); - for (size_t i = 0; i < variables.size(); ++i) { - CHECK(!AGInfo::IsNone(*variables[i]) && - AGInfo::IsVariable(variables[i]->entry_.node)) - << "Cannot differentiate with respect to the " << i+1 << "-th variable" - << " because it does not require gradient."; - xs.emplace_back(variables[i]->entry_); - x_grads.push_back(new NDArray()); - x_reqs.push_back(kWriteTo); - } - } else { - std::vector args = sym.ListInputs(Symbol::kReadOnlyArgs); - xs.reserve(args.size()); - x_grads.reserve(args.size()); - x_reqs.reserve(args.size()); - for (const auto& i : args) { - AGInfo& info = AGInfo::Get(i); - if (info.grad_req == kNullOp) continue; - xs.emplace_back(NodeEntry{i, 0, 0}); - x_grads.push_back(&info.out_grads[0]); - x_reqs.push_back(info.grad_req); - info.fresh_out_grad = true; - } - CHECK_GT(xs.size(), 0) - << "There are no inputs in computation graph that require gradients."; - } + GradientGraph gg = CreateGradientGraph(variables, graph.outputs); + // Run backward on the graph Graph g_graph = pass::MXGradient( - graph, graph.outputs, xs, ograd_entries, + graph, graph.outputs, gg.variable_nodes, ograd_entries, exec::AggregateGradient, nullptr, nullptr, zero_ops, "_copy"); - CHECK_EQ(g_graph.outputs.size(), xs.size()); + CHECK_EQ(g_graph.outputs.size(), gg.variable_nodes.size()); for (const auto& e : g_graph.outputs) { if (e.node->op() == nullptr) { auto node = Node::Create(); @@ -407,7 +421,7 @@ std::vector Imperative::Backward( } if (create_graph) { states.resize(num_forward_nodes); - nnvm::DFSVisit(sym.outputs, [&](const nnvm::NodePtr& n) { + nnvm::DFSVisit(graph.outputs, [&](const nnvm::NodePtr& n) { AGInfo& info = AGInfo::Get(n); states[idx.node_id(n.get())] = info.state; for (uint32_t i = 0; i < info.outputs.size(); ++i) { @@ -446,7 +460,7 @@ std::vector Imperative::Backward( } for (size_t i = num_forward_outputs; i < graph.outputs.size(); ++i) { size_t eid = idx.entry_id(graph.outputs[i]); - arrays[eid] = x_grads[i - num_forward_outputs]; + arrays[eid] = gg.gradients[i - num_forward_outputs]; ref_count[eid] = 1; } @@ -496,7 +510,7 @@ std::vector Imperative::Backward( } for (size_t i = num_forward_outputs; i < idx.outputs().size(); ++i) { size_t eid = idx.entry_id(idx.outputs()[i]); - array_reqs[eid] = x_reqs[i - num_forward_outputs]; + array_reqs[eid] = gg.op_req_types[i - num_forward_outputs]; } const auto& shapes = graph.GetAttr("shape"); @@ -541,14 +555,14 @@ std::vector Imperative::Backward( // Clear history if (!retain_graph) { - nnvm::DFSVisit(sym.outputs, [&](const nnvm::NodePtr& n) { + nnvm::DFSVisit(graph.outputs, [&](const nnvm::NodePtr& n) { AGInfo::Clear(n); n->inputs.clear(); }); } if (variables.size()) { - return x_grads; + return gg.gradients; } return {}; } From d68728a59555346be2d373f600edb93468577509 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 11:50:27 -0700 Subject: [PATCH 065/146] Update tvm to my repo --- .gitmodules | 2 +- 3rdparty/tvm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index e0ffec11bfd0..151b9fe39a86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/dmlc/tvm + url = https://github.com/larroy/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git diff --git a/3rdparty/tvm b/3rdparty/tvm index a706ad16f83d..c8f2302ac88b 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit a706ad16f83d810f4a6268cf604cece79c2c4791 +Subproject commit c8f2302ac88b47f7bbae178ddb6f6c857bcf8fbb From a600564395bb78ca9a4498e86b549e839af34919 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 12:00:35 -0700 Subject: [PATCH 066/146] udpate tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index c8f2302ac88b..0b8f6051680d 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit c8f2302ac88b47f7bbae178ddb6f6c857bcf8fbb +Subproject commit 0b8f6051680d7b474b6079fdd0407adcbe3d0897 From bb9123dc6e1604f5bbf7e36e6ff0d0bd318f55f0 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 12:15:45 -0700 Subject: [PATCH 067/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 0b8f6051680d..625f4f0f57a1 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 0b8f6051680d7b474b6079fdd0407adcbe3d0897 +Subproject commit 625f4f0f57a1c5a32ef013b778f25b736e7c1236 From 24fb41e6cc59cc507def7566cd083996a3743f24 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 12:19:47 -0700 Subject: [PATCH 068/146] Remove unnecesary ctor call of NodeEntry --- src/imperative/imperative.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index c02915e60d73..e0074f51f80e 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -165,14 +165,14 @@ void Imperative::GetBackwardDependency( node->inputs.clear(); node->inputs.reserve(num_inputs); for (size_t i = 0; i < num_inputs; ++i) { - node->inputs.emplace_back(nnvm::NodeEntry{nullptr, i, 0}); + node->inputs.emplace_back(nullptr, i, 0); } if (fgradient.count(node->op())) { std::vector ograd_entries; ograd_entries.reserve(num_outputs); for (size_t i = 0; i < num_outputs; ++i) { - ograd_entries.emplace_back(nnvm::NodeEntry{nullptr, i, 1}); + ograd_entries.emplace_back(nullptr, i, 1); } auto igrad_entries = fgradient[node->op()](node, ograd_entries); for (const auto& i : igrad_entries) { @@ -345,10 +345,10 @@ Imperative::GradientGraph Imperative::CreateGradientGraph(const std::vector Date: Mon, 29 Apr 2019 12:27:42 -0700 Subject: [PATCH 069/146] Fix emplace_back --- src/operator/operator_common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 732203f0e601..0cc25828e5b2 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -397,7 +397,7 @@ inline std::vector MakeGradNode( &inputs, &dict, &n); std::vector ret; for (uint32_t i = 0; i < p->num_outputs(); ++i) { - ret.emplace_back(nnvm::NodeEntry{p, i, 0}); + ret.emplace_back(p, i, 0); } return ret; } @@ -415,7 +415,7 @@ inline std::vector MakeZeroGradNodes( os << n->attrs.name << "_in" << i << "_backward"; } auto p = MakeNode("zeros_like", os.str(), {n->inputs[i]}, nullptr, &n); - ret.emplace_back(nnvm::NodeEntry{p, 0, 0}); + ret.emplace_back(p); } return ret; } From f6def8132a8feb6bf038f11bbcd5d54f2e3ef37c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 12:27:57 -0700 Subject: [PATCH 070/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 625f4f0f57a1..56514dda844c 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 625f4f0f57a1c5a32ef013b778f25b736e7c1236 +Subproject commit 56514dda844cbde28bf5727fb85c15470752d0d2 From 2472435a7c3d6cf89874fa261744e164282364df Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 12:31:09 -0700 Subject: [PATCH 071/146] Refactor CreateGradientVariableNodes --- include/mxnet/imperative.h | 4 ++-- src/imperative/imperative.cc | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index 5ed896b22715..41c2ab1d5854 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -179,8 +179,8 @@ class Imperative { const std::vector& ograds); class GradientGraph; - GradientGraph CreateGradientGraph(const std::vector& variables, - const std::vector& outputs); + GradientGraph CreateGradientVariableNodes(const std::vector &variables, + const std::vector &outputs); Imperative() { if (PreferBulkExecTrain()) backward_bulk_size_ = BulkExecMaxNodeTrainBwd(); diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index e0074f51f80e..63ba4c5cffd5 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -324,8 +324,8 @@ struct Imperative::GradientGraph { std::vector op_req_types; }; -Imperative::GradientGraph Imperative::CreateGradientGraph(const std::vector& variables, - const std::vector& outputs) { +Imperative::GradientGraph Imperative::CreateGradientVariableNodes(const std::vector &variables, + const std::vector &outputs) { GradientGraph gg; if (!variables.empty()) { gg.variable_nodes.reserve(variables.size()); @@ -381,7 +381,7 @@ std::vector Imperative::Backward( std::vector ograd_entries = CreateHeadGradients(outputs, ograds); // Get gradient graph - GradientGraph gg = CreateGradientGraph(variables, graph.outputs); + GradientGraph gg = CreateGradientVariableNodes(variables, graph.outputs); // Run backward on the graph Graph g_graph = pass::MXGradient( From f8a9c4bd056a82795d8b20d54a6f64c848207dc9 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 13:05:25 -0700 Subject: [PATCH 072/146] Fix errors and warnings --- CMakeLists.txt | 2 +- include/mxnet/imperative.h | 4 +- include/mxnet/io.h | 2 +- src/engine/stream_manager.h | 1 + src/imperative/imperative.cc | 115 +++++++++++++++--------------- src/imperative/imperative_utils.h | 6 +- src/io/iter_batchloader.h | 2 +- src/io/iter_libsvm.cc | 6 +- src/io/iter_prefetcher.h | 2 +- src/io/iter_sparse.h | 4 +- src/io/iter_sparse_batchloader.h | 6 +- src/io/iter_sparse_prefetcher.h | 6 +- 12 files changed, 80 insertions(+), 76 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d014e96ff77f..d5e128035489 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,7 +141,7 @@ else(MSVC) add_definitions(-DMSHADOW_USE_F16C=0) endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wno-unknown-pragmas -Wno-sign-compare") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wno-unused-parameter -Wno-unused-local-typedefs -Wno-implicit-fallthrough -Wno-unknown-pragmas -Wno-sign-compare -Werror=return-type") if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-braced-scalar-init") endif() diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index 41c2ab1d5854..f58d6370f0f9 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -178,8 +178,8 @@ class Imperative { static std::vector CreateHeadGradients(const std::vector& outputs, const std::vector& ograds); - class GradientGraph; - GradientGraph CreateGradientVariableNodes(const std::vector &variables, + class GradientVariableNodes; + GradientVariableNodes CreateGradientVariableNodes(const std::vector &variables, const std::vector &outputs); Imperative() { if (PreferBulkExecTrain()) diff --git a/include/mxnet/io.h b/include/mxnet/io.h index e18f03ed0ef3..2a352af16cf9 100644 --- a/include/mxnet/io.h +++ b/include/mxnet/io.h @@ -52,7 +52,7 @@ class IIterator : public dmlc::DataIter { /*! \brief move to next item */ virtual bool Next(void) = 0; /*! \brief get current data */ - virtual const DType &Value(void) const = 0; + virtual const DType& Value(void) const = 0; /*! \brief constructor */ virtual ~IIterator(void) {} /*! \brief store the name of each data, it could be used for making NDArrays */ diff --git a/src/engine/stream_manager.h b/src/engine/stream_manager.h index 42d03e55a275..7d69659767a0 100644 --- a/src/engine/stream_manager.h +++ b/src/engine/stream_manager.h @@ -102,6 +102,7 @@ RunContext StreamManager::GetRunContext( #endif // MXNET_USE_CUDA default: LOG(FATAL) << "Not Reached"; + break; } } return ret; diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 63ba4c5cffd5..f6dd538d3e3f 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -318,45 +318,46 @@ std::vector Imperative::CreateHeadGradients( return ograd_entries; } -struct Imperative::GradientGraph { +struct Imperative::GradientVariableNodes { std::vector variable_nodes; std::vector gradients; std::vector op_req_types; }; -Imperative::GradientGraph Imperative::CreateGradientVariableNodes(const std::vector &variables, +Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes(const std::vector &variables, const std::vector &outputs) { - GradientGraph gg; + GradientVariableNodes var_nodes; if (!variables.empty()) { - gg.variable_nodes.reserve(variables.size()); - gg.gradients.reserve(variables.size()); - gg.op_req_types.reserve(variables.size()); + var_nodes.variable_nodes.reserve(variables.size()); + var_nodes.gradients.reserve(variables.size()); + var_nodes.op_req_types.reserve(variables.size()); for (size_t i = 0; i < variables.size(); ++i) { CHECK(!AGInfo::IsNone(*variables[i]) && AGInfo::IsVariable(variables[i]->entry_.node)) << "Cannot differentiate with respect to the " << i+1 << "-th variable" << " because it does not require gradient."; - gg.variable_nodes.emplace_back(variables[i]->entry_); - gg.gradients.push_back(new NDArray()); - gg.op_req_types.push_back(kWriteTo); + var_nodes.variable_nodes.emplace_back(variables[i]->entry_); + var_nodes.gradients.push_back(new NDArray()); + var_nodes.op_req_types.push_back(kWriteTo); } } else { std::vector ro_nodes = nnvm::Symbol::ListInputs(Symbol::kReadOnlyArgs, outputs); - gg.variable_nodes.reserve(ro_nodes.size()); - gg.gradients.reserve(ro_nodes.size()); - gg.op_req_types.reserve(ro_nodes.size()); + var_nodes.variable_nodes.reserve(ro_nodes.size()); + var_nodes.gradients.reserve(ro_nodes.size()); + var_nodes.op_req_types.reserve(ro_nodes.size()); for (const auto& node : ro_nodes) { AGInfo& info = AGInfo::Get(node); if (info.grad_req != kNullOp) { - gg.variable_nodes.emplace_back(node); - gg.gradients.push_back(&info.out_grads[0]); - gg.op_req_types.push_back(info.grad_req); + var_nodes.variable_nodes.emplace_back(node); + var_nodes.gradients.push_back(&info.out_grads[0]); + var_nodes.op_req_types.push_back(info.grad_req); info.fresh_out_grad = true; } } - CHECK_GT(gg.variable_nodes.size(), 0) + CHECK_GT(var_nodes.variable_nodes.size(), 0) << "There are no inputs in computation graph that require gradients."; } + return var_nodes; } @@ -381,15 +382,16 @@ std::vector Imperative::Backward( std::vector ograd_entries = CreateHeadGradients(outputs, ograds); // Get gradient graph - GradientGraph gg = CreateGradientVariableNodes(variables, graph.outputs); + GradientVariableNodes gvar = CreateGradientVariableNodes(variables, graph.outputs); // Run backward on the graph - Graph g_graph = pass::MXGradient( - graph, graph.outputs, gg.variable_nodes, ograd_entries, + Graph gradient_graph = pass::MXGradient( + graph, graph.outputs, gvar.variable_nodes, ograd_entries, exec::AggregateGradient, nullptr, nullptr, zero_ops, "_copy"); - CHECK_EQ(g_graph.outputs.size(), gg.variable_nodes.size()); - for (const auto& e : g_graph.outputs) { + CHECK_EQ(gradient_graph.outputs.size(), gvar.variable_nodes.size()); + // TODO: move inside pass::MXGradient + for (const auto& e : gradient_graph.outputs) { if (e.node->op() == nullptr) { auto node = Node::Create(); node->attrs.op = copy_op; @@ -399,19 +401,20 @@ std::vector Imperative::Backward( graph.outputs.push_back(e); } } - const auto& idx = graph.indexed_graph(); + + const auto& indexed_graph = graph.indexed_graph(); // get number of nodes used in forward pass size_t num_forward_nodes = 0; size_t num_forward_entries = 0; for (size_t i = 0; i < num_forward_outputs; ++i) { num_forward_nodes = std::max( - num_forward_nodes, static_cast(idx.outputs()[i].node_id + 1)); + num_forward_nodes, static_cast(indexed_graph.outputs()[i].node_id + 1)); num_forward_entries = std::max( - num_forward_entries, static_cast(idx.entry_id(idx.outputs()[i])) + 1); + num_forward_entries, static_cast(indexed_graph.entry_id(indexed_graph.outputs()[i])) + 1); } // Allocate buffer - std::vector buff(idx.num_node_entries()); + std::vector buff(indexed_graph.num_node_entries()); std::vector ref_count(buff.size(), 0); std::vector states; std::vector arrays; @@ -423,11 +426,11 @@ std::vector Imperative::Backward( states.resize(num_forward_nodes); nnvm::DFSVisit(graph.outputs, [&](const nnvm::NodePtr& n) { AGInfo& info = AGInfo::Get(n); - states[idx.node_id(n.get())] = info.state; + states[indexed_graph.node_id(n.get())] = info.state; for (uint32_t i = 0; i < info.outputs.size(); ++i) { - CHECK(idx.exist(n.get())); - size_t nid = idx.node_id(n.get()); - size_t eid = idx.entry_id(nid, i); + CHECK(indexed_graph.exist(n.get())); + size_t nid = indexed_graph.node_id(n.get()); + size_t eid = indexed_graph.entry_id(nid, i); buff[eid] = info.outputs[i]; buff[eid].entry_ = NodeEntry{n, i, 0}; ref_count[eid] = 1; @@ -435,82 +438,82 @@ std::vector Imperative::Backward( }); for (auto& ograd_entry : ograd_entries) { AGInfo& info = AGInfo::Get(ograd_entry.node); - if (!idx.exist(ograd_entry.node.get())) continue; - size_t eid = idx.entry_id(ograd_entry); + if (!indexed_graph.exist(ograd_entry.node.get())) continue; + size_t eid = indexed_graph.entry_id(ograd_entry); buff[eid] = info.outputs[0]; buff[eid].entry_ = ograd_entry; } } else { states.reserve(num_forward_nodes); for (size_t i = 0; i < num_forward_nodes; ++i) { - const AGInfo& info = dmlc::get(idx[i].source->info); + const AGInfo& info = dmlc::get(indexed_graph[i].source->info); states.emplace_back(info.state); for (size_t j = 0; j < info.outputs.size(); ++j) { - size_t eid = idx.entry_id(i, j); + size_t eid = indexed_graph.entry_id(i, j); arrays[eid] = const_cast(&(info.outputs[j])); if (retain_graph || info.grad_req != kNullOp) ref_count[eid] = 1; } } for (auto& ograd_entry : ograd_entries) { - if (!idx.exist(ograd_entry.node.get())) continue; + if (!indexed_graph.exist(ograd_entry.node.get())) continue; AGInfo& info = AGInfo::Get(ograd_entry.node); - arrays[idx.entry_id(ograd_entry)] = &info.outputs[0]; + arrays[indexed_graph.entry_id(ograd_entry)] = &info.outputs[0]; } } for (size_t i = num_forward_outputs; i < graph.outputs.size(); ++i) { - size_t eid = idx.entry_id(graph.outputs[i]); - arrays[eid] = gg.gradients[i - num_forward_outputs]; + size_t eid = indexed_graph.entry_id(graph.outputs[i]); + arrays[eid] = gvar.gradients[i - num_forward_outputs]; ref_count[eid] = 1; } // Assign context - auto vctx = PlaceDevice(idx); + auto vctx = PlaceDevice(indexed_graph); // Infer shape type { std::pair node_range, entry_range; - node_range = {num_forward_nodes, idx.num_nodes()}; - entry_range = {num_forward_entries, idx.num_node_entries()}; + node_range = {num_forward_nodes, indexed_graph.num_nodes()}; + entry_range = {num_forward_entries, indexed_graph.num_node_entries()}; ShapeVector shapes; - shapes.reserve(idx.num_node_entries()); + shapes.reserve(indexed_graph.num_node_entries()); bool contain_unknown = false; for (const auto& i : arrays) shapes.emplace_back(i->shape()); CheckAndInferShape(&graph, std::move(shapes), false, node_range, entry_range, &contain_unknown); DTypeVector dtypes; - dtypes.reserve(idx.num_node_entries()); + dtypes.reserve(indexed_graph.num_node_entries()); for (const auto& i : arrays) dtypes.emplace_back(i->dtype()); CheckAndInferType(&graph, std::move(dtypes), false, node_range, entry_range); StorageTypeVector stypes; - stypes.reserve(idx.num_node_entries()); + stypes.reserve(indexed_graph.num_node_entries()); for (const auto& i : arrays) stypes.emplace_back(i->storage_type()); exec::DevMaskVector dev_mask; - dev_mask.reserve(idx.num_nodes()); + dev_mask.reserve(indexed_graph.num_nodes()); for (const auto& i : vctx) dev_mask.emplace_back(i.dev_mask()); CheckAndInferStorageType(&graph, std::move(dev_mask), std::move(stypes), false, node_range, entry_range); } // Calculate ref count - for (size_t i = num_forward_nodes; i < idx.num_nodes(); ++i) { - for (const auto& j : idx[i].inputs) { - ++ref_count[idx.entry_id(j)]; + for (size_t i = num_forward_nodes; i < indexed_graph.num_nodes(); ++i) { + for (const auto& j : indexed_graph[i].inputs) { + ++ref_count[indexed_graph.entry_id(j)]; } } // Assign reqs std::vector array_reqs(arrays.size(), kWriteTo); - for (size_t i = num_forward_entries; i < idx.num_node_entries(); ++i) { + for (size_t i = num_forward_entries; i < indexed_graph.num_node_entries(); ++i) { if (ref_count[i] == 0) array_reqs[i] = kNullOp; } - for (size_t i = num_forward_outputs; i < idx.outputs().size(); ++i) { - size_t eid = idx.entry_id(idx.outputs()[i]); - array_reqs[eid] = gg.op_req_types[i - num_forward_outputs]; + for (size_t i = num_forward_outputs; i < indexed_graph.outputs().size(); ++i) { + size_t eid = indexed_graph.entry_id(indexed_graph.outputs()[i]); + array_reqs[eid] = gvar.op_req_types[i - num_forward_outputs]; } const auto& shapes = graph.GetAttr("shape"); @@ -518,10 +521,10 @@ std::vector Imperative::Backward( const auto& stypes = graph.GetAttr("storage_type"); const auto& dispatch_modes = graph.GetAttr("dispatch_mode"); - for (size_t i = num_forward_nodes; i < idx.num_nodes(); ++i) { - auto num_outputs = idx[i].source->num_outputs(); + for (size_t i = num_forward_nodes; i < indexed_graph.num_nodes(); ++i) { + auto num_outputs = indexed_graph[i].source->num_outputs(); for (size_t j = 0; j < num_outputs; ++j) { - auto eid = idx.entry_id(i, j); + auto eid = indexed_graph.entry_id(i, j); if (!arrays[eid]->is_none()) continue; if (stypes[eid] == kDefaultStorage) { *arrays[eid] = NDArray(shapes[eid], vctx[i], true, dtypes[eid]); @@ -539,7 +542,7 @@ std::vector Imperative::Backward( int prev_bulk_size = Engine::Get()->set_bulk_size(backward_bulk_size_); try { - RunGraph(retain_graph, idx, arrays, num_forward_nodes, idx.num_nodes(), + RunGraph(retain_graph, indexed_graph, arrays, num_forward_nodes, indexed_graph.num_nodes(), std::move(array_reqs), std::move(ref_count), &states, dispatch_modes, is_recording()); } catch (const dmlc::Error& e) { @@ -562,7 +565,7 @@ std::vector Imperative::Backward( } if (variables.size()) { - return gg.gradients; + return gvar.gradients; } return {}; } diff --git a/src/imperative/imperative_utils.h b/src/imperative/imperative_utils.h index f0f60ae9c5a2..0b04b73f61d0 100644 --- a/src/imperative/imperative_utils.h +++ b/src/imperative/imperative_utils.h @@ -981,7 +981,7 @@ inline void CreateEngineOpSeg( if (stop && nid > seg_start) { auto& seg = (*opr_segs)[seg_start]; if (seg_execs.size()) { - seg = EngineOprSeg{false, nid}; + seg = EngineOprSeg{false, nid, nullptr}; seg.opr.reset(CreateEngineOp(default_ctx, seg_execs)); } else { seg = EngineOprSeg{true, nid, nullptr}; @@ -998,7 +998,7 @@ inline void CreateEngineOpSeg( seg_execs.clear(); seg_start = nid + 1; } else if (is_async) { - seg = EngineOprSeg{false, nid + 1}; + seg = EngineOprSeg{false, nid + 1, nullptr}; seg.opr.reset(CreateEngineOp(default_ctx, seg_execs)); seg_execs.clear(); seg_start = nid + 1; @@ -1008,7 +1008,7 @@ inline void CreateEngineOpSeg( if (end_nid > seg_start) { auto& seg = (*opr_segs)[seg_start]; if (seg_execs.size()) { - seg = EngineOprSeg{false, end_nid}; + seg = EngineOprSeg{false, end_nid, nullptr}; seg.opr.reset(CreateEngineOp(default_ctx, seg_execs)); } else { seg = EngineOprSeg{true, end_nid, nullptr}; diff --git a/src/io/iter_batchloader.h b/src/io/iter_batchloader.h index 279690b594e6..a1f2f6c48044 100644 --- a/src/io/iter_batchloader.h +++ b/src/io/iter_batchloader.h @@ -126,7 +126,7 @@ class BatchLoader : public IIterator { } return false; } - virtual const TBlobBatch &Value(void) const { + virtual const TBlobBatch& Value(void) const { return out_; } diff --git a/src/io/iter_libsvm.cc b/src/io/iter_libsvm.cc index 3decc7b33e04..27bb546fe069 100644 --- a/src/io/iter_libsvm.cc +++ b/src/io/iter_libsvm.cc @@ -144,16 +144,16 @@ class LibSVMIter: public SparseIIterator { return true; } - virtual const DataInst &Value(void) const { + virtual const DataInst& Value(void) const { return out_; } - virtual const NDArrayStorageType GetStorageType(bool is_data) const { + virtual NDArrayStorageType GetStorageType(bool is_data) const { if (is_data) return kCSRStorage; return param_.label_shape.Size() > 1 ? kCSRStorage : kDefaultStorage; } - virtual const mxnet::TShape GetShape(bool is_data) const { + virtual mxnet::TShape GetShape(bool is_data) const { if (is_data) return param_.data_shape; return param_.label_shape; } diff --git a/src/io/iter_prefetcher.h b/src/io/iter_prefetcher.h index fdd1d2b91925..29844ec2f568 100644 --- a/src/io/iter_prefetcher.h +++ b/src/io/iter_prefetcher.h @@ -131,7 +131,7 @@ class PrefetcherIter : public IIterator { } return iter.Next(&out_); } - virtual const DataBatch &Value(void) const { + virtual const DataBatch& Value(void) const { return *out_; } diff --git a/src/io/iter_sparse.h b/src/io/iter_sparse.h index 22b1836be419..bb61990ef04f 100644 --- a/src/io/iter_sparse.h +++ b/src/io/iter_sparse.h @@ -36,9 +36,9 @@ template class SparseIIterator : public IIterator { public: /*! \brief storage type of the data or label */ - virtual const NDArrayStorageType GetStorageType(bool is_data) const = 0; + virtual NDArrayStorageType GetStorageType(bool is_data) const = 0; /*! \brief shape of the data or label */ - virtual const mxnet::TShape GetShape(bool is_data) const = 0; + virtual mxnet::TShape GetShape(bool is_data) const = 0; }; // class SparseIIterator } // namespace mxnet diff --git a/src/io/iter_sparse_batchloader.h b/src/io/iter_sparse_batchloader.h index c0d856df89ec..07bd9d4e2f2b 100644 --- a/src/io/iter_sparse_batchloader.h +++ b/src/io/iter_sparse_batchloader.h @@ -100,15 +100,15 @@ class SparseBatchLoader : public BatchLoader, public SparseIIterator return false; } - virtual const TBlobBatch &Value(void) const { + virtual const TBlobBatch& Value(void) const { return BatchLoader::Value(); } - virtual const NDArrayStorageType GetStorageType(bool is_data) const { + virtual NDArrayStorageType GetStorageType(bool is_data) const { return sparse_base_->GetStorageType(is_data); } - virtual const mxnet::TShape GetShape(bool is_data) const { + virtual mxnet::TShape GetShape(bool is_data) const { mxnet::TShape inst_shape = sparse_base_->GetShape(is_data); std::vector shape_vec; shape_vec.push_back(param_.batch_size); diff --git a/src/io/iter_sparse_prefetcher.h b/src/io/iter_sparse_prefetcher.h index 3f06052b0292..8bb28681902a 100644 --- a/src/io/iter_sparse_prefetcher.h +++ b/src/io/iter_sparse_prefetcher.h @@ -126,15 +126,15 @@ class SparsePrefetcherIter : public PrefetcherIter { virtual bool Next(void) { return PrefetcherIter::Next(); } - virtual const DataBatch &Value(void) const { + virtual const DataBatch& Value(void) const { return PrefetcherIter::Value(); } - virtual const NDArrayStorageType GetStorageType(bool is_data) const { + virtual NDArrayStorageType GetStorageType(bool is_data) const { return sparse_loader_->GetStorageType(is_data); } - virtual const mxnet::TShape GetShape(bool is_data) const { + virtual mxnet::TShape GetShape(bool is_data) const { return sparse_loader_->GetShape(is_data); } From 5a1041d09d2f798a0392b5d2249f344837b4877a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 13:41:36 -0700 Subject: [PATCH 073/146] Improve documentation for auxiliary backward functions --- include/mxnet/imperative.h | 17 ++++++++++++----- src/imperative/imperative.cc | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index f58d6370f0f9..46ee667b5750 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -172,15 +172,22 @@ class Imperative { */ static std::vector CreateForwardGraph(const std::vector& outputs); /*! Create gradient nodes using output shapes and ctx. - * Gradient heads are set to 1 if they are not present (nullptr) + * Gradient heads are initialized to 1 if they are not present (nullptr) * @return vector of nodes */ - static std::vector CreateHeadGradients(const std::vector& outputs, - const std::vector& ograds); + static std::vector CreateHeadGradientNodes(const std::vector& outputs, + const std::vector& ograds); class GradientVariableNodes; - GradientVariableNodes CreateGradientVariableNodes(const std::vector &variables, - const std::vector &outputs); + /*! Create variable nodes. + * If variables is provided, gradient nodes are crated for them. Otherwise it uses read only + * inputs reachable from the outputs. + * @param variables + * @param outputs + * @return aux data structure with nodes and arrays for gradients + */ + GradientVariableNodes CreateGradientVariableNodes(const std::vector& variables, + const std::vector& outputs); Imperative() { if (PreferBulkExecTrain()) backward_bulk_size_ = BulkExecMaxNodeTrainBwd(); diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index f6dd538d3e3f..c9dbcb163fd1 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -296,15 +296,15 @@ std::vector Imperative::CreateForwardGraph(const std::vector Imperative::CreateHeadGradients( - const std::vector& outputs, - const std::vector& ograds) { +std::vector Imperative::CreateHeadGradientNodes( + const std::vector &outputs, + const std::vector &ograds) { using nnvm::NodeEntry; using nnvm::Node; std::vector ograd_entries; ograd_entries.reserve(ograds.size()); for (size_t i = 0; i < outputs.size(); ++i) { - ograd_entries.emplace_back(NodeEntry{Node::Create(), 0, 0}); + ograd_entries.emplace_back(Node::Create()); AGInfo &info = AGInfo::Create(ograd_entries.back().node); info.ctx = outputs[i]->ctx(); if (ograds[i] != nullptr) { @@ -341,11 +341,11 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes(const var_nodes.op_req_types.push_back(kWriteTo); } } else { - std::vector ro_nodes = nnvm::Symbol::ListInputs(Symbol::kReadOnlyArgs, outputs); - var_nodes.variable_nodes.reserve(ro_nodes.size()); - var_nodes.gradients.reserve(ro_nodes.size()); - var_nodes.op_req_types.reserve(ro_nodes.size()); - for (const auto& node : ro_nodes) { + std::vector input_ro_nodes = nnvm::Symbol::ListInputs(Symbol::kReadOnlyArgs, outputs); + var_nodes.variable_nodes.reserve(input_ro_nodes.size()); + var_nodes.gradients.reserve(input_ro_nodes.size()); + var_nodes.op_req_types.reserve(input_ro_nodes.size()); + for (const auto& node : input_ro_nodes) { AGInfo& info = AGInfo::Get(node); if (info.grad_req != kNullOp) { var_nodes.variable_nodes.emplace_back(node); @@ -378,10 +378,10 @@ std::vector Imperative::Backward( graph.outputs = CreateForwardGraph(outputs); const size_t num_forward_outputs = graph.outputs.size(); - // Prepare head gradients - std::vector ograd_entries = CreateHeadGradients(outputs, ograds); + // Prepare head gradient nodes + std::vector ograd_entries = CreateHeadGradientNodes(outputs, ograds); - // Get gradient graph + // Get variable nodes GradientVariableNodes gvar = CreateGradientVariableNodes(variables, graph.outputs); // Run backward on the graph From dde42b9d068a83a517cce750edc63a517ef93672 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 14:33:36 -0700 Subject: [PATCH 074/146] gvar -> gvars --- src/imperative/imperative.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index c9dbcb163fd1..548cde35046b 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -382,23 +382,24 @@ std::vector Imperative::Backward( std::vector ograd_entries = CreateHeadGradientNodes(outputs, ograds); // Get variable nodes - GradientVariableNodes gvar = CreateGradientVariableNodes(variables, graph.outputs); + GradientVariableNodes gvars = CreateGradientVariableNodes(variables, graph.outputs); // Run backward on the graph Graph gradient_graph = pass::MXGradient( - graph, graph.outputs, gvar.variable_nodes, ograd_entries, + graph, graph.outputs, gvars.variable_nodes, ograd_entries, exec::AggregateGradient, nullptr, nullptr, zero_ops, "_copy"); - CHECK_EQ(gradient_graph.outputs.size(), gvar.variable_nodes.size()); + CHECK_EQ(gradient_graph.outputs.size(), gvars.variable_nodes.size()); // TODO: move inside pass::MXGradient - for (const auto& e : gradient_graph.outputs) { - if (e.node->op() == nullptr) { + // Add backward nodes to the graph outputs + for (const auto& backward_node : gradient_graph.outputs) { + if (backward_node.node->op() == nullptr) { auto node = Node::Create(); node->attrs.op = copy_op; - node->inputs.push_back(e); + node->inputs.push_back(backward_node); graph.outputs.emplace_back(std::move(node)); } else { - graph.outputs.push_back(e); + graph.outputs.push_back(backward_node); } } @@ -463,7 +464,7 @@ std::vector Imperative::Backward( } for (size_t i = num_forward_outputs; i < graph.outputs.size(); ++i) { size_t eid = indexed_graph.entry_id(graph.outputs[i]); - arrays[eid] = gvar.gradients[i - num_forward_outputs]; + arrays[eid] = gvars.gradients[i - num_forward_outputs]; ref_count[eid] = 1; } @@ -513,7 +514,7 @@ std::vector Imperative::Backward( } for (size_t i = num_forward_outputs; i < indexed_graph.outputs().size(); ++i) { size_t eid = indexed_graph.entry_id(indexed_graph.outputs()[i]); - array_reqs[eid] = gvar.op_req_types[i - num_forward_outputs]; + array_reqs[eid] = gvars.op_req_types[i - num_forward_outputs]; } const auto& shapes = graph.GetAttr("shape"); @@ -565,7 +566,7 @@ std::vector Imperative::Backward( } if (variables.size()) { - return gvar.gradients; + return gvars.gradients; } return {}; } From 2adbdfd8b1e0e19a56b363b28d268f251635697f Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 15:22:44 -0700 Subject: [PATCH 075/146] Fix lint --- src/imperative/imperative.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 548cde35046b..1496ce5df7f1 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -324,8 +324,9 @@ struct Imperative::GradientVariableNodes { std::vector op_req_types; }; -Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes(const std::vector &variables, - const std::vector &outputs) { +Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes( + const std::vector &variables, + const std::vector &outputs) { GradientVariableNodes var_nodes; if (!variables.empty()) { var_nodes.variable_nodes.reserve(variables.size()); @@ -341,7 +342,9 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes(const var_nodes.op_req_types.push_back(kWriteTo); } } else { - std::vector input_ro_nodes = nnvm::Symbol::ListInputs(Symbol::kReadOnlyArgs, outputs); + std::vector input_ro_nodes = nnvm::Symbol::ListInputs( + Symbol::kReadOnlyArgs, + outputs); var_nodes.variable_nodes.reserve(input_ro_nodes.size()); var_nodes.gradients.reserve(input_ro_nodes.size()); var_nodes.op_req_types.reserve(input_ro_nodes.size()); @@ -427,7 +430,7 @@ std::vector Imperative::Backward( states.resize(num_forward_nodes); nnvm::DFSVisit(graph.outputs, [&](const nnvm::NodePtr& n) { AGInfo& info = AGInfo::Get(n); - states[indexed_graph.node_id(n.get())] = info.state; + states.at(indexed_graph.node_id(n.get())) = info.state; for (uint32_t i = 0; i < info.outputs.size(); ++i) { CHECK(indexed_graph.exist(n.get())); size_t nid = indexed_graph.node_id(n.get()); From 0a419b05d8df715cdf43e6a80be0a552f692c485 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 30 Apr 2019 15:16:34 -0700 Subject: [PATCH 076/146] Fix bug --- src/executor/exec_pass.h | 1 + src/imperative/imperative.cc | 20 ++++++++++++-------- src/nnvm/gradient.cc | 7 ++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/executor/exec_pass.h b/src/executor/exec_pass.h index 7e5130f4921c..f6030f854352 100644 --- a/src/executor/exec_pass.h +++ b/src/executor/exec_pass.h @@ -270,6 +270,7 @@ inline Graph MXGradient( if (copy_op_str != std::string()) { graph.attrs["copy_op"] = std::make_shared(std::move(copy_op_str)); } + /// @sa nnvm::pass::Gradient in gradient.cc return ApplyPass(std::move(graph), "MXGradient"); } } // namespace pass diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 1496ce5df7f1..353308fd282b 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -376,10 +376,9 @@ std::vector Imperative::Backward( static const std::vector zero_ops{Op::Get("zeros_like"), Op::Get("_zeros")}; static const Op* copy_op = Op::Get("_copy"); - // Construct forward graph Graph graph; graph.outputs = CreateForwardGraph(outputs); - const size_t num_forward_outputs = graph.outputs.size(); + // Prepare head gradient nodes std::vector ograd_entries = CreateHeadGradientNodes(outputs, ograds); @@ -392,15 +391,20 @@ std::vector Imperative::Backward( graph, graph.outputs, gvars.variable_nodes, ograd_entries, exec::AggregateGradient, nullptr, nullptr, zero_ops, "_copy"); + CHECK_EQ(gradient_graph.outputs.size(), gvars.variable_nodes.size()); + std::vector forward_outputs = graph.outputs; + const size_t num_forward_outputs = graph.outputs.size(); + // TODO: move inside pass::MXGradient - // Add backward nodes to the graph outputs for (const auto& backward_node : gradient_graph.outputs) { - if (backward_node.node->op() == nullptr) { + if (backward_node.node->is_variable()) { auto node = Node::Create(); node->attrs.op = copy_op; node->inputs.push_back(backward_node); graph.outputs.emplace_back(std::move(node)); + //AGInfo& info = AGInfo::Create(node); + //info.ctx = outputs[0]->ctx(); } else { graph.outputs.push_back(backward_node); } @@ -423,12 +427,12 @@ std::vector Imperative::Backward( std::vector states; std::vector arrays; arrays.reserve(buff.size()); - for (auto& buffered_array : buff) { + for (auto& buffered_array : buff) arrays.push_back(&buffered_array); - } + if (create_graph) { states.resize(num_forward_nodes); - nnvm::DFSVisit(graph.outputs, [&](const nnvm::NodePtr& n) { + nnvm::DFSVisit(forward_outputs, [&](const nnvm::NodePtr& n) { AGInfo& info = AGInfo::Get(n); states.at(indexed_graph.node_id(n.get())) = info.state; for (uint32_t i = 0; i < info.outputs.size(); ++i) { @@ -562,7 +566,7 @@ std::vector Imperative::Backward( // Clear history if (!retain_graph) { - nnvm::DFSVisit(graph.outputs, [&](const nnvm::NodePtr& n) { + nnvm::DFSVisit(forward_outputs, [&](const nnvm::NodePtr& n) { AGInfo::Clear(n); n->inputs.clear(); }); diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index 586027129a0b..2cde0321e373 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -190,7 +190,8 @@ Graph Gradient(Graph src) { if (grad_fun_map.contains(ptr->op())) { input_grads = grad_fun_map[ptr->op()](fwd_node, out_agg_grads); CHECK_EQ((*rit)->inputs.size(), input_grads.size()) - << "Gradient function not returning enough gradient"; + << "Gradient function not returning enough gradient, there should be as many gradients" + "as inputs returned."; } else if (CheckGradAllZero(out_agg_grads, zero_ops)) { for (size_t i = 0; i < fwd_node->num_inputs(); ++i) { std::ostringstream os; @@ -249,14 +250,14 @@ Graph Gradient(Graph src) { NodePtr copy_node = Node::Create(); std::ostringstream os; os << entry.sum.node->attrs.name << "_" << kv->second.first << "_copy"; - kv->second.first++; + ++kv->second.first; copy_node->attrs.op = copy_op; copy_node->attrs.name = os.str(); copy_node->inputs.emplace_back(entry.sum); if (copy_node->attrs.op->attr_parser != nullptr) { copy_node->attrs.op->attr_parser(&(copy_node->attrs)); } - unique_grads.emplace(NodeEntry{std::move(copy_node), 0, 0}, std::make_pair(1, counter)); + unique_grads.emplace(NodeEntry(std::move(copy_node)), std::make_pair(1, counter)); } } else { ret.outputs[counter] = entry.sum; From 82de0b11caa48aa343783f9602c41b519f3f0183 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 30 Apr 2019 15:17:33 -0700 Subject: [PATCH 077/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 56514dda844c..d9d5d256787f 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 56514dda844cbde28bf5727fb85c15470752d0d2 +Subproject commit d9d5d256787f4a0614042c6a343cf14db3cc4105 From ab1c6bcafc875ce04124e1671294bd5d1e1d508c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 30 Apr 2019 16:23:52 -0700 Subject: [PATCH 078/146] Fix lint --- src/imperative/imperative.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 353308fd282b..293ae0b48de4 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -396,15 +396,15 @@ std::vector Imperative::Backward( std::vector forward_outputs = graph.outputs; const size_t num_forward_outputs = graph.outputs.size(); - // TODO: move inside pass::MXGradient + // TODO(larroy): move inside pass::MXGradient for (const auto& backward_node : gradient_graph.outputs) { if (backward_node.node->is_variable()) { auto node = Node::Create(); node->attrs.op = copy_op; node->inputs.push_back(backward_node); graph.outputs.emplace_back(std::move(node)); - //AGInfo& info = AGInfo::Create(node); - //info.ctx = outputs[0]->ctx(); + // AGInfo& info = AGInfo::Create(node); + // info.ctx = outputs[0]->ctx(); } else { graph.outputs.push_back(backward_node); } @@ -418,7 +418,8 @@ std::vector Imperative::Backward( num_forward_nodes = std::max( num_forward_nodes, static_cast(indexed_graph.outputs()[i].node_id + 1)); num_forward_entries = std::max( - num_forward_entries, static_cast(indexed_graph.entry_id(indexed_graph.outputs()[i])) + 1); + num_forward_entries, static_cast(indexed_graph.entry_id( + indexed_graph.outputs()[i])) + 1); } // Allocate buffer From 696b25f717a5608a4c6dcb776fae4a60afabfc99 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 30 Apr 2019 17:13:13 -0700 Subject: [PATCH 079/146] remove -Wextra --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d5e128035489..a56d0b1c121f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,7 +141,7 @@ else(MSVC) add_definitions(-DMSHADOW_USE_F16C=0) endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wno-unused-parameter -Wno-unused-local-typedefs -Wno-implicit-fallthrough -Wno-unknown-pragmas -Wno-sign-compare -Werror=return-type") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wno-unknown-pragmas -Wno-sign-compare -Werror=return-type") if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-braced-scalar-init") endif() From 8d935f29930aea928c6d276f166eba2ef108312e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 30 Apr 2019 17:13:33 -0700 Subject: [PATCH 080/146] Add op name to graph nodes --- src/imperative/imperative.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 293ae0b48de4..f752f526a687 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -223,7 +223,7 @@ void Imperative::RecordOp( nnvm::NodePtr node = nnvm::Node::Create(); node->attrs = std::move(attrs); - node->attrs.name = "node_" + std::to_string(node_count_++); + node->attrs.name = "node_" + std::to_string(node_count_++) + "_" + node->attrs.op->name; AGInfo& info = AGInfo::Create(node); info.state = state; info.ctx = outputs[0]->ctx(); From b1c268b16b9139364126781218878a5036be9c65 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 5 Apr 2019 11:21:01 -0700 Subject: [PATCH 081/146] Fix build_ccache_wrappers: * Fix broken links * Make it idempotent fixes https://github.com/apache/incubator-mxnet/pull/13456 fixes https://github.com/apache/incubator-mxnet/issues/14117 fixes https://github.com/apache/incubator-mxnet/issues/11516 --- ci/docker/runtime_functions.sh | 49 ++++++++++++++-------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 091ffdf2551d..b9010a19e12c 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -66,37 +66,29 @@ build_ccache_wrappers() { # But in the beginning, we'll make this opt-in. In future, loads of processes like # the scala make step or numpy compilation and other pip package generations # could be heavily sped up by using ccache as well. - mkdir /tmp/ccache-redirects + mkdir -p /tmp/ccache-redirects export PATH=/tmp/ccache-redirects:$PATH - ln -s ccache /tmp/ccache-redirects/gcc - ln -s ccache /tmp/ccache-redirects/gcc-8 - ln -s ccache /tmp/ccache-redirects/g++ - ln -s ccache /tmp/ccache-redirects/g++-8 - ln -s ccache /tmp/ccache-redirects/nvcc - ln -s ccache /tmp/ccache-redirects/clang++-3.9 - ln -s ccache /tmp/ccache-redirects/clang-3.9 - ln -s ccache /tmp/ccache-redirects/clang++-5.0 - ln -s ccache /tmp/ccache-redirects/clang-5.0 - ln -s ccache /tmp/ccache-redirects/clang++-6.0 - ln -s ccache /tmp/ccache-redirects/clang-6.0 - ln -s ccache /usr/local/bin/gcc - ln -s ccache /usr/local/bin/gcc-8 - ln -s ccache /usr/local/bin/g++ - ln -s ccache /usr/local/bin/g++-8 - ln -s ccache /usr/local/bin/nvcc - ln -s ccache /usr/local/bin/clang++-3.9 - ln -s ccache /usr/local/bin/clang-3.9 - ln -s ccache /usr/local/bin/clang++-5.0 - ln -s ccache /usr/local/bin/clang-5.0 - ln -s ccache /usr/local/bin/clang++-6.0 - ln -s ccache /usr/local/bin/clang-6.0 - - export NVCC=ccache + CCACHE=`which ccache` + ln -sf $CCACHE /tmp/ccache-redirects/gcc + ln -sf $CCACHE /tmp/ccache-redirects/gcc-8 + ln -sf $CCACHE /tmp/ccache-redirects/g++ + ln -sf $CCACHE /tmp/ccache-redirects/g++-8 + ln -sf $CCACHE /tmp/ccache-redirects/nvcc + ln -sf $CCACHE /tmp/ccache-redirects/clang++-3.9 + ln -sf $CCACHE /tmp/ccache-redirects/clang-3.9 + ln -sf $CCACHE /tmp/ccache-redirects/clang++-5.0 + ln -sf $CCACHE /tmp/ccache-redirects/clang-5.0 + ln -sf $CCACHE /tmp/ccache-redirects/clang++-6.0 + ln -sf $CCACHE /tmp/ccache-redirects/clang-6.0 + # Doesn't work: https://github.com/ccache/ccache/issues/373 + #ln -sf $CCACHE /tmp/ccache-redirects/nvcc + #export NVCC="/tmp/ccache-redirects/nvcc" # Uncomment if you would like to debug CCache hit rates. # You can monitor using tail -f ccache-log - # export CCACHE_LOGFILE=/work/mxnet/ccache-log - # export CCACHE_DEBUG=1 + #export CCACHE_LOGFILE=/work/mxnet/ccache-log + #export CCACHE_LOGFILE=/tmp/ccache-log + #export CCACHE_DEBUG=1 } build_wheel() { @@ -667,8 +659,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ubuntu_gpu_cuda100_cudnn7() { set -ex - # unfortunately this build has problems in 3rdparty dependencies with ccache and make - # build_ccache_wrappers + build_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ From 73266328a473bd947d9415e246083f42dea94c1b Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 8 Feb 2019 13:55:02 +0100 Subject: [PATCH 082/146] Optimize move semantics of NodeEntry https://github.com/dmlc/tvm/pull/2576 Making copies of shared_ptr is very expensive, 100x more than moving. This PR reduces lock contention by using move semantics in NNVM nodes --- .gitmodules | 2 +- 3rdparty/tvm | 2 +- src/c_api/c_api_function.cc | 2 +- src/executor/graph_executor.cc | 8 ++++---- src/imperative/cached_op.cc | 4 ++-- src/imperative/imperative.cc | 2 +- src/nnvm/legacy_op_util.cc | 23 +++++++++++------------ src/operator/custom/custom.cc | 2 +- src/operator/nn/lrn.cc | 2 +- src/operator/operator_common.h | 2 +- src/operator/tensor/elemwise_sum.cc | 6 +++--- 11 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.gitmodules b/.gitmodules index e0ffec11bfd0..151b9fe39a86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/dmlc/tvm + url = https://github.com/larroy/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git diff --git a/3rdparty/tvm b/3rdparty/tvm index 0f053c82a747..e6df3d2fc099 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 +Subproject commit e6df3d2fc099291411308e983fa7edd2df8a669a diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc index 50f9b32d6e47..3adc581b50a5 100644 --- a/src/c_api/c_api_function.cc +++ b/src/c_api/c_api_function.cc @@ -56,7 +56,7 @@ std::vector Gradient( std::vector ret; for (uint32_t i = 0; i < g->num_outputs(); ++i) { - ret.emplace_back(nnvm::NodeEntry{g, i, 0}); + ret.emplace_back(std::move(g), i, 0); } return ret; diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index e726d29765ac..c94709da1748 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -157,7 +157,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { ng->attrs.op = Op::Get("_zeros_without_dtype"); ng->attrs.name = "zeros_without_dtype"; ng->attrs.op->attr_parser(&(ng->attrs)); - return nnvm::NodeEntry{ng, 0, 0}; + return nnvm::NodeEntry(std::move(ng), 0, 0); } // remove zero in the sum. at least keep 1. @@ -178,7 +178,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { sum_node->attrs.dict["num_args"] = std::to_string(v.size()); sum_node->attrs.op->attr_parser(&(sum_node->attrs)); sum_node->inputs = std::move(v); - return nnvm::NodeEntry{sum_node, 0, 0}; + return nnvm::NodeEntry(std::move(sum_node), 0, 0); } else { // use a stream line of plus instead nnvm::NodeEntry ret = v[0]; @@ -208,7 +208,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { x->attrs.op = ewise_plus_op; x->attrs.name = os.str(); x->inputs = {ret, v[i]}; - ret = nnvm::NodeEntry{x, 0, 0}; + ret = nnvm::NodeEntry(std::move(x), 0, 0); } // identity node is used to avoid exposure of dummy plus node // when its output get assigned to another space. @@ -257,7 +257,7 @@ nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, } if (!need_grad_) return g; for (size_t i = 0; i < g.outputs.size(); ++i) { - NodeEntry ngrad{nnvm::Node::Create(), 0, 0}; + NodeEntry ngrad; head_grad_entry_.emplace_back(AttrHint(ngrad, g.outputs[i])); head_grad_map_[ngrad.node.get()] = i; } diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 7a5ed21432d3..1601d0e0494f 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -160,7 +160,7 @@ CachedOp::CachedOp( { ograd_entries_.reserve(fwd_graph_.outputs.size()); for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { - ograd_entries_.emplace_back(NodeEntry{Node::Create(), 0, 0}); + ograd_entries_.emplace_back(); } std::vector xs; @@ -169,7 +169,7 @@ CachedOp::CachedOp( auto nid = idx.input_nodes()[i]; if (idx.mutable_input_nodes().count(nid)) continue; fwd_input_to_grad_output_[i] = xs.size(); - xs.emplace_back(NodeEntry{idx[nid].weak_ref.lock(), 0, 0}); + xs.emplace_back(idx[nid].weak_ref.lock(), 0, 0); } CHECK_GT(xs.size(), 0) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index b027de0a0f6f..1597eee9ca26 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -305,7 +305,7 @@ std::vector Imperative::Backward( std::vector ograd_entries; ograd_entries.reserve(ograds.size()); for (size_t i = 0; i < outputs.size(); ++i) { - ograd_entries.emplace_back(NodeEntry{Node::Create(), 0, 0}); + ograd_entries.emplace_back(); AGInfo& info = AGInfo::Create(ograd_entries.back().node); info.ctx = outputs[i]->ctx(); if (ograds[i] != nullptr) { diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 16ad0053e29a..e738a5a47695 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -321,9 +321,10 @@ inline std::vector OpPropGradient( const NodePtr& ptr, const std::vector& out_grads) { auto& prop = nnvm::get(ptr->attrs.parsed); - std::vector out_data(prop.outputs.size()); - for (uint32_t i = 0; i < out_data.size(); ++i) { - out_data[i] = NodeEntry{ptr, i, 0}; + std::vector out_data; + out_data.reserve(prop.outputs.size()); + for (size_t i = 0; i < out_data.size(); ++i) { + out_data.emplace_back(ptr, i, 0); } std::vector in_data( ptr->inputs.begin(), ptr->inputs.begin() + prop.arguments.size()); @@ -331,7 +332,7 @@ inline std::vector OpPropGradient( out_grads.begin(), out_grads.begin() + prop.ptr->NumVisibleOutputs()); auto inputs = prop.ptr->BackwardInputs(ograd, in_data, out_data); // add all the auxiliary data - for (uint32_t i = 0; i < prop.aux_states.size(); ++i) { + for (size_t i = 0; i < prop.aux_states.size(); ++i) { inputs.emplace_back(ptr->inputs[i + prop.arguments.size()]); } NodePtr gnode = Node::Create(); @@ -340,17 +341,15 @@ inline std::vector OpPropGradient( gnode->attrs = ptr->attrs; gnode->attrs.op = back_op; gnode->attrs.name = ptr->attrs.name + "_backward"; - std::vector in_grad(prop.arguments.size()); - for (uint32_t i = 0; i < prop.arguments.size(); ++i) { - in_grad[i] = NodeEntry{gnode, i, 0}; + std::vector in_grad; + in_grad.reserve(prop.arguments.size() + prop.aux_states.size()); + for (size_t i = 0; i < prop.arguments.size(); ++i) { + in_grad.emplace_back(gnode, i, 0); } // attach no gradient node to forbid gradient on aux_state if (prop.aux_states.size() != 0) { - NodePtr ng = Node::Create(); - ng->attrs.op = Op::Get("_NoGradient"); - ng->attrs.name = "NoGradient"; - for (uint32_t i = 0; i < prop.aux_states.size(); ++i) { - in_grad.emplace_back(NodeEntry{ng, 0, 0}); + for (size_t i = 0; i < prop.aux_states.size(); ++i) { + in_grad.emplace_back(Node::Create(Op::Get("_NoGradient"), "NoGradient"), 0, 0); } } return in_grad; diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 412bfa1bc3aa..544fdc3a8dce 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -224,7 +224,7 @@ std::vector Gradient( size_t i = static_cast(t); if (i >= params.num_outs + params.num_args) { uint32_t idx = static_cast(i-params.num_outs-params.num_args); - g->inputs.push_back(nnvm::NodeEntry{n, idx, 0}); + g->inputs.push_back(n, idx, 0); } else if (i >= params.num_outs) { g->inputs.push_back(n->inputs[i-params.num_outs]); } else { diff --git a/src/operator/nn/lrn.cc b/src/operator/nn/lrn.cc index b632e35b57fe..3a3ca59f2be1 100644 --- a/src/operator/nn/lrn.cc +++ b/src/operator/nn/lrn.cc @@ -77,7 +77,7 @@ struct LRNGrad { std::vector heads; heads.push_back(ograds[0]); // out_grad heads.push_back(n->inputs[lrn_enum::kData]); - heads.emplace_back(nnvm::NodeEntry{n, lrn_enum::kTmpNorm, 0}); + heads.emplace_back(n, lrn_enum::kTmpNorm, 0); return MakeGradNode(op_name, n, heads, n->attrs.dict); } }; diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 59f572211d0e..0328dfa7de26 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -447,7 +447,7 @@ inline std::vector MakeNonlossGradNode( p->inputs.insert(p->inputs.end(), inputs.begin(), inputs.end()); std::vector ret; for (uint32_t i = 0; i < p->num_outputs(); ++i) { - ret.emplace_back(nnvm::NodeEntry{p, i, 0}); + ret.emplace_back(std::move(p), i, 0); } return ret; } diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index f1ec8b5ad387..dec57633be22 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -49,12 +49,12 @@ std::vector ElementWiseSumGrad( nnvm::Op::Get("identity"); CHECK_EQ(ograds.size(), 1); std::vector ret; - nnvm::NodeEntry n_out{n, 0, 0}; - for (size_t i = 0; i < n->inputs.size(); i++) { + nnvm::NodeEntry n_out(n, 0, 0); + for (size_t i = 0; i < n->inputs.size(); ++i) { nnvm::NodePtr id_node = nnvm::Node::Create(); id_node->attrs.op = copy_op; id_node->inputs = {ograds[0]}; - ret.push_back(nnvm::NodeEntry{id_node, 0, 0}); + ret.emplace_back(id_node, 0, 0); } return ret; } From d81b79b3bab794d79ace0d05b8b69e8df97497d0 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 8 Feb 2019 15:35:20 +0100 Subject: [PATCH 083/146] Compile fix --- src/operator/custom/custom.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 544fdc3a8dce..63b42007317b 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -224,7 +224,7 @@ std::vector Gradient( size_t i = static_cast(t); if (i >= params.num_outs + params.num_args) { uint32_t idx = static_cast(i-params.num_outs-params.num_args); - g->inputs.push_back(n, idx, 0); + g->inputs.emplace_back(n, idx, 0); } else if (i >= params.num_outs) { g->inputs.push_back(n->inputs[i-params.num_outs]); } else { From 937f1a7f6a769b08c70b6552aa50934e2ae3d881 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sun, 10 Feb 2019 22:58:21 +0100 Subject: [PATCH 084/146] Fix --- src/operator/operator_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 0328dfa7de26..732203f0e601 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -447,7 +447,7 @@ inline std::vector MakeNonlossGradNode( p->inputs.insert(p->inputs.end(), inputs.begin(), inputs.end()); std::vector ret; for (uint32_t i = 0; i < p->num_outputs(); ++i) { - ret.emplace_back(std::move(p), i, 0); + ret.emplace_back(p, i, 0); } return ret; } From edf6ecd3da64e2d954b829f533f701c56d2547f7 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 11 Feb 2019 23:25:41 +0100 Subject: [PATCH 085/146] Fix crash, uninitialized Node ptr --- src/executor/graph_executor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index c94709da1748..5e5cb4251491 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -257,7 +257,7 @@ nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, } if (!need_grad_) return g; for (size_t i = 0; i < g.outputs.size(); ++i) { - NodeEntry ngrad; + NodeEntry ngrad(nnvm::Node::Create(), 0, 0); head_grad_entry_.emplace_back(AttrHint(ngrad, g.outputs[i])); head_grad_map_[ngrad.node.get()] = i; } From f72c29c69dc7b6bd1d694e90dad4dfc09e607e3c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 12 Feb 2019 00:09:53 +0100 Subject: [PATCH 086/146] Fix --- src/c_api/c_api_function.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc index 3adc581b50a5..3cd70379b68f 100644 --- a/src/c_api/c_api_function.cc +++ b/src/c_api/c_api_function.cc @@ -56,7 +56,7 @@ std::vector Gradient( std::vector ret; for (uint32_t i = 0; i < g->num_outputs(); ++i) { - ret.emplace_back(std::move(g), i, 0); + ret.emplace_back(g, i, 0); } return ret; From 66171ed39c690de85002075a6b8aca5d6026954c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 14 Mar 2019 18:43:53 -0700 Subject: [PATCH 087/146] Fix autograd crash --- src/imperative/imperative.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 1597eee9ca26..b027de0a0f6f 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -305,7 +305,7 @@ std::vector Imperative::Backward( std::vector ograd_entries; ograd_entries.reserve(ograds.size()); for (size_t i = 0; i < outputs.size(); ++i) { - ograd_entries.emplace_back(); + ograd_entries.emplace_back(NodeEntry{Node::Create(), 0, 0}); AGInfo& info = AGInfo::Create(ograd_entries.back().node); info.ctx = outputs[i]->ctx(); if (ograds[i] != nullptr) { From 18bef6657b554139acba8c7e6d85e4723d0b60c0 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 22:42:54 +0000 Subject: [PATCH 088/146] Fix clang tidy errors --- src/imperative/cached_op.cc | 2 +- src/imperative/imperative.cc | 2 +- src/operator/tensor/broadcast_reduce_op_value.cc | 4 ++-- src/operator/tensor/elemwise_unary_op_basic.cc | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 1601d0e0494f..90e8ee9d7c6b 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -118,7 +118,7 @@ CachedOp::CachedOp( if (_copy->attr_parser != nullptr) { _copy->attr_parser(&(copy_node->attrs)); } - fwd_graph_.outputs.push_back(NodeEntry{copy_node, 0, 0}); + fwd_graph_.outputs.emplace_back(copy_node, 0, 0); } else { dedup_out.insert({i, 0}); fwd_graph_.outputs.push_back(i); diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index b027de0a0f6f..a1c41ee0df6b 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -363,7 +363,7 @@ std::vector Imperative::Backward( auto node = Node::Create(); node->attrs.op = copy_op; node->inputs.push_back(e); - graph.outputs.push_back(NodeEntry{node, 0, 0}); + graph.outputs.emplace_back(node, 0, 0); } else { graph.outputs.push_back(e); } diff --git a/src/operator/tensor/broadcast_reduce_op_value.cc b/src/operator/tensor/broadcast_reduce_op_value.cc index f890963c2cf1..3a598dbcf374 100644 --- a/src/operator/tensor/broadcast_reduce_op_value.cc +++ b/src/operator/tensor/broadcast_reduce_op_value.cc @@ -287,11 +287,11 @@ NNVM_REGISTER_OP(broadcast_like) [](const nnvm::NodePtr& n, const std::vector& ograds) { if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); - auto lhs = MakeNonlossGradNode("_broadcast_backward", n, ograds, {}, + std::vector lhs = MakeNonlossGradNode("_broadcast_backward", n, ograds, {}, {{"keepdims", "true"}}); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); + lhs.emplace_back(ng, 0, 0); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index 5114a5d0dbe3..f9d79ebf893e 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -356,11 +356,11 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) "FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); - auto lhs = MakeGradNode("_backward_copy", n, ograds, + std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); + lhs.emplace_back(ng, 0, 0); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") @@ -495,11 +495,11 @@ Negative indices are supported, and `None` can be used for either `lhs_end` or ` "FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); - auto lhs = MakeGradNode("_backward_copy", n, ograds, + std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); + lhs.emplace_back(ng, 0, 0); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") From d941f0fe21d4a73754c93a18f2a582e0738fa7aa Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 22:44:30 +0000 Subject: [PATCH 089/146] Restore subrepo --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 151b9fe39a86..e0ffec11bfd0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/larroy/tvm + url = https://github.com/dmlc/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git From 1b13551e19dd28024b8af07a0cde6acc62d8bf21 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 23:48:30 +0000 Subject: [PATCH 090/146] restore tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index e6df3d2fc099..51785062553b 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit e6df3d2fc099291411308e983fa7edd2df8a669a +Subproject commit 51785062553b19a219b2639cd230b7da7455cd0d From e4b152e728d94ed90854c64b72d2a29b6147819d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 23:49:59 +0000 Subject: [PATCH 091/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 51785062553b..0f053c82a747 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 51785062553b19a219b2639cd230b7da7455cd0d +Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 From 6f57297465a8a865375dace8768a2c489a3b8869 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 18:21:43 -0700 Subject: [PATCH 092/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 0f053c82a747..2da23bd86905 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 +Subproject commit 2da23bd86905c3abec857a28b6135d069ea9515d From bc95ea4f01c4814696c83800001ffe558a99a381 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 10 Apr 2019 18:22:17 -0700 Subject: [PATCH 093/146] Update NDarray with NodeEntry constructors and refine initializer lists --- include/mxnet/ndarray.h | 52 +++++++++++++++++++------------ src/operator/elemwise_op_common.h | 4 +-- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 05d3fa45683e..c015044baa15 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -94,8 +94,11 @@ class NDArray { NDArray(const mxnet::TShape &shape, Context ctx, bool delay_alloc = false, int dtype = mshadow::default_type_flag) : ptr_(std::make_shared(shape, ctx, delay_alloc, dtype)), - shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_({nullptr, 0, 0}) { + shape_(shape), + dtype_(dtype), + storage_type_(kDefaultStorage), + entry_() + { } /*! \brief constructor for NDArray with storage type */ @@ -109,11 +112,13 @@ class NDArray { * \param ctx context of NDArray * \param dtype data type of this ndarray */ - explicit NDArray(Context ctx, int dtype = mshadow::default_type_flag) { - ptr_ = std::make_shared(mxnet::TShape(mshadow::Shape1(0)), ctx, true, dtype); - dtype_ = dtype; - storage_type_ = kDefaultStorage; - entry_ = {nullptr, 0, 0}; + explicit NDArray(Context ctx, int dtype = mshadow::default_type_flag) + : ptr_(std::make_shared(mxnet::TShape(mshadow::Shape1(0)), ctx, true, dtype)), + shape_(), + dtype_(dtype), + storage_type_(kDefaultStorage), + entry_() + { } /*! * \brief constructing a static NDArray that shares data with TBlob @@ -123,9 +128,11 @@ class NDArray { * \param dev_id the device id this tensor sits at */ NDArray(const TBlob &data, int dev_id) - : ptr_(std::make_shared(data, dev_id)), shape_(data.shape_), - dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_({nullptr, 0, 0}) { + : ptr_(std::make_shared(data, dev_id)), + shape_(data.shape_), + dtype_(data.type_flag_), + storage_type_(kDefaultStorage), + entry_() { } /*! @@ -137,20 +144,22 @@ class NDArray { * \param deleter the function pointer of custom deleter */ NDArray(const TBlob &data, int dev_id, const std::function& deleter) - : ptr_(new Chunk(data, dev_id), - [deleter](Chunk *p) { - deleter(); // call custom deleter - delete p; // delete Chunk object + : ptr_(new Chunk(data, dev_id), [deleter](Chunk *p) { + deleter(); // call custom deleter + delete p; // delete Chunk object }), shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_({nullptr, 0, 0}) { + entry_() { } /*! \brief create ndarray from shared memory */ NDArray(int shared_pid, int shared_id, const mxnet::TShape& shape, int dtype) - : ptr_(std::make_shared(shared_pid, shared_id, shape, dtype)), shape_(shape), - dtype_(dtype), storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { + : ptr_(std::make_shared(shared_pid, shared_id, shape, dtype)), + shape_(shape), + dtype_(dtype), + storage_type_(kDefaultStorage), + entry_() { } /*! @@ -165,8 +174,11 @@ class NDArray { */ NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, const TBlob &data, const std::vector &aux_data, int dev_id) - : ptr_(std::make_shared(stype, data, aux_data, dev_id)), shape_(shape), - dtype_(data.type_flag_), storage_type_(stype), entry_({nullptr, 0, 0}) { + : ptr_(std::make_shared(stype, data, aux_data, dev_id)), + shape_(shape), + dtype_(data.type_flag_), + storage_type_(stype), + entry_() { } /*! * \brief initialize the NDArray, assuming it is not assigned a meaningful shape before @@ -640,7 +652,7 @@ class NDArray { */ NDArray Detach() const { NDArray ret(*this); - ret.entry_ = nnvm::NodeEntry{nullptr, 0, 0}; + ret.entry_ = nnvm::NodeEntry(); return ret; } diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index 2edaa55540c1..6dae2dfa20c4 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -203,7 +203,7 @@ struct ElemwiseGradUseOut { std::vector heads; uint32_t n_out = n->num_outputs(); for (uint32_t i = 0; i < n_out; ++i) { - heads.emplace_back(nnvm::NodeEntry{n, i, 0}); + heads.emplace_back(n, i, 0); } return MakeNonlossGradNode(op_name, n, ograds, heads, n->attrs.dict); } @@ -220,7 +220,7 @@ struct ElemwiseGradUseInOut { } uint32_t n_out = n->num_outputs(); for (uint32_t i = 0; i < n_out; ++i) { - heads.emplace_back(nnvm::NodeEntry{n, i, 0}); + heads.emplace_back(n, i, 0); } return MakeGradNode(op_name, n, heads, n->attrs.dict); } From d026a0efda23b085cc08992a9184d0a0bfc3c31d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 11 Apr 2019 20:26:34 +0000 Subject: [PATCH 094/146] Fix lint --- include/mxnet/ndarray.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index c015044baa15..bb7f2b2f5b6d 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -97,8 +97,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() - { + entry_() { } /*! \brief constructor for NDArray with storage type */ @@ -117,8 +116,7 @@ class NDArray { shape_(), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() - { + entry_() { } /*! * \brief constructing a static NDArray that shares data with TBlob From b6fab773773a8ce5e66a8049465faa5140c275b7 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 12:54:22 -0700 Subject: [PATCH 095/146] fix --- src/imperative/cached_op.cc | 42 ++++++++++++++++++------------------- src/ndarray/ndarray.cc | 2 +- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 90e8ee9d7c6b..8f01fa134b6f 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -108,20 +108,20 @@ CachedOp::CachedOp( // construct forward graph { NodeEntryMap dedup_out; - for (const auto& i : sym.outputs) { - if (dedup_out.count(i)) { + for (const NodeEntry& nodeEntry : sym.outputs) { + if (dedup_out.count(nodeEntry)) { NodePtr copy_node = Node::Create(); copy_node->attrs.op = _copy; copy_node->attrs.name = - i.node->attrs.name + "_copy" + std::to_string(dedup_out[i]++); - copy_node->inputs.emplace_back(i); + nodeEntry.node->attrs.name + "_copy" + std::to_string(dedup_out[nodeEntry]++); + copy_node->inputs.emplace_back(nodeEntry); if (_copy->attr_parser != nullptr) { _copy->attr_parser(&(copy_node->attrs)); } fwd_graph_.outputs.emplace_back(copy_node, 0, 0); } else { - dedup_out.insert({i, 0}); - fwd_graph_.outputs.push_back(i); + dedup_out.emplace(nodeEntry); + fwd_graph_.outputs.push_back(nodeEntry); } } const auto& idx = fwd_graph_.indexed_graph(); @@ -143,14 +143,15 @@ CachedOp::CachedOp( // Set params { - const auto& idx = fwd_graph_.indexed_graph(); + const auto& indexed_graph = fwd_graph_.indexed_graph(); if (config_.data_indices.ndim() || config_.param_indices.ndim()) { CHECK_EQ(config_.data_indices.ndim() + config_.param_indices.ndim(), - idx.input_nodes().size()); + indexed_graph.input_nodes().size()); } else { std::vector tmp; - for (size_t i = 0; i < idx.input_nodes().size(); ++i) { - tmp.push_back(i); + tmp.reserve(indexed_graph.input_nodes().size()); + for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) { + tmp.emplace_back(i); } config_.data_indices.assign(tmp.begin(), tmp.end()); } @@ -158,21 +159,18 @@ CachedOp::CachedOp( // construct backward graph { - ograd_entries_.reserve(fwd_graph_.outputs.size()); - for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { - ograd_entries_.emplace_back(); - } - + ograd_entries_.resize(fwd_graph_.outputs.size()); std::vector xs; - const auto& idx = fwd_graph_.indexed_graph(); - for (size_t i = 0; i < idx.input_nodes().size(); ++i) { - auto nid = idx.input_nodes()[i]; - if (idx.mutable_input_nodes().count(nid)) continue; - fwd_input_to_grad_output_[i] = xs.size(); - xs.emplace_back(idx[nid].weak_ref.lock(), 0, 0); + const IndexedGraph& indexed_graph = fwd_graph_.indexed_graph(); + for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) { + const uint32_t node_id = indexed_graph.input_nodes()[i]; + if (indexed_graph.mutable_input_nodes().count(node_id)) + continue; + fwd_input_to_grad_output_.at(i) = xs.size(); + xs.emplace_back(std::move(indexed_graph[node_id].weak_ref.lock())); } - CHECK_GT(xs.size(), 0) + CHECK(!xs.empty()) << "There are no inputs in computation graph that require gradients."; grad_graph_ = pass::MXGradient( diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 0bfca8c10a1a..c143a5a947d7 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -53,7 +53,7 @@ namespace mxnet { NDArray::NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc, int dtype, std::vector aux_types, mxnet::ShapeVector aux_shapes, mxnet::TShape storage_shape) : shape_(shape), - dtype_(dtype), storage_type_(stype), entry_({nullptr, 0, 0}) { + dtype_(dtype), storage_type_(stype), entry_(nullptr, 0, 0) { // Assign default aux types if not given if (aux_types.size() == 0 && stype != kDefaultStorage) { From 91b99e8c33c300969bcde218911b2ef1236de0fb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 12:54:46 -0700 Subject: [PATCH 096/146] Revert "Restore subrepo" This reverts commit 3e6f56e5c055616b2983825d6254cac7d061167e. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e0ffec11bfd0..151b9fe39a86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/dmlc/tvm + url = https://github.com/larroy/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git From 69c6349366668df459761af7d97a29af052eaf55 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 12:55:32 -0700 Subject: [PATCH 097/146] Update tvm to my repo --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 2da23bd86905..9a2cf6cd9ce2 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 2da23bd86905c3abec857a28b6135d069ea9515d +Subproject commit 9a2cf6cd9ce2b8f5cebd1273042aea1111a19578 From 8fc23fe2c883842fed1acdc91bb69f5996f5839f Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 14:55:00 -0700 Subject: [PATCH 098/146] Fix --- include/mxnet/ndarray.h | 12 ++++++------ src/executor/graph_executor.cc | 1 + src/executor/infer_graph_attr_pass.cc | 3 ++- src/imperative/cached_op.cc | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index bb7f2b2f5b6d..13c83f2282a3 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -97,7 +97,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! \brief constructor for NDArray with storage type */ @@ -116,7 +116,7 @@ class NDArray { shape_(), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! * \brief constructing a static NDArray that shares data with TBlob @@ -130,7 +130,7 @@ class NDArray { shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! @@ -148,7 +148,7 @@ class NDArray { }), shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! \brief create ndarray from shared memory */ @@ -157,7 +157,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_() { + entry_(nullptr) { } /*! @@ -176,7 +176,7 @@ class NDArray { shape_(shape), dtype_(data.type_flag_), storage_type_(stype), - entry_() { + entry_(nullptr) { } /*! * \brief initialize the NDArray, assuming it is not assigned a meaningful shape before diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 5e5cb4251491..a1a61dcd6d18 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -162,6 +162,7 @@ nnvm::NodeEntry AggregateGradient(std::vector&& v) { // remove zero in the sum. at least keep 1. auto begin = std::remove_if(v.begin(), v.end(), [](const nnvm::NodeEntry& nodeEntry) { + CHECK(nodeEntry.node); return nodeEntry.node->op() == zeros_op || nodeEntry.node->op() == zeros_like_op; }); if (begin == v.begin()) ++begin; diff --git a/src/executor/infer_graph_attr_pass.cc b/src/executor/infer_graph_attr_pass.cc index fa7aee518486..fa5fa04f918a 100644 --- a/src/executor/infer_graph_attr_pass.cc +++ b/src/executor/infer_graph_attr_pass.cc @@ -628,7 +628,8 @@ nnvm::Graph InferShapeAttr(nnvm::Graph &&ret, } if (dispatch_mode_name) { for (size_t i = node_start; i < node_end; i++) { - if (dispatch_modes[i] == DispatchMode::kUndefined) ++num_unknown; + if (dispatch_modes[i] == DispatchMode::kUndefined) + ++num_unknown; } } ++i; diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 8f01fa134b6f..0189328b0097 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -120,7 +120,7 @@ CachedOp::CachedOp( } fwd_graph_.outputs.emplace_back(copy_node, 0, 0); } else { - dedup_out.emplace(nodeEntry); + dedup_out.emplace(nodeEntry, 0); fwd_graph_.outputs.push_back(nodeEntry); } } @@ -166,7 +166,7 @@ CachedOp::CachedOp( const uint32_t node_id = indexed_graph.input_nodes()[i]; if (indexed_graph.mutable_input_nodes().count(node_id)) continue; - fwd_input_to_grad_output_.at(i) = xs.size(); + fwd_input_to_grad_output_[i] = xs.size(); xs.emplace_back(std::move(indexed_graph[node_id].weak_ref.lock())); } From cdc08962e93714e25c9c4582c970ceea240158e0 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 13:39:46 -0700 Subject: [PATCH 099/146] Fix --- src/imperative/cached_op.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 0189328b0097..22d19fbe9c55 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -98,7 +98,7 @@ CachedOp::CachedOp( using namespace nnvm; using namespace imperative; static const std::vector zero_ops{Op::Get("zeros_like"), Op::Get("_zeros")}; - static const auto _copy = Op::Get("_copy"); + static const auto _copy_op = Op::Get("_copy"); config_.Init(flags); if (config_.static_shape) { @@ -107,18 +107,18 @@ CachedOp::CachedOp( // construct forward graph { - NodeEntryMap dedup_out; + NodeEntryMap dedup_out; for (const NodeEntry& nodeEntry : sym.outputs) { - if (dedup_out.count(nodeEntry)) { + if (dedup_out.find(nodeEntry) != dedup_out.end()) { NodePtr copy_node = Node::Create(); - copy_node->attrs.op = _copy; + copy_node->attrs.op = _copy_op; copy_node->attrs.name = nodeEntry.node->attrs.name + "_copy" + std::to_string(dedup_out[nodeEntry]++); copy_node->inputs.emplace_back(nodeEntry); - if (_copy->attr_parser != nullptr) { - _copy->attr_parser(&(copy_node->attrs)); + if (_copy_op->attr_parser != nullptr) { + _copy_op->attr_parser(&(copy_node->attrs)); } - fwd_graph_.outputs.emplace_back(copy_node, 0, 0); + fwd_graph_.outputs.emplace_back(std::move(copy_node)); } else { dedup_out.emplace(nodeEntry, 0); fwd_graph_.outputs.push_back(nodeEntry); @@ -197,7 +197,7 @@ CachedOp::CachedOp( } auto full_ref_count = fwd_graph_.GetAttr >("forward_ref_count"); - for (size_t i = 0; i < num_forward_entries; ++i) full_ref_count[i] += ref_count[i]; + for (size_t i = 0; i < num_forward_entries; ++i) full_ref_count.at(i) += ref_count[i]; fwd_graph_.attrs["full_ref_count"] = std::make_shared(std::move(full_ref_count)); From b7261126a020654d0e435eaccf56b1e19bd80953 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 13:45:26 -0700 Subject: [PATCH 100/146] Fix --- src/ndarray/ndarray.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index c143a5a947d7..79ac3e981ba0 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -171,7 +171,7 @@ nnvm::Symbol NDArray::get_autograd_symbol() const { #if MXNET_USE_MKLDNN == 1 NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) - : storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { + : storage_type_(kDefaultStorage), entry_(nullptr) { auto mem_desc = mem_pd.desc(); shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); dtype_ = get_mxnet_type(mem_desc.data.data_type); @@ -181,7 +181,7 @@ NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) } NDArray::NDArray(const std::shared_ptr &mkldnn_mem) - : storage_type_(kDefaultStorage), entry_({nullptr, 0, 0}) { + : storage_type_(kDefaultStorage), entry_(nullptr) { auto mem_pd = mkldnn_mem->get_primitive_desc(); auto mem_desc = mem_pd.desc(); shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); From d6bee4f673ad1fe72ba8f26b8827eae95471a11c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 14:30:05 -0700 Subject: [PATCH 101/146] Fix --- dev_menu.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index d439d8194f2a..6f1701fac5f3 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -99,15 +99,15 @@ def create_virtualenv(venv_exe, pyexe, venv) -> None: logging.warn("Skipping creation of virtualenv") return check_call([venv_exe, '-p', pyexe, venv]) - activate_this_py = os.path.join(venv, 'bin', 'activate_this.py') - # Activate virtualenv in this interpreter - exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) - check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) - check_call(['pip', 'install', '-r', 'tests/requirements.txt']) + # TODO: Activate virtualenv in this interpreter, this is not working as it is. + #activate_this_py = os.path.join(venv, 'bin', 'activate_this.py') + #exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) + #check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) + #check_call(['pip', 'install', '-r', 'tests/requirements.txt']) def create_virtualenv_default(): create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV) - logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV) + logging.info("You can use the virtualenv by executing 'source %s/bin/activate && pip install -e python'", DEFAULT_PYENV) COMMANDS = OrderedDict([ ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV), From deb9e2d3181ff5422e10ac939b88abb65e0c084e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 17:28:24 -0700 Subject: [PATCH 102/146] minor --- src/ndarray/ndarray.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 79ac3e981ba0..a8a34eaccac0 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -53,7 +53,7 @@ namespace mxnet { NDArray::NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc, int dtype, std::vector aux_types, mxnet::ShapeVector aux_shapes, mxnet::TShape storage_shape) : shape_(shape), - dtype_(dtype), storage_type_(stype), entry_(nullptr, 0, 0) { + dtype_(dtype), storage_type_(stype), entry_(nullptr) { // Assign default aux types if not given if (aux_types.size() == 0 && stype != kDefaultStorage) { From cdce4bfeef691d0e87c695698d6919990003e390 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 12 Apr 2019 18:49:19 -0700 Subject: [PATCH 103/146] Fixes --- include/mxnet/ndarray.h | 6 ++++-- src/imperative/cached_op.cc | 7 ++++++- src/imperative/imperative.cc | 2 +- src/nnvm/legacy_op_util.cc | 2 +- src/operator/custom/custom.cc | 4 ++-- src/operator/tensor/elemwise_sum.cc | 9 ++++----- src/operator/tensor/elemwise_unary_op_basic.cc | 10 ++++------ 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 13c83f2282a3..fecc26851b8c 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -82,7 +82,9 @@ class MKLDNNMemory; class NDArray { public: /*! \brief default constructor */ - NDArray() { + NDArray() + : entry_(nullptr) + { } /*! * \brief constructs a new dynamic NDArray @@ -650,7 +652,7 @@ class NDArray { */ NDArray Detach() const { NDArray ret(*this); - ret.entry_ = nnvm::NodeEntry(); + ret.entry_ = nnvm::NodeEntry(nullptr); return ret; } diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 22d19fbe9c55..acdfdd8fb6df 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -159,7 +159,12 @@ CachedOp::CachedOp( // construct backward graph { - ograd_entries_.resize(fwd_graph_.outputs.size()); + //ograd_entries_.resize(fwd_graph_.outputs.size()); + ograd_entries_.reserve(fwd_graph_.outputs.size()); + for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { + ograd_entries_.emplace_back(Node::Create()); + } + std::vector xs; const IndexedGraph& indexed_graph = fwd_graph_.indexed_graph(); for (size_t i = 0; i < indexed_graph.input_nodes().size(); ++i) { diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index a1c41ee0df6b..35c5172effc7 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -363,7 +363,7 @@ std::vector Imperative::Backward( auto node = Node::Create(); node->attrs.op = copy_op; node->inputs.push_back(e); - graph.outputs.emplace_back(node, 0, 0); + graph.outputs.emplace_back(std::move(node)); } else { graph.outputs.push_back(e); } diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index e738a5a47695..2c4bcec3d7ca 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -344,7 +344,7 @@ inline std::vector OpPropGradient( std::vector in_grad; in_grad.reserve(prop.arguments.size() + prop.aux_states.size()); for (size_t i = 0; i < prop.arguments.size(); ++i) { - in_grad.emplace_back(gnode, i, 0); + in_grad.emplace_back(std::move(gnode), i, 0); } // attach no gradient node to forbid gradient on aux_state if (prop.aux_states.size() != 0) { diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 63b42007317b..f6def8a08538 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -238,14 +238,14 @@ std::vector Gradient( std::vector ret; for (size_t i = 0; i < params.num_args; ++i) { - ret.emplace_back(nnvm::NodeEntry{g, static_cast(i), 0}); + ret.emplace_back(std::move(g), static_cast(i), 0); } if (params.num_auxs) { nnvm::NodePtr ng = nnvm::Node::Create(); ng->attrs.op = nnvm::Op::Get("_NoGradient"); ng->attrs.name = "NoGradient"; for (size_t i = 0; i < params.num_auxs; ++i) { - ret.emplace_back(nnvm::NodeEntry{ng, 0, 0}); + ret.emplace_back(std::move(ng), 0, 0); } } diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index dec57633be22..2ba4e309f3ac 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -49,12 +49,11 @@ std::vector ElementWiseSumGrad( nnvm::Op::Get("identity"); CHECK_EQ(ograds.size(), 1); std::vector ret; - nnvm::NodeEntry n_out(n, 0, 0); for (size_t i = 0; i < n->inputs.size(); ++i) { - nnvm::NodePtr id_node = nnvm::Node::Create(); - id_node->attrs.op = copy_op; - id_node->inputs = {ograds[0]}; - ret.emplace_back(id_node, 0, 0); + nnvm::NodePtr node = nnvm::Node::Create(); + node->attrs.op = copy_op; + node->inputs = {ograds[0]}; + ret.emplace_back(std::move(node), 0, 0); } return ret; } diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index f9d79ebf893e..58d441f18f83 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -358,9 +358,8 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); - auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", - {n->inputs[1]}, nullptr, &n); - lhs.emplace_back(ng, 0, 0); + lhs.emplace_back(MakeNode("zeros_like", n->attrs.name + "_rhs_backward", + {n->inputs[1]}, nullptr, &n)); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") @@ -497,9 +496,8 @@ Negative indices are supported, and `None` can be used for either `lhs_end` or ` if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); std::vector lhs = MakeGradNode("_backward_copy", n, ograds, std::unordered_map()); - auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", - {n->inputs[1]}, nullptr, &n); - lhs.emplace_back(ng, 0, 0); + lhs.emplace_back(MakeNode("zeros_like", n->attrs.name + "_rhs_backward", + {n->inputs[1]}, nullptr, &n)); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") From 9fb0b6874866df42778dfff8dd00600c55337377 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 13 Apr 2019 02:03:55 +0000 Subject: [PATCH 104/146] update tvm and dmlc-core --- 3rdparty/dmlc-core | 2 +- 3rdparty/tvm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 3ffea8694adf..82bf4c2e2af3 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 3ffea8694adf9c0363f9abbf162dc0e4a45b22c5 +Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 diff --git a/3rdparty/tvm b/3rdparty/tvm index 9a2cf6cd9ce2..3e244d496689 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 9a2cf6cd9ce2b8f5cebd1273042aea1111a19578 +Subproject commit 3e244d496689b8a96dfb7fea30b5242492aacd8f From 2d6c4d0d5f9df5b30e6a03087a88b4f7097aa733 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 13 Apr 2019 02:10:38 +0000 Subject: [PATCH 105/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 3e244d496689..b947674c79cb 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 3e244d496689b8a96dfb7fea30b5242492aacd8f +Subproject commit b947674c79cbc8bbe56712d1b48205cb3e43a4a2 From 0412d0744c01c5a3915e41f7a587c3c9aeb5e04d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 11 Apr 2019 21:55:01 +0000 Subject: [PATCH 106/146] Revert "update dmlc-core" This reverts commit fc51adb75348ad33c2ac685985940b5bd3f1ee99. --- 3rdparty/dmlc-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 82bf4c2e2af3..3ffea8694adf 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 +Subproject commit 3ffea8694adf9c0363f9abbf162dc0e4a45b22c5 From a9ab18ae0e9235495ef66719d36f782896b72043 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 15 Apr 2019 23:01:23 +0000 Subject: [PATCH 107/146] Update dmlc-core and tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index b947674c79cb..f4ca10c23195 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit b947674c79cbc8bbe56712d1b48205cb3e43a4a2 +Subproject commit f4ca10c231953f13a8cef82d9855e5d9bf043b9f From 75397cb36863188ffeea958c66e5028b35b3e641 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 15:40:15 -0700 Subject: [PATCH 108/146] Improve batch_norm with NodeEntry refactorings --- src/operator/nn/batch_norm.cc | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 622952cc4bc5..08d8839d507c 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -484,19 +484,20 @@ static inline bool BatchNormStorageType(const nnvm::NodeAttrs &attrs, std::vector BatchNormGrad(const nnvm::NodePtr& n, const std::vector& ograds) { std::vector out_data(n->num_outputs()); - for (uint32_t i = 0; i < out_data.size(); ++i) { - out_data[i] = nnvm::NodeEntry{n, i, 0}; + out_data.reserve(n->num_outputs()); + for (size_t i = 0; i < out_data.size(); ++i) { + out_data.emplace_back(n, i, 0); } std::vector heads; heads.reserve(8); - heads.push_back(ograds[0]); - heads.push_back(out_data[batchnorm::kMean]); - heads.push_back(out_data[batchnorm::kVar]); - heads.push_back(n->inputs[batchnorm::kData]); - heads.push_back(n->inputs[batchnorm::kGamma]); - heads.push_back(n->inputs[batchnorm::kBeta]); - heads.push_back(n->inputs[batchnorm::kInMovingMean]); - heads.push_back(n->inputs[batchnorm::kInMovingVar]); + heads.emplace_back(ograds[0]); + heads.emplace_back(out_data[batchnorm::kMean]); + heads.emplace_back(out_data[batchnorm::kVar]); + heads.emplace_back(n->inputs[batchnorm::kData]); + heads.emplace_back(n->inputs[batchnorm::kGamma]); + heads.emplace_back(n->inputs[batchnorm::kBeta]); + heads.emplace_back(n->inputs[batchnorm::kInMovingMean]); + heads.emplace_back(n->inputs[batchnorm::kInMovingVar]); nnvm::NodePtr gnode = nnvm::Node::Create(); gnode->inputs = std::move(heads); @@ -505,19 +506,17 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, gnode->attrs.op = nnvm::Op::Get("_backward_BatchNorm"); gnode->attrs.name = n->attrs.name + "_backward"; // The input of batchnorm - std::vector in_grad(5); - for (uint32_t i = 0; i < 3; ++i) { - in_grad[i] = nnvm::NodeEntry{gnode, i, 0}; - } - + std::vector in_grad; + in_grad.reserve(5); + for (size_t i = 0; i < 3; ++i) + in_grad.emplace_back(gnode, i, 0); // attach no gradient node to forbid gradient on aux_state nnvm::NodePtr ng = nnvm::Node::Create(); ng->attrs.op = Op::Get("_NoGradient"); ng->attrs.name = "NoGradient"; // the aux state of batchnorm - for (uint32_t i = 0; i < 2; ++i) { - in_grad[i + 3] = nnvm::NodeEntry{ng, 0, 0}; - } + for (size_t i = 3; i < 5; ++i) + in_grad.emplace_back(ng, 0, 0); return in_grad; } From 15cc946713bebe79ddc0069ef7bb77a22bbb8ecb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 15:40:44 -0700 Subject: [PATCH 109/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index f4ca10c23195..1acf86bbd080 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit f4ca10c231953f13a8cef82d9855e5d9bf043b9f +Subproject commit 1acf86bbd0809ebfe736ed045345719c24b7acb0 From 90604bbdd3699bcec3b534a21eb89509f54e635c Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 16:18:37 -0700 Subject: [PATCH 110/146] Fix bug introduced in batch_norm --- src/operator/nn/batch_norm.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 08d8839d507c..e5a68ebe227b 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -485,9 +485,8 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, const std::vector& ograds) { std::vector out_data(n->num_outputs()); out_data.reserve(n->num_outputs()); - for (size_t i = 0; i < out_data.size(); ++i) { + for (size_t i = 0; i < n->num_outputs(); ++i) out_data.emplace_back(n, i, 0); - } std::vector heads; heads.reserve(8); heads.emplace_back(ograds[0]); From 9428db97d41c751ff82fbd0578a66d2693ba40cb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:08:51 +0000 Subject: [PATCH 111/146] lint --- include/mxnet/ndarray.h | 3 +-- src/imperative/cached_op.cc | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index fecc26851b8c..0a3339b4a8a9 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -83,8 +83,7 @@ class NDArray { public: /*! \brief default constructor */ NDArray() - : entry_(nullptr) - { + : entry_(nullptr) { } /*! * \brief constructs a new dynamic NDArray diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index acdfdd8fb6df..16ba20d77338 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -159,11 +159,9 @@ CachedOp::CachedOp( // construct backward graph { - //ograd_entries_.resize(fwd_graph_.outputs.size()); ograd_entries_.reserve(fwd_graph_.outputs.size()); - for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) { + for (size_t i = 0; i < fwd_graph_.outputs.size(); ++i) ograd_entries_.emplace_back(Node::Create()); - } std::vector xs; const IndexedGraph& indexed_graph = fwd_graph_.indexed_graph(); From 674149a6fd538cd40020df4ec85b782cf183904d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:25:22 +0000 Subject: [PATCH 112/146] Add std::move --- src/operator/tensor/broadcast_reduce_op_value.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/operator/tensor/broadcast_reduce_op_value.cc b/src/operator/tensor/broadcast_reduce_op_value.cc index 3a598dbcf374..612d1f926650 100644 --- a/src/operator/tensor/broadcast_reduce_op_value.cc +++ b/src/operator/tensor/broadcast_reduce_op_value.cc @@ -286,12 +286,13 @@ NNVM_REGISTER_OP(broadcast_like) .set_attr("FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { - if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); + if (CheckGradAllZero(ograds)) + return MakeZeroGradNodes(n, ograds); std::vector lhs = MakeNonlossGradNode("_broadcast_backward", n, ograds, {}, - {{"keepdims", "true"}}); + {{"keepdims", "true"}}); auto ng = MakeNode("zeros_like", n->attrs.name + "_rhs_backward", {n->inputs[1]}, nullptr, &n); - lhs.emplace_back(ng, 0, 0); + lhs.emplace_back(std::move(ng)); return lhs; }) .add_argument("lhs", "NDArray-or-Symbol", "First input.") From d56d26edf44ba552bdad11fe7c1b317e2d7d7d14 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 16 Apr 2019 23:35:48 -0700 Subject: [PATCH 113/146] Fix bugs --- src/nnvm/legacy_op_util.cc | 4 ++-- src/operator/nn/batch_norm.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 2c4bcec3d7ca..25cc46ef7309 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -323,9 +323,9 @@ inline std::vector OpPropGradient( auto& prop = nnvm::get(ptr->attrs.parsed); std::vector out_data; out_data.reserve(prop.outputs.size()); - for (size_t i = 0; i < out_data.size(); ++i) { + for (size_t i = 0; i < prop.outputs.size(); ++i) out_data.emplace_back(ptr, i, 0); - } + std::vector in_data( ptr->inputs.begin(), ptr->inputs.begin() + prop.arguments.size()); std::vector ograd( diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index e5a68ebe227b..1e1fbb9e7abe 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -483,7 +483,7 @@ static inline bool BatchNormStorageType(const nnvm::NodeAttrs &attrs, std::vector BatchNormGrad(const nnvm::NodePtr& n, const std::vector& ograds) { - std::vector out_data(n->num_outputs()); + std::vector out_data; out_data.reserve(n->num_outputs()); for (size_t i = 0; i < n->num_outputs(); ++i) out_data.emplace_back(n, i, 0); From f0d1812db11d5bcfb19a23ab4ad9aa40fdf0237d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:18:42 -0700 Subject: [PATCH 114/146] Fix bug introduced in batch_norm --- src/operator/nn/batch_norm.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 1e1fbb9e7abe..1ade322f732f 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -489,14 +489,14 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, out_data.emplace_back(n, i, 0); std::vector heads; heads.reserve(8); - heads.emplace_back(ograds[0]); - heads.emplace_back(out_data[batchnorm::kMean]); - heads.emplace_back(out_data[batchnorm::kVar]); - heads.emplace_back(n->inputs[batchnorm::kData]); - heads.emplace_back(n->inputs[batchnorm::kGamma]); - heads.emplace_back(n->inputs[batchnorm::kBeta]); - heads.emplace_back(n->inputs[batchnorm::kInMovingMean]); - heads.emplace_back(n->inputs[batchnorm::kInMovingVar]); + heads.push_back(ograds[0]); + heads.push_back(out_data[batchnorm::kMean]); + heads.push_back(out_data[batchnorm::kVar]); + heads.push_back(n->inputs[batchnorm::kData]); + heads.push_back(n->inputs[batchnorm::kGamma]); + heads.push_back(n->inputs[batchnorm::kBeta]); + heads.push_back(n->inputs[batchnorm::kInMovingMean]); + heads.push_back(n->inputs[batchnorm::kInMovingVar]); nnvm::NodePtr gnode = nnvm::Node::Create(); gnode->inputs = std::move(heads); From 7dfb1db223747638466ccb40379c3901f11e06aa Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:26:53 -0700 Subject: [PATCH 115/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 1acf86bbd080..3073acda0f85 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 1acf86bbd0809ebfe736ed045345719c24b7acb0 +Subproject commit 3073acda0f854f320362138fd9165133cce64733 From 992f54a09cac4d8f560ee29e6560adb273fd2ea6 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:28:23 -0700 Subject: [PATCH 116/146] Update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 3073acda0f85..2db7717a0515 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 3073acda0f854f320362138fd9165133cce64733 +Subproject commit 2db7717a0515ae63c75ecb8aa841410d38d1f1d4 From dac9a33b7edbbd44806cb71c1c2e6f0e472ff7e1 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 00:29:20 -0700 Subject: [PATCH 117/146] Sync gradient.cc with tvm --- src/nnvm/gradient.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index 4927191a5964..c51704f82077 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -212,6 +212,8 @@ Graph Gradient(Graph src) { LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " << "because it didn't register FGradient attribute."; } + for(const auto& nodeEntry: input_grads) + CHECK(nodeEntry.node); auto git = input_grads.begin(); for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { auto& ge = output_grads[it->node.get()][it->index]; From f7c2b9ccc3d6ddab75a224845323926b7df23ff7 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 07:49:09 +0000 Subject: [PATCH 118/146] Fix bug --- src/nnvm/legacy_op_util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 25cc46ef7309..698666f94d90 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -344,7 +344,7 @@ inline std::vector OpPropGradient( std::vector in_grad; in_grad.reserve(prop.arguments.size() + prop.aux_states.size()); for (size_t i = 0; i < prop.arguments.size(); ++i) { - in_grad.emplace_back(std::move(gnode), i, 0); + in_grad.emplace_back(gnode, i, 0); } // attach no gradient node to forbid gradient on aux_state if (prop.aux_states.size() != 0) { From 6d4eae49ff60d05473af9f7eb816f6f516b2ace7 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 07:52:40 +0000 Subject: [PATCH 119/146] Use at --- dev_menu.py | 4 ++-- src/operator/nn/batch_norm.cc | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index 6f1701fac5f3..5a80ef0b59e4 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -113,7 +113,7 @@ def create_virtualenv_default(): ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV), [ CMake(), - create_virtualenv_default, + #create_virtualenv_default, ]), ('[Local] Python Unit tests', "./py3_venv/bin/nosetests -v tests/python/unittest/" @@ -209,7 +209,7 @@ def build(args) -> None: else: cmake = CMake() cmake() - create_virtualenv(venv_exe, pyexe, args.venv) + #create_virtualenv(venv_exe, pyexe, args.venv) def main(): logging.getLogger().setLevel(logging.INFO) diff --git a/src/operator/nn/batch_norm.cc b/src/operator/nn/batch_norm.cc index 1ade322f732f..dbb0f10561b9 100644 --- a/src/operator/nn/batch_norm.cc +++ b/src/operator/nn/batch_norm.cc @@ -489,14 +489,14 @@ std::vector BatchNormGrad(const nnvm::NodePtr& n, out_data.emplace_back(n, i, 0); std::vector heads; heads.reserve(8); - heads.push_back(ograds[0]); - heads.push_back(out_data[batchnorm::kMean]); - heads.push_back(out_data[batchnorm::kVar]); - heads.push_back(n->inputs[batchnorm::kData]); - heads.push_back(n->inputs[batchnorm::kGamma]); - heads.push_back(n->inputs[batchnorm::kBeta]); - heads.push_back(n->inputs[batchnorm::kInMovingMean]); - heads.push_back(n->inputs[batchnorm::kInMovingVar]); + heads.emplace_back(ograds.at(0)); + heads.emplace_back(out_data.at(batchnorm::kMean)); + heads.emplace_back(out_data.at(batchnorm::kVar)); + heads.emplace_back(n->inputs.at(batchnorm::kData)); + heads.emplace_back(n->inputs.at(batchnorm::kGamma)); + heads.emplace_back(n->inputs.at(batchnorm::kBeta)); + heads.emplace_back(n->inputs.at(batchnorm::kInMovingMean)); + heads.emplace_back(n->inputs[batchnorm::kInMovingVar]); nnvm::NodePtr gnode = nnvm::Node::Create(); gnode->inputs = std::move(heads); From 325591e3022c1395204ca9f5630dfd3f242c6e3e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:03:27 -0700 Subject: [PATCH 120/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 2db7717a0515..5922ccb588e5 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 2db7717a0515ae63c75ecb8aa841410d38d1f1d4 +Subproject commit 5922ccb588e5a9c09e51833c960489dc30df1db6 From 7c15e68be35dd6505647bd10bc12fcdc34bdcc35 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 08:29:04 +0000 Subject: [PATCH 121/146] lint --- src/nnvm/gradient.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index c51704f82077..badc3f9e2b85 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -212,7 +212,7 @@ Graph Gradient(Graph src) { LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " << "because it didn't register FGradient attribute."; } - for(const auto& nodeEntry: input_grads) + for (const auto& nodeEntry : input_grads) CHECK(nodeEntry.node); auto git = input_grads.begin(); for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { From 8b328fe5a21b91b84452c2de554c7b64bf8ea8de Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 08:41:52 +0000 Subject: [PATCH 122/146] Update subrepos --- 3rdparty/dmlc-core | 2 +- 3rdparty/tvm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 3ffea8694adf..82bf4c2e2af3 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 3ffea8694adf9c0363f9abbf162dc0e4a45b22c5 +Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 diff --git a/3rdparty/tvm b/3rdparty/tvm index 5922ccb588e5..824f236773e5 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 5922ccb588e5a9c09e51833c960489dc30df1db6 +Subproject commit 824f236773e503e86bd4218bd28dccf1ee970f99 From b661b0d638c58bdba4e48996d7e5d3946e018191 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 17 Apr 2019 01:46:17 -0700 Subject: [PATCH 123/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 824f236773e5..568235b75ce3 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 824f236773e503e86bd4218bd28dccf1ee970f99 +Subproject commit 568235b75ce38acf6d6660e4f1a75692fb0258cb From 982044cb909b1a6dba620b3cc2cf892042db17fa Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 18 Apr 2019 17:21:39 -0700 Subject: [PATCH 124/146] Fix moves leaving nodes uninitialized --- src/imperative/cached_op.cc | 2 +- src/operator/custom/custom.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imperative/cached_op.cc b/src/imperative/cached_op.cc index 16ba20d77338..47d4ac42006b 100644 --- a/src/imperative/cached_op.cc +++ b/src/imperative/cached_op.cc @@ -170,7 +170,7 @@ CachedOp::CachedOp( if (indexed_graph.mutable_input_nodes().count(node_id)) continue; fwd_input_to_grad_output_[i] = xs.size(); - xs.emplace_back(std::move(indexed_graph[node_id].weak_ref.lock())); + xs.emplace_back(indexed_graph[node_id].weak_ref.lock()); } CHECK(!xs.empty()) diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index f6def8a08538..77fe2e6e4b1c 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -238,14 +238,14 @@ std::vector Gradient( std::vector ret; for (size_t i = 0; i < params.num_args; ++i) { - ret.emplace_back(std::move(g), static_cast(i), 0); + ret.emplace_back(g, static_cast(i), 0); } if (params.num_auxs) { nnvm::NodePtr ng = nnvm::Node::Create(); ng->attrs.op = nnvm::Op::Get("_NoGradient"); ng->attrs.name = "NoGradient"; for (size_t i = 0; i < params.num_auxs; ++i) { - ret.emplace_back(std::move(ng), 0, 0); + ret.emplace_back(ng, 0, 0); } } From abc857bd73a14a90be4f80d3a6baf78523481286 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 18 Apr 2019 17:48:19 -0700 Subject: [PATCH 125/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index 568235b75ce3..cb7e33ca44ce 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 568235b75ce38acf6d6660e4f1a75692fb0258cb +Subproject commit cb7e33ca44cec59c896358ee69f0e5c26e5709f5 From 6016e2ca4d9c89d09f13f761c54292a79def7965 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 19 Apr 2019 12:09:20 -0700 Subject: [PATCH 126/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index cb7e33ca44ce..e031b39c5e1d 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit cb7e33ca44cec59c896358ee69f0e5c26e5709f5 +Subproject commit e031b39c5e1d4468b7db96f1db39dbb97e4beef8 From 25d9fac4004098a7d90634f5a4d4628b825e3d6e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 23:34:21 +0000 Subject: [PATCH 127/146] restore gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 151b9fe39a86..e0ffec11bfd0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/larroy/tvm + url = https://github.com/dmlc/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git From 3380e098d9a80e0439d02a113026bebe41ec53b9 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Apr 2019 23:39:19 +0000 Subject: [PATCH 128/146] restore tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index e031b39c5e1d..eed28f581553 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit e031b39c5e1d4468b7db96f1db39dbb97e4beef8 +Subproject commit eed28f581553525646a4312d377edd87fe827f42 From bea00f5bee473bc06b18fcedeedc37db6e71421a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 25 Apr 2019 00:03:37 +0000 Subject: [PATCH 129/146] Revert "Fix build_ccache_wrappers:" This reverts commit 381ff13753d12f71dc5f29355de9766bb9a6cc18. --- ci/docker/runtime_functions.sh | 49 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index b9010a19e12c..091ffdf2551d 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -66,29 +66,37 @@ build_ccache_wrappers() { # But in the beginning, we'll make this opt-in. In future, loads of processes like # the scala make step or numpy compilation and other pip package generations # could be heavily sped up by using ccache as well. - mkdir -p /tmp/ccache-redirects + mkdir /tmp/ccache-redirects export PATH=/tmp/ccache-redirects:$PATH - CCACHE=`which ccache` - ln -sf $CCACHE /tmp/ccache-redirects/gcc - ln -sf $CCACHE /tmp/ccache-redirects/gcc-8 - ln -sf $CCACHE /tmp/ccache-redirects/g++ - ln -sf $CCACHE /tmp/ccache-redirects/g++-8 - ln -sf $CCACHE /tmp/ccache-redirects/nvcc - ln -sf $CCACHE /tmp/ccache-redirects/clang++-3.9 - ln -sf $CCACHE /tmp/ccache-redirects/clang-3.9 - ln -sf $CCACHE /tmp/ccache-redirects/clang++-5.0 - ln -sf $CCACHE /tmp/ccache-redirects/clang-5.0 - ln -sf $CCACHE /tmp/ccache-redirects/clang++-6.0 - ln -sf $CCACHE /tmp/ccache-redirects/clang-6.0 - # Doesn't work: https://github.com/ccache/ccache/issues/373 - #ln -sf $CCACHE /tmp/ccache-redirects/nvcc - #export NVCC="/tmp/ccache-redirects/nvcc" + ln -s ccache /tmp/ccache-redirects/gcc + ln -s ccache /tmp/ccache-redirects/gcc-8 + ln -s ccache /tmp/ccache-redirects/g++ + ln -s ccache /tmp/ccache-redirects/g++-8 + ln -s ccache /tmp/ccache-redirects/nvcc + ln -s ccache /tmp/ccache-redirects/clang++-3.9 + ln -s ccache /tmp/ccache-redirects/clang-3.9 + ln -s ccache /tmp/ccache-redirects/clang++-5.0 + ln -s ccache /tmp/ccache-redirects/clang-5.0 + ln -s ccache /tmp/ccache-redirects/clang++-6.0 + ln -s ccache /tmp/ccache-redirects/clang-6.0 + ln -s ccache /usr/local/bin/gcc + ln -s ccache /usr/local/bin/gcc-8 + ln -s ccache /usr/local/bin/g++ + ln -s ccache /usr/local/bin/g++-8 + ln -s ccache /usr/local/bin/nvcc + ln -s ccache /usr/local/bin/clang++-3.9 + ln -s ccache /usr/local/bin/clang-3.9 + ln -s ccache /usr/local/bin/clang++-5.0 + ln -s ccache /usr/local/bin/clang-5.0 + ln -s ccache /usr/local/bin/clang++-6.0 + ln -s ccache /usr/local/bin/clang-6.0 + + export NVCC=ccache # Uncomment if you would like to debug CCache hit rates. # You can monitor using tail -f ccache-log - #export CCACHE_LOGFILE=/work/mxnet/ccache-log - #export CCACHE_LOGFILE=/tmp/ccache-log - #export CCACHE_DEBUG=1 + # export CCACHE_LOGFILE=/work/mxnet/ccache-log + # export CCACHE_DEBUG=1 } build_wheel() { @@ -659,7 +667,8 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ubuntu_gpu_cuda100_cudnn7() { set -ex - build_ccache_wrappers + # unfortunately this build has problems in 3rdparty dependencies with ccache and make + # build_ccache_wrappers make \ DEV=1 \ ENABLE_TESTCOVERAGE=1 \ From bf7cd9aa99f9a2dc6aea31a0b3127b504059fbe1 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 19 Apr 2019 13:52:20 -0700 Subject: [PATCH 130/146] readability --- src/nnvm/gradient.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index badc3f9e2b85..c4850d46de45 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -144,13 +144,13 @@ Graph Gradient(Graph src) { << "because it is unreachable from the outputs."; } - // construct mirror reduece memory strategy if needed + // construct mirror as memory reduction strategy if needed std::unordered_map mirror_map; if (mirror_fun != nullptr) { - for (const NodePtr& n : topo_order) { - if (mirror_fun(*n)) { + for (const NodePtr& node_ptr : topo_order) { + if (mirror_fun(*node_ptr)) { NodePtr new_node = Node::Create(); - *new_node = *n; + *new_node = *node_ptr; new_node->attrs.name += "_mirror"; for (auto& e : new_node->inputs) { e.node = mirror_map.at(e.node.get()); @@ -158,9 +158,9 @@ Graph Gradient(Graph src) { for (auto& n : new_node->control_deps) { n = mirror_map.at(n.get()); } - mirror_map[n.get()] = std::move(new_node); + mirror_map[node_ptr.get()] = std::move(new_node); } else { - mirror_map[n.get()] = n; + mirror_map[node_ptr.get()] = node_ptr; } } } From f55be152d8694da090300879bf03b6430c219d1a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 23 Apr 2019 11:01:07 -0700 Subject: [PATCH 131/146] Add checks to gradient.cc --- src/nnvm/gradient.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/nnvm/gradient.cc b/src/nnvm/gradient.cc index c4850d46de45..586027129a0b 100644 --- a/src/nnvm/gradient.cc +++ b/src/nnvm/gradient.cc @@ -186,7 +186,8 @@ Graph Gradient(Graph src) { if ((*rit)->inputs.size() != 0) { NodePtr fwd_node = (mirror_map.size() == 0 ? ptr : mirror_map.at(ptr.get())); std::vector input_grads; - if (grad_fun_map.count(ptr->op())) { + // Check for FGradient + if (grad_fun_map.contains(ptr->op())) { input_grads = grad_fun_map[ptr->op()](fwd_node, out_agg_grads); CHECK_EQ((*rit)->inputs.size(), input_grads.size()) << "Gradient function not returning enough gradient"; @@ -206,7 +207,7 @@ Graph Gradient(Graph src) { if (p->op()->attr_parser != nullptr) { p->op()->attr_parser(&(p->attrs)); } - input_grads.emplace_back(nnvm::NodeEntry{p, 0, 0}); + input_grads.emplace_back(p, 0, 0); } } else { LOG(FATAL) << "Operator " << fwd_node->op()->name << " is non-differentiable " @@ -215,13 +216,14 @@ Graph Gradient(Graph src) { for (const auto& nodeEntry : input_grads) CHECK(nodeEntry.node); auto git = input_grads.begin(); + CHECK((*rit)->inputs.size() <= input_grads.size()); for (auto it = (*rit)->inputs.begin(); it != (*rit)->inputs.end(); ++it, ++git) { - auto& ge = output_grads[it->node.get()][it->index]; + auto& output_grad_entry = output_grads[it->node.get()][it->index]; // if any of the backward op can do shape inference, the hint is not necessary. - if (finfer_shape.count(git->node->op())) { - ge.need_attr_hint = false; + if (finfer_shape.contains(git->node->op())) { + output_grad_entry.need_attr_hint = false; } - ge.grads.emplace_back(std::move(*git)); + output_grad_entry.grads.emplace_back(std::move(*git)); } } } From 3ddd9176cc0d858d75c3514ce83ed4a49e855296 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 29 Apr 2019 11:56:40 -0700 Subject: [PATCH 132/146] update tvm --- 3rdparty/tvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/tvm b/3rdparty/tvm index eed28f581553..a706ad16f83d 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit eed28f581553525646a4312d377edd87fe827f42 +Subproject commit a706ad16f83d810f4a6268cf604cece79c2c4791 From 7d4ad83a11908deb16e7007827e2a288c26a9904 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Fri, 3 May 2019 10:29:14 -0400 Subject: [PATCH 133/146] fix clojure tests --- contrib/clojure-package/test/good-test-ndarray-api.clj | 2 +- contrib/clojure-package/test/good-test-symbol-api.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/clojure-package/test/good-test-ndarray-api.clj b/contrib/clojure-package/test/good-test-ndarray-api.clj index 7554089d0ba0..f7f58f8f7c88 100644 --- a/contrib/clojure-package/test/good-test-ndarray-api.clj +++ b/contrib/clojure-package/test/good-test-ndarray-api.clj @@ -106,7 +106,7 @@ - Defined in src/operator/nn/batch_norm.cc:L574 + Defined in src/operator/nn/batch_norm.cc:L572 `data`: Input data to batch normalization `gamma`: gamma array diff --git a/contrib/clojure-package/test/good-test-symbol-api.clj b/contrib/clojure-package/test/good-test-symbol-api.clj index c7450f8eb5c1..3081304ebdb3 100644 --- a/contrib/clojure-package/test/good-test-symbol-api.clj +++ b/contrib/clojure-package/test/good-test-symbol-api.clj @@ -119,7 +119,7 @@ - Defined in src/operator/nn/batch_norm.cc:L574 + Defined in src/operator/nn/batch_norm.cc:L572 `data`: Input data to batch normalization (optional) `gamma`: gamma array (optional) From 1be6c4d865a159ce0d2dad2a8f557127c7d5fda3 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 6 May 2019 13:18:33 -0700 Subject: [PATCH 134/146] Fix warning --- tests/cpp/misc/libinfo_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cpp/misc/libinfo_test.cc b/tests/cpp/misc/libinfo_test.cc index 57f8f8d764c3..c3e5191e3c21 100644 --- a/tests/cpp/misc/libinfo_test.cc +++ b/tests/cpp/misc/libinfo_test.cc @@ -30,4 +30,5 @@ using namespace mxnet::features; TEST(RuntimeTest, RuntimeTestAll) { EXPECT_EQ(EnumNames::names.size(), MAX_FEATURES); const auto& features = LibInfo::getInstance()->getFeatures(); + EXPECT_FALSE(features.empty()); } From beeea60d298d37d296cd1c8de427eed93a3b5691 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 13 May 2019 12:45:08 -0700 Subject: [PATCH 135/146] Minor refactor --- src/imperative/imperative.cc | 9 +++------ src/operator/operator_common.h | 12 ++++++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index ea0d9f490299..e0ebd995291f 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -237,16 +237,13 @@ void Imperative::RecordOp( node->inputs.resize(inputs.size()); } - std::vector& save_inputs = *p_save_inputs; - std::vector& save_outputs = *p_save_outputs; - for (size_t i = 0; i < inputs.size(); ++i) { if (AGInfo::IsNone(*(inputs[i]))) { nnvm::NodeEntry entry{nnvm::Symbol::CreateVariable( "null" + std::to_string(variable_count_++)).outputs[0].node, 0, 0}; AGInfo& input_info = AGInfo::Create(entry.node); input_info.ctx = inputs[i]->ctx(); - if (save_inputs[i]) { + if ((*p_save_inputs)[i]) { input_info.outputs.emplace_back(*inputs[i]); } else { // Put a dummy array here since it will not be used. @@ -256,7 +253,7 @@ void Imperative::RecordOp( input_info.outputs.back().storage_type_ = inputs[i]->storage_type(); } inputs[i]->entry_ = std::move(entry); // assign last to prevent cyclic reference - } else if (save_inputs[i]) { + } else if ((*p_save_inputs)[i]) { AGInfo::Get(inputs[i]->entry_.node).outputs[inputs[i]->entry_.index] = inputs[i]->Detach(); } node->inputs[i] = inputs[i]->entry_; @@ -269,7 +266,7 @@ void Imperative::RecordOp( } for (size_t i = 0; i < outputs.size(); ++i) { - if (save_outputs[i]) { + if ((*p_save_outputs)[i]) { info.outputs.emplace_back(outputs[i]->Detach()); } else { // Put a dummy array here since it will not be used. diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 0cc25828e5b2..e21ca1d6ee68 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -425,10 +425,13 @@ inline std::vector MakeZeroGradNodes( inline bool CheckGradAllZero(const std::vector& ograds) { static const auto zero_op = nnvm::Op::Get("_zeros"); static const auto zero_like_op = nnvm::Op::Get("zeros_like"); - if (!ograds.size()) return false; + if (ograds.empty()) + return false; for (const auto& grad : ograds) { - if (!grad.node) return false; - if (grad.node->op() != zero_op && grad.node->op() != zero_like_op ) return false; + if (!grad.node) + return false; + if (grad.node->op() != zero_op && grad.node->op() != zero_like_op ) + return false; } return true; } @@ -440,7 +443,8 @@ inline std::vector MakeNonlossGradNode( const std::vector& ograds, const std::vector& inputs, const std::unordered_map& dict) { - if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds); + if (CheckGradAllZero(ograds)) + return MakeZeroGradNodes(n, ograds); auto p = MakeNode(op_name, n->attrs.name + "_backward", nullptr, &dict, &n); p->inputs.insert(p->inputs.end(), ograds.begin(), ograds.end()); From f11e36b1922281f3a185976e20109de014ec7782 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 13 May 2019 17:00:40 -0700 Subject: [PATCH 136/146] Add test for second order gradient --- tests/python/unittest/test_autograd.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 5b5aff31231b..86758bc4b23a 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -447,6 +447,21 @@ def test_gradient(): assert abs(x.grad.asscalar() - 2.71828175) < 1e-7 +def test_ag_grad(): + x = mx.nd.ones((3,3)) + y = mx.nd.ones((3,3)) + x.attach_grad() + y.attach_grad() + with mx.autograd.record(): + #z = x + y + z = x + y + x_grad_y_grad = mx.autograd.grad(z, x, create_graph=True, retain_graph=True) + print(x_grad_y_grad) + first_grad = nd.concat(*[x.reshape(-1) for x in x_grad_y_grad], dim=0) + fg_f = 2 * first_grad + second_grad = mx.autograd.grad(fg_f, [x,y], retain_graph=True) + + if __name__ == "__main__": import nose nose.runmodule() From 27c9131cc9e0569889c5ee86c9fec4e8eb706f6e Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 5 Jun 2019 11:38:15 -0700 Subject: [PATCH 137/146] CR --- include/mxnet/imperative.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index 46ee667b5750..8aa56aa735ad 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -48,7 +48,7 @@ class AGInfo { bool fresh_out_grad; AGInfo() : - grad_req(kNullOp), fresh_out_grad(false) {} + grad_req(kNullOp), fresh_out_grad(false) {} static void Clear(const nnvm::NodePtr& node) { if (node == nullptr || node->info.empty()) return; @@ -73,7 +73,7 @@ class AGInfo { static bool IsVariable(const nnvm::NodePtr& node) { AGInfo& info = Get(node); return info.grad_req != kNullOp && info.outputs.size() == 1 - && info.out_grads.size() == 1; + && info.out_grads.size() == 1; } }; From 67eb0b19007332776b76074eb2f8f7e1625cd19a Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 5 Jun 2019 11:42:30 -0700 Subject: [PATCH 138/146] pep8 --- tests/python/unittest/test_autograd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 86758bc4b23a..ee56b57e9728 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -392,6 +392,7 @@ def test_get_symbol(): y = x*x + 2*z - 1 assert len(get_symbol(y).list_arguments()) == 2 + @with_seed() def test_grad_with_stype(): def check_grad_with_stype(array_stype, grad_stype, expected_stype): @@ -411,6 +412,7 @@ def check_grad_with_stype(array_stype, grad_stype, expected_stype): # check the stype of the gradient when provided check_grad_with_stype(stype, grad_stype, grad_stype) + @with_seed() def test_sparse_dot_grad(): def check_sparse_dot_grad(rhs): @@ -434,6 +436,7 @@ def check_sparse_dot_grad(rhs): dns.attach_grad(stype='row_sparse') check_sparse_dot_grad(dns) + @with_seed() def test_gradient(): x = mx.nd.ones((1,)) From 74ec09f8c533c10a3ec857d3e1f94d686628297d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 5 Jun 2019 15:23:06 -0700 Subject: [PATCH 139/146] Remove test --- tests/python/unittest/test_autograd.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index ee56b57e9728..e04f9c4e389b 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -450,21 +450,6 @@ def test_gradient(): assert abs(x.grad.asscalar() - 2.71828175) < 1e-7 -def test_ag_grad(): - x = mx.nd.ones((3,3)) - y = mx.nd.ones((3,3)) - x.attach_grad() - y.attach_grad() - with mx.autograd.record(): - #z = x + y - z = x + y - x_grad_y_grad = mx.autograd.grad(z, x, create_graph=True, retain_graph=True) - print(x_grad_y_grad) - first_grad = nd.concat(*[x.reshape(-1) for x in x_grad_y_grad], dim=0) - fg_f = 2 * first_grad - second_grad = mx.autograd.grad(fg_f, [x,y], retain_graph=True) - - if __name__ == "__main__": import nose nose.runmodule() From 0afab8768c2f6754ae79250879d8f58ece31d940 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 5 Jun 2019 15:41:25 -0700 Subject: [PATCH 140/146] Restored submodules --- .gitmodules | 2 +- 3rdparty/dmlc-core | 2 +- 3rdparty/tvm | 2 +- src/imperative/imperative.cc | 8 +++++--- src/operator/nn/fully_connected-inl.h | 1 - src/operator/nn/fully_connected.cc | 18 ------------------ tests/python/unittest/test_gluon.py | 22 ++-------------------- 7 files changed, 10 insertions(+), 45 deletions(-) diff --git a/.gitmodules b/.gitmodules index 151b9fe39a86..e0ffec11bfd0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,7 +22,7 @@ branch = master [submodule "3rdparty/tvm"] path = 3rdparty/tvm - url = https://github.com/larroy/tvm + url = https://github.com/dmlc/tvm [submodule "3rdparty/onnx-tensorrt"] path = 3rdparty/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core index 3943914eed66..82bf4c2e2af3 160000 --- a/3rdparty/dmlc-core +++ b/3rdparty/dmlc-core @@ -1 +1 @@ -Subproject commit 3943914eed66470bd010df581e29e4dca4f7df6f +Subproject commit 82bf4c2e2af312b3d52513aa727483803a2f8734 diff --git a/3rdparty/tvm b/3rdparty/tvm index d9d5d256787f..a276b8a7d0cd 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit d9d5d256787f4a0614042c6a343cf14db3cc4105 +Subproject commit a276b8a7d0cd560e0579737a3954bd60050846e7 diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 72d88afe6167..a05bb9a8f5f4 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -339,9 +339,11 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes( var_nodes.op_req_types.push_back(kWriteTo); } } else { - std::vector input_ro_nodes = nnvm::Symbol::ListInputs( - Symbol::kReadOnlyArgs, - outputs); + nnvm::Symbol s; + s.outputs = outputs; + std::vector input_ro_nodes = s.ListInputs( + Symbol::kReadOnlyArgs + s.outputs = outputs;); var_nodes.variable_nodes.reserve(input_ro_nodes.size()); var_nodes.gradients.reserve(input_ro_nodes.size()); var_nodes.op_req_types.reserve(input_ro_nodes.size()); diff --git a/src/operator/nn/fully_connected-inl.h b/src/operator/nn/fully_connected-inl.h index f9ef3c3af66f..e4bb11f6bc56 100644 --- a/src/operator/nn/fully_connected-inl.h +++ b/src/operator/nn/fully_connected-inl.h @@ -237,7 +237,6 @@ void FullyConnectedGradCompute(const nnvm::NodeAttrs& attrs, } } - } // namespace op } // namespace mxnet namespace std { diff --git a/src/operator/nn/fully_connected.cc b/src/operator/nn/fully_connected.cc index 5c6994c747fe..a097357ef5a3 100644 --- a/src/operator/nn/fully_connected.cc +++ b/src/operator/nn/fully_connected.cc @@ -165,7 +165,6 @@ static bool FullyConnectedType(const nnvm::NodeAttrs& attrs, attrs, in_type, out_type, -1); } - struct FullyConnectedGrad { const char *op_name; std::vector operator()(const nnvm::NodePtr& n, @@ -177,22 +176,6 @@ struct FullyConnectedGrad { } }; - -std::vector FullyConnectedBackwardGrad( - const nnvm::NodePtr& n, - const std::vector& ograds) { - std::vector ret; - decltype(nnvm::NodeEntry::index) i = 0; - for (const auto& x : n->inputs) { - std::ostringstream os; - os << n->attrs.name << "_backward_" << i; - ret.emplace_back(nnvm::NodeEntry{MakeNode("zeros_like", os.str(), {x}, nullptr, &n), 0, 0}); - ++i; - } - return ret; -} - - inline static bool FCStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, DispatchMode* dispatch_mode, @@ -342,7 +325,6 @@ NNVM_REGISTER_OP(_backward_FullyConnected) .set_attr("FInplaceOption", [](const NodeAttrs& attrs){ return std::vector >{{1, 0}}; }) -.set_attr("FGradient", FullyConnectedBackwardGrad) .set_attr("FInferStorageType", BackwardFCStorageType) .set_attr_parser(ParamParser) #if MXNET_USE_MKLDNN == 1 diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index 00a51eebf481..08e4c52a5826 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -1,4 +1,4 @@ - # Licensed to the Apache Software Foundation (ASF) under one +# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file @@ -21,7 +21,7 @@ import mxnet as mx from mxnet import gluon from mxnet.gluon import nn -from mxnet.test_utils import assert_almost_equal, same +from mxnet.test_utils import assert_almost_equal from mxnet.ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID from common import (setup_module, with_seed, assertRaises, teardown, assert_raises_cudnn_not_satisfied) @@ -915,24 +915,6 @@ def test_sequential_warning(): assert len(w) == 1 -@with_seed() -def test_dense_backward(): - import mxnet.autograd as ag - import mxnet.ndarray as nd - x = nd.array([[1,2,3,400]]) - net = gluon.nn.Sequential() - with net.name_scope(): - net.add(gluon.nn.Dense(1, in_units=x.shape[1])) - net.initialize(mx.initializer.Constant(.5)) - params = [x.data() for x in net.collect_params().values()] - x.attach_grad() - with ag.record(): - y = net.forward(x) - y_grad = ag.grad(y, x, create_graph=True, retain_graph=True)[0] - y_grad.backward() - same(x.grad, nd.zeros(4)) - - @with_seed() def test_global_norm_clip(): stypes = ['default', 'row_sparse'] From 5d361871fb04da8e580734111f60a88057b478b7 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 25 Jul 2019 01:01:34 +0000 Subject: [PATCH 141/146] Fix build --- src/imperative/imperative.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index a81d79101931..e7cf4853f544 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -275,7 +275,7 @@ void Imperative::RecordOp( info.outputs.back().dtype_ = outputs[i]->dtype(); info.outputs.back().storage_type_ = outputs[i]->storage_type(); } - outputs[i]->entry_ = nnvm::NodeEntry{node, i, 0}; + outputs[i]->entry_ = nnvm::NodeEntry{node, static_cast(i), 0}; } } @@ -341,9 +341,7 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes( } else { nnvm::Symbol s; s.outputs = outputs; - std::vector input_ro_nodes = s.ListInputs( - Symbol::kReadOnlyArgs - s.outputs = outputs;); + std::vector input_ro_nodes = s.ListInputs(Symbol::kReadOnlyArgs); var_nodes.variable_nodes.reserve(input_ro_nodes.size()); var_nodes.gradients.reserve(input_ro_nodes.size()); var_nodes.op_req_types.reserve(input_ro_nodes.size()); From a551b2d6fc145efdeeab048742cc2440f6523d12 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Jul 2019 19:39:05 -0700 Subject: [PATCH 142/146] Fix dev_menu --- dev_menu.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index 5a80ef0b59e4..0b5db2505671 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -32,10 +32,12 @@ import yaml import shutil + DEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv') DEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3') DEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml') + class Confirm(object): def __init__(self, cmds): self.cmds = cmds @@ -51,6 +53,7 @@ def __call__(self): else: resp = input("Please answer yes or no: ") + class CMake(object): def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS, cmake_options_yaml_default='cmake/cmake_options.yml'): if os.path.exists(cmake_options_yaml): @@ -93,27 +96,38 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'): logging.info('Now building') check_call(shlex.split(build_cmd)) + def create_virtualenv(venv_exe, pyexe, venv) -> None: logging.info("Creating virtualenv in %s with python %s", venv, pyexe) if not (venv_exe and pyexe and venv): logging.warn("Skipping creation of virtualenv") return check_call([venv_exe, '-p', pyexe, venv]) - # TODO: Activate virtualenv in this interpreter, this is not working as it is. - #activate_this_py = os.path.join(venv, 'bin', 'activate_this.py') - #exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) - #check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) - #check_call(['pip', 'install', '-r', 'tests/requirements.txt']) + def create_virtualenv_default(): create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV) - logging.info("You can use the virtualenv by executing 'source %s/bin/activate && pip install -e python'", DEFAULT_PYENV) + logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV) + + +def provision_virtualenv(venv_path=DEFAULT_PYENV): + pip = os.path.join(venv_path, 'bin', 'pip') + if os.path.exists(pip): + # Install MXNet python bindigs + check_call([pip, 'install', '--upgrade', '--force-reinstall', '-e', 'python']) + # Install test dependencies + check_call([pip, 'install', '--upgrade', '--force-reinstall', '-r', os.path.join('tests', + 'requirements.txt')]) + else: + logging.warn("Can't find pip: '%s' not found", pip) + COMMANDS = OrderedDict([ ('[Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ({} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV), [ CMake(), - #create_virtualenv_default, + create_virtualenv_default, + provision_virtualenv, ]), ('[Local] Python Unit tests', "./py3_venv/bin/nosetests -v tests/python/unittest/" @@ -209,7 +223,8 @@ def build(args) -> None: else: cmake = CMake() cmake() - #create_virtualenv(venv_exe, pyexe, args.venv) + create_virtualenv_default() + provision_virtualenv() def main(): logging.getLogger().setLevel(logging.INFO) From 79a9ba6507d834c67ec96ca8748d1fafeb5beab9 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Jul 2019 19:44:34 -0700 Subject: [PATCH 143/146] CR --- include/mxnet/imperative.h | 8 ++-- include/mxnet/io.h | 2 +- src/imperative/imperative.cc | 66 ++++++++++++++------------------ src/io/iter_prefetcher.h | 2 +- src/io/iter_sparse_batchloader.h | 2 +- src/io/iter_sparse_prefetcher.h | 2 +- 6 files changed, 37 insertions(+), 45 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index c1538f73c76e..b4676ff9c9ac 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -170,7 +170,7 @@ class Imperative { * @param outputs source ndarrays * @return vector of nodes */ - static std::vector CreateForwardGraph(const std::vector& outputs); + static nnvm::Graph CreateGraph(const std::vector &outputs); /*! Create gradient nodes using output shapes and ctx. * Gradient heads are initialized to 1 if they are not present (nullptr) * @return vector of nodes @@ -178,7 +178,7 @@ class Imperative { static std::vector CreateHeadGradientNodes(const std::vector& outputs, const std::vector& ograds); - class GradientVariableNodes; + struct GradientVariableNodes; /*! Create variable nodes. * If variables is provided, gradient nodes are crated for them. Otherwise it uses read only * inputs reachable from the outputs. @@ -196,8 +196,8 @@ class Imperative { void GetBackwardDependency( const nnvm::NodePtr& node, size_t num_inputs, size_t num_outputs, - std::vector *p_save_inputs, - std::vector *p_save_outputs); + std::vector *save_inputs, + std::vector *save_outputs); /*! \brief indicate whether is training. */ #if DMLC_CXX11_THREAD_LOCAL static thread_local bool is_train_; diff --git a/include/mxnet/io.h b/include/mxnet/io.h index 2a352af16cf9..e18f03ed0ef3 100644 --- a/include/mxnet/io.h +++ b/include/mxnet/io.h @@ -52,7 +52,7 @@ class IIterator : public dmlc::DataIter { /*! \brief move to next item */ virtual bool Next(void) = 0; /*! \brief get current data */ - virtual const DType& Value(void) const = 0; + virtual const DType &Value(void) const = 0; /*! \brief constructor */ virtual ~IIterator(void) {} /*! \brief store the name of each data, it could be used for making NDArrays */ diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index e7cf4853f544..c5dd2f7fdda0 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -145,22 +145,16 @@ void Imperative::MarkVariables( } } - - - - void Imperative::GetBackwardDependency( const nnvm::NodePtr& node, size_t num_inputs, size_t num_outputs, - std::vector *p_save_inputs, - std::vector *p_save_outputs) { + std::vector *save_inputs, + std::vector *save_outputs) { static auto& fgradient = nnvm::Op::GetAttr("FGradient"); - std::vector& save_inputs = *p_save_inputs; - std::vector& save_outputs = *p_save_outputs; - save_inputs.resize(num_inputs); - save_outputs.resize(num_outputs); - std::fill(save_inputs.begin(), save_inputs.end(), false); - std::fill(save_outputs.begin(), save_outputs.end(), false); + save_inputs->resize(num_inputs); + save_outputs->resize(num_outputs); + std::fill(save_inputs->begin(), save_inputs->end(), false); + std::fill(save_outputs->begin(), save_outputs->end(), false); node->inputs.clear(); node->inputs.reserve(num_inputs); @@ -177,18 +171,18 @@ void Imperative::GetBackwardDependency( auto igrad_entries = fgradient[node->op()](node, ograd_entries); for (const auto& i : igrad_entries) { if (i.node == nullptr && i.version == 0) { - save_inputs[i.index] = true; + (*save_inputs)[i.index] = true; } else if (i.node == node) { - save_outputs[i.index] = true; + (*save_outputs)[i.index] = true; } } DFSVisit(igrad_entries, [&](const nnvm::NodePtr& gnode) { if (!gnode || gnode == node) return; for (const auto& i : gnode->inputs) { if (i.node == nullptr && i.version == 0) { - save_inputs[i.index] = true; + (*save_inputs)[i.index] = true; } else if (i.node == node) { - save_outputs[i.index] = true; + (*save_outputs)[i.index] = true; } } }); @@ -279,7 +273,8 @@ void Imperative::RecordOp( } } -std::vector Imperative::CreateForwardGraph(const std::vector& outputs) { +nnvm::Graph Imperative::CreateGraph(const std::vector &outputs) { + nnvm::Graph g; std::vector output_nodes; output_nodes.reserve(outputs.size()); for (const auto &i : outputs) { @@ -288,9 +283,9 @@ std::vector Imperative::CreateForwardGraph(const std::vectorentry_); + g.outputs.emplace_back(i->entry_); } - return output_nodes; + return g; } std::vector Imperative::CreateHeadGradientNodes( @@ -333,7 +328,7 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes( CHECK(!AGInfo::IsNone(*variables[i]) && AGInfo::IsVariable(variables[i]->entry_.node)) << "Cannot differentiate with respect to the " << i+1 << "-th variable" - << " because it does not require gradient."; + << " because it does not require gradient. Did you forget attach_grad()?"; var_nodes.variable_nodes.emplace_back(variables[i]->entry_); var_nodes.gradients.push_back(new NDArray()); var_nodes.op_req_types.push_back(kWriteTo); @@ -360,8 +355,6 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes( return var_nodes; } - - std::vector Imperative::Backward( const std::vector& outputs, const std::vector& ograds, @@ -373,9 +366,7 @@ std::vector Imperative::Backward( static const std::vector zero_ops{Op::Get("zeros_like"), Op::Get("_zeros")}; static const Op* copy_op = Op::Get("_copy"); - Graph graph; - graph.outputs = CreateForwardGraph(outputs); - + Graph graph = CreateGraph(outputs); // Prepare head gradient nodes std::vector ograd_entries = CreateHeadGradientNodes(outputs, ograds); @@ -405,7 +396,7 @@ std::vector Imperative::Backward( } } - const auto& indexed_graph = graph.indexed_graph(); + auto& indexed_graph = graph.indexed_graph(); // get number of nodes used in forward pass size_t num_forward_nodes = 0; size_t num_forward_entries = 0; @@ -429,19 +420,19 @@ std::vector Imperative::Backward( if (create_graph) { states.resize(num_forward_nodes); nnvm::DFSVisit(forward_outputs, [&](const nnvm::NodePtr& n) { - AGInfo& info = AGInfo::Get(n); + const AGInfo& info = AGInfo::Get(n); states.at(indexed_graph.node_id(n.get())) = info.state; - for (uint32_t i = 0; i < info.outputs.size(); ++i) { + for (size_t i = 0; i < info.outputs.size(); ++i) { CHECK(indexed_graph.exist(n.get())); - size_t nid = indexed_graph.node_id(n.get()); - size_t eid = indexed_graph.entry_id(nid, i); + const size_t nid = indexed_graph.node_id(n.get()); + const size_t eid = indexed_graph.entry_id(nid, i); buff[eid] = info.outputs[i]; - buff[eid].entry_ = NodeEntry{n, i, 0}; + buff[eid].entry_ = NodeEntry{n, static_cast(i), 0}; ref_count[eid] = 1; } }); for (auto& ograd_entry : ograd_entries) { - AGInfo& info = AGInfo::Get(ograd_entry.node); + const AGInfo& info = AGInfo::Get(ograd_entry.node); if (!indexed_graph.exist(ograd_entry.node.get())) continue; size_t eid = indexed_graph.entry_id(ograd_entry); buff[eid] = info.outputs[0]; @@ -450,13 +441,14 @@ std::vector Imperative::Backward( } else { states.reserve(num_forward_nodes); for (size_t i = 0; i < num_forward_nodes; ++i) { - const AGInfo& info = dmlc::get(indexed_graph[i].source->info); + // TODO(larroy): This is a code smell 💩 + AGInfo& info = const_cast(dmlc::get(indexed_graph[i].source->info)); states.emplace_back(info.state); for (size_t j = 0; j < info.outputs.size(); ++j) { - size_t eid = indexed_graph.entry_id(i, j); - arrays[eid] = const_cast(&(info.outputs[j])); - - if (retain_graph || info.grad_req != kNullOp) ref_count[eid] = 1; + const size_t eid = indexed_graph.entry_id(i, j); + arrays[eid] = &(info.outputs[j]); + if (retain_graph || info.grad_req != kNullOp) + ref_count[eid] = 1; } } for (auto& ograd_entry : ograd_entries) { diff --git a/src/io/iter_prefetcher.h b/src/io/iter_prefetcher.h index 29844ec2f568..fdd1d2b91925 100644 --- a/src/io/iter_prefetcher.h +++ b/src/io/iter_prefetcher.h @@ -131,7 +131,7 @@ class PrefetcherIter : public IIterator { } return iter.Next(&out_); } - virtual const DataBatch& Value(void) const { + virtual const DataBatch &Value(void) const { return *out_; } diff --git a/src/io/iter_sparse_batchloader.h b/src/io/iter_sparse_batchloader.h index 07bd9d4e2f2b..bd75b3ac1377 100644 --- a/src/io/iter_sparse_batchloader.h +++ b/src/io/iter_sparse_batchloader.h @@ -100,7 +100,7 @@ class SparseBatchLoader : public BatchLoader, public SparseIIterator return false; } - virtual const TBlobBatch& Value(void) const { + virtual const TBlobBatch &Value(void) const { return BatchLoader::Value(); } diff --git a/src/io/iter_sparse_prefetcher.h b/src/io/iter_sparse_prefetcher.h index 8bb28681902a..536f54fcddff 100644 --- a/src/io/iter_sparse_prefetcher.h +++ b/src/io/iter_sparse_prefetcher.h @@ -126,7 +126,7 @@ class SparsePrefetcherIter : public PrefetcherIter { virtual bool Next(void) { return PrefetcherIter::Next(); } - virtual const DataBatch& Value(void) const { + virtual const DataBatch &Value(void) const { return PrefetcherIter::Value(); } From 467fd0f25b77a7d5360146644710254a55fe0d03 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 24 Jul 2019 19:47:35 -0700 Subject: [PATCH 144/146] rename NDArray.entry_ to NDArray.autograd_ --- include/mxnet/imperative.h | 2 +- include/mxnet/ndarray.h | 18 +++++++++--------- src/imperative/imperative.cc | 26 +++++++++++++------------- src/ndarray/ndarray.cc | 14 +++++++------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/include/mxnet/imperative.h b/include/mxnet/imperative.h index b4676ff9c9ac..2976571a63cf 100644 --- a/include/mxnet/imperative.h +++ b/include/mxnet/imperative.h @@ -67,7 +67,7 @@ class AGInfo { } static bool IsNone(const NDArray& arr) { - return arr.entry_.node == nullptr || arr.entry_.node->info.empty(); + return arr.autograd_.node == nullptr || arr.autograd_.node->info.empty(); } static bool IsVariable(const nnvm::NodePtr& node) { diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index b2231e09719c..f20d438a43ad 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -85,7 +85,7 @@ class NDArray { public: /*! \brief default constructor */ NDArray() - : entry_(nullptr) { + : autograd_(nullptr) { } /*! * \brief constructs a new dynamic NDArray @@ -100,7 +100,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_(nullptr) { + autograd_(nullptr) { } /*! \brief constructor for NDArray with storage type */ @@ -119,7 +119,7 @@ class NDArray { shape_(), dtype_(dtype), storage_type_(kDefaultStorage), - entry_(nullptr) { + autograd_(nullptr) { } /*! * \brief constructing a static NDArray that shares data with TBlob @@ -133,7 +133,7 @@ class NDArray { shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_(nullptr) { + autograd_(nullptr) { } /*! @@ -151,7 +151,7 @@ class NDArray { }), shape_(data.shape_), dtype_(data.type_flag_), storage_type_(kDefaultStorage), - entry_(nullptr) { + autograd_(nullptr) { } /*! \brief create ndarray from shared memory */ @@ -160,7 +160,7 @@ class NDArray { shape_(shape), dtype_(dtype), storage_type_(kDefaultStorage), - entry_(nullptr) { + autograd_(nullptr) { } /*! @@ -179,7 +179,7 @@ class NDArray { shape_(shape), dtype_(data.type_flag_), storage_type_(stype), - entry_(nullptr) { + autograd_(nullptr) { } /*! * \brief initialize the NDArray, assuming it is not assigned a meaningful shape before @@ -657,7 +657,7 @@ class NDArray { */ NDArray Detach() const { NDArray ret(*this); - ret.entry_ = nnvm::NodeEntry(nullptr); + ret.autograd_ = nnvm::NodeEntry(nullptr); return ret; } @@ -1109,7 +1109,7 @@ class NDArray { /*! \brief storage type of data */ NDArrayStorageType storage_type_ = kUndefinedStorage; /*! \brief node entry for autograd */ - nnvm::NodeEntry entry_; + nnvm::NodeEntry autograd_; /*! * \brief internal TBlob * \note When user access tblob_ by some const methods like diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index c5dd2f7fdda0..355566538152 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -127,18 +127,18 @@ void Imperative::MarkVariables( for (size_t i = 0; i < variables.size(); ++i) { std::string str_c(std::to_string(variable_count_++)); { - variables[i]->entry_ = nnvm::NodeEntry{ + variables[i]->autograd_ = nnvm::NodeEntry{ nnvm::Symbol::CreateVariable("var" + str_c).outputs[0].node, 0, 0}; - AGInfo &info = AGInfo::Create(variables[i]->entry_.node); + AGInfo &info = AGInfo::Create(variables[i]->autograd_.node); info.outputs.emplace_back(variables[i]->Detach()); info.out_grads.emplace_back(gradients[i]->Detach()); info.grad_req = static_cast(grad_reqs[i]); info.ctx = variables[i]->ctx(); } { - gradients[i]->entry_ = nnvm::NodeEntry{ + gradients[i]->autograd_ = nnvm::NodeEntry{ nnvm::Symbol::CreateVariable("grad" + str_c).outputs[0].node, 0, 0}; - AGInfo &grad_info = AGInfo::Create(gradients[i]->entry_.node); + AGInfo &grad_info = AGInfo::Create(gradients[i]->autograd_.node); grad_info.outputs.emplace_back(gradients[i]->Detach()); grad_info.ctx = gradients[i]->ctx(); } @@ -246,11 +246,11 @@ void Imperative::RecordOp( input_info.outputs.back().dtype_ = inputs[i]->dtype(); input_info.outputs.back().storage_type_ = inputs[i]->storage_type(); } - inputs[i]->entry_ = std::move(entry); // assign last to prevent cyclic reference + inputs[i]->autograd_ = std::move(entry); // assign last to prevent cyclic reference } else if ((*p_save_inputs)[i]) { - AGInfo::Get(inputs[i]->entry_.node).outputs[inputs[i]->entry_.index] = inputs[i]->Detach(); + AGInfo::Get(inputs[i]->autograd_.node).outputs[inputs[i]->autograd_.index] = inputs[i]->Detach(); } - node->inputs[i] = inputs[i]->entry_; + node->inputs[i] = inputs[i]->autograd_; } for (auto output : outputs) { @@ -269,7 +269,7 @@ void Imperative::RecordOp( info.outputs.back().dtype_ = outputs[i]->dtype(); info.outputs.back().storage_type_ = outputs[i]->storage_type(); } - outputs[i]->entry_ = nnvm::NodeEntry{node, static_cast(i), 0}; + outputs[i]->autograd_ = nnvm::NodeEntry{node, static_cast(i), 0}; } } @@ -283,7 +283,7 @@ nnvm::Graph Imperative::CreateGraph(const std::vector &outputs) { << "You need to set is_recording to true or use autograd.record() to save " << "computational graphs for backward. If you want to differentiate the same " << "graph twice, you need to pass retain_graph=True to backward."; - g.outputs.emplace_back(i->entry_); + g.outputs.emplace_back(i->autograd_); } return g; } @@ -326,10 +326,10 @@ Imperative::GradientVariableNodes Imperative::CreateGradientVariableNodes( var_nodes.op_req_types.reserve(variables.size()); for (size_t i = 0; i < variables.size(); ++i) { CHECK(!AGInfo::IsNone(*variables[i]) && - AGInfo::IsVariable(variables[i]->entry_.node)) + AGInfo::IsVariable(variables[i]->autograd_.node)) << "Cannot differentiate with respect to the " << i+1 << "-th variable" << " because it does not require gradient. Did you forget attach_grad()?"; - var_nodes.variable_nodes.emplace_back(variables[i]->entry_); + var_nodes.variable_nodes.emplace_back(variables[i]->autograd_); var_nodes.gradients.push_back(new NDArray()); var_nodes.op_req_types.push_back(kWriteTo); } @@ -427,7 +427,7 @@ std::vector Imperative::Backward( const size_t nid = indexed_graph.node_id(n.get()); const size_t eid = indexed_graph.entry_id(nid, i); buff[eid] = info.outputs[i]; - buff[eid].entry_ = NodeEntry{n, static_cast(i), 0}; + buff[eid].autograd_ = NodeEntry{n, static_cast(i), 0}; ref_count[eid] = 1; } }); @@ -436,7 +436,7 @@ std::vector Imperative::Backward( if (!indexed_graph.exist(ograd_entry.node.get())) continue; size_t eid = indexed_graph.entry_id(ograd_entry); buff[eid] = info.outputs[0]; - buff[eid].entry_ = ograd_entry; + buff[eid].autograd_ = ograd_entry; } } else { states.reserve(num_forward_nodes); diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 78a827cf7e00..014988c7bd2e 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -53,7 +53,7 @@ namespace mxnet { NDArray::NDArray(const NDArrayStorageType stype, const mxnet::TShape &shape, Context ctx, bool delay_alloc, int dtype, std::vector aux_types, mxnet::ShapeVector aux_shapes, mxnet::TShape storage_shape) : shape_(shape), - dtype_(dtype), storage_type_(stype), entry_(nullptr) { + dtype_(dtype), storage_type_(stype), autograd_(nullptr) { // Assign default aux types if not given if (aux_types.size() == 0 && stype != kDefaultStorage) { @@ -152,7 +152,7 @@ void NDArray::Chunk::CheckAndAllocData(const mxnet::TShape &shape, int dtype) { NDArray NDArray::grad() const { if (AGInfo::IsNone(*this)) return NDArray(); - AGInfo& info = AGInfo::Get(entry_.node); + AGInfo& info = AGInfo::Get(autograd_.node); if (info.out_grads.size()) { CHECK_EQ(info.out_grads.size(), 1); return info.out_grads[0]; @@ -164,14 +164,14 @@ nnvm::Symbol NDArray::get_autograd_symbol() const { CHECK(!AGInfo::IsNone(*this)) << "NDArray is not part of a computation graph. Did you forget to turn on recording?"; nnvm::Symbol ret; - ret.outputs.emplace_back(entry_); + ret.outputs.emplace_back(autograd_); return ret; } #if MXNET_USE_MKLDNN == 1 NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) - : storage_type_(kDefaultStorage), entry_(nullptr) { + : storage_type_(kDefaultStorage), autograd_(nullptr) { auto mem_desc = mem_pd.desc(); shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); dtype_ = get_mxnet_type(mem_desc.data.data_type); @@ -181,7 +181,7 @@ NDArray::NDArray(mkldnn::memory::primitive_desc mem_pd) } NDArray::NDArray(const std::shared_ptr &mkldnn_mem) - : storage_type_(kDefaultStorage), entry_(nullptr) { + : storage_type_(kDefaultStorage), autograd_(nullptr) { auto mem_pd = mkldnn_mem->get_primitive_desc(); auto mem_desc = mem_pd.desc(); shape_ = mxnet::TShape(mem_desc.data.dims, mem_desc.data.dims + mem_desc.data.ndims); @@ -372,7 +372,7 @@ NDArray NDArray::FromDLPack(const DLManagedTensor* tensor, bool transient_handle bool NDArray::fresh_out_grad() const { if (AGInfo::IsNone(*this)) return false; - AGInfo& info = AGInfo::Get(entry_.node); + AGInfo& info = AGInfo::Get(autograd_.node); return info.fresh_out_grad; } @@ -380,7 +380,7 @@ bool NDArray::fresh_out_grad() const { void NDArray::set_fresh_out_grad(bool state) const { CHECK(!AGInfo::IsNone(*this)) << "NDArray has not been marked as a variable and does not have gradient state"; - AGInfo& info = AGInfo::Get(entry_.node); + AGInfo& info = AGInfo::Get(autograd_.node); info.fresh_out_grad = state; } From 0f262a7f6c8bd7fe1426c5348fbccbd19eb71a90 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Thu, 25 Jul 2019 13:33:30 -0700 Subject: [PATCH 145/146] Fix lint --- src/imperative/imperative.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 355566538152..bab5f7283d81 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -248,7 +248,8 @@ void Imperative::RecordOp( } inputs[i]->autograd_ = std::move(entry); // assign last to prevent cyclic reference } else if ((*p_save_inputs)[i]) { - AGInfo::Get(inputs[i]->autograd_.node).outputs[inputs[i]->autograd_.index] = inputs[i]->Detach(); + AGInfo::Get(inputs[i]->autograd_.node).outputs[inputs[i]->autograd_.index] = + inputs[i]->Detach(); } node->inputs[i] = inputs[i]->autograd_; } From 84a7654b4f243ca30e9f2204b9562b604987f641 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Fri, 23 Aug 2019 15:01:48 -0700 Subject: [PATCH 146/146] CR comments, validate grad_req is in range --- include/mxnet/op_attr_types.h | 3 ++- src/imperative/imperative.cc | 31 +++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/include/mxnet/op_attr_types.h b/include/mxnet/op_attr_types.h index 889b5028a460..ad061c993334 100644 --- a/include/mxnet/op_attr_types.h +++ b/include/mxnet/op_attr_types.h @@ -54,7 +54,8 @@ enum OpReqType { */ kWriteInplace, /*! \brief add to the provided space */ - kAddTo + kAddTo, + kOpReqTypeMax }; /*! diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 8ef2dc6a2779..fd80d72f99e5 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -126,22 +126,21 @@ void Imperative::MarkVariables( const std::vector& gradients) { for (size_t i = 0; i < variables.size(); ++i) { std::string str_c(std::to_string(variable_count_++)); - { - variables[i]->autograd_ = nnvm::NodeEntry{ - nnvm::Symbol::CreateVariable("var" + str_c).outputs[0].node, 0, 0}; - AGInfo &info = AGInfo::Create(variables[i]->autograd_.node); - info.outputs.emplace_back(variables[i]->Detach()); - info.out_grads.emplace_back(gradients[i]->Detach()); - info.grad_req = static_cast(grad_reqs[i]); - info.ctx = variables[i]->ctx(); - } - { - gradients[i]->autograd_ = nnvm::NodeEntry{ - nnvm::Symbol::CreateVariable("grad" + str_c).outputs[0].node, 0, 0}; - AGInfo &grad_info = AGInfo::Create(gradients[i]->autograd_.node); - grad_info.outputs.emplace_back(gradients[i]->Detach()); - grad_info.ctx = gradients[i]->ctx(); - } + // Add autograd storage for variables and link to the graph + variables[i]->autograd_ = nnvm::NodeEntry{ + nnvm::Symbol::CreateVariable("var" + str_c).outputs[0].node, 0, 0}; + AGInfo &info = AGInfo::Create(variables[i]->autograd_.node); + info.outputs.emplace_back(variables[i]->Detach()); + info.out_grads.emplace_back(gradients[i]->Detach()); + info.grad_req = static_cast(grad_reqs[i]); + CHECK(info.grad_req < kOpReqTypeMax) << "gradient update request out of range"; + info.ctx = variables[i]->ctx(); + // Handle gradients themselves + gradients[i]->autograd_ = nnvm::NodeEntry{ + nnvm::Symbol::CreateVariable("grad" + str_c).outputs[0].node, 0, 0}; + AGInfo &grad_info = AGInfo::Create(gradients[i]->autograd_.node); + grad_info.outputs.emplace_back(gradients[i]->Detach()); + grad_info.ctx = gradients[i]->ctx(); } }