From 45e1502e4b0cf8c38380417776c391464d89c6fb Mon Sep 17 00:00:00 2001 From: sxjscience Date: Sun, 14 Oct 2018 14:56:04 +0800 Subject: [PATCH 1/4] try to add support some ops --- .../tensor/elemwise_binary_op_basic.cc | 12 +++++++++- .../tensor/elemwise_unary_op_basic.cc | 8 ++++++- src/operator/tensor/elemwise_unary_op_trig.cc | 22 +++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/operator/tensor/elemwise_binary_op_basic.cc b/src/operator/tensor/elemwise_binary_op_basic.cc index 339290df8bf9..710ce5510236 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_op_basic.cc @@ -224,7 +224,17 @@ The storage type of ``elemwise_mul`` output depends on storage types of inputs return std::vector{ResourceRequest::kTempSpace}; }) .add_alias("_mul").add_alias("_Mul") -.set_attr("FGradient", ElemwiseGradUseIn{"_backward_mul"}); +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + auto lhs_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward_lhs", + {ograds[0], n->inputs[1]}, nullptr, &n); + auto rhs_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward_rhs", + {ograds[0], n->inputs[0]}, nullptr, &n); + std::vector ret; + ret.emplace_back(nnvm::NodeEntry{lhs_grad, 0, 0}); + ret.emplace_back(nnvm::NodeEntry{rhs_grad, 0, 0}); + return ret; + }); NNVM_REGISTER_OP(_backward_mul) .set_num_inputs(3) diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index 49ae976cfc2c..b11c1ebbcc28 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -623,7 +623,13 @@ The storage type of ``negative`` output depends upon the input storage type: - negative(csr) = csr )code") -.set_attr("FGradient", ElemwiseGradUseNone{"negative"}); +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + auto in_grad = MakeNode("negative", n->attrs.name + "_backward", {ograds[0]}, nullptr, &n); + std::vector ret; + ret.emplace_back(nnvm::NodeEntry{in_grad, 0, 0}); + return ret; + }); // reciprocal MXNET_OPERATOR_REGISTER_UNARY(reciprocal) diff --git a/src/operator/tensor/elemwise_unary_op_trig.cc b/src/operator/tensor/elemwise_unary_op_trig.cc index 288719f48a96..5de6de63c06d 100644 --- a/src/operator/tensor/elemwise_unary_op_trig.cc +++ b/src/operator/tensor/elemwise_unary_op_trig.cc @@ -44,7 +44,15 @@ The storage type of ``sin`` output depends upon the input storage type: - sin(csr) = csr )code" ADD_FILELINE) -.set_attr("FGradient", ElemwiseGradUseIn{ "_backward_sin" }); +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + auto x_grad = MakeNode("cos", n->attrs.name + "_mid_x_grad", {n->inputs[0]}, nullptr, &n); + auto in_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward", + {ograds[0], nnvm::NodeEntry{x_grad, 0, 0}}, nullptr, &n); + std::vector ret; + ret.emplace_back(nnvm::NodeEntry{in_grad, 0, 0}); + return ret; + }); MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU_DR(_backward_sin, unary_bwd); @@ -61,7 +69,17 @@ The input should be in radians (:math:`2\pi` rad equals 360 degrees). The storage type of ``cos`` output is always dense )code" ADD_FILELINE) -.set_attr("FGradient", ElemwiseGradUseIn{"_backward_cos"}); +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + auto x_grad = MakeNode("sin", n->attrs.name + "_mid_x_grad", {n->inputs[0]}, nullptr, &n); + auto neg_x_grad = MakeNode("negative", n->attrs.name + "_mid_neg_x_grad", + {nnvm::NodeEntry{x_grad, 0, 0}}, nullptr, &n); + auto in_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward", + {ograds[0], nnvm::NodeEntry{neg_x_grad, 0, 0}}, nullptr, &n); + std::vector ret; + ret.emplace_back(nnvm::NodeEntry{in_grad, 0, 0}); + return ret; + }); MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU(_backward_cos, unary_bwd); From 492e4cdd19db27c1a930ccb5f1ed6562d044fe8b Mon Sep 17 00:00:00 2001 From: Lin Yuan Date: Wed, 3 Apr 2019 10:38:04 -0700 Subject: [PATCH 2/4] add unit test for second order grad --- .../python/unittest/test_higher_order_grad.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/python/unittest/test_higher_order_grad.py diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py new file mode 100644 index 000000000000..696ac9fa8cab --- /dev/null +++ b/tests/python/unittest/test_higher_order_grad.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np +from mxnet import gluon, nd, autograd +from mxnet.test_utils import assert_almost_equal +from tests.python.unittest.common import with_seed + + +@with_seed() +def test_elemwise_mul(): + x = nd.array([1, 2, 3]) + y = nd.zeros(3) + x.attach_grad() + with autograd.record(): + y = nd.elemwise_mul(x, x) + y_grad = autograd.grad(y, x, create_graph=True, retain_graph=True)[0] + y_grad.backward() + expect_grad = nd.array([2, 2, 2]) + assert_almost_equal(expect_grad.asnumpy(), x.grad.asnumpy()) + + +@with_seed() +def test_sin(): + x = nd.array([1, 2, 3]) + x.attach_grad() + with autograd.record(): + y = nd.sin(x) + y_grad = autograd.grad(y, x, create_graph=True, retain_graph=True)[0] + y_grad.backward() + expect_grad = -nd.sin(x) + assert_almost_equal(expect_grad.asnumpy(), x.grad.asnumpy()) + + +@with_seed() +def test_cos(): + x = nd.array([1, 2, 3]) + x.attach_grad() + with autograd.record(): + y = nd.cos(x) + y_grad = autograd.grad(y, x, create_graph=True, retain_graph=True)[0] + y_grad.backward() + expect_grad = -nd.cos(x) + assert_almost_equal(expect_grad.asnumpy(), x.grad.asnumpy()) + + +if __name__ == '__main__': + import nose + nose.runmodule() From 45b334ebb571880388b924eba7629f63b4cd4a9d Mon Sep 17 00:00:00 2001 From: Lin Yuan Date: Wed, 3 Apr 2019 16:55:35 -0700 Subject: [PATCH 3/4] implement grad for relu and add unit test --- src/imperative/imperative.cc | 5 ++- .../tensor/elemwise_unary_op_basic.cc | 10 ++++- .../python/unittest/test_higher_order_grad.py | 41 +++++++++++++++---- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/imperative/imperative.cc b/src/imperative/imperative.cc index 3e5b3987522c..b07e761aa124 100644 --- a/src/imperative/imperative.cc +++ b/src/imperative/imperative.cc @@ -347,8 +347,9 @@ std::vector Imperative::Backward( x_reqs.push_back(info.grad_req); info.fresh_out_grad = true; } - CHECK_GT(xs.size(), 0) - << "There are no inputs in computation graph that require gradients."; + if (xs.empty()) { + LOG(WARNING) << "There are no inputs in computation graph that require gradients."; + } } Graph g_graph = pass::MXGradient( diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index 68654e3a116e..a16d3f2d89eb 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -83,7 +83,15 @@ The storage type of ``relu`` output depends upon the input storage type: - relu(csr) = csr )code" ADD_FILELINE) -.set_attr("FGradient", ElemwiseGradUseOut{"_backward_relu"}); +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + auto zero_node = MakeNode("zeros_like", n->attrs.name + "_relu_backward", {n->inputs[0]}, nullptr, &n); + auto x_grad = MakeNode("_greater", n->attrs.name + "_mid_x_grad", {n->inputs[0], nnvm::NodeEntry{zero_node, 0, 0}}, nullptr, &n); + auto in_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward", {ograds[0], nnvm::NodeEntry{x_grad, 0 , 0}}, nullptr, &n); + std::vector ret; + ret.emplace_back(nnvm::NodeEntry{in_grad, 0, 0}); + return ret; + }); MXNET_OPERATOR_REGISTER_BINARY_WITH_SPARSE_CPU(_backward_relu, unary_bwd); diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 696ac9fa8cab..4b6bce7f6a29 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -37,25 +37,50 @@ def test_elemwise_mul(): @with_seed() def test_sin(): + def sin(x): + return nd.sin(x) + x = nd.array([1, 2, 3]) - x.attach_grad() - with autograd.record(): - y = nd.sin(x) - y_grad = autograd.grad(y, x, create_graph=True, retain_graph=True)[0] - y_grad.backward() expect_grad = -nd.sin(x) - assert_almost_equal(expect_grad.asnumpy(), x.grad.asnumpy()) + check_second_order_unary(x, sin, expect_grad) @with_seed() def test_cos(): + def cos(x): + return nd.cos(x) + x = nd.array([1, 2, 3]) + expect_grad = -nd.cos(x) + check_second_order_unary(x, cos, expect_grad) + + +@with_seed() +def test_negative(): + def negative(x): + return nd.negative(x) + + x = nd.array([1, 2, 3]) + expect_grad = nd.zeros_like(x) + check_second_order_unary(x, negative, expect_grad) + + +@with_seed() +def test_relu(): + def relu(x): + return nd.relu(x) + + x = nd.array([1, 2, 3]) + expect_grad = nd.zeros_like(x) + check_second_order_unary(x, relu, expect_grad) + + +def check_second_order_unary(x, op, expect_grad): x.attach_grad() with autograd.record(): - y = nd.cos(x) + y = op(x) y_grad = autograd.grad(y, x, create_graph=True, retain_graph=True)[0] y_grad.backward() - expect_grad = -nd.cos(x) assert_almost_equal(expect_grad.asnumpy(), x.grad.asnumpy()) From 4dc0907a6cb636966d44018879c14ca4cfcf2a61 Mon Sep 17 00:00:00 2001 From: Lin Yuan Date: Fri, 5 Apr 2019 11:26:41 -0700 Subject: [PATCH 4/4] fix lint --- src/operator/tensor/elemwise_unary_op_basic.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/operator/tensor/elemwise_unary_op_basic.cc b/src/operator/tensor/elemwise_unary_op_basic.cc index a16d3f2d89eb..3f794966dc92 100644 --- a/src/operator/tensor/elemwise_unary_op_basic.cc +++ b/src/operator/tensor/elemwise_unary_op_basic.cc @@ -85,9 +85,12 @@ The storage type of ``relu`` output depends upon the input storage type: )code" ADD_FILELINE) .set_attr("FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { - auto zero_node = MakeNode("zeros_like", n->attrs.name + "_relu_backward", {n->inputs[0]}, nullptr, &n); - auto x_grad = MakeNode("_greater", n->attrs.name + "_mid_x_grad", {n->inputs[0], nnvm::NodeEntry{zero_node, 0, 0}}, nullptr, &n); - auto in_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward", {ograds[0], nnvm::NodeEntry{x_grad, 0 , 0}}, nullptr, &n); + auto zero_node = MakeNode("zeros_like", n->attrs.name + "_backward", + {n->inputs[0]}, nullptr, &n); + auto x_grad = MakeNode("_greater", n->attrs.name + "_mid_x_grad", + {n->inputs[0], nnvm::NodeEntry{zero_node, 0, 0}}, nullptr, &n); + auto in_grad = MakeNode("elemwise_mul", n->attrs.name + "_backward", + {ograds[0], nnvm::NodeEntry{x_grad, 0 , 0}}, nullptr, &n); std::vector ret; ret.emplace_back(nnvm::NodeEntry{in_grad, 0, 0}); return ret;