From bace0270988ba132da91f1be8eceec73cb32be50 Mon Sep 17 00:00:00 2001 From: Marisa Kirisame Date: Fri, 8 Mar 2019 19:14:03 -0800 Subject: [PATCH 1/5] do second order --- python/tvm/relay/op/_tensor_grad.py | 18 +++++++++++++++++- tests/python/relay/test_pass_gradient.py | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/python/tvm/relay/op/_tensor_grad.py b/python/tvm/relay/op/_tensor_grad.py index 173e97a00496..6b318890f61b 100644 --- a/python/tvm/relay/op/_tensor_grad.py +++ b/python/tvm/relay/op/_tensor_grad.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from ..expr import const from .op import register_gradient -from .transform import collapse_sum_like, where +from .transform import collapse_sum_like, broadcast_to_like, where from .tensor import exp, negative, power, less from .tensor import zeros_like, ones_like @@ -77,3 +77,19 @@ def divide_grad(orig, grad): x, y = orig.args return [collapse_sum_like(grad / y, x), collapse_sum_like(- (grad * orig / y), y)] + + +@register_gradient("zeros_like") +def zeros_like_grad(orig, grad): + """Returns [0]""" + return [orig] + +@register_gradient("ones_like") +def ones_like_grad(orig, grad): + """Returns [0]""" + return [zeros_like(orig.args[0])] + +@register_gradient("collapse_sum_like") +def collapse_sum_like_grad(orig, grad): + x, y = orig.args + return [broadcast_to_like(grad, x), zeros_like(y)] diff --git a/tests/python/relay/test_pass_gradient.py b/tests/python/relay/test_pass_gradient.py index 400941f12617..b5f55bf2ebce 100644 --- a/tests/python/relay/test_pass_gradient.py +++ b/tests/python/relay/test_pass_gradient.py @@ -190,6 +190,24 @@ def test_ref(): np.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) np.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) +def test_second_order(): + shape = (10, 10) + dtype = 'float32' + t = relay.TensorType(shape, dtype) + x = relay.var("x", t) + func = relay.Function([x], x * x) + back_func = relay.ir_pass.infer_type(gradient(func)) + y = relay.var("y", t) + back_func_adjusted = relay.Function([y], relay.TupleGetItem(relay.TupleGetItem(back_func(y), 1), 0)) + back_func_adjusted = relay.ir_pass.infer_type(back_func_adjusted) + back_back_func = relay.ir_pass.infer_type(gradient(back_func_adjusted)) + assert back_func.checked_type == relay.FuncType([t], relay.TupleType([t, relay.TupleType([t])])) + x_nd = rand(dtype, *shape) + ex = create_executor() + forward, (grad_x,) = ex.evaluate(back_back_func)(x_nd) + np.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) + np.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) + if __name__ == "__main__": test_id() test_add() @@ -200,3 +218,4 @@ def test_ref(): test_tuple() test_pow() test_ref() + test_second_order() From 33dd6d5bc13569c787ed7021a06e3e4df689c7cc Mon Sep 17 00:00:00 2001 From: Marisa Kirisame Date: Fri, 8 Mar 2019 19:39:11 -0800 Subject: [PATCH 2/5] add comment --- python/tvm/relay/op/_tensor_grad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tvm/relay/op/_tensor_grad.py b/python/tvm/relay/op/_tensor_grad.py index 6b318890f61b..0e796294e96c 100644 --- a/python/tvm/relay/op/_tensor_grad.py +++ b/python/tvm/relay/op/_tensor_grad.py @@ -91,5 +91,6 @@ def ones_like_grad(orig, grad): @register_gradient("collapse_sum_like") def collapse_sum_like_grad(orig, grad): + """Returns [broadcast_to_like(grad, x), 0]""" x, y = orig.args return [broadcast_to_like(grad, x), zeros_like(y)] From cbd6a359021ade7262424e36b7f9543a4ccda666 Mon Sep 17 00:00:00 2001 From: Marisa Kirisame Date: Sat, 9 Mar 2019 16:50:09 -0800 Subject: [PATCH 3/5] better name --- tests/python/relay/test_pass_gradient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/relay/test_pass_gradient.py b/tests/python/relay/test_pass_gradient.py index b5f55bf2ebce..1e3043bb8e61 100644 --- a/tests/python/relay/test_pass_gradient.py +++ b/tests/python/relay/test_pass_gradient.py @@ -190,7 +190,7 @@ def test_ref(): np.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) np.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) -def test_second_order(): +def test_square_second_order(): shape = (10, 10) dtype = 'float32' t = relay.TensorType(shape, dtype) @@ -218,4 +218,4 @@ def test_second_order(): test_tuple() test_pow() test_ref() - test_second_order() + test_square_second_order() From aa98c9af7188a714f51bafa9a24dd063fea78312 Mon Sep 17 00:00:00 2001 From: Marisa Kirisame Date: Fri, 22 Mar 2019 19:27:17 -0700 Subject: [PATCH 4/5] use tvm assert all close --- tests/python/relay/test_pass_gradient.py | 56 ++++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/python/relay/test_pass_gradient.py b/tests/python/relay/test_pass_gradient.py index 1e3043bb8e61..e9ff7799ca6a 100644 --- a/tests/python/relay/test_pass_gradient.py +++ b/tests/python/relay/test_pass_gradient.py @@ -20,8 +20,8 @@ def test_id(): ex = create_executor() x = rand(dtype, *shape) forward, (grad,) = ex.evaluate(back_func)(x) - np.testing.assert_allclose(forward.asnumpy(), x.asnumpy()) - np.testing.assert_allclose(grad.asnumpy(), np.ones_like(x.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), x.asnumpy()) + tvm.testing.assert_allclose(grad.asnumpy(), np.ones_like(x.asnumpy())) def test_add(): @@ -35,8 +35,8 @@ def test_add(): ex = create_executor() x = rand(dtype, *shape) forward, (grad,) = ex.evaluate(back_func)(x) - np.testing.assert_allclose(forward.asnumpy(), 2 * x.asnumpy()) - np.testing.assert_allclose(grad.asnumpy(), 2 * np.ones_like(x.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), 2 * x.asnumpy()) + tvm.testing.assert_allclose(grad.asnumpy(), 2 * np.ones_like(x.asnumpy())) def test_temp_add(): @@ -51,8 +51,8 @@ def test_temp_add(): ex = create_executor() x = rand(dtype, *shape) forward, (grad,) = ex.evaluate(back_func)(x) - np.testing.assert_allclose(forward.asnumpy(), 4 * x.asnumpy()) - np.testing.assert_allclose(grad.asnumpy(), 4 * np.ones_like(x.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), 4 * x.asnumpy()) + tvm.testing.assert_allclose(grad.asnumpy(), 4 * np.ones_like(x.asnumpy())) def test_sub(): @@ -66,8 +66,8 @@ def test_sub(): ex = create_executor() x = rand(dtype, *shape) forward, (grad,) = ex.evaluate(back_func)(x) - np.testing.assert_allclose(forward.asnumpy(), np.zeros_like(x.asnumpy())) - np.testing.assert_allclose(grad.asnumpy(), np.zeros_like(x.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), np.zeros_like(x.asnumpy())) + tvm.testing.assert_allclose(grad.asnumpy(), np.zeros_like(x.asnumpy())) def test_broadcast_add(): @@ -90,11 +90,11 @@ def test_broadcast_add(): relay.TupleType([t1, t2])])) ex = create_executor() forward, (grad_x, grad_y) = ex.evaluate(full_func)(x_nd, y_nd) - np.testing.assert_allclose(forward.asnumpy(), expected_forward) - np.testing.assert_allclose(grad_x.asnumpy(), - np.ones_like(expected_forward).sum(axis=2, keepdims=True)) - np.testing.assert_allclose(grad_y.asnumpy(), - np.ones_like(expected_forward).sum(axis=(0, 1), keepdims=True).squeeze(axis=0)) + tvm.testing.assert_allclose(forward.asnumpy(), expected_forward) + tvm.testing.assert_allclose(grad_x.asnumpy(), + np.ones_like(expected_forward).sum(axis=2, keepdims=True)) + tvm.testing.assert_allclose(grad_y.asnumpy(), + np.ones_like(expected_forward).sum(axis=(0, 1), keepdims=True).squeeze(axis=0)) def test_broadcast_subtract(): @@ -117,11 +117,11 @@ def test_broadcast_subtract(): relay.TupleType([t1, t2])])) ex = create_executor() forward, (grad_x, grad_y) = ex.evaluate(full_func)(x_nd, y_nd) - np.testing.assert_allclose(forward.asnumpy(), expected_forward) - np.testing.assert_allclose(grad_x.asnumpy(), - np.ones_like(expected_forward).sum(axis=2, keepdims=True)) - np.testing.assert_allclose(grad_y.asnumpy(), - -np.ones_like(expected_forward).sum(axis=(0, 1), keepdims=True).squeeze(axis=0)) + tvm.testing.assert_allclose(forward.asnumpy(), expected_forward) + tvm.testing.assert_allclose(grad_x.asnumpy(), + np.ones_like(expected_forward).sum(axis=2, keepdims=True)) + tvm.testing.assert_allclose(grad_y.asnumpy(), + -np.ones_like(expected_forward).sum(axis=(0, 1), keepdims=True).squeeze(axis=0)) def test_tuple(): @@ -147,10 +147,10 @@ def test_tuple(): expected_forward = x_np + y_np - z_np ex = create_executor() forward, (grad_x, grad_y, grad_z) = ex.evaluate(back_func)(x_nd, y_nd, z_nd) - np.testing.assert_allclose(forward.asnumpy(), expected_forward) - np.testing.assert_allclose(grad_x.asnumpy(), np.ones_like(grad_x.asnumpy())) - np.testing.assert_allclose(grad_y.asnumpy(), np.ones_like(grad_y.asnumpy())) - np.testing.assert_allclose(grad_z.asnumpy(), -1 * np.ones_like(grad_z.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), expected_forward) + tvm.testing.assert_allclose(grad_x.asnumpy(), np.ones_like(grad_x.asnumpy())) + tvm.testing.assert_allclose(grad_y.asnumpy(), np.ones_like(grad_y.asnumpy())) + tvm.testing.assert_allclose(grad_z.asnumpy(), -1 * np.ones_like(grad_z.asnumpy())) def test_pow(): @@ -168,8 +168,8 @@ def test_pow(): i_nd = rand(dtype, *shape) ex = create_executor(mod=mod) forward, (grad_i,) = ex.evaluate(back_func)(i_nd) - np.testing.assert_allclose(forward.asnumpy(), 8 * i_nd.asnumpy()) - np.testing.assert_allclose(grad_i.asnumpy(), 8 * np.ones_like(grad_i.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), 8 * i_nd.asnumpy()) + tvm.testing.assert_allclose(grad_i.asnumpy(), 8 * np.ones_like(grad_i.asnumpy())) def test_ref(): shape = (10, 10) @@ -187,8 +187,8 @@ def test_ref(): x_nd = rand(dtype, *shape) ex = create_executor() forward, (grad_x,) = ex.evaluate(back_func)(x_nd) - np.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) - np.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) + tvm.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) def test_square_second_order(): shape = (10, 10) @@ -205,8 +205,8 @@ def test_square_second_order(): x_nd = rand(dtype, *shape) ex = create_executor() forward, (grad_x,) = ex.evaluate(back_back_func)(x_nd) - np.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) - np.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) + tvm.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) + tvm.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) if __name__ == "__main__": test_id() From 84fa952dee9ffc9c29da465de1d9c32961894c6c Mon Sep 17 00:00:00 2001 From: Marisa Kirisame Date: Thu, 28 Mar 2019 11:52:10 -0700 Subject: [PATCH 5/5] refire ci --- tests/python/relay/test_pass_gradient.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/python/relay/test_pass_gradient.py b/tests/python/relay/test_pass_gradient.py index e9ff7799ca6a..690c82e5febe 100644 --- a/tests/python/relay/test_pass_gradient.py +++ b/tests/python/relay/test_pass_gradient.py @@ -171,6 +171,7 @@ def test_pow(): tvm.testing.assert_allclose(forward.asnumpy(), 8 * i_nd.asnumpy()) tvm.testing.assert_allclose(grad_i.asnumpy(), 8 * np.ones_like(grad_i.asnumpy())) + def test_ref(): shape = (10, 10) dtype = 'float32' @@ -190,6 +191,7 @@ def test_ref(): tvm.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) tvm.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) + def test_square_second_order(): shape = (10, 10) dtype = 'float32' @@ -208,6 +210,7 @@ def test_square_second_order(): tvm.testing.assert_allclose(forward.asnumpy(), 2 * x_nd.asnumpy()) tvm.testing.assert_allclose(grad_x.asnumpy(), 2 * np.ones_like(grad_x.asnumpy())) + if __name__ == "__main__": test_id() test_add()