From aa7b635d780f0aba7203b5f2bca5145b6403282f Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Wed, 1 Aug 2018 02:37:11 -0700 Subject: [PATCH 1/4] use assert_almost_equal, increase rtol, reduce matrix size --- tests/python/unittest/test_executor.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index 630cad87496d..445dc48ff426 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -18,13 +18,7 @@ import numpy as np import mxnet as mx from common import setup_module, with_seed, teardown - - -def reldiff(a, b): - diff = np.sum(np.abs(a - b)) - norm = np.sum(np.abs(a)) - reldiff = diff / norm - return reldiff +from mxnet.test_utils import assert_almost_equal def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): @@ -64,9 +58,9 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): out1 = uf(lhs_arr.asnumpy(), rhs_arr.asnumpy()) out3 = exec3.outputs[0].asnumpy() out4 = exec4.outputs[0].asnumpy() - assert reldiff(out1, out2) < 1e-6 - assert reldiff(out1, out3) < 1e-6 - assert reldiff(out1, out4) < 1e-6 + assert_almost_equal(out1, out2, rtol=1e-5, atol=1e-5) + assert_almost_equal(out1, out3, rtol=1e-5, atol=1e-5) + assert_almost_equal(out1, out4, rtol=1e-5, atol=1e-5) # test gradient out_grad = mx.nd.array(np.ones(out2.shape)) lhs_grad2, rhs_grad2 = gf(out_grad.asnumpy(), @@ -74,8 +68,8 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): rhs_arr.asnumpy()) executor.backward([out_grad]) - assert reldiff(lhs_grad.asnumpy(), lhs_grad2) < 1e-6 - assert reldiff(rhs_grad.asnumpy(), rhs_grad2) < 1e-6 + assert_almost_equal(lhs_grad.asnumpy(), lhs_grad2, rtol=1e-5, atol=1e-5) + assert_almost_equal(rhs_grad.asnumpy(), rhs_grad2, rtol=1e-5, atol=1e-5) @with_seed(0) @@ -118,12 +112,12 @@ def check_bind(disable_bulk_exec): check_bind(False) -@with_seed(0) +@with_seed() def test_dot(): nrepeat = 10 maxdim = 4 for repeat in range(nrepeat): - s =tuple(np.random.randint(1, 500, size=3)) + s =tuple(np.random.randint(1, 200, size=3)) check_bind_with_uniform(lambda x, y: np.dot(x, y), lambda g, x, y: (np.dot(g, y.T), np.dot(x.T, g)), 2, @@ -131,7 +125,7 @@ def test_dot(): rshape=(s[1], s[2]), sf = mx.symbol.dot) for repeat in range(nrepeat): - s =tuple(np.random.randint(1, 500, size=1)) + s =tuple(np.random.randint(1, 200, size=1)) check_bind_with_uniform(lambda x, y: np.dot(x, y), lambda g, x, y: (g * y, g * x), 2, From 26342b5791dfe546c8ed6c3a1326ee69ac3a8453 Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Wed, 1 Aug 2018 10:57:58 -0700 Subject: [PATCH 2/4] remove seed in test_bind --- tests/python/unittest/test_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index 445dc48ff426..43c88bacbda8 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -72,7 +72,7 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): assert_almost_equal(rhs_grad.asnumpy(), rhs_grad2, rtol=1e-5, atol=1e-5) -@with_seed(0) +@with_seed() def test_bind(): def check_bind(disable_bulk_exec): if disable_bulk_exec: From f8ba22a8eb5d880b0ab671c9811636595a460645 Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Wed, 1 Aug 2018 13:34:44 -0700 Subject: [PATCH 3/4] add seed 0 to test_bind, it is still flaky --- tests/python/unittest/test_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index 43c88bacbda8..445dc48ff426 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -72,7 +72,7 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): assert_almost_equal(rhs_grad.asnumpy(), rhs_grad2, rtol=1e-5, atol=1e-5) -@with_seed() +@with_seed(0) def test_bind(): def check_bind(disable_bulk_exec): if disable_bulk_exec: From 00922d4d9000b7aeb31e15863515704d3fbd36dd Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Wed, 1 Aug 2018 14:14:23 -0700 Subject: [PATCH 4/4] add comments for tracking --- tests/python/unittest/test_executor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index 445dc48ff426..3117f6646481 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -112,6 +112,8 @@ def check_bind(disable_bulk_exec): check_bind(False) +# @roywei: Removing fixed seed as flakiness in this test is fixed +# tracked at https://github.com/apache/incubator-mxnet/issues/11686 @with_seed() def test_dot(): nrepeat = 10