From 317d7501f317263bec1b145ab9f9ebb1d581caed Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Tue, 17 Jan 2023 21:59:41 +0300 Subject: [PATCH 01/19] add Bernoulli converter for onnx front-end --- python/tvm/relay/frontend/onnx.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/tvm/relay/frontend/onnx.py b/python/tvm/relay/frontend/onnx.py index ffd31317e9f5..13f765d40b8c 100644 --- a/python/tvm/relay/frontend/onnx.py +++ b/python/tvm/relay/frontend/onnx.py @@ -5582,6 +5582,35 @@ def _impl_v16(cls, inputs, attr, params): ) +class Bernoulli(OnnxOpConverter): + """Operator converter for Bernoulli""" + + @classmethod + def _impl_v15(cls, inputs, attr, params): + in_dtype = infer_type(inputs[0]).checked_type.dtype + assert in_dtype in [ + "float32", + "float64", + ], "Only float input tensor is currently supported." + # The data type for the elements of the output tensor. + # if not specified, we will use the data type of the input tensor + out_dtype = attr.get("dtype", None) + if out_dtype is None: + out_dtype = in_dtype + else: + out_dtype = get_type(out_dtype) + + seed = attr.get("seed", None) + if seed is None: + seed = np.random.randint(1e6) + else: + seed = int(seed) + + key = _random.threefry_key(seed) + _, uniform_nums = _op.random.uniform(key, infer_shape(inputs[0]), in_dtype) + return _op.cast(_op.less(uniform_nums, inputs[0]), out_dtype) + + class RandomNormal(OnnxOpConverter): """Operator converter for random_normal""" @@ -6348,6 +6377,7 @@ def _get_convert_map(opset): "QLinearGlobalAveragePool": QLinearGlobalAveragePool.get_converter(opset), "QLinearLeakyRelu": QLinearLeakyRelu.get_converter(opset), # Random number generation. + "Bernoulli": Bernoulli.get_converter(opset), "RandomNormal": RandomNormal.get_converter(opset), "RandomNormalLike": RandomNormalLike.get_converter(opset), "RandomUniform": RandomUniform.get_converter(opset), From 4fd035a1a077bd0df3e10476e81e957373a9866b Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 18 Jan 2023 21:11:29 +0300 Subject: [PATCH 02/19] test for bernoulli was implemented --- tests/python/frontend/onnx/test_forward.py | 95 ++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index f5b5f7c65cb5..7de524f74abd 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6707,6 +6707,101 @@ def verify_qlinearsigmoid(a_shape): verify_qlinearsigmoid([]) +@tvm.testing.parametrize_targets("llvm") +def test_random_bernoulli(target, dev): + """test_random_bernoulli""" + + def verify_bernoulli_with_ort( + shape, + in_dtype="float32", + out_dtype="int32", + seed=None, + out_shape=None, + target=target, + dev=dev, + use_vm=False, + opset=None, + freeze_params=False, + rtol=0.1, + atol=0.1, + opt_level=1, + convert_config=None, + ): + def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None): + onnx_itype = mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(in_dtype)] + onnx_otype = mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(out_dtype)] + node = helper.make_node( + "Bernoulli", + ["input"], + ["output"], + ) + dtype_attr = helper.make_attribute("dtype", onnx_otype) + node.attribute.append(dtype_attr) + if seed is not None: + seed_attr = helper.make_attribute("seed", seed) + node.attribute.append(seed_attr) + + graph = helper.make_graph( + [node], + "random_bernoulli_test", + inputs=[helper.make_tensor_value_info("input", onnx_itype, list(shape))], + outputs=[helper.make_tensor_value_info("output", onnx_otype, list(shape))], + ) + return helper.make_model(graph, producer_name="random_bernoulli_test") + + inputs = np.random.uniform(size=shape).astype(in_dtype) + model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) + if opset is not None: + model.opset_import[0].version = opset + + ort_out = get_onnxruntime_output(model, inputs) + if use_vm: + tvm_out = get_tvm_output_with_vm( + model, + inputs, + target, + dev, + opset=opset, + freeze_params=freeze_params, + convert_config=convert_config, + ) + else: + tvm_out = get_tvm_output( + model, + inputs, + target, + dev, + out_shape, + opset=opset, + opt_level=opt_level, + convert_config=convert_config, + ) + + if not isinstance(tvm_out, list): + tvm_out = [tvm_out] + if not isinstance(ort_out, list): + ort_out = [ort_out] + for tvm_val, ort_val in zip(tvm_out, ort_out): + tvm.testing.assert_allclose(ort_val.mean(), tvm_val.mean(), rtol=rtol, atol=atol) + tvm.testing.assert_allclose(np.std(ort_val), np.std(tvm_val), rtol=rtol, atol=atol) + assert ort_val.dtype == tvm_val.dtype + + # Simple test + verify_bernoulli_with_ort([100]) + + # Floating output type + verify_bernoulli_with_ort([100], out_dtype="float32") + + # FDouble input type + verify_bernoulli_with_ort([100], in_dtype="float64") + + # Test N-D tensor generation + verify_bernoulli_with_ort([2, 4, 100, 100]) + + # Test with seed + verify_bernoulli_with_ort([100], seed=np.random.randint(1e6)) + + @tvm.testing.parametrize_targets("llvm") def test_random_uniform(target, dev): """test_random_uniform""" From 0b099b0829083c0de6d082da4abfb57eb0556866 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Mon, 23 Jan 2023 10:37:34 +0300 Subject: [PATCH 03/19] fix tuple split. update test for stability with different seed on ort and tvm sides --- python/tvm/relay/frontend/onnx.py | 3 ++- tests/python/frontend/onnx/test_forward.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/python/tvm/relay/frontend/onnx.py b/python/tvm/relay/frontend/onnx.py index 13f765d40b8c..98c9a34b4089 100644 --- a/python/tvm/relay/frontend/onnx.py +++ b/python/tvm/relay/frontend/onnx.py @@ -5607,7 +5607,8 @@ def _impl_v15(cls, inputs, attr, params): seed = int(seed) key = _random.threefry_key(seed) - _, uniform_nums = _op.random.uniform(key, infer_shape(inputs[0]), in_dtype) + inter_outputs = _op.random.uniform(key, infer_shape(inputs[0]), in_dtype) + _, uniform_nums = _expr.TupleWrapper(inter_outputs, 2) return _op.cast(_op.less(uniform_nums, inputs[0]), out_dtype) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 7de524f74abd..5fc19ed7945e 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6750,7 +6750,11 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) return helper.make_model(graph, producer_name="random_bernoulli_test") inputs = np.random.uniform(size=shape).astype(in_dtype) - model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) + if seed is None: + ort_seed = None + else: + ort_seed = float(seed) + model = get_bernoulli_model(shape, in_dtype, out_dtype, ort_seed) if opset is not None: model.opset_import[0].version = opset @@ -6787,19 +6791,19 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) assert ort_val.dtype == tvm_val.dtype # Simple test - verify_bernoulli_with_ort([100]) + verify_bernoulli_with_ort([1000]) # Floating output type - verify_bernoulli_with_ort([100], out_dtype="float32") + verify_bernoulli_with_ort([1000], out_dtype="float32") - # FDouble input type - verify_bernoulli_with_ort([100], in_dtype="float64") + # Double input type + verify_bernoulli_with_ort([1000], in_dtype="float64") # Test N-D tensor generation verify_bernoulli_with_ort([2, 4, 100, 100]) # Test with seed - verify_bernoulli_with_ort([100], seed=np.random.randint(1e6)) + verify_bernoulli_with_ort([1000], seed=np.random.randint(1e6)) @tvm.testing.parametrize_targets("llvm") From d326ce69b21c8f2c9d01231ee28457b45eb3bff3 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 09:15:22 +0300 Subject: [PATCH 04/19] check that output values are 0 or 1 --- tests/python/frontend/onnx/test_forward.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 5fc19ed7945e..833e5fe90e3c 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6785,6 +6785,11 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) tvm_out = [tvm_out] if not isinstance(ort_out, list): ort_out = [ort_out] + # check that values are 0 or 1 + for tvm_val in tvm_out: + tvm_flat_val = tvm_val.flatten() + for i in range(len(tvm_flat_val)): + assert tvm_flat_val[i] == 0 or tvm_flat_val[i] == 1 for tvm_val, ort_val in zip(tvm_out, ort_out): tvm.testing.assert_allclose(ort_val.mean(), tvm_val.mean(), rtol=rtol, atol=atol) tvm.testing.assert_allclose(np.std(ort_val), np.std(tvm_val), rtol=rtol, atol=atol) From 09b79cc956e4734593b25777c9704ebbe7c2054b Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 09:16:16 +0300 Subject: [PATCH 05/19] remove std check as meaningless --- tests/python/frontend/onnx/test_forward.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 833e5fe90e3c..2c8598be68c2 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6792,7 +6792,6 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) assert tvm_flat_val[i] == 0 or tvm_flat_val[i] == 1 for tvm_val, ort_val in zip(tvm_out, ort_out): tvm.testing.assert_allclose(ort_val.mean(), tvm_val.mean(), rtol=rtol, atol=atol) - tvm.testing.assert_allclose(np.std(ort_val), np.std(tvm_val), rtol=rtol, atol=atol) assert ort_val.dtype == tvm_val.dtype # Simple test From f8e7b0df05b6f48e3adbceb32e9875a591bfdf31 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 09:54:50 +0300 Subject: [PATCH 06/19] calculate theoretical mean and compare with result, remove ort for comparison. clean code --- tests/python/frontend/onnx/test_forward.py | 39 +++++++--------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 2c8598be68c2..f39f86bb67de 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6711,7 +6711,7 @@ def verify_qlinearsigmoid(a_shape): def test_random_bernoulli(target, dev): """test_random_bernoulli""" - def verify_bernoulli_with_ort( + def verify_bernoulli( shape, in_dtype="float32", out_dtype="int32", @@ -6720,12 +6720,10 @@ def verify_bernoulli_with_ort( target=target, dev=dev, use_vm=False, - opset=None, freeze_params=False, rtol=0.1, atol=0.1, opt_level=1, - convert_config=None, ): def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None): onnx_itype = mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(in_dtype)] @@ -6738,7 +6736,7 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) dtype_attr = helper.make_attribute("dtype", onnx_otype) node.attribute.append(dtype_attr) if seed is not None: - seed_attr = helper.make_attribute("seed", seed) + seed_attr = helper.make_attribute("seed", float(seed)) node.attribute.append(seed_attr) graph = helper.make_graph( @@ -6750,24 +6748,15 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) return helper.make_model(graph, producer_name="random_bernoulli_test") inputs = np.random.uniform(size=shape).astype(in_dtype) - if seed is None: - ort_seed = None - else: - ort_seed = float(seed) - model = get_bernoulli_model(shape, in_dtype, out_dtype, ort_seed) - if opset is not None: - model.opset_import[0].version = opset + model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) - ort_out = get_onnxruntime_output(model, inputs) if use_vm: tvm_out = get_tvm_output_with_vm( model, inputs, target, dev, - opset=opset, freeze_params=freeze_params, - convert_config=convert_config, ) else: tvm_out = get_tvm_output( @@ -6776,38 +6765,34 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) target, dev, out_shape, - opset=opset, opt_level=opt_level, - convert_config=convert_config, ) if not isinstance(tvm_out, list): tvm_out = [tvm_out] - if not isinstance(ort_out, list): - ort_out = [ort_out] - # check that values are 0 or 1 + ideal_mean = np.sum(inputs) for tvm_val in tvm_out: + # check that values are 0 or 1 tvm_flat_val = tvm_val.flatten() for i in range(len(tvm_flat_val)): assert tvm_flat_val[i] == 0 or tvm_flat_val[i] == 1 - for tvm_val, ort_val in zip(tvm_out, ort_out): - tvm.testing.assert_allclose(ort_val.mean(), tvm_val.mean(), rtol=rtol, atol=atol) - assert ort_val.dtype == tvm_val.dtype + # check that mean value is close to the theoretical one + tvm.testing.assert_allclose(ideal_mean, tvm_val.mean(), rtol=rtol, atol=atol) # Simple test - verify_bernoulli_with_ort([1000]) + verify_bernoulli([1000]) # Floating output type - verify_bernoulli_with_ort([1000], out_dtype="float32") + verify_bernoulli([1000], out_dtype="float32") # Double input type - verify_bernoulli_with_ort([1000], in_dtype="float64") + verify_bernoulli([1000], in_dtype="float64") # Test N-D tensor generation - verify_bernoulli_with_ort([2, 4, 100, 100]) + verify_bernoulli([2, 4, 100, 100]) # Test with seed - verify_bernoulli_with_ort([1000], seed=np.random.randint(1e6)) + verify_bernoulli([1000], seed=np.random.randint(1e6)) @tvm.testing.parametrize_targets("llvm") From 8962ab7dfdfda57547246019fdfe48b66eeef062 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 10:06:52 +0300 Subject: [PATCH 07/19] add customized input as arg --- tests/python/frontend/onnx/test_forward.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index f39f86bb67de..b79c1e3fbae0 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6712,18 +6712,17 @@ def test_random_bernoulli(target, dev): """test_random_bernoulli""" def verify_bernoulli( - shape, + inputs=None, + shape=[], in_dtype="float32", out_dtype="int32", seed=None, - out_shape=None, target=target, dev=dev, use_vm=False, freeze_params=False, rtol=0.1, atol=0.1, - opt_level=1, ): def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None): onnx_itype = mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(in_dtype)] @@ -6747,7 +6746,12 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) ) return helper.make_model(graph, producer_name="random_bernoulli_test") - inputs = np.random.uniform(size=shape).astype(in_dtype) + if inputs is None: + assert len(shape) != 0 + inputs = np.random.uniform(size=shape).astype(in_dtype) + else: + shape = inputs.shape + in_dtype=inputs.dtype model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) if use_vm: @@ -6764,8 +6768,6 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) inputs, target, dev, - out_shape, - opt_level=opt_level, ) if not isinstance(tvm_out, list): From dd99e6f2ac652ce1c3e6bcfaefec2589f78971db Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 10:15:26 +0300 Subject: [PATCH 08/19] add test with input sequence of 0 and 1 --- tests/python/frontend/onnx/test_forward.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index b79c1e3fbae0..67a2a6356101 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6723,6 +6723,7 @@ def verify_bernoulli( freeze_params=False, rtol=0.1, atol=0.1, + in_out_equal=False, ): def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None): onnx_itype = mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(in_dtype)] @@ -6778,8 +6779,11 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) tvm_flat_val = tvm_val.flatten() for i in range(len(tvm_flat_val)): assert tvm_flat_val[i] == 0 or tvm_flat_val[i] == 1 - # check that mean value is close to the theoretical one - tvm.testing.assert_allclose(ideal_mean, tvm_val.mean(), rtol=rtol, atol=atol) + if in_out_equal: + tvm.testing.assert_allclose(inputs, tvm_val) + else: + # check that mean value is close to the theoretical one + tvm.testing.assert_allclose(ideal_mean, tvm_val.mean(), rtol=rtol, atol=atol) # Simple test verify_bernoulli([1000]) @@ -6796,6 +6800,10 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) # Test with seed verify_bernoulli([1000], seed=np.random.randint(1e6)) + # Test input sequence of 0 and 1 + inputs = np.random.randint(2, size=[2, 4, 100, 100]) + verify_bernoulli(inputs, in_out_equal=True) + @tvm.testing.parametrize_targets("llvm") def test_random_uniform(target, dev): From 5a4724f90c81ecfa9aec487e77cb8e1334da4810 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 10:33:38 +0300 Subject: [PATCH 09/19] pylint fix --- tests/python/frontend/onnx/test_forward.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 67a2a6356101..c5c4e4a876f6 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6752,7 +6752,7 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) inputs = np.random.uniform(size=shape).astype(in_dtype) else: shape = inputs.shape - in_dtype=inputs.dtype + in_dtype = inputs.dtype model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) if use_vm: From fa105a60471ab2b44f6f4800e30d6781fbe523f0 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 13:36:57 +0300 Subject: [PATCH 10/19] fix inputs-shape issue --- tests/python/frontend/onnx/test_forward.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index c5c4e4a876f6..a7458055f7d7 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6786,19 +6786,19 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) tvm.testing.assert_allclose(ideal_mean, tvm_val.mean(), rtol=rtol, atol=atol) # Simple test - verify_bernoulli([1000]) + verify_bernoulli(shape=[1000]) # Floating output type - verify_bernoulli([1000], out_dtype="float32") + verify_bernoulli(shape=[1000], out_dtype="float32") # Double input type - verify_bernoulli([1000], in_dtype="float64") + verify_bernoulli(shape=[1000], in_dtype="float64") # Test N-D tensor generation - verify_bernoulli([2, 4, 100, 100]) + verify_bernoulli(shape=[2, 4, 100, 100]) # Test with seed - verify_bernoulli([1000], seed=np.random.randint(1e6)) + verify_bernoulli(shape=[1000], seed=np.random.randint(1e6)) # Test input sequence of 0 and 1 inputs = np.random.randint(2, size=[2, 4, 100, 100]) From e99d6c941ed6e23f43fb96d7e2d5315ac5d3f6f6 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 15:10:55 +0300 Subject: [PATCH 11/19] add binomial test --- tests/python/frontend/onnx/test_forward.py | 47 ++++++++++++++-------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index a7458055f7d7..3c5c51aec7f3 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6771,19 +6771,36 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) dev, ) - if not isinstance(tvm_out, list): - tvm_out = [tvm_out] - ideal_mean = np.sum(inputs) - for tvm_val in tvm_out: - # check that values are 0 or 1 - tvm_flat_val = tvm_val.flatten() - for i in range(len(tvm_flat_val)): - assert tvm_flat_val[i] == 0 or tvm_flat_val[i] == 1 - if in_out_equal: - tvm.testing.assert_allclose(inputs, tvm_val) - else: - # check that mean value is close to the theoretical one - tvm.testing.assert_allclose(ideal_mean, tvm_val.mean(), rtol=rtol, atol=atol) + if isinstance(tvm_out, list): + tvm_out = tvm_out[0] + ideal_mean = np.mean(inputs) + # check that values are 0 or 1 + tvm_flat = tvm_out.flatten() + for i in range(len(tvm_flat)): + assert tvm_flat[i] == 0 or tvm_flat[i] == 1 + if in_out_equal: + tvm.testing.assert_allclose(inputs, tvm_out) + else: + # check that mean value is close to the theoretical one by binomial test + bnm_test_res = scipy.stats.binomtest( + int(k=np.sum(tvm_flat)), n=len(tvm_flat), p=ideal_mean + ) + assert bnm_test_res.pvalue >= 1e-6 + + # Test input sequence of 0 and 1 + inputs = np.random.randint(2, size=[10000]) + verify_bernoulli(inputs, in_out_equal=True) + + # Binomial test input with 0.5 values + val_num = 10000 + arr = [0.5] * val_num + inputs = np.array(arr) + verify_bernoulli(inputs) + + # Binomial test input with 0.1 values + arr = [0.1] * val_num + inputs = np.array(arr) + verify_bernoulli(inputs) # Simple test verify_bernoulli(shape=[1000]) @@ -6800,10 +6817,6 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) # Test with seed verify_bernoulli(shape=[1000], seed=np.random.randint(1e6)) - # Test input sequence of 0 and 1 - inputs = np.random.randint(2, size=[2, 4, 100, 100]) - verify_bernoulli(inputs, in_out_equal=True) - @tvm.testing.parametrize_targets("llvm") def test_random_uniform(target, dev): From 863cecba3f2ecbb960f531938eb3ce62887c01bb Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 16:51:34 +0300 Subject: [PATCH 12/19] fix input type --- tests/python/frontend/onnx/test_forward.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 3c5c51aec7f3..e29e8a11f663 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6788,18 +6788,18 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) assert bnm_test_res.pvalue >= 1e-6 # Test input sequence of 0 and 1 - inputs = np.random.randint(2, size=[10000]) + inputs = np.random.randint(2, size=[10000]).astype("float32") verify_bernoulli(inputs, in_out_equal=True) # Binomial test input with 0.5 values val_num = 10000 arr = [0.5] * val_num - inputs = np.array(arr) + inputs = np.array(arr).astype("float32") verify_bernoulli(inputs) # Binomial test input with 0.1 values arr = [0.1] * val_num - inputs = np.array(arr) + inputs = np.array(arr).astype("float32") verify_bernoulli(inputs) # Simple test From 53d66baecc600f24e681830dfac9ffd3430e1c04 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Wed, 25 Jan 2023 19:30:45 +0300 Subject: [PATCH 13/19] small fix --- tests/python/frontend/onnx/test_forward.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index e29e8a11f663..7012429ce9be 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6783,7 +6783,7 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) else: # check that mean value is close to the theoretical one by binomial test bnm_test_res = scipy.stats.binomtest( - int(k=np.sum(tvm_flat)), n=len(tvm_flat), p=ideal_mean + k=np.sum(tvm_flat, dtype="int32"), n=len(tvm_flat), p=ideal_mean ) assert bnm_test_res.pvalue >= 1e-6 From 6314fa34e3e61f02e19264ebdb73039860d4a0dd Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Fri, 27 Jan 2023 08:44:07 +0300 Subject: [PATCH 14/19] update 0-1 check --- tests/python/frontend/onnx/test_forward.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 7012429ce9be..b033ab537f15 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6776,8 +6776,7 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) ideal_mean = np.mean(inputs) # check that values are 0 or 1 tvm_flat = tvm_out.flatten() - for i in range(len(tvm_flat)): - assert tvm_flat[i] == 0 or tvm_flat[i] == 1 + assert np.array_equal(tvm_flat, tvm_flat.astype("bool")) if in_out_equal: tvm.testing.assert_allclose(inputs, tvm_out) else: From 163e1013f7dbaa2b2db7c46ec70adc26d2e8e541 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Fri, 27 Jan 2023 08:45:56 +0300 Subject: [PATCH 15/19] init arrays in numpy style --- tests/python/frontend/onnx/test_forward.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index b033ab537f15..88bbf6b6279b 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6792,13 +6792,11 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) # Binomial test input with 0.5 values val_num = 10000 - arr = [0.5] * val_num - inputs = np.array(arr).astype("float32") + inputs = np.ones([val_num], dtype="float32") * 0.5 verify_bernoulli(inputs) # Binomial test input with 0.1 values - arr = [0.1] * val_num - inputs = np.array(arr).astype("float32") + inputs = np.ones([val_num], dtype="float32") * 0.1 verify_bernoulli(inputs) # Simple test From c19fc4926af09ea599a1dfba25d47d08b05fc0d6 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Fri, 27 Jan 2023 09:01:39 +0300 Subject: [PATCH 16/19] check result determinism for fixed seed --- tests/python/frontend/onnx/test_forward.py | 48 +++++++++++++++++----- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 88bbf6b6279b..3c9eee2a7ccd 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6711,7 +6711,7 @@ def verify_qlinearsigmoid(a_shape): def test_random_bernoulli(target, dev): """test_random_bernoulli""" - def verify_bernoulli( + def _get_tvm_output( inputs=None, shape=[], in_dtype="float32", @@ -6721,9 +6721,6 @@ def verify_bernoulli( dev=dev, use_vm=False, freeze_params=False, - rtol=0.1, - atol=0.1, - in_out_equal=False, ): def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None): onnx_itype = mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype(in_dtype)] @@ -6756,7 +6753,7 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) if use_vm: - tvm_out = get_tvm_output_with_vm( + return get_tvm_output_with_vm( model, inputs, target, @@ -6764,13 +6761,37 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) freeze_params=freeze_params, ) else: - tvm_out = get_tvm_output( + return get_tvm_output( model, inputs, target, dev, ) + def verify_bernoulli( + inputs=None, + shape=[], + in_dtype="float32", + out_dtype="int32", + seed=None, + target=target, + dev=dev, + use_vm=False, + freeze_params=False, + in_out_equal=False, + ): + tvm_out = _get_tvm_output( + inputs, + shape, + in_dtype, + out_dtype, + seed, + target, + dev, + use_vm, + freeze_params, + ) + if isinstance(tvm_out, list): tvm_out = tvm_out[0] ideal_mean = np.mean(inputs) @@ -6800,19 +6821,26 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) verify_bernoulli(inputs) # Simple test - verify_bernoulli(shape=[1000]) + verify_bernoulli(shape=[val_num]) # Floating output type - verify_bernoulli(shape=[1000], out_dtype="float32") + verify_bernoulli(shape=[val_num], out_dtype="float32") # Double input type - verify_bernoulli(shape=[1000], in_dtype="float64") + verify_bernoulli(shape=[val_num], in_dtype="float64") # Test N-D tensor generation verify_bernoulli(shape=[2, 4, 100, 100]) # Test with seed - verify_bernoulli(shape=[1000], seed=np.random.randint(1e6)) + verify_bernoulli(shape=[val_num], seed=np.random.randint(1e6)) + + # Test result determinism with the same seeds + inputs = np.random.uniform(size=[val_num]) + fixed_seed=np.random.randint(1e6) + tvm_out_1 = _get_tvm_output(inputs, seed=fixed_seed) + tvm_out_2 = _get_tvm_output(inputs, seed=fixed_seed) + tvm.testing.assert_allclose(tvm_out_1, tvm_out_2) @tvm.testing.parametrize_targets("llvm") From 00295c9154dc22e76c49a00795124bffc081f0c8 Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Fri, 27 Jan 2023 10:46:03 +0300 Subject: [PATCH 17/19] fix inputs issue --- tests/python/frontend/onnx/test_forward.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 3c9eee2a7ccd..5fa5e43509cf 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6712,9 +6712,7 @@ def test_random_bernoulli(target, dev): """test_random_bernoulli""" def _get_tvm_output( - inputs=None, - shape=[], - in_dtype="float32", + inputs, out_dtype="int32", seed=None, target=target, @@ -6744,12 +6742,8 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) ) return helper.make_model(graph, producer_name="random_bernoulli_test") - if inputs is None: - assert len(shape) != 0 - inputs = np.random.uniform(size=shape).astype(in_dtype) - else: - shape = inputs.shape - in_dtype = inputs.dtype + shape = inputs.shape + in_dtype = inputs.dtype model = get_bernoulli_model(shape, in_dtype, out_dtype, seed) if use_vm: @@ -6780,10 +6774,12 @@ def verify_bernoulli( freeze_params=False, in_out_equal=False, ): + if inputs is None: + assert len(shape) != 0 + inputs = np.random.uniform(size=shape).astype(in_dtype) + tvm_out = _get_tvm_output( inputs, - shape, - in_dtype, out_dtype, seed, target, @@ -6837,7 +6833,7 @@ def verify_bernoulli( # Test result determinism with the same seeds inputs = np.random.uniform(size=[val_num]) - fixed_seed=np.random.randint(1e6) + fixed_seed = np.random.randint(1e6) tvm_out_1 = _get_tvm_output(inputs, seed=fixed_seed) tvm_out_2 = _get_tvm_output(inputs, seed=fixed_seed) tvm.testing.assert_allclose(tvm_out_1, tvm_out_2) From e91364b6845c53f950c5c9c8b5da18239721c7ce Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Fri, 27 Jan 2023 12:13:29 +0300 Subject: [PATCH 18/19] modify binomial test --- tests/python/frontend/onnx/test_forward.py | 37 +++++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 5fa5e43509cf..54e2a23be81f 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6762,6 +6762,18 @@ def get_bernoulli_model(shape, in_dtype="float32", out_dtype="int32", seed=None) dev, ) + def binom_test(input, ideal_mean, threshold=0.05): + # This test is strictly appropriate when input probabilities are all identical. + # In that case, it should lead to flaky failures in only one run in a million (p>=1e-6). + # The test should be over-conservative when input probabilities are not identical. + # (i.e., It should have a rate of flaky failures lower than one run in a million.) + # If this test starts repeatedly throwing flaky failures, consult a statistician + # in addition to your regular debugging. + bnm_test_res = scipy.stats.binomtest( + k=np.sum(input, dtype="int32"), n=len(input), p=ideal_mean + ) + return bnm_test_res.pvalue > threshold + def verify_bernoulli( inputs=None, shape=[], @@ -6790,7 +6802,6 @@ def verify_bernoulli( if isinstance(tvm_out, list): tvm_out = tvm_out[0] - ideal_mean = np.mean(inputs) # check that values are 0 or 1 tvm_flat = tvm_out.flatten() assert np.array_equal(tvm_flat, tvm_flat.astype("bool")) @@ -6798,10 +6809,26 @@ def verify_bernoulli( tvm.testing.assert_allclose(inputs, tvm_out) else: # check that mean value is close to the theoretical one by binomial test - bnm_test_res = scipy.stats.binomtest( - k=np.sum(tvm_flat, dtype="int32"), n=len(tvm_flat), p=ideal_mean - ) - assert bnm_test_res.pvalue >= 1e-6 + ideal_mean = np.mean(inputs) + repeats = 3 + check = False + for i in range(repeats): + if binom_test(tvm_flat, ideal_mean): + check = True + break + else: + # repeat with new seed + seed = np.random.randint(1e6) + tvm_flat = _get_tvm_output( + inputs, + out_dtype, + seed, + target, + dev, + use_vm, + freeze_params, + ).flatten() + assert check, "Binomial test failed" # Test input sequence of 0 and 1 inputs = np.random.randint(2, size=[10000]).astype("float32") From 3c85a2e351d173b2a555c815656b8177442972eb Mon Sep 17 00:00:00 2001 From: Valery Chernov Date: Fri, 27 Jan 2023 13:09:46 +0300 Subject: [PATCH 19/19] pylint fix --- tests/python/frontend/onnx/test_forward.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 54e2a23be81f..559587e2a597 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -6820,14 +6820,14 @@ def verify_bernoulli( # repeat with new seed seed = np.random.randint(1e6) tvm_flat = _get_tvm_output( - inputs, - out_dtype, - seed, - target, - dev, - use_vm, - freeze_params, - ).flatten() + inputs, + out_dtype, + seed, + target, + dev, + use_vm, + freeze_params, + ).flatten() assert check, "Binomial test failed" # Test input sequence of 0 and 1