From ab4adc6e4c6742b942a5e9ca521dc19dc7e04f78 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 08:55:15 +0900
Subject: [PATCH 01/13] support strides in onnx slice op

---
 python/tvm/relay/frontend/onnx.py          | 26 ++++++++++++++----
 tests/python/frontend/onnx/test_forward.py | 32 +++++++++++++++++++---
 2 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/python/tvm/relay/frontend/onnx.py b/python/tvm/relay/frontend/onnx.py
index 5f31724ec3ea..74ac74e67620 100644
--- a/python/tvm/relay/frontend/onnx.py
+++ b/python/tvm/relay/frontend/onnx.py
@@ -1024,6 +1024,9 @@ def _impl_v10(cls, inputs, attr, params):
         attrs = {"starts": inputs[1], "ends": inputs[2]}
         if len(inputs) >= 4:
             attrs["axes"] = inputs[3]
+        if len(inputs) >= 5:
+            attrs["steps"] = inputs[4]
+
         attrs = {k: (v, get_name(v)) for (k, v) in attrs.items()}
         attrs = {
             k: params[v[1]].asnumpy()
@@ -1033,12 +1036,23 @@ def _impl_v10(cls, inputs, attr, params):
         }
 
         # Update the starts and ends according to axes if required.
-        if "axes" in attrs:
-            if max(attrs["axes"] + 1) != len(attrs["axes"]):
-                new_starts, new_ends, _ = cls._common(attrs["starts"], attrs["ends"], attrs["axes"])
-                attrs["starts"] = new_starts
-                attrs["ends"] = new_ends
-        return _op.strided_slice(inputs[0], begin=list(attrs["starts"]), end=list(attrs["ends"]))
+        if "axes" in attrs and max(attrs["axes"] + 1) != len(attrs["axes"]):
+            new_starts, new_ends, _ = cls._common(attrs["starts"], attrs["ends"], attrs["axes"])
+            attrs["starts"] = new_starts
+            attrs["ends"] = new_ends
+
+        begins = list(attrs["starts"])
+        ends = list(attrs["ends"])
+        strides = [1] * len(begins)
+
+        if "steps" in attrs:
+            steps = list(attrs["steps"])
+            axes = attrs["axes"]
+            assert len(steps) == len(axes)
+            for axis, step in zip(axes, steps):
+                strides[axis] = step
+
+        return _op.strided_slice(inputs[0], begin=begins, end=ends, strides=strides)
 
 
 class Gather(OnnxOpConverter):
diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index 894a6b6d40ce..c786b351484a 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -14,6 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import io
 import numpy as np
 import math
 import onnx
@@ -525,8 +526,7 @@ def _test_slice_iteration_v1(indata, outdata, starts, ends, axes=None):
 
     for target, ctx in tvm.testing.enabled_targets():
         tvm_out = get_tvm_output(model, indata, target, ctx, outdata.shape, "float32", opset=1)
-
-    tvm.testing.assert_allclose(outdata, tvm_out)
+        tvm.testing.assert_allclose(outdata, tvm_out)
 
 
 def _test_slice_iteration_v10(indata, outdata, **attrs):
@@ -616,8 +616,7 @@ def add_noop_to_input_attr(attr_name, attr):
 
     for target, ctx in tvm.testing.enabled_targets():
         tvm_out = get_tvm_output(model, indata, target, ctx, outdata.shape, "float32", opset=10)
-
-    tvm.testing.assert_allclose(outdata, tvm_out)
+        tvm.testing.assert_allclose(outdata, tvm_out)
 
 
 @tvm.testing.uses_gpu
@@ -681,6 +680,31 @@ def test_slice():
         x, x, starts=(0, 0), ends=(9223372036854775807, 9223372036854775807), axes=(0, 3)
     )
 
+    def test_slice_with_strides():
+        class SliceWithStrides(torch.nn.Module):
+            def forward(self, x):
+                return x[..., 0::2] + x[..., 1::2]
+
+        onnx_io = io.BytesIO()
+        torch.onnx.export(
+            SliceWithStrides(),
+            (torch.randn(1, 4),),
+            onnx_io,
+            input_names=["x"],
+            output_names=["y"],
+            opset_version=10,
+            enable_onnx_checker=True,
+        )
+        model = onnx.load_model_from_string(onnx_io.getvalue())
+
+        for target, ctx in tvm.testing.enabled_targets():
+            x = np.random.uniform(size=(1, 4)).astype("float32")
+            tvm_out = get_tvm_output(model, x, target, ctx, (1, 2), "float32")
+            onnx_out = get_onnxruntime_output(model, x, "float32")
+            tvm.testing.assert_allclose(onnx_out, tvm_out)
+
+    test_slice_with_strides()
+
 
 def _test_onnx_op_elementwise(inshape, outfunc, npargs, dtype, opname, kwargs):
     indata = np.random.uniform(-1, 1, size=inshape).astype(dtype)

From 75bc5eb0c452e516633e217e13842c33750a17ef Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 09:08:04 +0900
Subject: [PATCH 02/13] refactor test

---
 tests/python/frontend/onnx/test_forward.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index c786b351484a..3a840c5f13ec 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -112,10 +112,13 @@ def get_onnxruntime_output(model, inputs, dtype="float32"):
     return ort_out
 
 
-def verify_onnx_forward_impl(graph_file, data_shape, out_shape):
+def verify_onnx_forward_impl(graph_file_or_bytes, data_shape, out_shape):
     dtype = "float32"
-    x = np.random.uniform(size=data_shape)
-    model = onnx.load_model(graph_file)
+    x = np.random.uniform(size=data_shape).astype(dtype)
+    if isinstance(graph_file_or_bytes, bytes):
+        model = onnx.load_model_from_string(graph_file_or_bytes)
+    else:
+        model = onnx.load_model(graph_file_or_bytes)
     c2_out = get_onnxruntime_output(model, x, dtype)
     for target, ctx in tvm.testing.enabled_targets():
         tvm_out = get_tvm_output(model, x, target, ctx, out_shape, dtype)
@@ -695,13 +698,8 @@ def forward(self, x):
             opset_version=10,
             enable_onnx_checker=True,
         )
-        model = onnx.load_model_from_string(onnx_io.getvalue())
 
-        for target, ctx in tvm.testing.enabled_targets():
-            x = np.random.uniform(size=(1, 4)).astype("float32")
-            tvm_out = get_tvm_output(model, x, target, ctx, (1, 2), "float32")
-            onnx_out = get_onnxruntime_output(model, x, "float32")
-            tvm.testing.assert_allclose(onnx_out, tvm_out)
+        verify_onnx_forward_impl(onnx_io.getvalue(), (1, 4), (1, 2))
 
     test_slice_with_strides()
 

From 659c28c0edf766536d13db23faa17297db5b876d Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 09:29:48 +0900
Subject: [PATCH 03/13] more refactoring

---
 tests/python/frontend/onnx/test_forward.py | 68 ++++++----------------
 1 file changed, 19 insertions(+), 49 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index 3a840c5f13ec..ee272bc1ce52 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -112,17 +112,13 @@ def get_onnxruntime_output(model, inputs, dtype="float32"):
     return ort_out
 
 
-def verify_onnx_forward_impl(graph_file_or_bytes, data_shape, out_shape):
+def verify_onnx_forward_impl(model, data_shape, out_shape=None, rtol=1e-5, atol=1e-5):
     dtype = "float32"
     x = np.random.uniform(size=data_shape).astype(dtype)
-    if isinstance(graph_file_or_bytes, bytes):
-        model = onnx.load_model_from_string(graph_file_or_bytes)
-    else:
-        model = onnx.load_model(graph_file_or_bytes)
-    c2_out = get_onnxruntime_output(model, x, dtype)
+    ort_out = get_onnxruntime_output(model, x, dtype)
     for target, ctx in tvm.testing.enabled_targets():
         tvm_out = get_tvm_output(model, x, target, ctx, out_shape, dtype)
-        tvm.testing.assert_allclose(c2_out, tvm_out, rtol=1e-5, atol=1e-5)
+        tvm.testing.assert_allclose(ort_out, tvm_out, rtol=rtol, atol=atol)
 
 
 def make_constant_node(name, data_type, dims, vals):
@@ -165,8 +161,7 @@ def test_reshape():
     for target, ctx in tvm.testing.enabled_targets():
         x = np.random.uniform(size=in_shape).astype("int32")
         tvm_out = get_tvm_output(model, x, target, ctx, ref_shape, "float32")
-
-    tvm.testing.assert_allclose(ref_shape, tvm_out.shape)
+        tvm.testing.assert_allclose(ref_shape, tvm_out.shape)
 
 
 @tvm.testing.uses_gpu
@@ -197,8 +192,7 @@ def _test_expand(name, data, shape, ref_data):
 
         for target, ctx in tvm.testing.enabled_targets():
             tvm_out = get_tvm_output(model, data, target, ctx, ref_data.shape, "float32")
-
-        tvm.testing.assert_allclose(ref_data, tvm_out)
+            tvm.testing.assert_allclose(ref_data, tvm_out)
 
     in_shape = (3, 1)
     shape = (3, 4)
@@ -225,11 +219,7 @@ def verify_depth_to_space(inshape, outshape, mode, blockSize):
 
     model = helper.make_model(graph, producer_name="depth_to_space_test")
 
-    for target, ctx in tvm.testing.enabled_targets():
-        x = np.random.uniform(size=inshape).astype("float32")
-        tvm_out = get_tvm_output(model, x, target, ctx, outshape, "float32")
-        onnx_out = get_onnxruntime_output(model, x, "float32")
-        tvm.testing.assert_allclose(onnx_out, tvm_out)
+    verify_onnx_forward_impl(model, inshape, outshape)
 
 
 @tvm.testing.uses_gpu
@@ -252,11 +242,7 @@ def verify_space_to_depth(inshape, outshape, blockSize):
 
     model = helper.make_model(graph, producer_name="space_to_depth_test")
 
-    for target, ctx in tvm.testing.enabled_targets():
-        x = np.random.uniform(size=inshape).astype("float32")
-        tvm_out = get_tvm_output(model, x, target, ctx, outshape, "float32")
-        onnx_out = get_onnxruntime_output(model, x, "float32")
-        tvm.testing.assert_allclose(onnx_out, tvm_out)
+    verify_onnx_forward_impl(model, inshape, outshape)
 
 
 @tvm.testing.uses_gpu
@@ -297,8 +283,7 @@ def test_shape():
     for target, ctx in tvm.testing.enabled_targets():
         x = np.random.uniform(size=in_shape).astype("int32")
         tvm_out = get_tvm_output(model, x, target, ctx, ref_shape, "int32")
-
-    tvm.testing.assert_allclose(ref_shape, tvm_out)
+        tvm.testing.assert_allclose(ref_shape, tvm_out)
 
 
 def _test_power_iteration(x_shape, y_shape):
@@ -354,8 +339,7 @@ def test_squeeze():
     for target, ctx in tvm.testing.enabled_targets():
         x = np.random.uniform(size=in_shape).astype("float32")
         tvm_out = get_tvm_output(model, x, target, ctx, out_shape, "float32")
-
-    tvm.testing.assert_allclose(out_shape, tvm_out.shape)
+        tvm.testing.assert_allclose(out_shape, tvm_out.shape)
 
 
 @tvm.testing.uses_gpu
@@ -379,8 +363,7 @@ def test_flatten():
     for target, ctx in tvm.testing.enabled_targets():
         x = np.random.uniform(size=in_shape).astype("int32")
         tvm_out = get_tvm_output(model, x, target, ctx, ref_shape, "float32")
-
-    tvm.testing.assert_allclose(ref_shape, tvm_out.shape)
+        tvm.testing.assert_allclose(ref_shape, tvm_out.shape)
 
 
 @tvm.testing.uses_gpu
@@ -402,8 +385,7 @@ def test_unsqueeze():
     for target, ctx in tvm.testing.enabled_targets():
         x = np.random.uniform(size=in_shape).astype("float32")
         tvm_out = get_tvm_output(model, x, target, ctx, out_shape, "float32")
-
-    tvm.testing.assert_allclose(out_shape, tvm_out.shape)
+        tvm.testing.assert_allclose(out_shape, tvm_out.shape)
 
 
 def verify_gather(in_shape, indices, axis, dtype):
@@ -454,6 +436,7 @@ def verify_gatherelements(in_shape, indices, axis):
         outputs=[helper.make_tensor_value_info("output", TensorProto.FLOAT, list(in_shape))],
     )
     model = helper.make_model(graph, producer_name="gather_elements_test")
+
     onnx_out = get_onnxruntime_output(model, [x, indices])
 
     for target, ctx in tvm.testing.enabled_targets():
@@ -698,8 +681,8 @@ def forward(self, x):
             opset_version=10,
             enable_onnx_checker=True,
         )
-
-        verify_onnx_forward_impl(onnx_io.getvalue(), (1, 4), (1, 2))
+        model = onnx.load_model_from_string(onnx_io.getvalue())
+        verify_onnx_forward_impl(model, (1, 4), (1, 2))
 
     test_slice_with_strides()
 
@@ -721,8 +704,7 @@ def _test_onnx_op_elementwise(inshape, outfunc, npargs, dtype, opname, kwargs):
 
     for target, ctx in tvm.testing.enabled_targets():
         tvm_out = get_tvm_output(model, indata, target, ctx, outdata.shape, dtype)
-
-    tvm.testing.assert_allclose(outdata, tvm_out)
+        tvm.testing.assert_allclose(outdata, tvm_out)
 
 
 @tvm.testing.uses_gpu
@@ -764,11 +746,7 @@ def test_clip_min_max_as_inputs():
     )
     model = helper.make_model(graph, producer_name="clip_test")
 
-    indata = np.random.uniform(-1, 7, size=input_shape).astype("float32")
-    onnx_out = get_onnxruntime_output(model, indata, "float32")
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, indata, target, ctx, input_shape, "float32")
-    tvm.testing.assert_allclose(onnx_out, tvm_out)
+    verify_onnx_forward_impl(model, input_shape, input_shape)
 
 
 @tvm.testing.uses_gpu
@@ -793,8 +771,7 @@ def _test_finite_ops(inshape, outfunc, npargs, dtype, opname, kwargs):
 
     for target, ctx in tvm.testing.enabled_targets():
         tvm_out = get_tvm_output(model, indata, target, ctx, outdata.shape, dtype)
-
-    tvm.testing.assert_allclose(outdata, tvm_out)
+        tvm.testing.assert_allclose(outdata, tvm_out)
 
 
 @tvm.testing.uses_gpu
@@ -1596,10 +1573,7 @@ def verify_reduce_func(func, data, axis, keepdims):
 
     model = helper.make_model(graph, producer_name="reduce_test")
 
-    onnx_out = get_onnxruntime_output(model, data, "float32")
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, data, target, ctx, outshape, "float32")
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl(model, data.shape, outshape)
 
 
 @tvm.testing.uses_gpu
@@ -1922,11 +1896,7 @@ def check_torch_conversion(model, input_size):
     # Set verbose=True for more output
     torch.onnx.export(model(), dummy_input, file_name, export_params=True, verbose=False)
     onnx_model = onnx.load(file_name)
-    for target, ctx in tvm.testing.enabled_targets():
-        input_data = np.random.uniform(size=input_size).astype("int32")
-        c2_out = get_onnxruntime_output(onnx_model, input_data)
-        tvm_out = get_tvm_output(onnx_model, input_data, target, ctx)
-        tvm.testing.assert_allclose(c2_out, tvm_out)
+    verify_onnx_forward_impl(onnx_model, input_size)
 
 
 @tvm.testing.uses_gpu

From 3284cb8323b3878db26fc5cc4056529d31a33cbc Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 10:03:19 +0900
Subject: [PATCH 04/13] more refactoring

---
 tests/python/frontend/onnx/test_forward.py | 141 +++++++--------------
 1 file changed, 49 insertions(+), 92 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index ee272bc1ce52..d886af7790cc 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -105,20 +105,39 @@ def get_onnxruntime_output(model, inputs, dtype="float32"):
 
     rep = onnxruntime.backend.prepare(model, "CPU")
     if isinstance(inputs, list) and len(inputs) > 1:
-        ort_out = rep.run(inputs)
+        return rep.run(inputs)
+    elif isinstance(inputs, list) and len(inputs) == 1:
+        inp = inputs[0]
     else:
-        x = inputs.astype(dtype)
-        ort_out = rep.run(x)[0]
-    return ort_out
+        inp = inputs
 
+    return rep.run(inp.astype(dtype))[0]
+
+
+def verify_onnx_forward_impl_with_inputs(
+    model, inputs, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5
+):
+    def flatten(out):
+        if isinstance(out, list) and len(out) == 1:
+            out = out[0]
+        if isinstance(out, np.ndarray):
+            return out.flatten()
+        return out
+
+    ort_out = get_onnxruntime_output(model, inputs, dtype)
 
-def verify_onnx_forward_impl(model, data_shape, out_shape=None, rtol=1e-5, atol=1e-5):
-    dtype = "float32"
-    x = np.random.uniform(size=data_shape).astype(dtype)
-    ort_out = get_onnxruntime_output(model, x, dtype)
     for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, x, target, ctx, out_shape, dtype)
-        tvm.testing.assert_allclose(ort_out, tvm_out, rtol=rtol, atol=atol)
+        tvm_out = get_tvm_output(model, inputs, target, ctx, out_shape, dtype)
+        tvm.testing.assert_allclose(flatten(ort_out), flatten(tvm_out), rtol=rtol, atol=atol)
+
+
+def verify_onnx_forward_impl(
+    model, input_shapes, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5
+):
+    inputs = [np.random.uniform(size=ishape).astype(dtype) for ishape in input_shapes]
+    verify_onnx_forward_impl_with_inputs(
+        model, inputs, out_shape=out_shape, dtype=dtype, rtol=rtol, atol=atol
+    )
 
 
 def make_constant_node(name, data_type, dims, vals):
@@ -219,7 +238,7 @@ def verify_depth_to_space(inshape, outshape, mode, blockSize):
 
     model = helper.make_model(graph, producer_name="depth_to_space_test")
 
-    verify_onnx_forward_impl(model, inshape, outshape)
+    verify_onnx_forward_impl(model, [inshape], outshape)
 
 
 @tvm.testing.uses_gpu
@@ -242,7 +261,7 @@ def verify_space_to_depth(inshape, outshape, blockSize):
 
     model = helper.make_model(graph, producer_name="space_to_depth_test")
 
-    verify_onnx_forward_impl(model, inshape, outshape)
+    verify_onnx_forward_impl(model, [inshape], outshape)
 
 
 @tvm.testing.uses_gpu
@@ -437,11 +456,7 @@ def verify_gatherelements(in_shape, indices, axis):
     )
     model = helper.make_model(graph, producer_name="gather_elements_test")
 
-    onnx_out = get_onnxruntime_output(model, [x, indices])
-
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, [x, indices], target, ctx, onnx_out[0].shape)
-        tvm.testing.assert_allclose(onnx_out[0], tvm_out)
+    verify_onnx_forward_impl_with_inputs(model, [x, indices])
 
 
 @tvm.testing.uses_gpu
@@ -478,11 +493,7 @@ def verify_scatter(in_shape, indices, axis):
         outputs=[helper.make_tensor_value_info("output", TensorProto.FLOAT, list(in_shape))],
     )
     model = helper.make_model(graph, producer_name="scatter_test")
-    onnx_out = get_onnxruntime_output(model, [x, indices, updates])
-
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, [x, indices, updates], target, ctx, onnx_out[0].shape)
-        tvm.testing.assert_allclose(onnx_out[0], tvm_out)
+    verify_onnx_forward_impl_with_inputs(model, [x, indices, updates])
 
 
 @tvm.testing.uses_gpu
@@ -682,7 +693,7 @@ def forward(self, x):
             enable_onnx_checker=True,
         )
         model = onnx.load_model_from_string(onnx_io.getvalue())
-        verify_onnx_forward_impl(model, (1, 4), (1, 2))
+        verify_onnx_forward_impl(model, [(1, 4)], (1, 2))
 
     test_slice_with_strides()
 
@@ -746,7 +757,7 @@ def test_clip_min_max_as_inputs():
     )
     model = helper.make_model(graph, producer_name="clip_test")
 
-    verify_onnx_forward_impl(model, input_shape, input_shape)
+    verify_onnx_forward_impl(model, [input_shape], input_shape)
 
 
 @tvm.testing.uses_gpu
@@ -1573,7 +1584,7 @@ def verify_reduce_func(func, data, axis, keepdims):
 
     model = helper.make_model(graph, producer_name="reduce_test")
 
-    verify_onnx_forward_impl(model, data.shape, outshape)
+    verify_onnx_forward_impl(model, [data.shape], outshape)
 
 
 @tvm.testing.uses_gpu
@@ -1811,15 +1822,7 @@ def verify_prelu(x_shape, a_shape):
 
         model = helper.make_model(graph, producer_name="prelu_test")
 
-        indata = np.random.uniform(-10, 10, x_shape).astype(np.float32)
-        slopedata = np.random.uniform(-10, 10, a_shape).astype(np.float32)
-        onnx_out = get_onnxruntime_output(model, [indata, slopedata])
-
-        for target, ctx in [("llvm", tvm.cpu())]:
-            tvm_out = get_tvm_output(
-                model, [indata, slopedata], target, ctx, list(x_shape), output_dtype="float32"
-            )
-            tvm.testing.assert_allclose(onnx_out[0], tvm_out, rtol=1e-05, atol=1e-05)
+        verify_onnx_forward_impl(model, [x_shape, a_shape], list(x_shape))
 
     verify_prelu([3, 4, 5, 6], [1, 4, 1, 1])
     verify_prelu([1, 8, 5, 6], [1, 8, 1, 1])
@@ -1896,7 +1899,7 @@ def check_torch_conversion(model, input_size):
     # Set verbose=True for more output
     torch.onnx.export(model(), dummy_input, file_name, export_params=True, verbose=False)
     onnx_model = onnx.load(file_name)
-    verify_onnx_forward_impl(onnx_model, input_size)
+    verify_onnx_forward_impl(onnx_model, [input_size])
 
 
 @tvm.testing.uses_gpu
@@ -2236,18 +2239,9 @@ def verify_batch_norm(in_shape):
         )
 
         model = helper.make_model(graph, producer_name="batchnorm_test")
-
-        for target, ctx in tvm.testing.enabled_targets():
-            x = np.random.uniform(size=in_shape).astype("float32")
-            scale = np.random.uniform(size=in_shape[1]).astype("float32")
-            b = np.random.uniform(size=in_shape[1]).astype("float32")
-            mean = np.random.uniform(size=in_shape[1]).astype("float32")
-            var = np.random.uniform(size=in_shape[1]).astype("float32")
-            onnx_out = get_onnxruntime_output(model, [x, scale, b, mean, var], "float32")[0]
-            tvm_out = get_tvm_output(
-                model, [x, scale, b, mean, var], target, ctx, in_shape, "float32"
-            )
-            tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+        # X, scale, b, mean, var
+        inshapes = [in_shape, in_shape[1], in_shape[1], in_shape[1], in_shape[1]]
+        verify_onnx_forward_impl(model, inshapes, in_shape)
 
     verify_batch_norm([1, 3, 224, 224])
     verify_batch_norm([1, 3, 24, 24])
@@ -2280,19 +2274,9 @@ def verify_batch_norm_dynamic_subgraph(in_shape, o_shape):
         )
 
         model = helper.make_model(graph, producer_name="batchnorm_test")
-
-        for target, ctx in tvm.testing.enabled_targets():
-            x = np.random.uniform(size=in_shape).astype("float32")
-            inp = np.random.uniform(size=o_shape).astype("float32")
-            scale = np.random.uniform(size=in_shape[1]).astype("float32")
-            b = np.random.uniform(size=in_shape[1]).astype("float32")
-            mean = np.random.uniform(size=in_shape[1]).astype("float32")
-            var = np.random.uniform(size=in_shape[1]).astype("float32")
-            onnx_out = get_onnxruntime_output(model, [x, inp, scale, b, mean, var], "float32")[0]
-            tvm_out = get_tvm_output(
-                model, [x, inp, scale, b, mean, var], target, ctx, in_shape, "float32"
-            )
-            tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+        # X, inp, scale, b, mean, var
+        inshapes = [in_shape, o_shape, in_shape[1], in_shape[1], in_shape[1], in_shape[1]]
+        verify_onnx_forward_impl(model, inshapes, in_shape)
 
     verify_batch_norm_dynamic_subgraph([16, 16, 10, 10], [160, 160])
 
@@ -2356,12 +2340,7 @@ def verify_conv(
 
     model = helper.make_model(graph, producer_name="conv_test")
 
-    for target, ctx in tvm.testing.enabled_targets():
-        x = np.random.uniform(size=x_shape).astype("float32")
-        W = np.random.uniform(size=w_shape).astype("float32")
-        tvm_out = get_tvm_output(model, [x, W], target, ctx, y_shape)
-        onnx_out = get_onnxruntime_output(model, [x, W], "float32")[0]
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl(model, [x_shape, w_shape], y_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2468,13 +2447,7 @@ def verify_convtranspose(x_shape, w_shape, y_shape, p):
     )
 
     model = helper.make_model(graph, producer_name="convtranspose_trest")
-
-    for target, ctx in tvm.testing.enabled_targets():
-        x = np.random.uniform(size=x_shape).astype("float32")
-        W = np.random.uniform(size=w_shape).astype("float32")
-        tvm_out = get_tvm_output(model, [x, W], target, ctx, y_shape)
-        onnx_out = get_onnxruntime_output(model, [x, W], "float32")[0]
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl(model, [x_shape, w_shape], y_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2540,11 +2513,7 @@ def verify_pooling(x_shape, kernel_shape, strides, pads, out_shape, mode, auto_p
     )
 
     model = helper.make_model(graph, producer_name="pooling_test")
-
-    for target, ctx in tvm.testing.enabled_targets():
-        onnx_out = get_onnxruntime_output(model, x_np, "float32")
-        tvm_out = get_tvm_output(model, [x_np], target, ctx, out_shape)
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl(model, [x_shape], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2649,12 +2618,7 @@ def verify_mod(x_shape, y_shape, fmod, out_shape, dtype="float32"):
         outputs=[helper.make_tensor_value_info("z", onnx_dtype, list(out_shape))],
     )
     model = helper.make_model(graph, producer_name="mod_test")
-
-    onnx_out = get_onnxruntime_output(model, [x_np, y_np], dtype)[0]
-
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, [x_np, y_np], target, ctx, out_shape)
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl_with_inputs(model, [x_np, y_np], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2723,9 +2687,6 @@ def test_xor():
 
 
 def verify_max_roi_pool(x_shape, rois_shape, pooled_shape, spatial_scale, out_shape):
-    x_np = np.random.uniform(size=x_shape).astype("float32")
-    rois_np = np.random.uniform(size=rois_shape).astype("float32")
-
     if spatial_scale is None:
         pool_node = helper.make_node(
             "MaxRoiPool", inputs=["x", "rois"], outputs=["y"], pooled_shape=pooled_shape
@@ -2750,11 +2711,7 @@ def verify_max_roi_pool(x_shape, rois_shape, pooled_shape, spatial_scale, out_sh
     )
 
     model = helper.make_model(graph, producer_name="pool_test")
-
-    onnx_out = get_onnxruntime_output(model, [x_np, rois_np], "float32")[0]
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, [x_np, rois_np], target, ctx, out_shape)
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl(model, [x_shape, rois_shape], out_shape)
 
 
 @tvm.testing.uses_gpu

From b8c9250becf9a1484f1b0742cce2aeb9540a9679 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 10:19:08 +0900
Subject: [PATCH 05/13] more refactor

---
 tests/python/frontend/onnx/test_forward.py | 29 ++++------------------
 1 file changed, 5 insertions(+), 24 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index d886af7790cc..aabc2ef989a7 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -2734,8 +2734,6 @@ def test_max_roi_pool():
 
 
 def verify_lppool(x_shape, kernel_shape, p, strides, pads, out_shape, auto_pad="NOTSET"):
-    x_np = np.random.uniform(size=x_shape).astype("float32")
-
     if pads is None:
         pool_node = helper.make_node(
             "LpPool",
@@ -2765,11 +2763,7 @@ def verify_lppool(x_shape, kernel_shape, p, strides, pads, out_shape, auto_pad="
     )
 
     model = helper.make_model(graph, producer_name="lppool_test")
-
-    for target, ctx in tvm.testing.enabled_targets():
-        onnx_out = get_onnxruntime_output(model, x_np, "float32")
-        tvm_out = get_tvm_output(model, [x_np], target, ctx, out_shape)
-        tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-5, atol=1e-5)
+    verify_onnx_forward_impl(model, [x_shape], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -3177,12 +3171,7 @@ def verify(ishape, oshape, scales, mode, coord_trans):
 
         model = helper.make_model(graph, producer_name="resize_test")
 
-        for target, ctx in tvm.testing.enabled_targets():
-            x = np.random.uniform(size=ishape).astype("float32")
-            onnx_out = get_onnxruntime_output(model, x, "float32")
-            tvm_out = get_tvm_output(model, x, target, ctx, oshape, "float32", opset=11)
-
-            tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-05, atol=1e-05)
+        verify_onnx_forward_impl(model, [ishape], oshape)
 
     # upsampling
     verify([1, 16, 32, 32], [1, 16, 64, 64], [], "nearest", "asymmetric")
@@ -3327,17 +3316,9 @@ def verify_roi_align(
         np_rois = np.random.uniform(size=[num_roi, 4]).astype("float32") * input_dims[2]
         np_batch_indicies = np.random.randint(low=0, high=input_dims[0], size=num_roi)
 
-        onnx_out = get_onnxruntime_output(model, [np_data, np_rois, np_batch_indicies])
-        for target, ctx in [("llvm", tvm.cpu())]:
-            tvm_out = get_tvm_output(
-                model,
-                [np_data, np_rois, np_batch_indicies],
-                target,
-                ctx,
-                output_dims,
-                output_dtype="float32",
-            )
-            tvm.testing.assert_allclose(onnx_out[0], tvm_out, rtol=1e-05, atol=1e-05)
+        verify_onnx_forward_impl_with_inputs(
+            model, [np_data, np_rois, np_batch_indicies], output_dims
+        )
 
     verify_roi_align((1, 4, 16, 16), 32, 7, 7, sampling_ratio=0, spatial_scale=1.0)
     verify_roi_align((4, 4, 16, 32), 32, 7, 7, sampling_ratio=0, spatial_scale=1.0)

From f7f03a7899b3106858500b1b8f084c45dd1073b5 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 10:19:52 +0900
Subject: [PATCH 06/13] rename

---
 tests/python/frontend/onnx/test_forward.py | 48 ++++++++++------------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index aabc2ef989a7..a1e4037fb222 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -114,7 +114,7 @@ def get_onnxruntime_output(model, inputs, dtype="float32"):
     return rep.run(inp.astype(dtype))[0]
 
 
-def verify_onnx_forward_impl_with_inputs(
+def verify_with_ort_with_inputs(
     model, inputs, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5
 ):
     def flatten(out):
@@ -131,11 +131,9 @@ def flatten(out):
         tvm.testing.assert_allclose(flatten(ort_out), flatten(tvm_out), rtol=rtol, atol=atol)
 
 
-def verify_onnx_forward_impl(
-    model, input_shapes, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5
-):
+def verify_with_ort(model, input_shapes, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5):
     inputs = [np.random.uniform(size=ishape).astype(dtype) for ishape in input_shapes]
-    verify_onnx_forward_impl_with_inputs(
+    verify_with_ort_with_inputs(
         model, inputs, out_shape=out_shape, dtype=dtype, rtol=rtol, atol=atol
     )
 
@@ -238,7 +236,7 @@ def verify_depth_to_space(inshape, outshape, mode, blockSize):
 
     model = helper.make_model(graph, producer_name="depth_to_space_test")
 
-    verify_onnx_forward_impl(model, [inshape], outshape)
+    verify_with_ort(model, [inshape], outshape)
 
 
 @tvm.testing.uses_gpu
@@ -261,7 +259,7 @@ def verify_space_to_depth(inshape, outshape, blockSize):
 
     model = helper.make_model(graph, producer_name="space_to_depth_test")
 
-    verify_onnx_forward_impl(model, [inshape], outshape)
+    verify_with_ort(model, [inshape], outshape)
 
 
 @tvm.testing.uses_gpu
@@ -456,7 +454,7 @@ def verify_gatherelements(in_shape, indices, axis):
     )
     model = helper.make_model(graph, producer_name="gather_elements_test")
 
-    verify_onnx_forward_impl_with_inputs(model, [x, indices])
+    verify_with_ort_with_inputs(model, [x, indices])
 
 
 @tvm.testing.uses_gpu
@@ -493,7 +491,7 @@ def verify_scatter(in_shape, indices, axis):
         outputs=[helper.make_tensor_value_info("output", TensorProto.FLOAT, list(in_shape))],
     )
     model = helper.make_model(graph, producer_name="scatter_test")
-    verify_onnx_forward_impl_with_inputs(model, [x, indices, updates])
+    verify_with_ort_with_inputs(model, [x, indices, updates])
 
 
 @tvm.testing.uses_gpu
@@ -693,7 +691,7 @@ def forward(self, x):
             enable_onnx_checker=True,
         )
         model = onnx.load_model_from_string(onnx_io.getvalue())
-        verify_onnx_forward_impl(model, [(1, 4)], (1, 2))
+        verify_with_ort(model, [(1, 4)], (1, 2))
 
     test_slice_with_strides()
 
@@ -757,7 +755,7 @@ def test_clip_min_max_as_inputs():
     )
     model = helper.make_model(graph, producer_name="clip_test")
 
-    verify_onnx_forward_impl(model, [input_shape], input_shape)
+    verify_with_ort(model, [input_shape], input_shape)
 
 
 @tvm.testing.uses_gpu
@@ -1584,7 +1582,7 @@ def verify_reduce_func(func, data, axis, keepdims):
 
     model = helper.make_model(graph, producer_name="reduce_test")
 
-    verify_onnx_forward_impl(model, [data.shape], outshape)
+    verify_with_ort(model, [data.shape], outshape)
 
 
 @tvm.testing.uses_gpu
@@ -1822,7 +1820,7 @@ def verify_prelu(x_shape, a_shape):
 
         model = helper.make_model(graph, producer_name="prelu_test")
 
-        verify_onnx_forward_impl(model, [x_shape, a_shape], list(x_shape))
+        verify_with_ort(model, [x_shape, a_shape], list(x_shape))
 
     verify_prelu([3, 4, 5, 6], [1, 4, 1, 1])
     verify_prelu([1, 8, 5, 6], [1, 8, 1, 1])
@@ -1899,7 +1897,7 @@ def check_torch_conversion(model, input_size):
     # Set verbose=True for more output
     torch.onnx.export(model(), dummy_input, file_name, export_params=True, verbose=False)
     onnx_model = onnx.load(file_name)
-    verify_onnx_forward_impl(onnx_model, [input_size])
+    verify_with_ort(onnx_model, [input_size])
 
 
 @tvm.testing.uses_gpu
@@ -2241,7 +2239,7 @@ def verify_batch_norm(in_shape):
         model = helper.make_model(graph, producer_name="batchnorm_test")
         # X, scale, b, mean, var
         inshapes = [in_shape, in_shape[1], in_shape[1], in_shape[1], in_shape[1]]
-        verify_onnx_forward_impl(model, inshapes, in_shape)
+        verify_with_ort(model, inshapes, in_shape)
 
     verify_batch_norm([1, 3, 224, 224])
     verify_batch_norm([1, 3, 24, 24])
@@ -2276,7 +2274,7 @@ def verify_batch_norm_dynamic_subgraph(in_shape, o_shape):
         model = helper.make_model(graph, producer_name="batchnorm_test")
         # X, inp, scale, b, mean, var
         inshapes = [in_shape, o_shape, in_shape[1], in_shape[1], in_shape[1], in_shape[1]]
-        verify_onnx_forward_impl(model, inshapes, in_shape)
+        verify_with_ort(model, inshapes, in_shape)
 
     verify_batch_norm_dynamic_subgraph([16, 16, 10, 10], [160, 160])
 
@@ -2340,7 +2338,7 @@ def verify_conv(
 
     model = helper.make_model(graph, producer_name="conv_test")
 
-    verify_onnx_forward_impl(model, [x_shape, w_shape], y_shape)
+    verify_with_ort(model, [x_shape, w_shape], y_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2447,7 +2445,7 @@ def verify_convtranspose(x_shape, w_shape, y_shape, p):
     )
 
     model = helper.make_model(graph, producer_name="convtranspose_trest")
-    verify_onnx_forward_impl(model, [x_shape, w_shape], y_shape)
+    verify_with_ort(model, [x_shape, w_shape], y_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2513,7 +2511,7 @@ def verify_pooling(x_shape, kernel_shape, strides, pads, out_shape, mode, auto_p
     )
 
     model = helper.make_model(graph, producer_name="pooling_test")
-    verify_onnx_forward_impl(model, [x_shape], out_shape)
+    verify_with_ort(model, [x_shape], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2618,7 +2616,7 @@ def verify_mod(x_shape, y_shape, fmod, out_shape, dtype="float32"):
         outputs=[helper.make_tensor_value_info("z", onnx_dtype, list(out_shape))],
     )
     model = helper.make_model(graph, producer_name="mod_test")
-    verify_onnx_forward_impl_with_inputs(model, [x_np, y_np], out_shape)
+    verify_with_ort_with_inputs(model, [x_np, y_np], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2711,7 +2709,7 @@ def verify_max_roi_pool(x_shape, rois_shape, pooled_shape, spatial_scale, out_sh
     )
 
     model = helper.make_model(graph, producer_name="pool_test")
-    verify_onnx_forward_impl(model, [x_shape, rois_shape], out_shape)
+    verify_with_ort(model, [x_shape, rois_shape], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -2763,7 +2761,7 @@ def verify_lppool(x_shape, kernel_shape, p, strides, pads, out_shape, auto_pad="
     )
 
     model = helper.make_model(graph, producer_name="lppool_test")
-    verify_onnx_forward_impl(model, [x_shape], out_shape)
+    verify_with_ort(model, [x_shape], out_shape)
 
 
 @tvm.testing.uses_gpu
@@ -3171,7 +3169,7 @@ def verify(ishape, oshape, scales, mode, coord_trans):
 
         model = helper.make_model(graph, producer_name="resize_test")
 
-        verify_onnx_forward_impl(model, [ishape], oshape)
+        verify_with_ort(model, [ishape], oshape)
 
     # upsampling
     verify([1, 16, 32, 32], [1, 16, 64, 64], [], "nearest", "asymmetric")
@@ -3316,9 +3314,7 @@ def verify_roi_align(
         np_rois = np.random.uniform(size=[num_roi, 4]).astype("float32") * input_dims[2]
         np_batch_indicies = np.random.randint(low=0, high=input_dims[0], size=num_roi)
 
-        verify_onnx_forward_impl_with_inputs(
-            model, [np_data, np_rois, np_batch_indicies], output_dims
-        )
+        verify_with_ort_with_inputs(model, [np_data, np_rois, np_batch_indicies], output_dims)
 
     verify_roi_align((1, 4, 16, 16), 32, 7, 7, sampling_ratio=0, spatial_scale=1.0)
     verify_roi_align((4, 4, 16, 32), 32, 7, 7, sampling_ratio=0, spatial_scale=1.0)

From 2b0e61d17c77a797167ae9783dba89059aa25760 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 10:33:14 +0900
Subject: [PATCH 07/13] add more test for slice

---
 tests/python/frontend/onnx/test_forward.py | 30 +++++++++++++---------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index a1e4037fb222..ab5c0a3c9a89 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -680,18 +680,24 @@ class SliceWithStrides(torch.nn.Module):
             def forward(self, x):
                 return x[..., 0::2] + x[..., 1::2]
 
-        onnx_io = io.BytesIO()
-        torch.onnx.export(
-            SliceWithStrides(),
-            (torch.randn(1, 4),),
-            onnx_io,
-            input_names=["x"],
-            output_names=["y"],
-            opset_version=10,
-            enable_onnx_checker=True,
-        )
-        model = onnx.load_model_from_string(onnx_io.getvalue())
-        verify_with_ort(model, [(1, 4)], (1, 2))
+        class SliceWithStrides2(torch.nn.Module):
+            def forward(self, x):
+                return x[0::2, 0::2] + x[1::2, 1::2]
+
+        for module in [SliceWithStrides, SliceWithStrides2]:
+            ishape = (4, 4)
+            onnx_io = io.BytesIO()
+            torch.onnx.export(
+                module(),
+                (torch.randn(ishape),),
+                onnx_io,
+                input_names=["x"],
+                output_names=["y"],
+                opset_version=10,
+                enable_onnx_checker=True,
+            )
+            model = onnx.load_model_from_string(onnx_io.getvalue())
+            verify_with_ort(model, [ishape])
 
     test_slice_with_strides()
 

From 3e806d8ee3876d0386e03a69d3959c39cf9d970c Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 13:18:56 +0900
Subject: [PATCH 08/13] handle vm and cpu only tests

---
 tests/python/frontend/onnx/test_forward.py | 59 +++++++++++++++++-----
 1 file changed, 45 insertions(+), 14 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index ab5c0a3c9a89..429d320e21a5 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -53,8 +53,7 @@ def get_tvm_output_with_vm(graph_def, input_data, target, ctx, opset=None):
     mod, params = relay.frontend.from_onnx(graph_def, shape_dict, opset=opset)
 
     ex = relay.create_executor("vm", mod=mod, ctx=ctx, target=target)
-    indata = tvm.nd.array(input_data)
-    result = ex.evaluate()(indata)
+    result = ex.evaluate()(*input_data)
     return result.asnumpy()
 
 
@@ -110,12 +109,19 @@ def get_onnxruntime_output(model, inputs, dtype="float32"):
         inp = inputs[0]
     else:
         inp = inputs
-
     return rep.run(inp.astype(dtype))[0]
 
 
 def verify_with_ort_with_inputs(
-    model, inputs, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5
+    model,
+    inputs,
+    out_shape=None,
+    targets=None,
+    use_vm=False,
+    opset=None,
+    dtype="float32",
+    rtol=1e-5,
+    atol=1e-5,
 ):
     def flatten(out):
         if isinstance(out, list) and len(out) == 1:
@@ -126,15 +132,42 @@ def flatten(out):
 
     ort_out = get_onnxruntime_output(model, inputs, dtype)
 
-    for target, ctx in tvm.testing.enabled_targets():
-        tvm_out = get_tvm_output(model, inputs, target, ctx, out_shape, dtype)
+    if targets is None:
+        targets = [tgt for (tgt, _) in tvm.testing.enabled_targets()]
+
+    for target in targets:
+        ctx = tvm.context(target, 0)
+
+        if use_vm:
+            tvm_out = get_tvm_output_with_vm(model, inputs, target, ctx, opset=opset)
+        else:
+            tvm_out = get_tvm_output(model, inputs, target, ctx, out_shape, dtype, opset=opset)
+
         tvm.testing.assert_allclose(flatten(ort_out), flatten(tvm_out), rtol=rtol, atol=atol)
 
 
-def verify_with_ort(model, input_shapes, out_shape=None, dtype="float32", rtol=1e-5, atol=1e-5):
+def verify_with_ort(
+    model,
+    input_shapes,
+    out_shape=None,
+    targets=None,
+    use_vm=False,
+    opset=None,
+    dtype="float32",
+    rtol=1e-5,
+    atol=1e-5,
+):
     inputs = [np.random.uniform(size=ishape).astype(dtype) for ishape in input_shapes]
     verify_with_ort_with_inputs(
-        model, inputs, out_shape=out_shape, dtype=dtype, rtol=rtol, atol=atol
+        model,
+        inputs,
+        out_shape=out_shape,
+        targets=targets,
+        use_vm=use_vm,
+        opset=opset,
+        dtype=dtype,
+        rtol=rtol,
+        atol=atol,
     )
 
 
@@ -3175,7 +3208,7 @@ def verify(ishape, oshape, scales, mode, coord_trans):
 
         model = helper.make_model(graph, producer_name="resize_test")
 
-        verify_with_ort(model, [ishape], oshape)
+        verify_with_ort(model, [ishape], oshape, opset=11)
 
     # upsampling
     verify([1, 16, 32, 32], [1, 16, 64, 64], [], "nearest", "asymmetric")
@@ -3208,11 +3241,9 @@ def verify_nonzero(indata, outdata, dtype):
 
         model = helper.make_model(graph, producer_name="nonzero_test")
 
-        onnx_out = get_onnxruntime_output(model, indata, dtype)
-
-        for target, ctx in [("llvm", tvm.cpu())]:
-            tvm_out = get_tvm_output_with_vm(model, indata, target, ctx, opset=9)
-            tvm.testing.assert_allclose(onnx_out, tvm_out, rtol=1e-05, atol=1e-05)
+        verify_with_ort_with_inputs(
+            model, [indata], targets=["llvm"], dtype="int64", use_vm=True, opset=9
+        )
 
     input_data = np.array([[1, 0], [1, 1]], dtype=np.int64)
     result = np.array((np.nonzero(input_data)))  # expected output [[0, 1, 1], [0, 0, 1]]

From 04ff460fe334feefee9fb5ea77ad4ce2018cff46 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 13:24:27 +0900
Subject: [PATCH 09/13] minor fix

---
 tests/python/frontend/onnx/test_forward.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index 429d320e21a5..46d9187b3861 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -54,7 +54,9 @@ def get_tvm_output_with_vm(graph_def, input_data, target, ctx, opset=None):
 
     ex = relay.create_executor("vm", mod=mod, ctx=ctx, target=target)
     result = ex.evaluate()(*input_data)
-    return result.asnumpy()
+    if isinstance(result, tvm.runtime.NDArray):
+        return result.asnumpy()
+    return [r.asnumpy() for r in result]
 
 
 def get_tvm_output(
@@ -2313,7 +2315,7 @@ def verify_batch_norm_dynamic_subgraph(in_shape, o_shape):
         model = helper.make_model(graph, producer_name="batchnorm_test")
         # X, inp, scale, b, mean, var
         inshapes = [in_shape, o_shape, in_shape[1], in_shape[1], in_shape[1], in_shape[1]]
-        verify_with_ort(model, inshapes, in_shape)
+        verify_with_ort(model, inshapes, in_shape, use_vm=False)
 
     verify_batch_norm_dynamic_subgraph([16, 16, 10, 10], [160, 160])
 
@@ -3208,7 +3210,7 @@ def verify(ishape, oshape, scales, mode, coord_trans):
 
         model = helper.make_model(graph, producer_name="resize_test")
 
-        verify_with_ort(model, [ishape], oshape, opset=11)
+        verify_with_ort(model, [ishape], oshape, use_vm=False, opset=11)
 
     # upsampling
     verify([1, 16, 32, 32], [1, 16, 64, 64], [], "nearest", "asymmetric")

From 65ca3648dd342903ce6715e33135fb2de8b08880 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 13:48:39 +0900
Subject: [PATCH 10/13] remove pytorch dependency from slice test

---
 tests/python/frontend/onnx/test_forward.py | 52 +++++++++-------------
 1 file changed, 22 insertions(+), 30 deletions(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index 46d9187b3861..286ea8081306 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -14,17 +14,14 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import io
 import numpy as np
 import math
 import onnx
 from onnx import helper, TensorProto, mapping
 import torch
 import torchvision
-from tvm import topi
 import tvm.topi.testing
 import tvm
-from tvm import te
 from tvm import relay
 from tvm.contrib import graph_runtime
 import scipy
@@ -563,6 +560,7 @@ def _test_slice_iteration_v10(indata, outdata, **attrs):
     starts = attrs["starts"]
     ends = attrs["ends"]
     axes = None if "axes" not in attrs else attrs["axes"]
+    steps = None if "steps" not in attrs else attrs["steps"]
     starts = np.asarray(starts)
     ends = np.asarray(ends)
     inputs = [
@@ -619,8 +617,8 @@ def add_noop_to_input_attr(attr_name, attr):
             return [ref_node, ref_node2, reshape1_node, reshape2_node]
 
     slice_inputs = []
-    for attr_name in ["starts", "ends", "axes"]:
-        if attr_name == "axes" and not axes:
+    for attr_name in ["starts", "ends", "axes", "steps"]:
+        if attr_name not in attrs:
             continue
         if "add_noop_to_input_attrs" in attrs and attr_name in attrs["add_noop_to_input_attrs"]:
             nodes.extend(add_noop_to_input_attr(attr_name, attrs[attr_name]))
@@ -632,6 +630,13 @@ def add_noop_to_input_attr(attr_name, attr):
         axes = np.asarray(axes)
         inputs.append(helper.make_tensor_value_info("axes", TensorProto.INT32, list(axes.shape)))
         initializer.append(helper.make_tensor("axes", TensorProto.INT32, list(axes.shape), axes))
+
+    if steps:
+        assert axes is not None and len(axes) == len(steps)
+        steps = np.asarray(steps)
+        inputs.append(helper.make_tensor_value_info("steps", TensorProto.INT32, list(axes.shape)))
+        initializer.append(helper.make_tensor("steps", TensorProto.INT32, list(steps.shape), steps))
+
     y = helper.make_node("Slice", ["data", *slice_inputs], ["out"])
 
     nodes.append(y)
@@ -710,31 +715,18 @@ def test_slice():
         x, x, starts=(0, 0), ends=(9223372036854775807, 9223372036854775807), axes=(0, 3)
     )
 
-    def test_slice_with_strides():
-        class SliceWithStrides(torch.nn.Module):
-            def forward(self, x):
-                return x[..., 0::2] + x[..., 1::2]
-
-        class SliceWithStrides2(torch.nn.Module):
-            def forward(self, x):
-                return x[0::2, 0::2] + x[1::2, 1::2]
-
-        for module in [SliceWithStrides, SliceWithStrides2]:
-            ishape = (4, 4)
-            onnx_io = io.BytesIO()
-            torch.onnx.export(
-                module(),
-                (torch.randn(ishape),),
-                onnx_io,
-                input_names=["x"],
-                output_names=["y"],
-                opset_version=10,
-                enable_onnx_checker=True,
-            )
-            model = onnx.load_model_from_string(onnx_io.getvalue())
-            verify_with_ort(model, [ishape])
-
-    test_slice_with_strides()
+    x = np.random.randn(4, 4).astype(np.float32)
+    _test_slice_iteration_v10(
+        x, x[:, 1::2], starts=(1,), ends=(9223372036854775807,), axes=(1,), steps=(2,)
+    )
+    _test_slice_iteration_v10(
+        x,
+        x[0::1, 1::2],
+        starts=(0, 1),
+        ends=(9223372036854775807, 9223372036854775807),
+        axes=(0, 1),
+        steps=(1, 2),
+    )
 
 
 def _test_onnx_op_elementwise(inshape, outfunc, npargs, dtype, opname, kwargs):

From 9bbedc6495b19f536ec306de4b5f17dce953253b Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 13:51:55 +0900
Subject: [PATCH 11/13] use explicit end value for one test

---
 tests/python/frontend/onnx/test_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index 286ea8081306..32dcd73c4484 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -723,7 +723,7 @@ def test_slice():
         x,
         x[0::1, 1::2],
         starts=(0, 1),
-        ends=(9223372036854775807, 9223372036854775807),
+        ends=(4, 4),
         axes=(0, 1),
         steps=(1, 2),
     )

From 17d179119cd139e64f3a60faa65458660c072b61 Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 14:03:40 +0900
Subject: [PATCH 12/13] minor fix

---
 tests/python/frontend/onnx/test_forward.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index 32dcd73c4484..c50deaa70905 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -1615,7 +1615,7 @@ def verify_reduce_func(func, data, axis, keepdims):
 
     model = helper.make_model(graph, producer_name="reduce_test")
 
-    verify_with_ort(model, [data.shape], outshape)
+    verify_with_ort_with_inputs(model, [data], outshape)
 
 
 @tvm.testing.uses_gpu

From 820c0572a621b041677c6f63e629d43d0f05a56d Mon Sep 17 00:00:00 2001
From: masa <masa@pop-os.localdomain>
Date: Sun, 13 Sep 2020 18:06:53 +0900
Subject: [PATCH 13/13] use int32 input for imagenet test

---
 tests/python/frontend/onnx/test_forward.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py
index c50deaa70905..81c8e77f0537 100644
--- a/tests/python/frontend/onnx/test_forward.py
+++ b/tests/python/frontend/onnx/test_forward.py
@@ -1930,7 +1930,8 @@ def check_torch_conversion(model, input_size):
     # Set verbose=True for more output
     torch.onnx.export(model(), dummy_input, file_name, export_params=True, verbose=False)
     onnx_model = onnx.load(file_name)
-    verify_with_ort(onnx_model, [input_size])
+    input_data = np.random.uniform(size=input_size).astype("int32")
+    verify_with_ort_with_inputs(onnx_model, [input_data])
 
 
 @tvm.testing.uses_gpu