From 9b7d23f0b2cc6304f73ea16dfd1f2b6298a31aa5 Mon Sep 17 00:00:00 2001 From: Eldritch Cheese Date: Wed, 11 Jan 2023 14:04:26 -0600 Subject: [PATCH 1/4] [Test][Topi] Use binary fractions for crop_and_divide unit tests The `crop_and_resize` operator uses floating-point arithmetic to determine whether an index is within a view-box. This can cause the use of `extrapolation_value` to depend on target-dependent rounding differences. For example, this issue was initially noticed on Vulkan during debugging of https://github.com/apache/tvm/pull/13530, and was the result of computing `0.2*223.0 + 0.8*223.0 < 223.0`. If all intermediates are cast to float32, the left-hand side evaluates to `223.00002`. If intermediates are kept at a higher precision, the left-hand side evaluates to `223.0`. The floating-point indexing can't be removed, because the operator must match the API defined by TensorFlow's operator implementation. The TensorFlow documentation for [`CropAndResize`](https://www.tensorflow.org/api_docs/cc/class/tensorflow/ops/crop-and-resize) does not specify behavior in these cases, nor do the current TensorFlow unit tests check cases of rounding error. Since the TensorFlow unit tests only use binary fractions for the `boxes` argument, which largely avoids the rounding issue, this commit updates the TVM unit tests to avoid depending on floating-point precision. --- tests/python/relay/test_op_level5.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/python/relay/test_op_level5.py b/tests/python/relay/test_op_level5.py index 10d0ea0d6d26..08e45eefb6a8 100644 --- a/tests/python/relay/test_op_level5.py +++ b/tests/python/relay/test_op_level5.py @@ -238,12 +238,14 @@ def test_crop_and_resize(self, target, dev, executor_kind, layout, interpolate_m if layout == "NHWC": img_shape = (10, 224, 224, 3) - boxes = np.array([[0.1, 0.2, 0.8, 0.7], [0.2, 0, 1, 0.6]]).astype("float32") + boxes = np.array([[0.125, 0.25, 0.8125, 0.71875], [0.25, 0, 1, 0.625]]).astype( + "float32" + ) box_indices = np.array([1, 0]).astype("int32") crop_size = np.array([20, 30]).astype("int32") elif layout == "NCHW": img_shape = (5, 3, 255, 255) - boxes = np.array([[0, 0, 1, 1], [0.2, 0.1, 1, 0.9]]).astype("float32") + boxes = np.array([[0, 0, 1, 1], [0.25, 0.125, 1, 0.9375]]).astype("float32") box_indices = np.array([0, 1]).astype("int32") crop_size = np.array([30, 30]).astype("int32") else: From 04410eeaee4fa906f80ef6ea1656582280a4185d Mon Sep 17 00:00:00 2001 From: Eldritch Cheese Date: Wed, 25 Jan 2023 08:14:42 -0600 Subject: [PATCH 2/4] Use seeded random data for unit test --- tests/python/relay/test_op_level5.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/relay/test_op_level5.py b/tests/python/relay/test_op_level5.py index 08e45eefb6a8..136668dbde53 100644 --- a/tests/python/relay/test_op_level5.py +++ b/tests/python/relay/test_op_level5.py @@ -251,6 +251,7 @@ def test_crop_and_resize(self, target, dev, executor_kind, layout, interpolate_m else: raise ValueError(f"Unknown layout: {layout}") + np.random.seed(0) image_data = np.random.uniform(size=img_shape).astype("float32") ref_res = tvm.topi.testing.crop_and_resize_python( From 4f920e075003df1b83bf3b88b514d6d33e9b6a78 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Tue, 21 Feb 2023 09:30:44 -0600 Subject: [PATCH 3/4] Add epsilon offset to avoid depending on floating-point behavior --- tests/python/relay/test_op_level5.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/python/relay/test_op_level5.py b/tests/python/relay/test_op_level5.py index 136668dbde53..47f90647500b 100644 --- a/tests/python/relay/test_op_level5.py +++ b/tests/python/relay/test_op_level5.py @@ -235,17 +235,23 @@ def test_crop_and_resize(self, target, dev, executor_kind, layout, interpolate_m pytest.xfail("Known failing case for these parameters") extrapolation_value = 0.0 + eps = 1e-4 if layout == "NHWC": img_shape = (10, 224, 224, 3) - boxes = np.array([[0.125, 0.25, 0.8125, 0.71875], [0.25, 0, 1, 0.625]]).astype( - "float32" - ) + boxes = np.array( + [ + [0.125 + eps, 0.25 + eps, 0.8125 - eps, 0.71875 - eps], + [0.25 + eps, 0 + eps, 1 - eps, 0.625 - eps], + ] + ).astype("float32") box_indices = np.array([1, 0]).astype("int32") crop_size = np.array([20, 30]).astype("int32") elif layout == "NCHW": img_shape = (5, 3, 255, 255) - boxes = np.array([[0, 0, 1, 1], [0.25, 0.125, 1, 0.9375]]).astype("float32") + boxes = np.array( + [[0, 0, 1, 1], [0.25 + eps, 0.125 + eps, 1 - eps, 0.9375 - eps]] + ).astype("float32") box_indices = np.array([0, 1]).astype("int32") crop_size = np.array([30, 30]).astype("int32") else: From aad6f0e43e2d1b1b9bf28c3688a3976870aff2ff Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Fri, 31 Mar 2023 09:59:13 -0500 Subject: [PATCH 4/4] Use randomly-generated boxes for unit tests This mimics the example usage of `tf.image.crop_and_resize`, whose API this operator is intended to follow. Using any boxes with edges representable as integer fractions has the potential for the in-bounds check to be impacted by rounding error (e.g. `0.2*x + 0.8*x < x`). Unfortunately, there's no way to remove this possibility altogether without changing the API, such as accepting the box location as an integer fraction, rather than a `float32`, but that would break compatibility. To avoid the risk of a flaky unit test based on the specific random boxes used, the PRNG is seeded prior to generation. --- tests/python/relay/test_op_level5.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/python/relay/test_op_level5.py b/tests/python/relay/test_op_level5.py index 47f90647500b..db910661ca2c 100644 --- a/tests/python/relay/test_op_level5.py +++ b/tests/python/relay/test_op_level5.py @@ -235,29 +235,24 @@ def test_crop_and_resize(self, target, dev, executor_kind, layout, interpolate_m pytest.xfail("Known failing case for these parameters") extrapolation_value = 0.0 + + np.random.seed(0) + eps = 1e-4 if layout == "NHWC": img_shape = (10, 224, 224, 3) - boxes = np.array( - [ - [0.125 + eps, 0.25 + eps, 0.8125 - eps, 0.71875 - eps], - [0.25 + eps, 0 + eps, 1 - eps, 0.625 - eps], - ] - ).astype("float32") + boxes = np.random.uniform(size=(2, 4)).astype("float32") box_indices = np.array([1, 0]).astype("int32") crop_size = np.array([20, 30]).astype("int32") elif layout == "NCHW": img_shape = (5, 3, 255, 255) - boxes = np.array( - [[0, 0, 1, 1], [0.25 + eps, 0.125 + eps, 1 - eps, 0.9375 - eps]] - ).astype("float32") + boxes = np.random.uniform(size=(2, 4)).astype("float32") box_indices = np.array([0, 1]).astype("int32") crop_size = np.array([30, 30]).astype("int32") else: raise ValueError(f"Unknown layout: {layout}") - np.random.seed(0) image_data = np.random.uniform(size=img_shape).astype("float32") ref_res = tvm.topi.testing.crop_and_resize_python(