From 3dd4d80163b71eb18ac3ec6399f88192b0bc1371 Mon Sep 17 00:00:00 2001 From: Andrei Hutu Date: Thu, 21 Mar 2024 16:37:00 +0000 Subject: [PATCH 1/4] [AOT][Testing] Improve output mismatch information on test failure Enhanced AOT test harness to include overall mismatch percentage and the individual mismatch positions from the output tensor for debugging test failures. Both of these are still gated behind `print_output_on_mismatch == True`. I also added tests to check for the presence and correctness of this new debug information. Sample output: ``` Element [Position]: Actual, Reference ------------------------------------- Element [0, 8, 8, 7]: 521.846313, 521.847412 Element [0, 9, 8, 51]: 478.874359, 478.875549 Element [0, 9, 9, 6]: 462.901001, 462.899658 Mismatched elements: 3 / 16384 (0.02%) ... ``` --- python/tvm/testing/aot.py | 26 ++++++++-- .../python/relay/aot/test_aot_test_harness.py | 52 ++++++++++++++++++- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/python/tvm/testing/aot.py b/python/tvm/testing/aot.py index 8d74f545a3c2..ffa51d1b478a 100644 --- a/python/tvm/testing/aot.py +++ b/python/tvm/testing/aot.py @@ -459,19 +459,35 @@ def _emit_main_compare( if print_output_on_mismatch: main_file.write( - f"int mismatch = 0;" - f'printf("Actual, Reference\\n");\n' + f"{{\n" + f"int mismatch = 0;\n" + f"int out_ndim = {outputs[key].ndim};\n" + f"int out_shape[] = {{{','.join(map(str, outputs[key].shape))}}};\n" + f"int out_indices[out_ndim];\n" + f'printf("Element [Position]: Actual, Reference\\n");\n' + f'printf("-------------------------------------\\n");\n' f"for (int i = 0; i<{data_length_var_name}; i++) {{\n" f"\tif ({comparison_function}({actual_data_name}[i]-" f"{expected_data_name}[i]) > {tolerance}) {{\n" - f'\t\tprintf("{value_format_specifier}, {value_format_specifier}\\n"' + f"\t\tint flat_index = i;\n" + f"\t\tfor (int j = out_ndim - 1; j >= 0; j--){{\n" + f"\t\t\tout_indices[j] = flat_index % out_shape[j];\n" + f"\t\t\tflat_index /= out_shape[j];\n" + f"\t\t}}\n" + f'\t\tprintf("Element [%d", out_indices[0]);\n' + f"\t\tfor (int j = 1; j < out_ndim; j++)\n" + f'\t\t\tprintf(", %d", out_indices[j]);\n' + f'\t\tprintf("]: {value_format_specifier}, {value_format_specifier}\\n"' f", {actual_data_name}[i], {expected_data_name}[i]);\n" - f"\t\tmismatch = 1;\n" + f"\t\tmismatch += 1;\n" f"\t}}\n" f"}}" - f"if (mismatch == 1) {{\n" + f"if (mismatch >= 1) {{\n" + f"\tfloat percent_mismatched = ((float) mismatch) / ((float) {data_length_var_name}) * 100;\n" + f'\tprintf("\\nMismatched elements: %d / %zu (%.2f%%)\\n", mismatch, {data_length_var_name}, percent_mismatched);\n' f'\tprintf("{AOT_FAILURE_TOKEN}\\n");\n' f"\treturn -1;\n" + f"}}\n" f"}}" ) else: diff --git a/tests/python/relay/aot/test_aot_test_harness.py b/tests/python/relay/aot/test_aot_test_harness.py index 8ec9506f9f65..3d10f15d4ab4 100644 --- a/tests/python/relay/aot/test_aot_test_harness.py +++ b/tests/python/relay/aot/test_aot_test_harness.py @@ -46,7 +46,57 @@ def test_output_on_mismatch_option(): ).astype(dtype) } - msg = ".*Actual, Reference\n2.000000, 0.000000\nAOT_TEST_FAILURE.*" + msg = ".*Actual, Reference(\n|.)*2.000000, 0.000000(\n|.)*AOT_TEST_FAILURE.*" + with pytest.raises(RuntimeError, match=msg): + compile_and_run( + AOTTestModel(module=tvm.IRModule.from_expr(func), inputs={}, outputs=outputs), + test_runner, + interface_api, + use_unpacked_api, + print_output_on_mismatch=True, + ) + + +def test_output_position_on_mismatch(): + """ + Test the mismatch position output for the print_output_on_mismatch option. + """ + interface_api = "packed" + use_unpacked_api = True + test_runner = AOTTestRunner() + dtype = "float32" + + x = np.zeros(shape=(2, 2), dtype=dtype) + x[-1, -1] = 1 + func = relay.Function([], relay.const(x, dtype=dtype)) + outputs = {"output": np.zeros(shape=(2, 2), dtype=dtype)} + + msg = ".*Element \\[1, 1\\]:.*" + with pytest.raises(RuntimeError, match=msg): + compile_and_run( + AOTTestModel(module=tvm.IRModule.from_expr(func), inputs={}, outputs=outputs), + test_runner, + interface_api, + use_unpacked_api, + print_output_on_mismatch=True, + ) + + +def test_mismatch_percentage(): + """ + Test the mismatch percentage for the print_output_on_mismatch option. + """ + interface_api = "packed" + use_unpacked_api = True + test_runner = AOTTestRunner() + dtype = "float32" + + x = np.zeros(shape=(8,), dtype=dtype) + x[0] = 1 + func = relay.Function([], relay.const(x, dtype=dtype)) + outputs = {"output": np.zeros(shape=(8,), dtype=dtype)} + + msg = ".*Mismatched elements: 1 / 8 \\(12.50%\\).*" with pytest.raises(RuntimeError, match=msg): compile_and_run( AOTTestModel(module=tvm.IRModule.from_expr(func), inputs={}, outputs=outputs), From 70a56cbf176f74e10436c0a9c3d6975c2ff9d486 Mon Sep 17 00:00:00 2001 From: Andrei Hutu Date: Fri, 22 Mar 2024 09:30:25 +0000 Subject: [PATCH 2/4] Fix linting --- python/tvm/testing/aot.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/tvm/testing/aot.py b/python/tvm/testing/aot.py index ffa51d1b478a..5b9a95db165b 100644 --- a/python/tvm/testing/aot.py +++ b/python/tvm/testing/aot.py @@ -483,8 +483,10 @@ def _emit_main_compare( f"\t}}\n" f"}}" f"if (mismatch >= 1) {{\n" - f"\tfloat percent_mismatched = ((float) mismatch) / ((float) {data_length_var_name}) * 100;\n" - f'\tprintf("\\nMismatched elements: %d / %zu (%.2f%%)\\n", mismatch, {data_length_var_name}, percent_mismatched);\n' + f"\tfloat percent_mismatched = ((float) mismatch) /" + f"((float) {data_length_var_name}) * 100;\n" + f'\tprintf("\\nMismatched elements: %d / %zu (%.2f%%)\\n"' + f", mismatch, {data_length_var_name}, percent_mismatched);\n" f'\tprintf("{AOT_FAILURE_TOKEN}\\n");\n' f"\treturn -1;\n" f"}}\n" From add4a8e8b5a95b56e8a4202a787f8e9ba2398555 Mon Sep 17 00:00:00 2001 From: Andrei Hutu Date: Fri, 22 Mar 2024 16:32:18 +0000 Subject: [PATCH 3/4] Replace multiple strings with one multiline string --- python/tvm/testing/aot.py | 66 ++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/python/tvm/testing/aot.py b/python/tvm/testing/aot.py index 5b9a95db165b..fa1af5df31f9 100644 --- a/python/tvm/testing/aot.py +++ b/python/tvm/testing/aot.py @@ -459,38 +459,40 @@ def _emit_main_compare( if print_output_on_mismatch: main_file.write( - f"{{\n" - f"int mismatch = 0;\n" - f"int out_ndim = {outputs[key].ndim};\n" - f"int out_shape[] = {{{','.join(map(str, outputs[key].shape))}}};\n" - f"int out_indices[out_ndim];\n" - f'printf("Element [Position]: Actual, Reference\\n");\n' - f'printf("-------------------------------------\\n");\n' - f"for (int i = 0; i<{data_length_var_name}; i++) {{\n" - f"\tif ({comparison_function}({actual_data_name}[i]-" - f"{expected_data_name}[i]) > {tolerance}) {{\n" - f"\t\tint flat_index = i;\n" - f"\t\tfor (int j = out_ndim - 1; j >= 0; j--){{\n" - f"\t\t\tout_indices[j] = flat_index % out_shape[j];\n" - f"\t\t\tflat_index /= out_shape[j];\n" - f"\t\t}}\n" - f'\t\tprintf("Element [%d", out_indices[0]);\n' - f"\t\tfor (int j = 1; j < out_ndim; j++)\n" - f'\t\t\tprintf(", %d", out_indices[j]);\n' - f'\t\tprintf("]: {value_format_specifier}, {value_format_specifier}\\n"' - f", {actual_data_name}[i], {expected_data_name}[i]);\n" - f"\t\tmismatch += 1;\n" - f"\t}}\n" - f"}}" - f"if (mismatch >= 1) {{\n" - f"\tfloat percent_mismatched = ((float) mismatch) /" - f"((float) {data_length_var_name}) * 100;\n" - f'\tprintf("\\nMismatched elements: %d / %zu (%.2f%%)\\n"' - f", mismatch, {data_length_var_name}, percent_mismatched);\n" - f'\tprintf("{AOT_FAILURE_TOKEN}\\n");\n' - f"\treturn -1;\n" - f"}}\n" - f"}}" + f""" + {{ + int mismatch = 0; + int out_ndim = {outputs[key].ndim}; + int out_shape[] = {{{','.join(map(str, outputs[key].shape))}}}; + int out_indices[out_ndim]; + printf("Element [Position]: Actual, Reference\\n"); + printf("-------------------------------------\\n"); + for (int i = 0; i<{data_length_var_name}; i++) {{ + if ({comparison_function}({actual_data_name}[i] - + {expected_data_name}[i]) > {tolerance}) {{ + int flat_index = i; + for (int j = out_ndim - 1; j >= 0; j--){{ + out_indices[j] = flat_index % out_shape[j]; + flat_index /= out_shape[j]; + }} + printf("Element [%d", out_indices[0]); + for (int j = 1; j < out_ndim; j++) + printf(", %d", out_indices[j]); + printf("]: {value_format_specifier}, {value_format_specifier}\\n", + {actual_data_name}[i], {expected_data_name}[i]); + mismatch += 1; + }} + }} + if (mismatch >= 1) {{ + float percent_mismatched = + ((float) mismatch) / ((float) {data_length_var_name}) * 100; + printf("\\nMismatched elements: %d / %zu (%.2f%%)\\n", + mismatch, {data_length_var_name}, percent_mismatched); + printf("{AOT_FAILURE_TOKEN}\\n"); + return -1; + }} + }} + """ ) else: main_file.write( From 74c9fddb5c27cb4a66aefcb75d25e9f3edb5e3a6 Mon Sep 17 00:00:00 2001 From: Andrei Hutu Date: Fri, 22 Mar 2024 17:27:08 +0000 Subject: [PATCH 4/4] Fix linting --- python/tvm/testing/aot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tvm/testing/aot.py b/python/tvm/testing/aot.py index fa1af5df31f9..0aba24716423 100644 --- a/python/tvm/testing/aot.py +++ b/python/tvm/testing/aot.py @@ -484,7 +484,7 @@ def _emit_main_compare( }} }} if (mismatch >= 1) {{ - float percent_mismatched = + float percent_mismatched = ((float) mismatch) / ((float) {data_length_var_name}) * 100; printf("\\nMismatched elements: %d / %zu (%.2f%%)\\n", mismatch, {data_length_var_name}, percent_mismatched);