From 6d49bd3737aa696fe4933b2a5c187f0ea0921841 Mon Sep 17 00:00:00 2001 From: Matthew Van Schellebeeck Date: Wed, 30 Nov 2022 20:47:52 -0500 Subject: [PATCH 1/2] Rollback ubuntu version --- .github/workflows/rust.yml | 2 +- .../core/tests/sqllogictests/parse_test.py | 89 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 datafusion/core/tests/sqllogictests/parse_test.py diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cdde8aa64c8da..e6ab64ddef8e1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -296,7 +296,7 @@ jobs: test-datafusion-pyarrow: name: cargo test pyarrow (amd64) needs: [linux-build-lib] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 container: image: amd64/rust env: diff --git a/datafusion/core/tests/sqllogictests/parse_test.py b/datafusion/core/tests/sqllogictests/parse_test.py new file mode 100644 index 0000000000000..66de607cbc560 --- /dev/null +++ b/datafusion/core/tests/sqllogictests/parse_test.py @@ -0,0 +1,89 @@ +import re + +pattern = re.compile( + r"async fn (.*?)\(\) [.\s\S\t]*?let sql =\s*([.\s\S\t]*?);[.\s\S\t]*?let expected =\s*([.\s\S\t]*?);" +) + +TEST_BASE_PATH = "../../datafusion/core/tests/sql" +TEST_FILES = [ + "aggregates.rs", +] +IGNORE_FUNCTIONS = [ + "csv_query_approx_percentile_cont", + "csv_query_approx_percentile_cont_with_weight", + "csv_query_approx_percentile_cont_with_histogram_bins", +] + + +def construct_sqllogictest(function_name, column_count, sql_query, result): + return ( + "# " + + function_name + + "\nquery " + + ("I" * column_count) + + "\n" + + sql_query + + "\n" + + result + ) + + +def strip_formatting(result: str) -> str: + lines = result.split("\n") + if "vec!" in result: + lines = lines[4:-2] + lines = [line.replace('"', "").replace(",", "") for line in lines] + lines = [ + ",".join(re.findall("(.*?)\|", line)[1:]).replace(" ", "") for line in lines + ] + + result = [] + length = 0 + for line in lines: + res = [] + split = line.split(",") + length = max(length, len(split)) + for x in split: + res.append(x) + # if x == "": + # res.append(" ") + # else: + # res.append(x) + result.append(" ".join(res)) + return (length, "\n".join(result).rstrip()) + + +count = 0 +for test_file in TEST_FILES: + with open("output.txt", "a") as the_file: + with open(f"{TEST_BASE_PATH}/{test_file}", "r") as file: + file_string = file.read() + split = file_string.split("#[tokio::test]") + sqllogic_test = None + for i in split[1:]: + result = re.search(pattern, i) + if not result: + sqllogic_test = "# could not parse function" + # try parse function name + result = re.search(r"async fn (.*?)\(\) [.\s\S\t]*?", i) + if result: + sqllogic_test = "# could not parse function " + result.group(1) + else: + sqllogic_test = "# UNKNOWN" + else: + count += 1 + function_name = result.group(1) + sql_query = result.group(2).replace('"', "") + (length, expected_result) = strip_formatting(result.group(3)) + # print(f"function_name: {function_name}") + # print(f"sql_query: {sql_query}") + # print(f"expected_result (formatted): {expected_result}") + # print(f"expected_result: {result.group(3)}") + sqllogic_test = construct_sqllogictest( + function_name, + length, + sql_query, + expected_result, + ) + + the_file.write(sqllogic_test + "\n\n") From 0b9d3981ea7ec7d1b39edb782dd04b4aa34ca65d Mon Sep 17 00:00:00 2001 From: Matthew Van Schellebeeck Date: Wed, 30 Nov 2022 20:49:04 -0500 Subject: [PATCH 2/2] Remove file --- .../core/tests/sqllogictests/parse_test.py | 89 ------------------- 1 file changed, 89 deletions(-) delete mode 100644 datafusion/core/tests/sqllogictests/parse_test.py diff --git a/datafusion/core/tests/sqllogictests/parse_test.py b/datafusion/core/tests/sqllogictests/parse_test.py deleted file mode 100644 index 66de607cbc560..0000000000000 --- a/datafusion/core/tests/sqllogictests/parse_test.py +++ /dev/null @@ -1,89 +0,0 @@ -import re - -pattern = re.compile( - r"async fn (.*?)\(\) [.\s\S\t]*?let sql =\s*([.\s\S\t]*?);[.\s\S\t]*?let expected =\s*([.\s\S\t]*?);" -) - -TEST_BASE_PATH = "../../datafusion/core/tests/sql" -TEST_FILES = [ - "aggregates.rs", -] -IGNORE_FUNCTIONS = [ - "csv_query_approx_percentile_cont", - "csv_query_approx_percentile_cont_with_weight", - "csv_query_approx_percentile_cont_with_histogram_bins", -] - - -def construct_sqllogictest(function_name, column_count, sql_query, result): - return ( - "# " - + function_name - + "\nquery " - + ("I" * column_count) - + "\n" - + sql_query - + "\n" - + result - ) - - -def strip_formatting(result: str) -> str: - lines = result.split("\n") - if "vec!" in result: - lines = lines[4:-2] - lines = [line.replace('"', "").replace(",", "") for line in lines] - lines = [ - ",".join(re.findall("(.*?)\|", line)[1:]).replace(" ", "") for line in lines - ] - - result = [] - length = 0 - for line in lines: - res = [] - split = line.split(",") - length = max(length, len(split)) - for x in split: - res.append(x) - # if x == "": - # res.append(" ") - # else: - # res.append(x) - result.append(" ".join(res)) - return (length, "\n".join(result).rstrip()) - - -count = 0 -for test_file in TEST_FILES: - with open("output.txt", "a") as the_file: - with open(f"{TEST_BASE_PATH}/{test_file}", "r") as file: - file_string = file.read() - split = file_string.split("#[tokio::test]") - sqllogic_test = None - for i in split[1:]: - result = re.search(pattern, i) - if not result: - sqllogic_test = "# could not parse function" - # try parse function name - result = re.search(r"async fn (.*?)\(\) [.\s\S\t]*?", i) - if result: - sqllogic_test = "# could not parse function " + result.group(1) - else: - sqllogic_test = "# UNKNOWN" - else: - count += 1 - function_name = result.group(1) - sql_query = result.group(2).replace('"', "") - (length, expected_result) = strip_formatting(result.group(3)) - # print(f"function_name: {function_name}") - # print(f"sql_query: {sql_query}") - # print(f"expected_result (formatted): {expected_result}") - # print(f"expected_result: {result.group(3)}") - sqllogic_test = construct_sqllogictest( - function_name, - length, - sql_query, - expected_result, - ) - - the_file.write(sqllogic_test + "\n\n")