From bdfdeba4aa82694bf673162308c3a53f3d11ccca Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:26:55 +0800 Subject: [PATCH] [fix](regression-test) Make `test_decimal256_outfile_csv` export the data to S3 rather than local file system (#42211) Because it is difficult to check out the outfile files when export data to local file system, it's better to export data to S3. --- .../decimalv3/test_decimal256_outfile_csv.out | 21 +++++ .../test_decimal256_outfile_csv.groovy | 85 ++++++++----------- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out b/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out index 6dd74d5daa541a..e30101a08e657b 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out +++ b/regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out @@ -20,3 +20,24 @@ 4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000 9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000 +-- !select_tvf1 -- +\N \N \N +-9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000 +-4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000 +-99999999999999999999999999999.999999999 1.0000000000 1.00000000000 +0.000000000 0.0000000000 0.00000000000 +1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999 +1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999 +2.000000000 499999999999999999999999999999999999999999999999999999999999999999.9999999999 49999999999999999999999999999999999999999999999999999999999999999.99999999999 +3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333 +3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333 +4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999 +4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999 +5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333 +5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333 +6.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999 +7.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999 +99999999999999999999999999999.999999999 1.0000000000 1.00000000000 +4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000 +9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000 + diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy index 79ff824ef00bb5..736ff970f1c9a9 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy @@ -22,36 +22,15 @@ import java.nio.file.Files import java.nio.file.Paths suite("test_decimal256_outfile_csv") { - StringBuilder strBuilder = new StringBuilder() - strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword) - strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe") - - String command = strBuilder.toString() - def process = command.toString().execute() - def code = process.waitFor() - def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - def out = process.getText() - logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def response = parseJson(out.trim()) - assertEquals(response.code, 0) - assertEquals(response.msg, "success") - def configJson = response.data.rows - boolean enableOutfileToLocal = false - for (Object conf: configJson) { - assert conf instanceof Map - if (((Map) conf).get("Name").toLowerCase() == "enable_outfile_to_local") { - enableOutfileToLocal = ((Map) conf).get("Value").toLowerCase() == "true" - } - } - if (!enableOutfileToLocal) { - logger.warn("Please set enable_outfile_to_local to true to run test_outfile") - return - } - sql "set enable_nereids_planner = true;" sql "set enable_decimal256 = true;" + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName"); + sql "DROP TABLE IF EXISTS `test_decimal256_outfile_csv`" sql """ CREATE TABLE IF NOT EXISTS `test_decimal256_outfile_csv` ( @@ -105,31 +84,37 @@ suite("test_decimal256_outfile_csv") { SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3; """ - def uuid = UUID.randomUUID().toString() - def outFileNamePrefix = """test_decimal256_outfile_csv_${uuid}_""" - def outFilePath = """/tmp/${outFileNamePrefix}""" - FilenameFilter filter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith(outFileNamePrefix); - } - }; + + def outFilePath = "${bucket}/outfile/csv/test_decimal256_outfile_csv/exp_" + + def outfile_to_S3 = { export_table_name, foramt -> + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY k1 + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${foramt} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + return res[0][3] + } + try { logger.info("outfile: " + outFilePath) - sql """ - SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3 INTO OUTFILE "file://${outFilePath}" properties("column_separator" = ","); - """ - File path = new File("/tmp/") - File[] files = path.listFiles(filter) - assert files.length == 1 - List outLines = Files.readAllLines(Paths.get(files[0].getAbsolutePath()), StandardCharsets.UTF_8); - assert outLines.size() == 19 + def outfile_url = outfile_to_S3("test_decimal256_outfile_csv", "csv") + + qt_select_tvf1 """ SELECT * FROM S3 ( + "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.csv", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "csv", + "region" = "${region}" + ); + """ } finally { - File path = new File("/tmp/") - if (path.exists()) { - for (File f: path.listFiles(filter)) { - f.delete(); - } - } } } \ No newline at end of file