Skip to content

[fix](regression-test) Make test_decimal256_outfile_csv export the data to S3 rather than local file system #42211 #42271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
Oct 22, 2024
Merged

[fix](regression-test) Make test_decimal256_outfile_csv export the data to S3 rather than local file system #42211 #42271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,24 @@
4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000

-- !select_tvf1 --
\N \N \N
-9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-99999999999999999999999999999.999999999 1.0000000000 1.00000000000
0.000000000 0.0000000000 0.00000000000
1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
2.000000000 499999999999999999999999999999999999999999999999999999999999999999.9999999999 49999999999999999999999999999999999999999999999999999999999999999.99999999999
3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
6.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999
7.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999
99999999999999999999999999999.999999999 1.0000000000 1.00000000000
4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000

85 changes: 35 additions & 50 deletions regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,36 +22,15 @@ import java.nio.file.Files
import java.nio.file.Paths

suite("test_decimal256_outfile_csv") {
StringBuilder strBuilder = new StringBuilder()
strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe")

String command = strBuilder.toString()
def process = command.toString().execute()
def code = process.waitFor()
def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
def out = process.getText()
logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
def response = parseJson(out.trim())
assertEquals(response.code, 0)
assertEquals(response.msg, "success")
def configJson = response.data.rows
boolean enableOutfileToLocal = false
for (Object conf: configJson) {
assert conf instanceof Map
if (((Map<String, String>) conf).get("Name").toLowerCase() == "enable_outfile_to_local") {
enableOutfileToLocal = ((Map<String, String>) conf).get("Value").toLowerCase() == "true"
}
}
if (!enableOutfileToLocal) {
logger.warn("Please set enable_outfile_to_local to true to run test_outfile")
return
}

sql "set enable_nereids_planner = true;"
sql "set enable_decimal256 = true;"

String ak = getS3AK()
String sk = getS3SK()
String s3_endpoint = getS3Endpoint()
String region = getS3Region()
String bucket = context.config.otherConfigs.get("s3BucketName");

sql "DROP TABLE IF EXISTS `test_decimal256_outfile_csv`"
sql """
CREATE TABLE IF NOT EXISTS `test_decimal256_outfile_csv` (
Expand Down Expand Up @@ -105,31 +84,37 @@ suite("test_decimal256_outfile_csv") {
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3;
"""

def uuid = UUID.randomUUID().toString()
def outFileNamePrefix = """test_decimal256_outfile_csv_${uuid}_"""
def outFilePath = """/tmp/${outFileNamePrefix}"""
FilenameFilter filter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith(outFileNamePrefix);
}
};

def outFilePath = "${bucket}/outfile/csv/test_decimal256_outfile_csv/exp_"

def outfile_to_S3 = { export_table_name, foramt ->
// select ... into outfile ...
def res = sql """
SELECT * FROM ${export_table_name} t ORDER BY k1
INTO OUTFILE "s3://${outFilePath}"
FORMAT AS ${foramt}
PROPERTIES (
"s3.endpoint" = "${s3_endpoint}",
"s3.region" = "${region}",
"s3.secret_key"="${sk}",
"s3.access_key" = "${ak}"
);
"""
return res[0][3]
}

try {
logger.info("outfile: " + outFilePath)
sql """
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3 INTO OUTFILE "file://${outFilePath}" properties("column_separator" = ",");
"""
File path = new File("/tmp/")
File[] files = path.listFiles(filter)
assert files.length == 1
List<String> outLines = Files.readAllLines(Paths.get(files[0].getAbsolutePath()), StandardCharsets.UTF_8);
assert outLines.size() == 19
def outfile_url = outfile_to_S3("test_decimal256_outfile_csv", "csv")

qt_select_tvf1 """ SELECT * FROM S3 (
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
"region" = "${region}"
);
"""
} finally {
File path = new File("/tmp/")
if (path.exists()) {
for (File f: path.listFiles(filter)) {
f.delete();
}
}
}
}