From 7c17628e4c1bf9cd5f790ee0de2a957dab5e178a Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 1 Jul 2024 19:06:36 +0800 Subject: [PATCH 1/2] [case](restart_fe) add demo case for restart_fe test --- regression-test/data/javaudf_p0/null/test.out | 26 ++++++++ .../suites/javaudf_p0/null/load.groovy | 66 +++++++++++++++++++ .../suites/javaudf_p0/null/test.groovy | 24 +++++++ 3 files changed, 116 insertions(+) create mode 100644 regression-test/data/javaudf_p0/null/test.out create mode 100644 regression-test/suites/javaudf_p0/null/load.groovy create mode 100644 regression-test/suites/javaudf_p0/null/test.groovy diff --git a/regression-test/data/javaudf_p0/null/test.out b/regression-test/data/javaudf_p0/null/test.out new file mode 100644 index 00000000000000..dc59ed24afdcdb --- /dev/null +++ b/regression-test/data/javaudf_p0/null/test.out @@ -0,0 +1,26 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !select -- +\N + +-- !select -- +\N +\N +\N +\N +\N +\N +\N +\N +\N + diff --git a/regression-test/suites/javaudf_p0/null/load.groovy b/regression-test/suites/javaudf_p0/null/load.groovy new file mode 100644 index 00000000000000..add19203bb71e8 --- /dev/null +++ b/regression-test/suites/javaudf_p0/null/load.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite('test_javaudf_null_load', 'p0,restart_fe') { + // In order to cover the scenario that udf can be used normally after fe restart. + // We devided the origin case test_javaudf_null.groovy into two parts, load and query, this case is the first part. + // run load and query -> restart fe -> run query again, + // by this way, we can cover the scenario. + def tableName = 'test_javaudf_null' + def jarPath = """${context.file.parent}/../jars/java-udf-case-jar-with-dependencies.jar""" + scp_udf_file_to_all_be(jarPath) + + log.info("Jar path: ${jarPath}".toString()) + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """DROP FUNCTION IF EXISTS java_udf_null_test(int);""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 9; i ++) { + sb.append(""" + (${i}), + """) + } + sb.append(""" + (${i}) + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE FUNCTION java_udf_null_test(int) RETURNS int PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.NullTest", + "type"="JAVA_UDF" + ); """ +} diff --git a/regression-test/suites/javaudf_p0/null/test.groovy b/regression-test/suites/javaudf_p0/null/test.groovy new file mode 100644 index 00000000000000..2c67c3e7ead495 --- /dev/null +++ b/regression-test/suites/javaudf_p0/null/test.groovy @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_javaudf_null_query', 'p0,restart_fe') { + def tableName = 'test_javaudf_null' + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + qt_select ''' SELECT java_udf_null_test(1) result; ''' + qt_select """ SELECT java_udf_null_test(user_id) result FROM ${tableName} ORDER BY result; """ +} From c6125c23a3072fa245a1ebde93930bfcab1070f5 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 3 Jul 2024 20:20:36 +0800 Subject: [PATCH 2/2] 2 --- regression-test/README.md | 11 +++ .../suites/auth_up_down_p0/load.groovy | 2 +- .../test_grant_revoke_auth.groovy | 2 +- .../javaudf_p0/test_javaudf_null.groovy | 71 ------------------- 4 files changed, 13 insertions(+), 73 deletions(-) delete mode 100644 regression-test/suites/javaudf_p0/test_javaudf_null.groovy diff --git a/regression-test/README.md b/regression-test/README.md index 849fc3681e7858..f856f2141a69b3 100644 --- a/regression-test/README.md +++ b/regression-test/README.md @@ -19,6 +19,7 @@ under the License. # 新加case注意事项 +## 常规 case 1. 变量名前要写 def，否则是全局变量，并行跑的 case 的时候可能被其他 case 影响。 Problematic code: @@ -65,3 +66,13 @@ under the License. sql """sync""" sql """select count(*) from table """ ``` + +6. UDF 的 case，需要把对应的 jar 包拷贝到所有 BE 机器上。 + + [示例](https://github.com/apache/doris/blob/master/regression-test/suites/javaudf_p0/test_javaudf_case.groovy#L27) + + +## 兼容性 case +指重启 FE 测试或升级测试中，在初始集群上创建的资源或规则，在集群重启或升级后也能正常使用，比如权限、UDF等。 +这些 case 需要拆分成两个文件，load.groovy 和 xxxx.groovy，放到一个文件夹中并加上 `restart_fe` 组标签，[示例](https://github.com/apache/doris/pull/37118)。 + diff --git a/regression-test/suites/auth_up_down_p0/load.groovy b/regression-test/suites/auth_up_down_p0/load.groovy index fd1405e58e947b..1cc43ba6a54a5c 100644 --- a/regression-test/suites/auth_up_down_p0/load.groovy +++ b/regression-test/suites/auth_up_down_p0/load.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_upgrade_downgrade_prepare_auth","p0,auth") { +suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") { String user1 = 'test_upgrade_downgrade_compatibility_auth_user1' String user2 = 'test_upgrade_downgrade_compatibility_auth_user2' diff --git a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy index ff9588b7c481b0..8828be6652827a 100644 --- a/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy +++ b/regression-test/suites/auth_up_down_p0/test_grant_revoke_auth.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_upgrade_downgrade_compatibility_auth","p0,auth") { +suite("test_upgrade_downgrade_compatibility_auth","p0,auth,restart_fe") { sql """ADMIN SET FRONTEND CONFIG ('experimental_enable_workload_group' = 'true');""" sql """set experimental_enable_pipeline_engine = true;""" diff --git a/regression-test/suites/javaudf_p0/test_javaudf_null.groovy b/regression-test/suites/javaudf_p0/test_javaudf_null.groovy deleted file mode 100644 index a710b1cb93273b..00000000000000 --- a/regression-test/suites/javaudf_p0/test_javaudf_null.groovy +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.nio.file.Paths - -suite("test_javaudf_null") { - def tableName = "test_javaudf_null" - def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" - scp_udf_file_to_all_be(jarPath) - - log.info("Jar path: ${jarPath}".toString()) - try { - sql """ DROP TABLE IF EXISTS ${tableName} """ - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `user_id` INT NOT NULL COMMENT "" - ) - DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); - """ - StringBuilder sb = new StringBuilder() - int i = 1 - for (; i < 9; i ++) { - sb.append(""" - (${i}), - """) - } - sb.append(""" - (${i}) - """) - sql """ INSERT INTO ${tableName} VALUES - ${sb.toString()} - """ - qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ - - File path = new File(jarPath) - if (!path.exists()) { - throw new IllegalStateException("""${jarPath} doesn't exist! """) - } - - sql """ CREATE FUNCTION java_udf_null_test(int) RETURNS int PROPERTIES ( - "file"="file://${jarPath}", - "symbol"="org.apache.doris.udf.NullTest", - "type"="JAVA_UDF" - ); """ - - qt_select """ SELECT java_udf_null_test(1) result; """ - qt_select """ SELECT java_udf_null_test(user_id) result FROM ${tableName} ORDER BY result; """ - - } finally { - try_sql("DROP FUNCTION IF EXISTS java_udf_null_test(int);") - try_sql("DROP TABLE IF EXISTS ${tableName}") - } -}