Skip to content

branch-3.0: [fix](nereids)Fix dlog1, trim, extract_url_parameter and parse_url FE constant calculate bug. #49074 #49225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Mar 20, 2025
Merged

branch-3.0: [fix](nereids)Fix dlog1, trim, extract_url_parameter and parse_url FE constant calculate bug. #49074 #49225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,6 @@
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dexp;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dfloor;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DigitalMasking;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dlog1;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dlog10;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Domain;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DomainWithoutWww;
Expand Down Expand Up @@ -624,7 +623,6 @@ public class BuiltinScalarFunctions implements FunctionHelper {
scalar(Dexp.class, "dexp"),
scalar(Dfloor.class, "dfloor"),
scalar(DigitalMasking.class, "digital_masking"),
scalar(Dlog1.class, "dlog1"),
scalar(Dlog10.class, "dlog10"),
scalar(Domain.class, "domain"),
scalar(DomainWithoutWww.class, "domain_without_www"),
Expand Down Expand Up @@ -757,7 +755,7 @@ public class BuiltinScalarFunctions implements FunctionHelper {
scalar(Length.class, "length"),
scalar(Crc32.class, "crc32"),
scalar(Like.class, "like"),
scalar(Ln.class, "ln"),
scalar(Ln.class, "ln", "dlog1"),
scalar(Locate.class, "locate"),
scalar(Log.class, "log"),
scalar(Log10.class, "log10"),
Expand Down
11 changes: 0 additions & 11 deletions ...va/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1064,17 +1064,6 @@ public static Expression dexp(DoubleLiteral first) {
return checkOutputBoundary(new DoubleLiteral(exp));
}

/**
* dlog1
*/
@ExecFunction(name = "dlog1")
public static Expression dlog1(DoubleLiteral first) {
if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true)) {
return new NullLiteral(DoubleType.INSTANCE);
}
return checkOutputBoundary(new DoubleLiteral(Math.log1p(first.getValue())));
}

/**
* dlog10
*/
Expand Down
67 changes: 54 additions & 13 deletions ...ava/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,13 +144,17 @@ private static String trimImpl(String first, String second, boolean left, boolea
if (left) {
do {
result = afterReplace;
afterReplace = result.replaceFirst("^" + second, "");
if (result.startsWith(second)) {
afterReplace = result.substring(second.length());
}
} while (!afterReplace.equals(result));
}
if (right) {
do {
result = afterReplace;
afterReplace = result.replaceFirst(second + "$", "");
if (result.endsWith(second)) {
afterReplace = result.substring(0, result.length() - second.length());
}
} while (!afterReplace.equals(result));
}
return result;
Expand Down Expand Up @@ -843,39 +847,71 @@ public static Expression parseurl(StringLikeLiteral first, StringLikeLiteral sec
StringBuilder sb = new StringBuilder();
switch (second.getValue().toUpperCase()) {
case "PROTOCOL":
sb.append(uri.getScheme()); // e.g., http, https
String scheme = uri.getScheme();
if (scheme == null) {
return new NullLiteral(first.getDataType());
}
sb.append(scheme); // e.g., http, https
break;
case "HOST":
sb.append(uri.getHost()); // e.g., www.example.com
String host = uri.getHost();
if (host == null) {
return new NullLiteral(first.getDataType());
}
sb.append(host); // e.g., www.example.com
break;
case "PATH":
sb.append(uri.getPath()); // e.g., /page
String path = uri.getPath();
if (path == null) {
return new NullLiteral(first.getDataType());
}
sb.append(path); // e.g., /page
break;
case "REF":
try {
sb.append(uri.toURL().getRef()); // e.g., /page
String ref = uri.toURL().getRef();
if (ref == null) {
return new NullLiteral(first.getDataType());
}
sb.append(ref); // e.g., /page
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
break;
case "AUTHORITY":
sb.append(uri.getAuthority()); // e.g., param1=value1&param2=value2
String authority = uri.getAuthority();
if (authority == null) {
return new NullLiteral(first.getDataType());
}
sb.append(authority); // e.g., param1=value1&param2=value2
break;
case "FILE":
try {
sb.append(uri.toURL().getFile()); // e.g., param1=value1&param2=value2
String file = uri.toURL().getFile();
if (file == null) {
return new NullLiteral(first.getDataType());
}
sb.append(file); // e.g., param1=value1&param2=value2
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
break;
case "QUERY":
sb.append(uri.getQuery()); // e.g., param1=value1&param2=value2
String query = uri.getQuery();
if (query == null) {
return new NullLiteral(first.getDataType());
}
sb.append(query); // e.g., param1=value1&param2=value2
break;
case "PORT":
sb.append(uri.getPort());
break;
case "USERINFO":
sb.append(uri.getUserInfo()); // e.g., user:pass
String userInfo = uri.getUserInfo();
if (userInfo == null) {
return new NullLiteral(first.getDataType());
}
sb.append(userInfo); // e.g., user:pass
break;
default:
throw new RuntimeException("Valid URL parts are 'PROTOCOL', 'HOST', "
Expand Down Expand Up @@ -943,10 +979,15 @@ public static Expression extractUrlParameter(StringLikeLiteral first, StringLike
if (first.getValue() == null || first.getValue().indexOf('?') == -1) {
return castStringLikeLiteral(first, "");
}
URI uri;
try {
uri = new URI(first.getValue());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}

String[] urlParts = first.getValue().split("\\?", -1);
if (urlParts.length > 1) {
String query = urlParts[1];
String query = uri.getQuery();
if (query != null) {
String[] pairs = query.split("&", -1);

for (String pair : pairs) {
Expand Down
68 changes: 0 additions & 68 deletions ...core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Dlog1.java
View file
Open in desktop

This file was deleted.

5 changes: 0 additions & 5 deletions ...c/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dexp;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dfloor;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DigitalMasking;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dlog1;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Dlog10;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Domain;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DomainWithoutWww;
Expand Down Expand Up @@ -1111,10 +1110,6 @@ default R visitDfloor(Dfloor dfloor, C context) {
return visitScalarFunction(dfloor, context);
}

default R visitDlog1(Dlog1 dlog1, C context) {
return visitScalarFunction(dlog1, context);
}

default R visitDlog10(Dlog10 dlog10, C context) {
return visitScalarFunction(dlog10, context);
}
Expand Down
47 changes: 31 additions & 16 deletions ...n-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ suite("fold_constant_numeric_arithmatic") {
sql "set enable_fold_constant_by_be=false"

//Abs function cases
testFoldConst("SELECT ABS(1)")
testFoldConst("SELECT ABS(0)")
testFoldConst("SELECT ABS(-1)")
testFoldConst("SELECT ABS(1.5)")
testFoldConst("SELECT ABS(-1.5)")
testFoldConst("SELECT ABS(1E308)")
testFoldConst("SELECT ABS(1)")
testFoldConst("SELECT ABS(0)")
testFoldConst("SELECT ABS(-1)")
testFoldConst("SELECT ABS(1.5)")
testFoldConst("SELECT ABS(-1.5)")
testFoldConst("SELECT ABS(1E308)")
testFoldConst("SELECT ABS(-1E308)")
testFoldConst("SELECT ABS(NULL)") // NULL handling
testFoldConst("SELECT ABS('')") // Empty string handling
Expand Down Expand Up @@ -69,9 +69,9 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT ATAN(1) AS atan_case_1") //atan(1) = π/4
testFoldConst("SELECT ATAN(0) AS atan_case_2") //atan(0) = 0
testFoldConst("SELECT ATAN(-1) AS atan_case_3") //atan(-1)
testFoldConst("SELECT ATAN(1.5)")
testFoldConst("SELECT ATAN(-1.5)")
testFoldConst("SELECT ATAN(1E308)")
testFoldConst("SELECT ATAN(1.5)")
testFoldConst("SELECT ATAN(-1.5)")
testFoldConst("SELECT ATAN(1E308)")
testFoldConst("SELECT ATAN(-1E308)")
testFoldConst("SELECT ATAN(NULL)") // NULL handling
testFoldConst("SELECT ATAN(PI())") // PI input
Expand All @@ -84,9 +84,9 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT ATAN2(0, 1) AS atan2_case_2") //atan2(0, 1) = 0
testFoldConst("SELECT ATAN2(1, 0) AS atan2_case_3") //atan2(1, 0) = π/2
testFoldConst("SELECT ATAN2(0, 0) AS atan2_case_exception") //undefined (returns NULL or error)
testFoldConst("SELECT ATAN2(1.5, 1.5)")
testFoldConst("SELECT ATAN2(-1.5, 1.5)")
testFoldConst("SELECT ATAN2(1E308, 1E308)")
testFoldConst("SELECT ATAN2(1.5, 1.5)")
testFoldConst("SELECT ATAN2(-1.5, 1.5)")
testFoldConst("SELECT ATAN2(1E308, 1E308)")
testFoldConst("SELECT ATAN2(-1E308, 1E308)")
testFoldConst("SELECT ATAN2(NULL, 1)") // NULL y
testFoldConst("SELECT ATAN2(1, NULL)") // NULL x
Expand All @@ -98,7 +98,7 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT BIN(16) AS bin_case_2") //bin(16) = 10000
testFoldConst("SELECT BIN(255) AS bin_case_3") //bin(255)
testFoldConst("SELECT BIN(-1) AS bin_case_exception") //returns NULL or error in some databases
testFoldConst("SELECT BIN(1E308)")
testFoldConst("SELECT BIN(1E308)")
testFoldConst("SELECT BIN(-1E308)")
testFoldConst("SELECT BIN(0)") // Zero case
testFoldConst("SELECT BIN(NULL)") // NULL handling
Expand All @@ -112,7 +112,7 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT BIT_COUNT(16) AS bitcount_case_2") //bitcount(16) = 1
testFoldConst("SELECT BIT_COUNT(255) AS bitcount_case_3") //bitcount(255) = 8
testFoldConst("SELECT BIT_COUNT(-1) AS bitcount_case_exception")
testFoldConst("SELECT BIT_COUNT(1E308)")
testFoldConst("SELECT BIT_COUNT(1E308)")
testFoldConst("SELECT BIT_COUNT(-1E308)")
testFoldConst("SELECT BIT_COUNT(0)") // Zero case
testFoldConst("SELECT BIT_COUNT(NULL)") // NULL handling
Expand All @@ -138,7 +138,7 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT CEIL(-3.4) AS ceil_case_2")
testFoldConst("SELECT CEIL(5.0) AS ceil_case_3")
testFoldConst("SELECT CEIL(1E308) AS ceil_case_overflow")
testFoldConst("SELECT CEIL(1E308)")
testFoldConst("SELECT CEIL(1E308)")
testFoldConst("SELECT CEIL(-1E308)")
testFoldConst("SELECT CEIL(NULL)") // NULL handling
testFoldConst("SELECT CEIL(0)") // Zero case
Expand All @@ -157,7 +157,7 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT COALESCE(NULL, NULL, 7) AS coalesce_case_2")
testFoldConst("SELECT COALESCE(3, 5) AS coalesce_case_3")
testFoldConst("SELECT COALESCE(NULL, NULL) AS coalesce_case_4")
testFoldConst("SELECT COALESCE(1E308)")
testFoldConst("SELECT COALESCE(1E308)")
testFoldConst("SELECT COALESCE(-1E308)")
testFoldConst("SELECT COALESCE(NULL, NULL, NULL)") // All NULL
testFoldConst("SELECT COALESCE('', NULL, 'test')") // Empty string
Expand Down Expand Up @@ -316,6 +316,21 @@ suite("fold_constant_numeric_arithmatic") {
testFoldConst("SELECT LN(0.1)") // Small decimal
testFoldConst("SELECT LN(100)") // Larger number

//dlog1 function cases
testFoldConst("SELECT dlog1(1)")
testFoldConst("SELECT dlog1(2.71828)")
testFoldConst("SELECT dlog1(10)")
testFoldConst("SELECT dlog1(1e10)")
testFoldConst("SELECT dlog1(0.1)")
testFoldConst("SELECT dlog1(0.001)")
testFoldConst("SELECT dlog1(1e-10)")
testFoldConst("SELECT dlog1(1e308)")
testFoldConst("SELECT dlog1(1e-308)")
testFoldConst("SELECT dlog1(0)")
testFoldConst("SELECT dlog1(-1)")
testFoldConst("SELECT dlog1(-10)")
testFoldConst("SELECT dlog1(NULL)")

//Log function cases
testFoldConst("SELECT log(100, 10), log(8, 2), log(1000, 10)")
testFoldConst("SELECT LOG(NULL, 10)") // NULL number
Expand Down
Loading
Loading