From a85beb177398ae13f8c1b13057e38c006e44d2d6 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:47:49 +0800 Subject: [PATCH] [opt](Nereids) do not fallback if nereids failed because timeout (#39499) pick from master #39499 since legacy planner will cost more time to plan, fallback will be worse than throw exception directly --- .../exceptions/DoNotFallbackException.java | 27 +++++++++++++++ .../jobs/scheduler/SimpleJobScheduler.java | 3 +- .../org/apache/doris/qe/SessionVariable.java | 2 +- .../org/apache/doris/qe/StmtExecutor.java | 18 ++++++++-- .../nereids_p0/test_timeout_fallback.groovy | 33 +++++++++++++++++++ 5 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DoNotFallbackException.java create mode 100644 regression-test/suites/nereids_p0/test_timeout_fallback.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DoNotFallbackException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DoNotFallbackException.java new file mode 100644 index 00000000000000..b6253f52c6b5df --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DoNotFallbackException.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.exceptions; + +/** + * Exception for can not fall back error in Nereids. + */ +public class DoNotFallbackException extends RuntimeException { + public DoNotFallbackException(String msg) { + super(msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java index ec751bdab2d446..32a82127e6dbe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.jobs.scheduler; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.DoNotFallbackException; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.qe.SessionVariable; @@ -36,7 +37,7 @@ public void executeJobPool(ScheduleContext scheduleContext) { if (sessionVariable.enableNereidsTimeout && context.getStatementContext().getStopwatch().elapsed(TimeUnit.MILLISECONDS) > sessionVariable.nereidsTimeoutSecond * 1000L) { - throw new RuntimeException( + throw new DoNotFallbackException( "Nereids cost too much time ( > " + sessionVariable.nereidsTimeoutSecond + "s )"); } Job job = pool.pop(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 40a808533f83b5..ea0c40942436aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1031,7 +1031,7 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { public boolean enableNewShuffleHashMethod = true; @VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true) - public int nereidsTimeoutSecond = 5; + public int nereidsTimeoutSecond = 30; @VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG) public boolean enablePushDownNoGroupAgg = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 7bb4ee6c2af62d..02450f424c0219 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -126,6 +126,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.DoNotFallbackException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.minidump.MinidumpUtils; @@ -490,8 +491,13 @@ public void execute(TUniqueId queryId) throws Exception { MinidumpUtils.saveMinidumpString(context.getMinidump(), DebugUtil.printId(context.queryId())); } // try to fall back to legacy planner - LOG.debug("nereids cannot process statement\n" + originStmt.originStmt - + "\n because of " + e.getMessage(), e); + LOG.debug("nereids cannot process statement\n{}\n because of {}", + originStmt.originStmt, e.getMessage(), e); + if (e instanceof NereidsException + && ((NereidsException) e).getException() instanceof DoNotFallbackException) { + LOG.warn("Analyze failed. {}", context.getQueryIdentifier(), e); + throw new AnalysisException(e.getMessage()); + } if (e instanceof NereidsException && !context.getSessionVariable().enableFallbackToOriginalPlanner) { LOG.warn("Analyze failed. {}", context.getQueryIdentifier(), e); @@ -599,6 +605,11 @@ private void executeByNereids(TUniqueId queryId) throws Exception { syncJournalIfNeeded(); try { ((Command) logicalPlan).run(context, this); + } catch (DoNotFallbackException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Command({}) process failed.", originStmt.originStmt, e); + } + throw new NereidsException("Command(" + originStmt.originStmt + ") process failed.", e); } catch (QueryStateException e) { LOG.debug("DDL statement(" + originStmt.originStmt + ") process failed.", e); context.setState(e.getQueryState()); @@ -631,6 +642,9 @@ private void executeByNereids(TUniqueId queryId) throws Exception { try { planner.plan(parsedStmt, context.getSessionVariable().toThrift()); checkBlockRules(); + } catch (DoNotFallbackException e) { + LOG.warn("Nereids plan query failed:\n{}", originStmt.originStmt, e); + throw new NereidsException("Command(" + originStmt.originStmt + ") process failed.", e); } catch (Exception e) { LOG.debug("Nereids plan query failed:\n{}", originStmt.originStmt); throw new NereidsException(new AnalysisException("Unexpected exception: " + e.getMessage(), e)); diff --git a/regression-test/suites/nereids_p0/test_timeout_fallback.groovy b/regression-test/suites/nereids_p0/test_timeout_fallback.groovy new file mode 100644 index 00000000000000..52ceda85635c54 --- /dev/null +++ b/regression-test/suites/nereids_p0/test_timeout_fallback.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timeout_fallback") { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=true" + sql "set enable_nereids_timeout=true" + sql "set nereids_timeout_second=-1" + + test { + sql "select 1" + exception "Nereids cost too much time" + } + + test { + sql "explain select 1" + exception "Nereids cost too much time" + } +}