Skip to content

[improvement](planner) unset common fields to reduce plan thrift size #12495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 4 commits into from
Sep 9, 2022
Merged

[improvement](planner) unset common fields to reduce plan thrift size #12495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
48fa18f
[improvment](planner) unset common fields to reduce plan thrift size
morningman Sep 8, 2022
f2f6bd9
[improvment](planner) unset common fields to reduce plan thrift size
morningman Sep 8, 2022
7b640d3
remove verify
morningman Sep 8, 2022
6f3b4e0
remove verify
morningman Sep 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,17 +294,21 @@ public String debugString() {
* and that all rhs exprs are analyzed.
*/
private void verify() {
for (int i = 0; i < lhs.size(); ++i) {
for (int j = i + 1; j < lhs.size(); ++j) {
if (lhs.get(i).equals(lhs.get(j))) {
if (LOG.isTraceEnabled()) {
LOG.trace("verify: smap=" + this.debugString());
// This method is very very time consuming, especially when planning large complex query.
// So disable it by default.
if (LOG.isDebugEnabled()) {
for (int i = 0; i < lhs.size(); ++i) {
for (int j = i + 1; j < lhs.size(); ++j) {
if (lhs.get(i).equals(lhs.get(j))) {
if (LOG.isTraceEnabled()) {
LOG.trace("verify: smap=" + this.debugString());
}
// TODO(zc): partition by k1, order by k1, there is failed.
// Preconditions.checkState(false);
}
// TODO(zc): partition by k1, order by k1, there is failed.
// Preconditions.checkState(false);
}
Preconditions.checkState(!checkAnalyzed || rhs.get(i).isAnalyzed());
}
Preconditions.checkState(!checkAnalyzed || rhs.get(i).isAnalyzed());
}
}

Expand Down
20 changes: 5 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,6 @@ public class Coordinator {
// copied from TQueryExecRequest; constant across all fragments
private final TDescriptorTable descTable;

private final Set<Long> alreadySentBackendIds = Sets.newHashSet();

// Why do we use query global?
// When `NOW()` function is in sql, we need only one now(),
// but, we execute `NOW()` distributed.
Expand Down Expand Up @@ -396,7 +394,6 @@ public void clearExportStatus() {
}
this.exportFiles.clear();
this.needCheckBackendExecStates.clear();
this.alreadySentBackendIds.clear();
} finally {
lock.unlock();
}
Expand Down Expand Up @@ -748,9 +745,6 @@ private void waitRpc(List<Pair<BackendExecStates, Future<PExecPlanFragmentResult
} finally {
pair.first.scopedSpan.endSpan();
}

// succeed to send the plan fragment, update the "alreadySentBackendIds"
alreadySentBackendIds.add(pair.first.beId);
}
}

Expand Down Expand Up @@ -2081,15 +2075,11 @@ public BackendExecState(PlanFragmentId fragmentId, int instanceId, int profileFr
* This information can be obtained from the cache of BE.
*/
public void unsetFields() {
if (alreadySentBackendIds.contains(backend.getId())) {
this.rpcParams.unsetDescTbl();
this.rpcParams.unsetCoord();
this.rpcParams.unsetQueryGlobals();
this.rpcParams.unsetResourceInfo();
this.rpcParams.setIsSimplifiedParam(true);
} else {
this.rpcParams.setIsSimplifiedParam(false);
}
this.rpcParams.unsetDescTbl();
this.rpcParams.unsetCoord();
this.rpcParams.unsetQueryGlobals();
this.rpcParams.unsetResourceInfo();
this.rpcParams.setIsSimplifiedParam(true);
}

// update profile.
Expand Down