database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvment](planner) unset common fields to reduce plan thrift size (#12495)

Browse Source 
1. For query with 1656 union, the plan thrift size will be reduced from 400MB+ to 2MB.
This optimization is introduced from #4904, but lost after #9720

2. Disable ExprSubstitutionMap.verify when debug is disable.
So that the plan time of query with 1656 union will be reduced from 20s to 2s
This commit is contained in:
Mingyu Chen
2022-09-09 09:02:45 +08:00
committed by GitHub
parent d2a23a4cf9
commit e84272ed43
2 changed files with 17 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
View File

@ -294,17 +294,21 @@ public final class ExprSubstitutionMap {
* and that all rhs exprs are analyzed.
*/
private void verify() {
for (int i = 0; i < lhs.size(); ++i) {
for (int j = i + 1; j < lhs.size(); ++j) {
if (lhs.get(i).equals(lhs.get(j))) {
if (LOG.isTraceEnabled()) {
LOG.trace("verify: smap=" + this.debugString());
// This method is very very time consuming, especially when planning large complex query.
// So disable it by default.
if (LOG.isDebugEnabled()) {
for (int i = 0; i < lhs.size(); ++i) {
for (int j = i + 1; j < lhs.size(); ++j) {
if (lhs.get(i).equals(lhs.get(j))) {
if (LOG.isTraceEnabled()) {
LOG.trace("verify: smap=" + this.debugString());
}
// TODO(zc): partition by k1, order by k1, there is failed.
// Preconditions.checkState(false);
}
// TODO(zc): partition by k1, order by k1, there is failed.
// Preconditions.checkState(false);
}
Preconditions.checkState(!checkAnalyzed || rhs.get(i).isAnalyzed());
}
Preconditions.checkState(!checkAnalyzed || rhs.get(i).isAnalyzed());
}
}

20
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -155,8 +155,6 @@ public class Coordinator {
// copied from TQueryExecRequest; constant across all fragments
private final TDescriptorTable descTable;
private final Set<Long> alreadySentBackendIds = Sets.newHashSet();
// Why do we use query global?
// When `NOW()` function is in sql, we need only one now(),
// but, we execute `NOW()` distributed.
@ -396,7 +394,6 @@ public class Coordinator {
}
this.exportFiles.clear();
this.needCheckBackendExecStates.clear();
this.alreadySentBackendIds.clear();
} finally {
lock.unlock();
}
@ -748,9 +745,6 @@ public class Coordinator {
} finally {
pair.first.scopedSpan.endSpan();
}
// succeed to send the plan fragment, update the "alreadySentBackendIds"
alreadySentBackendIds.add(pair.first.beId);
}
}
@ -2081,15 +2075,11 @@ public class Coordinator {
* This information can be obtained from the cache of BE.
*/
public void unsetFields() {
if (alreadySentBackendIds.contains(backend.getId())) {
this.rpcParams.unsetDescTbl();
this.rpcParams.unsetCoord();
this.rpcParams.unsetQueryGlobals();
this.rpcParams.unsetResourceInfo();
this.rpcParams.setIsSimplifiedParam(true);
} else {
this.rpcParams.setIsSimplifiedParam(false);
}
this.rpcParams.unsetDescTbl();
this.rpcParams.unsetCoord();
this.rpcParams.unsetQueryGlobals();
this.rpcParams.unsetResourceInfo();
this.rpcParams.setIsSimplifiedParam(true);
}
// update profile.