database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](shuffle) enable strict consistency dml by default (#32958)

Browse Source
This commit is contained in:
Kaijie Chen
2024-05-08 09:50:34 +08:00
committed by yiguolei
parent 05260d6d47
commit 400105a921
5 changed files with 45 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java
View File

@ -21,8 +21,10 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
@ -199,6 +201,11 @@ public class PhysicalOlapTableSink<CHILD_TYPE extends Plan> extends PhysicalTabl
if (targetTable.isPartitionDistributed()) {
DistributionInfo distributionInfo = targetTable.getDefaultDistributionInfo();
if (distributionInfo instanceof HashDistributionInfo) {
// Do not enable shuffle for duplicate key tables when its tablet num is less than threshold.
if (targetTable.getKeysType() == KeysType.DUP_KEYS
&& distributionInfo.getBucketNum() < Config.min_tablets_for_dup_table_shuffle) {
return PhysicalProperties.ANY;
}
return PhysicalProperties.TABLET_ID_SHUFFLE;
} else if (distributionInfo instanceof RandomDistributionInfo) {
return PhysicalProperties.ANY;

10
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -893,7 +893,7 @@ public class SessionVariable implements Serializable, Writable {
public boolean enableNereidsDmlWithPipeline = true;
@VariableMgr.VarAttr(name = ENABLE_STRICT_CONSISTENCY_DML, needForward = true)
public boolean enableStrictConsistencyDml = false;
public boolean enableStrictConsistencyDml = true;
@VariableMgr.VarAttr(name = ENABLE_VECTORIZED_ENGINE, varType = VariableAnnotation.EXPERIMENTAL_ONLINE)
public boolean enableVectorizedEngine = true;
@ -3457,6 +3457,14 @@ public class SessionVariable implements Serializable, Writable {
this.dumpNereidsMemo = dumpNereidsMemo;
}
public boolean isEnableStrictConsistencyDml() {
return this.enableStrictConsistencyDml;
}
public void setEnableStrictConsistencyDml(boolean value) {
this.enableStrictConsistencyDml = value;
}
public void disableStrictConsistencyDmlOnce() throws DdlException {
if (!enableStrictConsistencyDml) {
return;

2
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -3127,6 +3127,8 @@ public class StmtExecutor {
try {
if (sessionVariable.isEnableNereidsPlanner()) {
try {
// disable shuffle for http stream (only 1 sink)
sessionVariable.disableStrictConsistencyDmlOnce();
httpStreamParams = generateHttpStreamNereidsPlan(queryId);
} catch (NereidsException | ParseException e) {
if (context.getMinidump() != null && context.getMinidump().toString(4) != null) {