database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[pick](mtmv)Pick some pr to 21 #39998 #40332 #40485 (#40843)

Browse Source 
## Proposed changes
pr: https://github.com/apache/doris/pull/40485
commitId: 18a374f6

pr: https://github.com/apache/doris/pull/40332
commitId: cd2062ae

pr: https://github.com/apache/doris/pull/39998
commitId: 948fb2ab
This commit is contained in:
seawinde
2024-09-15 23:48:33 +08:00
committed by GitHub
parent b52b572ade
commit fe9f4fdd95
11 changed files with 199 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -250,6 +250,19 @@ public class MTMV extends OlapTable {
}
}
public boolean isUseForRewrite() {
readMvLock();
try {
if (!StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE))) {
return Boolean.valueOf(mvProperties.get(PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE));
}
// default is true
return true;
} finally {
readMvUnlock();
}
}
public int getRefreshPartitionNum() {
readMvLock();
try {

3
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -176,6 +176,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION =
"enable_nondeterministic_function";
public static final String PROPERTIES_USE_FOR_REWRITE =
"use_for_rewrite";
public static final String PROPERTIES_EXCLUDED_TRIGGER_TABLES = "excluded_trigger_tables";
public static final String PROPERTIES_REFRESH_PARTITION_NUM = "refresh_partition_num";
public static final String PROPERTIES_WORKLOAD_GROUP = "workload_group";

15
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
View File

@ -38,7 +38,8 @@ public class MTMVPropertyUtil {
PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT,
PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT,
PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT,
PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION
PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION,
PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE
);
public static void analyzeProperty(String key, String value) {
@ -65,6 +66,10 @@ public class MTMVPropertyUtil {
analyzePartitionSyncLimit(value);
break;
case PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION:
analyzeBooleanProperty(value, PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION);
break;
case PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE:
analyzeBooleanProperty(value, PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE);
break;
default:
throw new AnalysisException("illegal key:" + key);
@ -138,4 +143,12 @@ public class MTMVPropertyUtil {
}
}
private static void analyzeBooleanProperty(String propertyValue, String propertyName) {
if (StringUtils.isEmpty(propertyValue)) {
return;
}
if (!"true".equalsIgnoreCase(propertyValue) && !"false".equalsIgnoreCase(propertyValue)) {
throw new AnalysisException(String.format("valid property %s fail", propertyName));
}
}
}

29
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -485,6 +485,13 @@ public class NereidsPlanner extends Planner {
public String getExplainString(ExplainOptions explainOptions) {
ExplainLevel explainLevel = getExplainLevel(explainOptions);
String plan = "";
String mvSummary = "";
if (this.getPhysicalPlan() != null && cascadesContext != null) {
mvSummary = cascadesContext.getMaterializationContexts().isEmpty() ? "" :
"\n\n========== MATERIALIZATIONS ==========\n"
+ MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
this.getPhysicalPlan());
}
switch (explainLevel) {
case PARSED_PLAN:
plan = parsedPlan.treeString();
@ -496,22 +503,16 @@ public class NereidsPlanner extends Planner {
plan = rewrittenPlan.treeString();
break;
case OPTIMIZED_PLAN:
plan = "cost = " + cost + "\n" + optimizedPlan.treeString();
plan = "cost = " + cost + "\n" + optimizedPlan.treeString() + mvSummary;
break;
case SHAPE_PLAN:
plan = optimizedPlan.shape("");
break;
case MEMO_PLAN:
StringBuilder materializationStringBuilder = new StringBuilder();
materializationStringBuilder.append("materializationContexts:").append("\n");
for (MaterializationContext ctx : cascadesContext.getMaterializationContexts()) {
materializationStringBuilder.append("\n").append(ctx).append("\n");
}
plan = cascadesContext.getMemo().toString()
+ "\n\n========== OPTIMIZED PLAN ==========\n"
+ optimizedPlan.treeString()
+ "\n\n========== MATERIALIZATIONS ==========\n"
+ materializationStringBuilder;
+ mvSummary;
break;
case ALL_PLAN:
plan = "========== PARSED PLAN "
@ -525,18 +526,20 @@ public class NereidsPlanner extends Planner {
+ rewrittenPlan.treeString() + "\n\n"
+ "========== OPTIMIZED PLAN "
+ getTimeMetricString(SummaryProfile::getPrettyNereidsOptimizeTime) + " ==========\n"
+ optimizedPlan.treeString();
+ optimizedPlan.treeString() + "\n\n";
plan += mvSummary;
break;
default:
plan = super.getExplainString(explainOptions)
+ MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
this.getPhysicalPlan());
plan = super.getExplainString(explainOptions);
plan += mvSummary;
if (statementContext != null) {
if (statementContext.isHasUnknownColStats()) {
plan += "\n\nStatistics\n planed with unknown column statistics\n";
plan += "\n\n\n========== STATISTICS ==========\n";
plan += "planed with unknown column statistics\n";
}
}
}
if (statementContext != null) {
if (!statementContext.getHints().isEmpty()) {
String hint = getHintExplainString(statementContext.getHints());

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -107,8 +107,8 @@ public class InitMaterializationContextHook implements PlannerHook {
Set<TableIf> usedTables) {
Set<MTMV> availableMTMVs = getAvailableMTMVs(usedTables, cascadesContext);
if (availableMTMVs.isEmpty()) {
LOG.debug(String.format("Enable materialized view rewrite but availableMTMVs is empty, current queryId "
+ "is %s", cascadesContext.getConnectContext().getQueryIdentifier()));
LOG.debug("Enable materialized view rewrite but availableMTMVs is empty, current queryId "
+ "is {}", cascadesContext.getConnectContext().getQueryIdentifier());
return ImmutableList.of();
}
List<MaterializationContext> asyncMaterializationContext = new ArrayList<>();
@ -116,6 +116,13 @@ public class InitMaterializationContextHook implements PlannerHook {
MTMVCache mtmvCache = null;
try {
mtmvCache = materializedView.getOrGenerateCache(cascadesContext.getConnectContext());
// If mv property use_for_rewrite is set false, should not partition in
// query rewrite by materialized view
if (!materializedView.isUseForRewrite()) {
LOG.debug("mv doesn't part in query rewrite process because "
+ "use_for_rewrite is false, mv is {}", materializedView.getName());
continue;
}
if (mtmvCache == null) {
continue;
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -403,7 +403,7 @@ public abstract class MaterializationContext {
}
private static String generateIdentifierName(List<String> qualifiers) {
return String.join("#", qualifiers);
return String.join(".", qualifiers);
}
@Override

18
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -374,6 +374,20 @@ public class MaterializedViewUtils {
+ "but now is %s", relation.getClass().getSimpleName()));
return null;
}
SlotReference contextPartitionColumn = getContextPartitionColumn(context);
if (contextPartitionColumn == null) {
context.addFailReason(String.format("mv partition column is not from table when relation check, "
+ "mv partition column is %s", context.getMvPartitionColumn()));
return null;
}
// Check the table which mv partition column belonged to is same as the current check relation or not
if (!((LogicalCatalogRelation) relation).getTable().getFullQualifiers().equals(
contextPartitionColumn.getTable().map(TableIf::getFullQualifiers).orElse(ImmutableList.of()))) {
context.addFailReason(String.format("mv partition column name is not belonged to current check , "
+ "table, current table is %s",
((LogicalCatalogRelation) relation).getTable().getFullQualifiers()));
return null;
}
LogicalCatalogRelation logicalCatalogRelation = (LogicalCatalogRelation) relation;
TableIf table = logicalCatalogRelation.getTable();
// if self join, self join can not partition track now, remove the partition column correspondingly
@ -402,10 +416,6 @@ public class MaterializedViewUtils {
return null;
}
Set<Column> partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns());
SlotReference contextPartitionColumn = getContextPartitionColumn(context);
if (contextPartitionColumn == null) {
return null;
}
Column mvReferenceColumn = contextPartitionColumn.getColumn().get();
Expr definExpr = mvReferenceColumn.getDefineExpr();
if (definExpr instanceof SlotRef) {

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
View File

@ -359,7 +359,9 @@ public class CreateMTMVInfo {
List<Expression> functionCollectResult = MaterializedViewUtils.extractNondeterministicFunction(plan);
if (!CollectionUtils.isEmpty(functionCollectResult)) {
throw new AnalysisException(String.format(
"can not contain invalid expression, the expression is %s",
"can not contain nonDeterministic expression, the expression is %s. "
+ "Should add 'enable_nondeterministic_function' = 'true' property "
+ "when create materialized view if you know the property real meaning entirely",
functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(","))));
}
}

20
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
View File

@ -251,6 +251,26 @@ public class MaterializedViewUtilsTest extends TestWithFeService {
connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION");
}
// Test when join both side are all partition table and partition column name is same
@Test
public void joinPartitionNameSameTest() {
PlanChecker.from(connectContext)
.checkExplain("select t1.upgrade_day, t2.batch_no, count(*) "
+ "from test2 t2 join test1 t1 on "
+ "t1.upgrade_day = t2.upgrade_day "
+ "group by t1.upgrade_day, t2.batch_no;",
nereidsPlanner -> {
Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
RelatedTableInfo relatedTableInfo =
MaterializedViewUtils.getRelatedTableInfo("upgrade_day", null,
rewrittenPlan, nereidsPlanner.getCascadesContext());
checkRelatedTableInfo(relatedTableInfo,
"test1",
"upgrade_day",
true);
});
}
@Test
public void getRelatedTableInfoWhenAutoPartitionTest() {
PlanChecker.from(connectContext)