diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index ecd02302ac..8a740395a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -41,6 +41,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; /** * Materialized view is performed to materialize the results of query. @@ -568,6 +569,11 @@ public class CreateMaterializedViewStmt extends DdlStmt { return new StringBuilder().append(MATERIALIZED_VIEW_NAME_PREFIX).append(name).toString(); } + public static String mvColumnBuilder(Optional functionName, String sourceColumnName) { + return functionName.map(s -> mvAggregateColumnBuilder(s, sourceColumnName)) + .orElseGet(() -> mvColumnBuilder(sourceColumnName)); + } + public static String mvColumnBreaker(String name) { if (name.startsWith(MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) { // mva_SUM__k2 -> k2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index e2f04c6ea6..ac5c05eb26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -634,9 +634,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slotList = new ImmutableList.Builder() - .addAll(olapScan.getOutput()) - .build(); + List slotList = olapScan.getOutput(); Set deferredMaterializedExprIds = Collections.emptySet(); if (olapScan.getMutableState(PhysicalOlapScan.DEFERRED_MATERIALIZED_SLOTS).isPresent()) { deferredMaterializedExprIds = (Set) (olapScan @@ -644,6 +642,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slotList = new ImmutableList.Builder() .addAll(schemaScan.getOutput()) - .addAll(schemaScan.getNonUserVisibleOutput()) .build(); TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); tupleDescriptor.setTable(table); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java index 324a0b95ce..cea674ce03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProj import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionOptimization; import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; import org.apache.doris.nereids.rules.rewrite.logical.AdjustNullable; import org.apache.doris.nereids.rules.rewrite.logical.AggScalarSubQueryToWindowFunction; @@ -263,13 +264,15 @@ public class NereidsRewriter extends BatchRewriteJob { topic("MV optimization", topDown( - // TODO: enable this rule after https://github.com/apache/doris/issues/18263 is fixed - // new SelectMaterializedIndexWithAggregate(), + new SelectMaterializedIndexWithAggregate(), new SelectMaterializedIndexWithoutAggregate(), new PushdownFilterThroughProject(), new MergeProjects(), new PruneOlapScanTablet() - ) + ), + custom(RuleType.COLUMN_PRUNING, ColumnPruning::new), + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new) ), // this rule batch must keep at the end of rewrite to do some plan check diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java index cb9584086d..4a08b8a7a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java @@ -38,7 +38,6 @@ import java.util.Set; */ public class LogicalProperties { protected final Supplier> outputSupplier; - protected final Supplier> nonUserVisibleOutputSupplier; protected final Supplier> outputExprIdsSupplier; protected final Supplier> outputSetSupplier; protected final Supplier> outputMapSupplier; @@ -59,9 +58,6 @@ public class LogicalProperties { this.outputSupplier = Suppliers.memoize( Objects.requireNonNull(outputSupplier, "outputSupplier can not be null") ); - this.nonUserVisibleOutputSupplier = Suppliers.memoize( - Objects.requireNonNull(nonUserVisibleOutputSupplier, "nonUserVisibleOutputSupplier can not be null") - ); this.outputExprIdsSupplier = Suppliers.memoize( () -> this.outputSupplier.get().stream().map(NamedExpression::getExprId).map(Id.class::cast) .collect(ImmutableList.toImmutableList()) @@ -83,10 +79,6 @@ public class LogicalProperties { return outputSupplier.get(); } - public List getNonUserVisibleOutput() { - return nonUserVisibleOutputSupplier.get(); - } - public Set getOutputSet() { return outputSetSupplier.get(); } @@ -104,14 +96,13 @@ public class LogicalProperties { } public LogicalProperties withOutput(List output) { - return new LogicalProperties(Suppliers.ofInstance(output), nonUserVisibleOutputSupplier); + return new LogicalProperties(Suppliers.ofInstance(output)); } @Override public String toString() { return "LogicalProperties{" + "\noutputSupplier=" + outputSupplier.get() - + "\nnonUserVisibleOutputSupplier=" + nonUserVisibleOutputSupplier.get() + "\noutputExprIdsSupplier=" + outputExprIdsSupplier.get() + "\noutputSetSupplier=" + outputSetSupplier.get() + "\noutputMapSupplier=" + outputMapSupplier.get() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index cc864990fb..9927b7d5df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -38,7 +38,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * some check need to do after analyze whole plan. @@ -58,7 +57,7 @@ public class CheckAfterRewrite extends OneAnalysisRuleFactory { .flatMap(expr -> expr.getInputSlots().stream()) .collect(Collectors.toSet()); Set childrenOutput = plan.children().stream() - .flatMap(child -> Stream.concat(child.getOutput().stream(), child.getNonUserVisibleOutput().stream())) + .flatMap(child -> child.getOutput().stream()) .map(NamedExpression::getExprId) .collect(Collectors.toSet()); notFromChildren = notFromChildren.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 1829e7734b..ed1cbaf731 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -58,6 +58,7 @@ public class LogicalOlapScanToPhysicalOlapScan extends OneImplementationRuleFact olapScan.getSelectedPartitionIds(), convertDistribution(olapScan), olapScan.getPreAggStatus(), + olapScan.getOutputByMvIndex(olapScan.getTable().getBaseIndexId()), Optional.empty(), olapScan.getLogicalProperties()) ).toRule(RuleType.LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE); @@ -79,6 +80,31 @@ public class LogicalOlapScanToPhysicalOlapScan extends OneImplementationRuleFact || olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { // TODO if a mv is selected, we ignore base table's distributionInfo for now // need improve this to handle the case if mv's distributionInfo is the same as base table + if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + List output = olapScan.getOutput(); + List baseOutput = olapScan.getOutputByMvIndex(olapScan.getTable().getBaseIndexId()); + List hashColumns = Lists.newArrayList(); + for (int i = 0; i < output.size(); i++) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + if (((SlotReference) output.get(i)).getColumn().get().getNameWithoutMvPrefix() + .equals(column.getName())) { + hashColumns.add(output.get(i).getExprId()); + } + } + } + if (hashColumns.size() != hashDistributionInfo.getDistributionColumns().size()) { + for (int i = 0; i < baseOutput.size(); i++) { + for (Column column : hashDistributionInfo.getDistributionColumns()) { + if (((SlotReference) baseOutput.get(i)).getColumn().get().equals(column)) { + hashColumns.add(baseOutput.get(i).getExprId()); + } + } + } + } + return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), + olapScan.getSelectedIndexId(), Sets.newHashSet(olapScan.getSelectedPartitionIds())); + } return DistributionSpecAny.INSTANCE; } HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java index ef00c2811b..59949c8b24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java @@ -17,10 +17,14 @@ package org.apache.doris.nereids.rules.mv; +import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -30,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; @@ -40,16 +45,26 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmapWithCheck; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; +import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -76,23 +91,66 @@ public abstract class AbstractSelectMaterializedIndexRule { protected boolean containAllRequiredColumns( MaterializedIndex index, LogicalOlapScan scan, - Set requiredScanOutput) { + Set requiredScanOutput, + Set requiredExpr) { OlapTable table = scan.getTable(); - Set requiredColumnNames = requiredScanOutput.stream() - .map(s -> normalizeName(s.getName())) - .collect(Collectors.toSet()); + Set requiredMvColumnNames = requiredScanOutput.stream() + .map(s -> normalizeName(Column.getNameWithoutMvPrefix(s.getName()))) + .collect(Collectors.toSet()); Set mvColNames = table.getSchemaByIndexId(index.getId(), true).stream() - .map(c -> normalizeName(c.getNameWithoutMvPrefix())) + .map(c -> normalizeName(parseMvColumnToSql(c.getNameWithoutMvPrefix()))) .collect(Collectors.toSet()); - table.getSchemaByIndexId(index.getId(), true).stream() - .forEach(column -> mvColNames.add(normalizeName(column.getName()))); + return mvColNames.containsAll(requiredMvColumnNames) + || requiredExpr.stream() + .map(AbstractSelectMaterializedIndexRule::removeCastAndAlias) + .filter(e -> !containsAllColumn(e, mvColNames)) + .collect(Collectors.toSet()).isEmpty(); + } - return mvColNames - .containsAll(requiredColumnNames); + public static String parseMvColumnToSql(String mvName) { + return new NereidsParser().parseExpression( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvName)).toSql(); + } + + public static String parseMvColumnToMvName(String mvName, Optional aggTypeName) { + return CreateMaterializedViewStmt.mvColumnBuilder(aggTypeName, + new NereidsParser().parseExpression( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvName)).toSql()); + } + + protected static Expression removeCastAndAlias(Expression expression) { + List children = new ArrayList<>(); + for (Expression child : expression.children()) { + children.add(removeCastAndAlias(child)); + } + if (expression instanceof Cast) { + return ((Cast) expression.withChildren(children)).child(); + } + if (expression instanceof Alias) { + return ((Alias) expression.withChildren(children)).child(); + } + return children.isEmpty() ? expression : expression.withChildren(children); + } + + protected static boolean containsAllColumn(Expression expression, Set mvColumnNames) { + if (mvColumnNames.contains(Column.getNameWithoutMvPrefix(expression.toSql()))) { + return true; + } + if (expression.children().isEmpty()) { + return false; + } + boolean childContain = true; + for (Expression child : expression.children()) { + if (child instanceof Literal) { + continue; + } + childContain &= containsAllColumn(child, mvColumnNames); + } + return childContain; } /** @@ -103,6 +161,10 @@ public abstract class AbstractSelectMaterializedIndexRule { List candidates, LogicalOlapScan scan, Set predicates) { + if (candidates.isEmpty()) { + return scan.getTable().getBaseIndexId(); + } + OlapTable table = scan.getTable(); // Scan slot exprId -> slot name Map exprIdToName = scan.getOutput() @@ -293,7 +355,7 @@ public abstract class AbstractSelectMaterializedIndexRule { } public static String normalizeName(String name) { - return name.replace("`", ""); + return name.replace("`", "").toLowerCase(); } public static Expression slotToCaseWhen(Expression expression) { @@ -317,4 +379,236 @@ public abstract class AbstractSelectMaterializedIndexRule { } return aggregateFunction.withChildren(slot).toSql(); } + + protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { + Map baseSlotToMvSlot = new HashMap<>(); + Map mvNameToMvSlot = new HashMap<>(); + if (mvPlan.getSelectedIndexId() == mvPlan.getTable().getBaseIndexId()) { + return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot); + } + for (Slot mvSlot : mvPlan.getOutputByMvIndex(mvPlan.getSelectedIndexId())) { + boolean isPushed = false; + for (Slot baseSlot : mvPlan.getOutput()) { + if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) { + continue; + } + if (baseSlot.toSql().equalsIgnoreCase( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker( + normalizeName(mvSlot.getName())))) { + baseSlotToMvSlot.put(baseSlot, mvSlot); + isPushed = true; + break; + } + } + if (!isPushed) { + if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) { + mvNameToMvSlot.put(normalizeName( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvSlot.getName())), + mvSlot); + } + mvNameToMvSlot.put(normalizeName(mvSlot.getName()), mvSlot); + } + } + return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot); + } + + /** SlotContext */ + protected static class SlotContext { + // base index Slot to selected mv Slot + public final Map baseSlotToMvSlot; + + // selected mv Slot name to mv Slot + public final Map mvNameToMvSlot; + + public SlotContext(Map baseSlotToMvSlot, Map mvNameToMvSlot) { + this.baseSlotToMvSlot = ImmutableMap.copyOf(baseSlotToMvSlot); + this.mvNameToMvSlot = ImmutableMap.copyOf(mvNameToMvSlot); + } + } + + /** + * ReplaceExpressions + * Notes: For the sum type, the original column and the mv column may have inconsistent types, + * but the be side will not check the column type, so it can work normally + */ + protected static class ReplaceExpressions extends DefaultPlanVisitor { + private final SlotContext slotContext; + + public ReplaceExpressions(SlotContext slotContext) { + this.slotContext = slotContext; + } + + public Plan replace(Plan plan, LogicalOlapScan scan) { + if (scan.getSelectedIndexId() == scan.getTable().getBaseIndexId()) { + return plan; + } + return plan.accept(this, null); + } + + @Override + public LogicalAggregate visitLogicalAggregate(LogicalAggregate agg, Void ctx) { + Plan child = agg.child(0).accept(this, ctx); + List groupByExprs = agg.getGroupByExpressions(); + List newGroupByExprs = groupByExprs.stream() + .map(expr -> new ReplaceExpressionWithMvColumn(slotContext).replace(expr)) + .collect(ImmutableList.toImmutableList()); + + List outputExpressions = agg.getOutputExpressions(); + List newOutputExpressions = outputExpressions.stream() + .map(expr -> (NamedExpression) new ReplaceExpressionWithMvColumn(slotContext).replace(expr)) + .collect(ImmutableList.toImmutableList()); + + return agg.withNormalized(newGroupByExprs, newOutputExpressions, child); + } + + @Override + public LogicalRepeat visitLogicalRepeat(LogicalRepeat repeat, Void ctx) { + Plan child = repeat.child(0).accept(this, ctx); + List> groupingSets = repeat.getGroupingSets(); + ImmutableList.Builder> newGroupingExprs = ImmutableList.builder(); + for (List expressions : groupingSets) { + newGroupingExprs.add(expressions.stream() + .map(expr -> new ReplaceExpressionWithMvColumn(slotContext).replace(expr)) + .collect(ImmutableList.toImmutableList()) + ); + } + + List outputExpressions = repeat.getOutputExpressions(); + List newOutputExpressions = outputExpressions.stream() + .map(expr -> (NamedExpression) new ReplaceExpressionWithMvColumn(slotContext).replace(expr)) + .collect(ImmutableList.toImmutableList()); + + return repeat.withNormalizedExpr(newGroupingExprs.build(), newOutputExpressions, child); + } + + @Override + public LogicalFilter visitLogicalFilter(LogicalFilter filter, Void ctx) { + Plan child = filter.child(0).accept(this, ctx); + Set newConjuncts = ImmutableSet.copyOf(ExpressionUtils.extractConjunction( + new ReplaceExpressionWithMvColumn(slotContext).replace(filter.getPredicate()))); + + return filter.withConjunctsAndChild(newConjuncts, child); + } + + @Override + public LogicalProject visitLogicalProject(LogicalProject project, Void ctx) { + Plan child = project.child(0).accept(this, ctx); + List projects = project.getProjects(); + List newProjects = projects.stream() + .map(expr -> (NamedExpression) new ReplaceExpressionWithMvColumn(slotContext).replace(expr)) + .collect(ImmutableList.toImmutableList()); + + return project.withProjectsAndChild(newProjects, child); + } + + @Override + public LogicalOlapScan visitLogicalOlapScan(LogicalOlapScan scan, Void ctx) { + return scan.withLogicalProperties(Optional.empty()); + } + } + + /** + * ReplaceExpressionWithMvColumn + */ + protected static class ReplaceExpressionWithMvColumn extends DefaultExpressionRewriter { + // base index Slot to selected mv Slot + private final Map baseSlotToMvSlot; + + // selected mv Slot name to mv Slot + private final Map mvNameToMvSlot; + + public ReplaceExpressionWithMvColumn(SlotContext slotContext) { + this.baseSlotToMvSlot = ImmutableMap.copyOf(slotContext.baseSlotToMvSlot); + this.mvNameToMvSlot = ImmutableMap.copyOf(slotContext.mvNameToMvSlot); + } + + public Expression replace(Expression expression) { + return expression.accept(this, null); + } + + @Override + public Expression visit(Expression expr, Void context) { + if (notUseMv() || org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(expr.toSql())) { + return expr; + } else if (checkExprIsMvColumn(expr)) { + return mvNameToMvSlot.get( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql())); + } else { + expr = super.visit(expr, context); + return expr; + } + } + + @Override + public Expression visitSlotReference(SlotReference slotReference, Void context) { + if (baseSlotToMvSlot.containsKey(slotReference)) { + return baseSlotToMvSlot.get(slotReference); + } + return slotReference; + } + + @Override + public Expression visitAggregateFunction(AggregateFunction aggregateFunction, Void context) { + String childrenName = aggregateFunction.children() + .stream() + .map(Expression::toSql) + .collect(Collectors.joining(", ")); + String mvName = org.apache.doris.analysis.CreateMaterializedViewStmt.mvAggregateColumnBuilder( + aggregateFunction.getName(), childrenName); + if (mvNameToMvSlot.containsKey(mvName)) { + return aggregateFunction.withChildren(mvNameToMvSlot.get(mvName)); + } else if (mvNameToMvSlot.containsKey(childrenName)) { + // aggRewrite eg: bitmap_union_count -> bitmap_union + return aggregateFunction.withChildren(mvNameToMvSlot.get(childrenName)); + } + return visit(aggregateFunction, context); + } + + @Override + public Expression visitAlias(Alias alias, Void context) { + if (mvNameToMvSlot.containsKey(alias.toSlot().toSql())) { + return mvNameToMvSlot.get(alias.toSlot().toSql()); + } + return visit(alias, context); + } + + @Override + public Expression visitScalarFunction(ScalarFunction scalarFunction, Void context) { + List newChildrenWithoutCast = scalarFunction.children().stream() + .map(child -> { + if (child instanceof Cast) { + return ((Cast) child).child(); + } + return child; + }).collect(ImmutableList.toImmutableList()); + Expression newScalarFunction = scalarFunction.withChildren(newChildrenWithoutCast); + if (checkExprIsMvColumn(newScalarFunction)) { + return mvNameToMvSlot.get( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBuilder(newScalarFunction.toSql())); + } + return visit(scalarFunction, context); + } + + private boolean notUseMv() { + return baseSlotToMvSlot.isEmpty() && mvNameToMvSlot.isEmpty(); + } + + private boolean checkExprIsMvColumn(Expression expr) { + return mvNameToMvSlot.containsKey( + org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql())); + } + } + + protected List generateProjectsAlias( + List oldProjects, SlotContext slotContext) { + return oldProjects.stream().map(e -> { + if (slotContext.baseSlotToMvSlot.containsKey(e.toSlot())) { + return new Alias(e.getExprId(), slotContext.baseSlotToMvSlot.get(e.toSlot()), e.getName()); + } + if (slotContext.mvNameToMvSlot.containsKey(e.toSql())) { + return new Alias(e.getExprId(), slotContext.mvNameToMvSlot.get(e.toSql()), e.getName()); + } + return e; + }).collect(ImmutableList.toImmutableList()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java index a0dfb600f1..7015fc78cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java @@ -33,16 +33,19 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnionAgg; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHash; import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; @@ -64,6 +67,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.collect.Streams; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -88,71 +93,95 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial return ImmutableList.of( // only agg above scan // Aggregate(Scan) - logicalAggregate(logicalOlapScan().when(this::shouldSelectIndex)).then(agg -> { + logicalAggregate(logicalOlapScan().when(this::shouldSelectIndex)).thenApply(ctx -> { + LogicalAggregate agg = ctx.root; LogicalOlapScan scan = agg.child(); SelectResult result = select( scan, agg.getInputSlots(), ImmutableSet.of(), extractAggFunctionAndReplaceSlot(agg, Optional.empty()), - agg.getGroupByExpressions()); + agg.getGroupByExpressions(), + new HashSet<>(agg.getExpressions())); + + LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) - ); + return new ReplaceExpressions(slotContext) + .replace(agg.withChildren(mvPlan), mvPlan); } else { - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput( + agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + mvPlan + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_SCAN), // filter could push down scan. // Aggregate(Filter(Scan)) logicalAggregate(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex))) - .then(agg -> { + .thenApply(ctx -> { + LogicalAggregate> agg = ctx.root; LogicalFilter filter = agg.child(); LogicalOlapScan scan = filter.child(); ImmutableSet requiredSlots = ImmutableSet.builder() .addAll(agg.getInputSlots()) .addAll(filter.getInputSlots()) .build(); + ImmutableSet requiredExpr = ImmutableSet.builder() + .addAll(agg.getExpressions()) + .addAll(filter.getExpressions()) + .build(); SelectResult result = select( scan, requiredSlots, filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.empty()), - agg.getGroupByExpressions() + agg.getGroupByExpressions(), + requiredExpr ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren(filter.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) - )); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren(filter.withChildren(mvPlan)), mvPlan)); } else { - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.empty(), Optional.empty(), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - // Note that no need to replace slots in the filter, because the slots to - // replace are value columns, which shouldn't appear in filters. - filter.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.empty(), Optional.empty(), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + // Note that no need to replace slots in the filter, + // because the slots to + // replace are value columns, which shouldn't appear in filters. + filter.withChildren(mvPlan) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_SCAN), // column pruning or other projections such as alias, etc. // Aggregate(Project(Scan)) logicalAggregate(logicalProject(logicalOlapScan().when(this::shouldSelectIndex))) - .then(agg -> { + .thenApply(ctx -> { + LogicalAggregate> agg = ctx.root; LogicalProject project = agg.child(); LogicalOlapScan scan = project.child(); SelectResult result = select( @@ -162,113 +191,162 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial extractAggFunctionAndReplaceSlot(agg, Optional.of(project)), ExpressionUtils.replace(agg.getGroupByExpressions(), - project.getAliasToProducer()) + project.getAliasToProducer()), + collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren( - project.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) - ) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + project.withProjectsAndChild( + generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), + mvPlan)), mvPlan)); } else { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); LogicalProject newProject = new LogicalProject<>( - newProjectList, + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - newProject - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + newProject + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_SCAN), // filter could push down and project. // Aggregate(Project(Filter(Scan))) logicalAggregate(logicalProject(logicalFilter(logicalOlapScan() - .when(this::shouldSelectIndex)))).then(agg -> { + .when(this::shouldSelectIndex)))).thenApply(ctx -> { + LogicalAggregate>> agg = ctx.root; LogicalProject> project = agg.child(); LogicalFilter filter = project.child(); LogicalOlapScan scan = filter.child(); Set requiredSlots = Stream.concat( project.getInputSlots().stream(), filter.getInputSlots().stream()) .collect(Collectors.toSet()); + ImmutableSet requiredExpr = ImmutableSet.builder() + .addAll(collectRequireExprWithAggAndProject( + agg.getExpressions(), project.getProjects())) + .addAll(filter.getExpressions()) + .build(); SelectResult result = select( scan, requiredSlots, filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.of(project)), ExpressionUtils.replace(agg.getGroupByExpressions(), - project.getAliasToProducer()) + project.getAliasToProducer()), + requiredExpr ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren(project.withChildren(filter.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) - ))); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + project.withProjectsAndChild( + generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), + filter.withChildren(mvPlan))), mvPlan)); } else { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>(newProjectList, - filter.withChildren(scan.withMaterializedIndexSelected(result.preAggStatus, - result.indexId))); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + filter.withChildren(mvPlan)); - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - newProject - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + newProject + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_FILTER_SCAN), // filter can't push down // Aggregate(Filter(Project(Scan))) logicalAggregate(logicalFilter(logicalProject(logicalOlapScan() - .when(this::shouldSelectIndex)))).then(agg -> { + .when(this::shouldSelectIndex)))).thenApply(ctx -> { + LogicalAggregate>> agg = ctx.root; LogicalFilter> filter = agg.child(); LogicalProject project = filter.child(); LogicalOlapScan scan = project.child(); + ImmutableSet requiredExpr = ImmutableSet.builder() + .addAll(collectRequireExprWithAggAndProject( + agg.getExpressions(), project.getProjects())) + .addAll(collectRequireExprWithAggAndProject( + filter.getExpressions(), project.getProjects())) + .build(); SelectResult result = select( scan, project.getInputSlots(), - ImmutableSet.of(), + filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.of(project)), ExpressionUtils.replace(agg.getGroupByExpressions(), - project.getAliasToProducer()) + project.getAliasToProducer()), + requiredExpr ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren(filter.withChildren(project.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) - ))); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + filter.withChildren( + project.withProjectsAndChild( + generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), + mvPlan))), mvPlan)); } else { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>(newProjectList, - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan); - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - filter.withChildren(newProject) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + filter.withChildren(newProject) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_PROJECT_SCAN), // only agg above scan // Aggregate(Repeat(Scan)) - logicalAggregate(logicalRepeat(logicalOlapScan().when(this::shouldSelectIndex))).then(agg -> { + logicalAggregate(logicalRepeat(logicalOlapScan().when(this::shouldSelectIndex))).thenApply(ctx -> { + LogicalAggregate> agg = ctx.root; LogicalRepeat repeat = agg.child(); LogicalOlapScan scan = repeat.child(); SelectResult result = select( @@ -276,28 +354,41 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial agg.getInputSlots(), ImmutableSet.of(), extractAggFunctionAndReplaceSlot(agg, Optional.empty()), - nonVirtualGroupByExprs(agg)); + nonVirtualGroupByExprs(agg), + new HashSet<>(agg.getExpressions())); + + LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren( - repeat.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) + ), mvPlan)); } else { - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput( + agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN), // filter could push down scan. // Aggregate(Repeat(Filter(Scan))) logicalAggregate(logicalRepeat(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex)))) - .then(agg -> { + .thenApply(ctx -> { + LogicalAggregate>> agg = ctx.root; LogicalRepeat> repeat = agg.child(); LogicalFilter filter = repeat.child(); LogicalOlapScan scan = filter.child(); @@ -305,42 +396,58 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial .addAll(agg.getInputSlots()) .addAll(filter.getInputSlots()) .build(); + ImmutableSet requiredExpr = ImmutableSet.builder() + .addAll(agg.getExpressions()) + .addAll(filter.getExpressions()) + .build(); SelectResult result = select( scan, requiredSlots, filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.empty()), - nonVirtualGroupByExprs(agg) + nonVirtualGroupByExprs(agg), + requiredExpr ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren( - repeat.withChildren( - filter.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, - result.indexId)) - )); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + filter.withChildren(mvPlan) + )), mvPlan)); } else { - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.empty(), Optional.empty(), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - // Not that no need to replace slots in the filter, because the slots to replace - // are value columns, which shouldn't appear in filters. - repeat.withChildren(filter.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, - result.indexId))) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.empty(), Optional.empty(), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + // Not that no need to replace slots in the filter, + // because the slots to replace + // are value columns, which shouldn't appear in filters. + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + filter.withChildren(mvPlan)) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_SCAN), // column pruning or other projections such as alias, etc. // Aggregate(Repeat(Project(Scan))) logicalAggregate(logicalRepeat(logicalProject(logicalOlapScan().when(this::shouldSelectIndex)))) - .then(agg -> { + .thenApply(ctx -> { + LogicalAggregate>> agg = ctx.root; LogicalRepeat> repeat = agg.child(); LogicalProject project = repeat.child(); LogicalOlapScan scan = project.child(); @@ -351,38 +458,53 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial extractAggFunctionAndReplaceSlot(agg, Optional.of(project)), ExpressionUtils.replace(nonVirtualGroupByExprs(agg), - project.getAliasToProducer()) + project.getAliasToProducer()), + collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren( - repeat.withChildren( - project.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, - result.indexId) - )) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + project.withProjectsAndChild( + generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), + mvPlan)) + ), mvPlan)); } else { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); LogicalProject newProject = new LogicalProject<>( - newProjectList, - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withChildren(newProject) - ); + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + mvPlan); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs( + mvPlan, repeat.getOutputs()), newProject) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_SCAN), // filter could push down and project. // Aggregate(Repeat(Project(Filter(Scan)))) logicalAggregate(logicalRepeat(logicalProject(logicalFilter(logicalOlapScan() - .when(this::shouldSelectIndex))))).then(agg -> { + .when(this::shouldSelectIndex))))).thenApply(ctx -> { + LogicalAggregate>>> agg = ctx.root; LogicalRepeat>> repeat = agg.child(); LogicalProject> project = repeat.child(); LogicalFilter filter = project.child(); @@ -390,72 +512,124 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Set requiredSlots = Stream.concat( project.getInputSlots().stream(), filter.getInputSlots().stream()) .collect(Collectors.toSet()); + ImmutableSet requiredExpr = ImmutableSet.builder() + .addAll(collectRequireExprWithAggAndProject( + agg.getExpressions(), project.getProjects())) + .addAll(filter.getExpressions()) + .build(); SelectResult result = select( scan, requiredSlots, filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.of(project)), ExpressionUtils.replace(nonVirtualGroupByExprs(agg), - project.getAliasToProducer()) + project.getAliasToProducer()), + requiredExpr ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren(repeat.withChildren(project.withChildren(filter.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)) - ))); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + project.withProjectsAndChild( + generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), + filter.withChildren( + mvPlan + )))), + mvPlan)); } else { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>(newProjectList, - filter.withChildren(scan.withMaterializedIndexSelected(result.preAggStatus, - result.indexId))); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + filter.withChildren(mvPlan)); - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withChildren(newProject) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs( + mvPlan, repeat.getOutputs()), newProject) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_FILTER_SCAN), // filter can't push down // Aggregate(Repeat(Filter(Project(Scan)))) logicalAggregate(logicalRepeat(logicalFilter(logicalProject(logicalOlapScan() - .when(this::shouldSelectIndex))))).then(agg -> { + .when(this::shouldSelectIndex))))).thenApply(ctx -> { + LogicalAggregate>>> agg = ctx.root; LogicalRepeat>> repeat = agg.child(); LogicalFilter> filter = repeat.child(); LogicalProject project = filter.child(); LogicalOlapScan scan = project.child(); + ImmutableSet requiredExpr = ImmutableSet.builder() + .addAll(collectRequireExprWithAggAndProject( + agg.getExpressions(), project.getProjects())) + .addAll(collectRequireExprWithAggAndProject( + filter.getExpressions(), project.getProjects())) + .build(); SelectResult result = select( scan, project.getInputSlots(), - ImmutableSet.of(), + filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.of(project)), ExpressionUtils.replace(nonVirtualGroupByExprs(agg), - project.getAliasToProducer()) + project.getAliasToProducer()), + requiredExpr ); + LogicalOlapScan mvPlan = + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.exprRewriteMap.isEmpty()) { - return agg.withChildren(repeat.withChildren(filter.withChildren(project.withChildren( - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)) - ))); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + agg.withChildren( + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + filter.withChildren( + project.withProjectsAndChild( + generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), + mvPlan + )))), + mvPlan)); } else { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>(newProjectList, + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); - return new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withChildren(filter.withChildren(newProject)) - ); + return new LogicalProject( + generateProjectsAlias(agg.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + repeat.withAggOutputAndChild( + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + filter.withChildren(newProject)) + ), mvPlan)); } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_PROJECT_SCAN) ); @@ -478,7 +652,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Set requiredScanOutput, Set predicates, List aggregateFunctions, - List groupingExprs) { + List groupingExprs, + Set requiredExpr) { // remove virtual slot for grouping sets. Set nonVirtualRequiredScanOutput = requiredScanOutput.stream() .filter(slot -> !(slot instanceof VirtualSlotReference)) @@ -500,19 +675,18 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial default: throw new RuntimeException("Not supported keys type: " + scan.getTable().getKeysType()); } - if (table.isDupKeysOrMergeOnWrite()) { - Map> indexesGroupByIsBaseOrNot = table.getVisibleIndex() - .stream() - .collect(Collectors.groupingBy(index -> index.getId() == table.getBaseIndexId())); + Map> indexesGroupByIsBaseOrNot = table.getVisibleIndex() + .stream() + .collect(Collectors.groupingBy(index -> index.getId() == table.getBaseIndexId())); + if (table.isDupKeysOrMergeOnWrite()) { // Duplicate-keys table could use base index and indexes that pre-aggregation status is on. - Set candidatesWithoutRewriting = Stream.concat( - indexesGroupByIsBaseOrNot.get(true).stream(), + Set candidatesWithoutRewriting = indexesGroupByIsBaseOrNot.getOrDefault(false, ImmutableList.of()) .stream() .filter(index -> checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs).isOn()) - ).collect(ImmutableSet.toImmutableSet()); + .collect(Collectors.toSet()); // try to rewrite bitmap, hll by materialized index columns. List candidatesWithRewriting = indexesGroupByIsBaseOrNot.getOrDefault(false, @@ -532,11 +706,12 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial List haveAllRequiredColumns = Streams.concat( candidatesWithoutRewriting.stream() - .filter(index -> containAllRequiredColumns(index, scan, nonVirtualRequiredScanOutput)), + .filter(index -> containAllRequiredColumns( + index, scan, nonVirtualRequiredScanOutput, requiredExpr)), candidatesWithRewriting .stream() .filter(aggRewriteResult -> containAllRequiredColumns(aggRewriteResult.index, scan, - aggRewriteResult.requiredScanOutput)) + aggRewriteResult.requiredScanOutput, requiredExpr)) .map(aggRewriteResult -> aggRewriteResult.index) ).collect(Collectors.toList()); @@ -552,26 +727,42 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial return new SelectResult(scan.getPreAggStatus(), scan.getTable().getBaseIndexId(), new ExprRewriteMap()); } - final PreAggStatus preAggStatus; - if (preAggEnabledByHint(scan)) { - // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. - preAggStatus = PreAggStatus.on(); - } else { - // Only checking pre-aggregation status by base index is enough for aggregate-keys and - // unique-keys OLAP table. - // Because the schemas in non-base materialized index are subsets of the schema of base index. - preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates, + + Set candidatesWithoutRewriting = new HashSet<>(); + + for (MaterializedIndex index : indexesGroupByIsBaseOrNot.getOrDefault(false, ImmutableList.of())) { + final PreAggStatus preAggStatus; + if (preAggEnabledByHint(scan)) { + preAggStatus = PreAggStatus.on(); + } else { + preAggStatus = checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs); + } + + if (preAggStatus.isOn()) { + candidatesWithoutRewriting.add(index); + } } - if (preAggStatus.isOff()) { + SelectResult baseIndexSelectResult = new SelectResult( + checkPreAggStatus(scan, scan.getTable().getBaseIndexId(), + predicates, aggregateFunctions, groupingExprs), + scan.getTable().getBaseIndexId(), new ExprRewriteMap()); + if (candidatesWithoutRewriting.isEmpty()) { // return early if pre agg status if off. - return new SelectResult(preAggStatus, scan.getTable().getBaseIndexId(), new ExprRewriteMap()); + return baseIndexSelectResult; } else { - List rollupsWithAllRequiredCols = table.getVisibleIndex().stream() - .filter(index -> containAllRequiredColumns(index, scan, nonVirtualRequiredScanOutput)) + List rollupsWithAllRequiredCols = + Stream.concat(candidatesWithoutRewriting.stream(), + indexesGroupByIsBaseOrNot.get(true).stream()) + .filter(index -> containAllRequiredColumns( + index, scan, nonVirtualRequiredScanOutput, requiredExpr)) .collect(Collectors.toList()); - return new SelectResult(preAggStatus, selectBestIndex(rollupsWithAllRequiredCols, scan, predicates), - new ExprRewriteMap()); + + long selectedIndex = selectBestIndex(rollupsWithAllRequiredCols, scan, predicates); + if (selectedIndex == scan.getTable().getBaseIndexId()) { + return baseIndexSelectResult; + } + return new SelectResult(PreAggStatus.on(), selectedIndex, new ExprRewriteMap()); } } } @@ -687,24 +878,24 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial @Override public PreAggStatus visitMax(Max max, CheckContext context) { - return checkAggFunc(max, AggregateType.MAX, extractSlotId(max.child()), context, true); + return checkAggFunc(max, AggregateType.MAX, context, true); } @Override public PreAggStatus visitMin(Min min, CheckContext context) { - return checkAggFunc(min, AggregateType.MIN, extractSlotId(min.child()), context, true); + return checkAggFunc(min, AggregateType.MIN, context, true); } @Override public PreAggStatus visitSum(Sum sum, CheckContext context) { - return checkAggFunc(sum, AggregateType.SUM, extractSlotId(sum.child()), context, false); + return checkAggFunc(sum, AggregateType.SUM, context, false); } @Override public PreAggStatus visitCount(Count count, CheckContext context) { if (count.isDistinct() && count.arity() == 1) { - Optional exprIdOpt = extractSlotId(count.child(0)); - if (exprIdOpt.isPresent() && context.checkKeyColContainsKey(exprIdOpt.get())) { + Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(count.child(0)); + if (slotOpt.isPresent() && context.keyNameToColumn.containsKey(normalizeName(slotOpt.get().toSql()))) { return PreAggStatus.on(); } } @@ -719,7 +910,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial expr = expr.child(0); } Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(expr); - if (slotOpt.isPresent() && context.checkValueColContainsKey(slotOpt.get().getExprId())) { + if (slotOpt.isPresent() && context.valueNameToColumn.containsKey(normalizeName(slotOpt.get().toSql()))) { return PreAggStatus.on(); } else { return PreAggStatus.off("invalid bitmap_union_count: " + bitmapUnionCount.toSql()); @@ -729,110 +920,94 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial @Override public PreAggStatus visitHllUnionAgg(HllUnionAgg hllUnionAgg, CheckContext context) { Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(hllUnionAgg.child()); - if (slotOpt.isPresent() && context.checkValueColContainsKey(slotOpt.get().getExprId())) { + if (slotOpt.isPresent() && context.valueNameToColumn.containsKey(normalizeName(slotOpt.get().toSql()))) { return PreAggStatus.on(); } else { return PreAggStatus.off("invalid hll_union_agg: " + hllUnionAgg.toSql()); } } + @Override + public PreAggStatus visitGroupConcat(GroupConcat groupConcat, CheckContext context) { + return checkAggFunc(groupConcat, AggregateType.SUM, context, false); + } + private PreAggStatus checkAggFunc( AggregateFunction aggFunc, AggregateType matchingAggType, - Optional exprIdOpt, CheckContext ctx, boolean canUseKeyColumn) { - return exprIdOpt.map(exprId -> { - if (ctx.checkKeyColContainsKey(exprId)) { - if (canUseKeyColumn) { - return PreAggStatus.on(); - } else { - Column column = ctx.getKeyColumn(exprId); - return PreAggStatus.off(String.format("Aggregate function %s contains key column %s.", - aggFunc.toSql(), column.getName())); - } - } else if (ctx.checkValueColContainsKey(exprId)) { - AggregateType aggType = ctx.getValueColumn(exprId).getAggregationType(); - if (aggType == matchingAggType) { - return PreAggStatus.on(); - } else { - return PreAggStatus.off(String.format("Aggregate operator don't match, aggregate function: %s" - + ", column aggregate type: %s", aggFunc.toSql(), aggType)); - } - } else { - return PreAggStatus.off(String.format("Slot(%s) in %s is neither key column nor value column.", - exprId, aggFunc.toSql())); - } - }).orElse(PreAggStatus.off(String.format("Input of aggregate function %s should be slot or cast on slot.", - aggFunc.toSql()))); - } + String childNameWithFuncName = ctx.isBaseIndex() + ? normalizeName(aggFunc.child(0).toSql()) + : normalizeName(CreateMaterializedViewStmt.mvColumnBuilder( + matchingAggType, normalizeName(aggFunc.child(0).toSql()))); - // TODO: support more type of expressions, such as case when. - private Optional extractSlotId(Expression expr) { - return ExpressionUtils.isSlotOrCastOnSlot(expr); + boolean contains = containsAllColumn( + removeCastAndAlias(aggFunc.child(0)), ctx.keyNameToColumn.keySet()); + if (ctx.keyNameToColumn.containsKey(childNameWithFuncName) || contains) { + if (canUseKeyColumn || (!ctx.isBaseIndex() && contains)) { + return PreAggStatus.on(); + } else { + Column column = ctx.keyNameToColumn.get(childNameWithFuncName); + return PreAggStatus.off(String.format("Aggregate function %s contains key column %s.", + aggFunc.toSql(), column == null ? "empty column" : column.getName())); + } + } else if (ctx.valueNameToColumn.containsKey(childNameWithFuncName)) { + AggregateType aggType = ctx.valueNameToColumn.get(childNameWithFuncName).getAggregationType(); + if (aggType == matchingAggType) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off(String.format("Aggregate operator don't match, aggregate function: %s" + + ", column aggregate type: %s", aggFunc.toSql(), aggType)); + } + } else { + return PreAggStatus.off(String.format("Slot(%s) in %s is neither key column nor value column.", + childNameWithFuncName, aggFunc.toSql())); + } } } private static class CheckContext { - public final Map exprIdToKeyColumn; - public final Map exprIdToValueColumn; - public final LogicalOlapScan scan; - public final Map baseExprIdToIndexExprId; + public final Map keyNameToColumn; + + public final Map valueNameToColumn; + + public final long index; public CheckContext(LogicalOlapScan scan, long indexId) { this.scan = scan; // map> - Map> nameToColumnGroupingByIsKey + Map> baseNameToColumnGroupingByIsKey = scan.getTable().getSchemaByIndexId(indexId) .stream() .collect(Collectors.groupingBy( Column::isKey, - Collectors.toMap(c -> normalizeName(c.getNameWithoutMvPrefix()), Function.identity()))); - Map keyNameToColumn = nameToColumnGroupingByIsKey.get(true); - Map valueNameToColumn = nameToColumnGroupingByIsKey.getOrDefault(false, ImmutableMap.of()); - Map nameToExprId = - scan.getOutputByMvIndex(indexId).stream() - .collect(Collectors.toMap( - c -> normalizeName(Column.getNameWithoutMvPrefix(c.getName())), - NamedExpression::getExprId - )); - this.exprIdToKeyColumn = keyNameToColumn.entrySet() + Collectors.toMap(c -> normalizeName(parseMvColumnToSql(c.getName())), Function.identity(), + (v1, v2) -> v1))); + Map> mvNameToColumnGroupingByIsKey + = scan.getTable().getSchemaByIndexId(indexId) .stream() - .collect(Collectors.toMap( - e -> nameToExprId.get(e.getKey()), - Map.Entry::getValue) - ); - this.exprIdToValueColumn = valueNameToColumn.entrySet() - .stream() - .collect(Collectors.toMap( - e -> nameToExprId.get(e.getKey()), - Map.Entry::getValue) - ); - this.baseExprIdToIndexExprId = scan.getOutput().stream() - .filter(s -> nameToExprId.containsKey(s.getName())) - .collect(Collectors.toMap(Slot::getExprId, - e -> nameToExprId.get(e.getName()))); + .collect(Collectors.groupingBy( + Column::isKey, + Collectors.toMap(c -> normalizeName( + parseMvColumnToMvName(c.getNameWithoutMvPrefix(), + c.isAggregated() ? Optional.of(c.getAggregationType().name()) : Optional.empty())), + Function.identity()))); + this.keyNameToColumn = mvNameToColumnGroupingByIsKey.getOrDefault(true, new HashMap<>()); + for (String name : baseNameToColumnGroupingByIsKey.getOrDefault(true, new HashMap<>()).keySet()) { + this.keyNameToColumn.putIfAbsent(name, baseNameToColumnGroupingByIsKey.get(true).get(name)); + } + this.valueNameToColumn = mvNameToColumnGroupingByIsKey.getOrDefault(false, new HashMap<>()); + for (String key : baseNameToColumnGroupingByIsKey.getOrDefault(false, ImmutableMap.of()).keySet()) { + this.valueNameToColumn.putIfAbsent(key, baseNameToColumnGroupingByIsKey.get(false).get(key)); + } + this.index = indexId; } - public boolean checkKeyColContainsKey(ExprId exprId) { - return baseExprIdToIndexExprId.containsKey(exprId) - && exprIdToKeyColumn.containsKey(baseExprIdToIndexExprId.get(exprId)); - } - - public boolean checkValueColContainsKey(ExprId exprId) { - return baseExprIdToIndexExprId.containsKey(exprId) - && exprIdToValueColumn.containsKey(baseExprIdToIndexExprId.get(exprId)); - } - - public Column getKeyColumn(ExprId exprId) { - return exprIdToKeyColumn.getOrDefault(exprId, exprIdToKeyColumn.get(baseExprIdToIndexExprId.get(exprId))); - } - - public Column getValueColumn(ExprId exprId) { - return exprIdToValueColumn.getOrDefault( - exprId, exprIdToValueColumn.get(baseExprIdToIndexExprId.get(exprId))); + public boolean isBaseIndex() { + return index == scan.getTable().getBaseIndexId(); } } @@ -864,9 +1039,9 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial return exprs.stream() .map(expr -> expr.getInputSlots() .stream() - .filter(slot -> ctx.checkValueColContainsKey(slot.getExprId())) + .filter(slot -> ctx.valueNameToColumn.containsKey(normalizeName(slot.toSql()))) .findAny() - .map(slot -> Pair.of(expr, ctx.getValueColumn(slot.getExprId()))) + .map(slot -> Pair.of(expr, ctx.valueNameToColumn.get(normalizeName(slot.toSql())))) ) .filter(Optional::isPresent) .findAny() @@ -987,19 +1162,19 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(count.child(0)); // count distinct a value column. - if (slotOpt.isPresent() && !context.checkContext.checkKeyColContainsKey( - slotOpt.get().getExprId())) { - String bitmapUnionColumn = CreateMaterializedViewStmt + if (slotOpt.isPresent() && !context.checkContext.keyNameToColumn.containsKey( + normalizeName(slotOpt.get().toSql()))) { + String bitmapUnionColumn = normalizeName(CreateMaterializedViewStmt .mvColumnBuilder(AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder( - spliceAggFunctionWithSlot(count, slotOpt.get()))); + spliceAggFunctionWithSlot(count, slotOpt.get())))); Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionColumn); // has bitmap_union column - if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { - Slot bitmapUnionSlot = context.checkContext.scan.getNonUserVisibleOutput() + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { + Slot bitmapUnionSlot = context.checkContext.scan.getOutputByMvIndex(context.checkContext.index) .stream() - .filter(s -> bitmapUnionColumn.equals(normalizeName(s.getName()))) + .filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .get(); @@ -1015,18 +1190,18 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(count.child(0)); // count a value column. - if (slotOpt.isPresent() && !context.checkContext.checkKeyColContainsKey( - slotOpt.get().getExprId())) { - String countColumn = CreateMaterializedViewStmt + if (slotOpt.isPresent() && !context.checkContext.keyNameToColumn.containsKey( + normalizeName(slotOpt.get().toSql()))) { + String countColumn = normalizeName(CreateMaterializedViewStmt .mvColumnBuilder(AggregateType.SUM, - CreateMaterializedViewStmt.mvColumnBuilder(slotToCaseWhen(slotOpt.get()).toSql())); + CreateMaterializedViewStmt.mvColumnBuilder(slotToCaseWhen(slotOpt.get()).toSql()))); Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(countColumn); // has bitmap_union_count column - if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { - Slot countSlot = context.checkContext.scan.getNonUserVisibleOutput() + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { + Slot countSlot = context.checkContext.scan.getOutputByMvIndex(context.checkContext.index) .stream() - .filter(s -> countColumn.equals(normalizeName(s.getName()))) + .filter(s -> countColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .get(); @@ -1050,18 +1225,19 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial ToBitmap toBitmap = (ToBitmap) bitmapUnionCount.child(); Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(toBitmap.child()); if (slotOpt.isPresent()) { - String bitmapUnionCountColumn = CreateMaterializedViewStmt + String bitmapUnionCountColumn = normalizeName(CreateMaterializedViewStmt .mvColumnBuilder(AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder( - spliceScalarFunctionWithSlot(toBitmap, slotOpt.get()))); + spliceScalarFunctionWithSlot(toBitmap, slotOpt.get())))); Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionCountColumn); // has bitmap_union_count column - if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { - Slot bitmapUnionCountSlot = context.checkContext.scan.getNonUserVisibleOutput() + Slot bitmapUnionCountSlot = context.checkContext.scan + .getOutputByMvIndex(context.checkContext.index) .stream() - .filter(s -> bitmapUnionCountColumn.equals(normalizeName(s.getName()))) + .filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .get(); @@ -1072,6 +1248,33 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial return newBitmapUnionCount; } } + } else if (bitmapUnionCount.child() instanceof BitmapHash) { + BitmapHash bitmapHash = (BitmapHash) bitmapUnionCount.child(); + Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(bitmapHash.child()); + if (slotOpt.isPresent()) { + String bitmapUnionCountColumn = normalizeName(CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.BITMAP_UNION, + CreateMaterializedViewStmt.mvColumnBuilder( + spliceScalarFunctionWithSlot(bitmapHash, slotOpt.get())))); + + Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionCountColumn); + // has bitmap_union_count column + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { + + Slot bitmapUnionCountSlot = context.checkContext.scan + .getOutputByMvIndex(context.checkContext.index) + .stream() + .filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName()))) + .findFirst() + .get(); + + context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot); + context.exprRewriteMap.projectExprMap.put(bitmapHash, bitmapUnionCountSlot); + BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot); + context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount); + return newBitmapUnionCount; + } + } } return bitmapUnionCount; @@ -1086,17 +1289,17 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial HllHash hllHash = (HllHash) hllUnion.child(); Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(hllHash.child()); if (slotOpt.isPresent()) { - String hllUnionColumn = CreateMaterializedViewStmt + String hllUnionColumn = normalizeName(CreateMaterializedViewStmt .mvColumnBuilder(AggregateType.HLL_UNION, CreateMaterializedViewStmt.mvColumnBuilder( - spliceScalarFunctionWithSlot(hllHash, slotOpt.get()))); + spliceScalarFunctionWithSlot(hllHash, slotOpt.get())))); Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(hllUnionColumn); // has hll_union column - if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { - Slot hllUnionSlot = context.checkContext.scan.getNonUserVisibleOutput() + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { + Slot hllUnionSlot = context.checkContext.scan.getOutputByMvIndex(context.checkContext.index) .stream() - .filter(s -> hllUnionColumn.equals(normalizeName(s.getName()))) + .filter(s -> hllUnionColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .get(); @@ -1121,17 +1324,17 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial HllHash hllHash = (HllHash) hllUnionAgg.child(); Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(hllHash.child()); if (slotOpt.isPresent()) { - String hllUnionColumn = CreateMaterializedViewStmt + String hllUnionColumn = normalizeName(CreateMaterializedViewStmt .mvColumnBuilder(AggregateType.HLL_UNION, CreateMaterializedViewStmt.mvColumnBuilder( - spliceScalarFunctionWithSlot(hllHash, slotOpt.get()))); + spliceScalarFunctionWithSlot(hllHash, slotOpt.get())))); Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(hllUnionColumn); // has hll_union column - if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { - Slot hllUnionSlot = context.checkContext.scan.getNonUserVisibleOutput() + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { + Slot hllUnionSlot = context.checkContext.scan.getOutputByMvIndex(context.checkContext.index) .stream() - .filter(s -> hllUnionColumn.equals(normalizeName(s.getName()))) + .filter(s -> hllUnionColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .get(); @@ -1154,19 +1357,19 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial public Expression visitNdv(Ndv ndv, RewriteContext context) { Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(ndv.child(0)); // ndv on a value column. - if (slotOpt.isPresent() && !context.checkContext.checkKeyColContainsKey( - slotOpt.get().getExprId())) { - String hllUnionColumn = CreateMaterializedViewStmt + if (slotOpt.isPresent() && !context.checkContext.keyNameToColumn.containsKey( + normalizeName(slotOpt.get().toSql()))) { + String hllUnionColumn = normalizeName(CreateMaterializedViewStmt .mvColumnBuilder(AggregateType.HLL_UNION, CreateMaterializedViewStmt.mvColumnBuilder( - spliceAggFunctionWithSlot(ndv, slotOpt.get()))); + spliceAggFunctionWithSlot(ndv, slotOpt.get())))); Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(hllUnionColumn); // has hll_union column - if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { - Slot hllUnionSlot = context.checkContext.scan.getNonUserVisibleOutput() + if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { + Slot hllUnionSlot = context.checkContext.scan.getOutputByMvIndex(context.checkContext.index) .stream() - .filter(s -> hllUnionColumn.equals(normalizeName(s.getName()))) + .filter(s -> hllUnionColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .get(); @@ -1246,7 +1449,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Map projectMap) { return project.getProjects().stream() .map(expr -> (NamedExpression) ExpressionUtils.replace(expr, projectMap)) - .collect(ImmutableList.toImmutableList()); + .collect(Collectors.toList()); } private List nonVirtualGroupByExprs(LogicalAggregate agg) { @@ -1254,4 +1457,52 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial .filter(expr -> !(expr instanceof VirtualSlotReference)) .collect(ImmutableList.toImmutableList()); } + + /** + * Put all the slots provided by mv into the project, + * and cannot simply replace them in the subsequent ReplaceExpressionWithMvColumn rule, + * because one base column in mv may correspond to multiple mv columns: eg. k2 -> max_k2/min_k2/sum_k2 + * TODO: Do not add redundant columns + */ + private List generateNewOutputsWithMvOutputs( + LogicalOlapScan mvPlan, List outputs) { + if (mvPlan.getSelectedIndexId() == mvPlan.getTable().getBaseIndexId()) { + return outputs; + } + return ImmutableList.builder() + .addAll(mvPlan.getOutputByMvIndex(mvPlan.getSelectedIndexId())) + .addAll(outputs.stream() + .filter(s -> !(s instanceof Slot)) + .collect(ImmutableList.toImmutableList())) + .addAll(outputs.stream() + .filter(SlotNotFromChildren.class::isInstance) + .collect(ImmutableList.toImmutableList())) + .build(); + } + + /** + * eg: select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t; + * +--LogicalAggregate[88] ( groupByExpr=[t#4], outputExpr=[t#4, sum(abs((k2#1 + 1))) AS `sum(abs(k2 + 1))`#5]) + * +--LogicalProject[87] ( distinct=false, projects=[(abs(k1#0) + 1) AS `t`#4, k2#1]) + * +--LogicalOlapScan() + * t -> abs(k1#0) + 1 + */ + private Set collectRequireExprWithAggAndProject( + List aggExpressions, List projectExpressions) { + Map exprIdToExpression = projectExpressions.stream() + .collect(Collectors.toMap(NamedExpression::getExprId, e -> { + if (e instanceof Alias) { + return ((Alias) e).child(); + } + return e; + })); + return aggExpressions.stream() + .map(e -> { + if ((e instanceof NamedExpression) + && exprIdToExpression.containsKey(((NamedExpression) e).getExprId())) { + return exprIdToExpression.get(((NamedExpression) e).getExprId()); + } + return e; + }).collect(ImmutableSet.toImmutableSet()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java index 012fb01534..c5baf8f3f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java @@ -24,25 +24,22 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Select materialized index, i.e., both for rollup and materialized view when aggregate is not present. @@ -63,47 +60,95 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater // project with pushdown filter. // Project(Filter(Scan)) logicalProject(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex))) - .then(project -> { + .thenApply(ctx -> { + LogicalProject> project = ctx.root; LogicalFilter filter = project.child(); LogicalOlapScan scan = filter.child(); - return project.withChildren(filter.withChildren( - select(scan, project::getInputSlots, filter::getConjuncts))); + + LogicalOlapScan mvPlan = select( + scan, project::getInputSlots, filter::getConjuncts, + Stream.concat(filter.getExpressions().stream(), + project.getExpressions().stream()).collect(ImmutableSet.toImmutableSet())); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + + return new LogicalProject( + generateProjectsAlias(project.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + project.withChildren(filter.withChildren(mvPlan)), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_PROJECT_FILTER_SCAN), // project with filter that cannot be pushdown. // Filter(Project(Scan)) logicalFilter(logicalProject(logicalOlapScan().when(this::shouldSelectIndex))) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter> filter = ctx.root; LogicalProject project = filter.child(); LogicalOlapScan scan = project.child(); - return filter.withChildren(project.withChildren( - select(scan, project::getInputSlots, ImmutableSet::of) - )); + + LogicalOlapScan mvPlan = select( + scan, project::getInputSlots, ImmutableSet::of, + new HashSet<>(project.getExpressions())); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + + return new LogicalProject( + generateProjectsAlias(project.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + filter.withChildren(project.withChildren(mvPlan)), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_FILTER_PROJECT_SCAN), // scan with filters could be pushdown. // Filter(Scan) logicalFilter(logicalOlapScan().when(this::shouldSelectIndex)) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter filter = ctx.root; LogicalOlapScan scan = filter.child(); - return filter.withChildren(select(scan, filter::getOutputSet, filter::getConjuncts)); + LogicalOlapScan mvPlan = select( + scan, filter::getOutputSet, filter::getConjuncts, + new HashSet<>(filter.getExpressions())); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + + return new LogicalProject( + generateProjectsAlias(scan.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalProject(mvPlan.getOutput(), filter.withChildren(mvPlan)), mvPlan)); }) .toRule(RuleType.MATERIALIZED_INDEX_FILTER_SCAN), // project and scan. // Project(Scan) logicalProject(logicalOlapScan().when(this::shouldSelectIndex)) - .then(project -> { + .thenApply(ctx -> { + LogicalProject project = ctx.root; LogicalOlapScan scan = project.child(); - return project.withChildren( - select(scan, project::getInputSlots, ImmutableSet::of)); + + LogicalOlapScan mvPlan = select( + scan, project::getInputSlots, ImmutableSet::of, + new HashSet<>(project.getExpressions())); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + + return new LogicalProject( + generateProjectsAlias(project.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + project.withChildren(mvPlan), mvPlan)); }) .toRule(RuleType.MATERIALIZED_INDEX_PROJECT_SCAN), // only scan. logicalOlapScan() .when(this::shouldSelectIndex) - .then(scan -> select(scan, scan::getOutputSet, ImmutableSet::of)) + .thenApply(ctx -> { + LogicalOlapScan scan = ctx.root; + + LogicalOlapScan mvPlan = select( + scan, scan::getOutputSet, ImmutableSet::of, + scan.getOutputSet()); + SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + + return new LogicalProject( + generateProjectsAlias(mvPlan.getOutput(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalProject(mvPlan.getOutput(), mvPlan), mvPlan)); + }) .toRule(RuleType.MATERIALIZED_INDEX_SCAN) ); } @@ -116,10 +161,11 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater * @param predicatesSupplier Supplier to get pushdown predicates. * @return Result scan node. */ - private LogicalPlan select( + private LogicalOlapScan select( LogicalOlapScan scan, Supplier> requiredScanOutputSupplier, - Supplier> predicatesSupplier) { + Supplier> predicatesSupplier, + Set requiredExpr) { OlapTable table = scan.getTable(); long baseIndexId = table.getBaseIndexId(); KeysType keysType = scan.getTable().getKeysType(); @@ -138,16 +184,13 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater if (scan.getTable().isDupKeysOrMergeOnWrite()) { // Set pre-aggregation to `on` to keep consistency with legacy logic. List candidates = scan.getTable().getVisibleIndex().stream() + .filter(index -> index.getId() != baseIndexId) .filter(index -> !indexHasAggregate(index, scan)) .filter(index -> containAllRequiredColumns(index, scan, - requiredScanOutputSupplier.get())) + requiredScanOutputSupplier.get(), requiredExpr)) .collect(Collectors.toList()); long bestIndex = selectBestIndex(candidates, scan, predicatesSupplier.get()); - if (bestIndex == baseIndexId) { - return scan.withMaterializedIndexSelected(PreAggStatus.on(), bestIndex); - } else { - return createProjectForMv(scan.withMaterializedIndexSelected(PreAggStatus.on(), bestIndex)); - } + return scan.withMaterializedIndexSelected(PreAggStatus.on(), bestIndex); } else { final PreAggStatus preAggStatus; if (preAggEnabledByHint(scan)) { @@ -163,17 +206,18 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater // No aggregate on scan. // So only base index and indexes that have all the keys could be used. List candidates = table.getVisibleIndex().stream() - .filter(index -> index.getId() == baseIndexId - || table.getKeyColumnsByIndexId(index.getId()).size() == baseIndexKeySize) - .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get())) + .filter(index -> table.getKeyColumnsByIndexId(index.getId()).size() == baseIndexKeySize) + .filter(index -> containAllRequiredColumns( + index, scan, requiredScanOutputSupplier.get(), + predicatesSupplier.get())) .collect(Collectors.toList()); if (candidates.size() == 1) { // `candidates` only have base index. return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); } else { - return createProjectForMv(scan.withMaterializedIndexSelected(preAggStatus, - selectBestIndex(candidates, scan, predicatesSupplier.get()))); + return scan.withMaterializedIndexSelected(preAggStatus, + selectBestIndex(candidates, scan, predicatesSupplier.get())); } } } @@ -183,28 +227,4 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater .stream() .anyMatch(Column::isAggregated); } - - private LogicalProject createProjectForMv(LogicalOlapScan scan) { - Preconditions.checkArgument(scan.getSelectedIndexId() != scan.getTable().getBaseIndexId()); - List mvSlots = scan.getOutputByMvIndex(scan.getSelectedIndexId()); - List baseSlots = scan.getOutputByMvIndex(scan.getTable().getBaseIndexId()); - List aliases = Lists.newArrayList(); - List baseColumnNames = mvSlots.stream() - .map(slot -> org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(slot.getName())) - .collect(Collectors.toList()); - boolean isMvName = org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlots.get(0).getName()); - for (int i = 0; i < baseColumnNames.size(); ++i) { - for (Slot slot : baseSlots) { - if (((SlotReference) slot).getColumn().get().getName() - .equals(baseColumnNames.get(i))) { - aliases.add( - new Alias(slot.getExprId(), isMvName ? mvSlots.get(i) : slot, baseColumnNames.get(i))); - break; - } - } - } - return new LogicalProject(aliases, - isMvName ? scan.withOutput(scan.getOutputByMvIndex(scan.getSelectedIndexId())) - : scan); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 309ffd7bcc..1c1452b399 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -88,6 +88,12 @@ public class SlotReference extends Slot { column.isAllowNull(), qualifier, column); } + public static SlotReference fromColumn(Column column, String name, List qualifier) { + DataType dataType = DataType.fromCatalogType(column.getType()); + return new SlotReference(StatementScopeIdGenerator.newExprId(), name, dataType, + column.isAllowNull(), qualifier, column); + } + @Override public String getName() { return name; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 93a7b9ba11..a273daf114 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -164,11 +164,6 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla return getLogicalProperties().getOutput(); } - @Override - public List getNonUserVisibleOutput() { - return getLogicalProperties().getNonUserVisibleOutput(); - } - @Override public Set getOutputSet() { return getLogicalProperties().getOutputSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index 1708878fa4..bc85e146bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -25,8 +25,6 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.MutableState; import org.apache.doris.nereids.util.MutableState.MultiMutableState; -import com.google.common.collect.ImmutableList; - import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -92,11 +90,6 @@ public class FakePlan implements Plan { return new ArrayList<>(); } - @Override - public List getNonUserVisibleOutput() { - return ImmutableList.of(); - } - @Override public String treeString() { return "DUMMY"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index da6eb24d43..108474799a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -88,8 +88,6 @@ public interface Plan extends TreeNode { */ List getOutput(); - List getNonUserVisibleOutput(); - /** * Get output slot set of the plan. */ @@ -117,10 +115,6 @@ public interface Plan extends TreeNode { throw new IllegalStateException("Not support compute output for " + getClass().getName()); } - default List computeNonUserVisibleOutput() { - return ImmutableList.of(); - } - String treeString(); default Plan withOutput(List output) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java index 556e989fae..48c372068b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java @@ -108,11 +108,6 @@ public abstract class Command extends AbstractPlan implements LogicalPlan { throw new RuntimeException("Command do not implement getOutput"); } - @Override - public List getNonUserVisibleOutput() { - throw new RuntimeException("Command do not implement getNonUserVisibleOutput"); - } - @Override public String treeString() { throw new RuntimeException("Command do not implement treeString"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java index 704a0dd222..15ea90e909 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java @@ -60,7 +60,7 @@ public abstract class AbstractLogicalPlan extends AbstractPlan implements Logica if (hasUnboundChild || hasUnboundExpression()) { return UnboundLogicalProperties.INSTANCE; } else { - return new LogicalProperties(this::computeOutput, this::computeNonUserVisibleOutput); + return new LogicalProperties(this::computeOutput); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 8ddcdca828..749538ec34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -130,4 +130,8 @@ public class LogicalFilter extends LogicalUnary withLogicalProperties(Optional logicalProperties) { return new LogicalFilter<>(conjuncts, Optional.empty(), logicalProperties, child()); } + + public LogicalFilter withConjunctsAndChild(Set conjuncts, Plan child) { + return new LogicalFilter<>(conjuncts, child); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index ed7fa920e1..9aabd2547b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -26,6 +26,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.rules.mv.AbstractSelectMaterializedIndexRule; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.ObjectId; @@ -74,7 +75,13 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, */ private final PreAggStatus preAggStatus; - private final Map mvNameToSlot; + /** + * When the Slotreference is generated through fromColumn, + * the exprId will be generated incrementally, + * causing the slotId of the base to change when the output is recalculated. + * This structure is responsible for storing the generated Slotreference + */ + private final Map cacheSlotWithSlotName; /////////////////////////////////////////////////////////////////////////// // Members for tablet ids. @@ -140,7 +147,8 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, Optional groupExpression, Optional logicalProperties, List selectedPartitionIds, boolean partitionPruned, List selectedTabletIds, long selectedIndexId, boolean indexSelected, - PreAggStatus preAggStatus, List partitions, List hints, Map mvNameToSlot) { + PreAggStatus preAggStatus, List partitions, + List hints, Map cacheSlotWithSlotName) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); @@ -155,7 +163,7 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, .filter(partitionId -> this.getTable().getPartition(partitionId).hasData()).collect( Collectors.toList()); this.hints = Objects.requireNonNull(hints, "hints can not be null"); - this.mvNameToSlot = Objects.requireNonNull(mvNameToSlot, "mvNameToSlot can not be null"); + this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName, "mvNameToSlot can not be null"); } public List getSelectedPartitionIds() { @@ -200,7 +208,7 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, && Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected) && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds) && Objects.equals(hints, ((LogicalOlapScan) o).hints) - && Objects.equals(mvNameToSlot, ((LogicalOlapScan) o).mvNameToSlot); + && Objects.equals(cacheSlotWithSlotName, ((LogicalOlapScan) o).cacheSlotWithSlotName); } @Override @@ -209,45 +217,50 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, selectedPartitionIds, partitionPruned, selectedIndexId, indexSelected, selectedTabletIds, - hints, mvNameToSlot); + hints); } @Override public LogicalOlapScan withGroupExpression(Optional groupExpression) { return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, mvNameToSlot); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName); } @Override public LogicalOlapScan withLogicalProperties(Optional logicalProperties) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, mvNameToSlot); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, mvNameToSlot); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, - indexId, true, preAgg, manuallySpecifiedPartitions, hints, mvNameToSlot); + indexId, true, preAgg, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, mvNameToSlot); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, mvNameToSlot); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName); } @Override @@ -284,6 +297,9 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, @Override public List computeOutput() { + if (selectedIndexId != ((OlapTable) table).getBaseIndexId()) { + return getOutputByMvIndex(selectedIndexId); + } List otherColumns = new ArrayList<>(); if (!Util.showHiddenColumns() && getTable().hasDeleteSign() && !ConnectContext.get().getSessionVariable() @@ -291,18 +307,14 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, otherColumns.add(getTable().getDeleteSignColumn()); } return Stream.concat(table.getBaseSchema().stream(), otherColumns.stream()) - .map(col -> SlotReference.fromColumn(col, qualified())) - .collect(ImmutableList.toImmutableList()); - } - - @Override - public List computeNonUserVisibleOutput() { - OlapTable olapTable = (OlapTable) table; - return olapTable.getVisibleIndexIdToMeta().values() - .stream() - .filter(index -> index.getIndexId() != ((OlapTable) table).getBaseIndexId()) - .flatMap(index -> index.getSchema().stream()) - .map(this::generateUniqueSlot) + .map(col -> { + if (cacheSlotWithSlotName.containsKey(col.getName())) { + return cacheSlotWithSlotName.get(col.getName()); + } + Slot slot = SlotReference.fromColumn(col, qualified()); + cacheSlotWithSlotName.put(col.getName(), slot); + return slot; + }) .collect(ImmutableList.toImmutableList()); } @@ -311,23 +323,28 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, * and create a new slotReference for the slot that has not appeared in the materialized view. */ public List getOutputByMvIndex(long indexId) { - if (indexId == ((OlapTable) table).getBaseIndexId()) { - return getOutput(); - } - OlapTable olapTable = (OlapTable) table; + // PhysicalStorageLayerAggregateTest has no visible index + if (-1 == indexId) { + return olapTable.getIndexMetaByIndexId(indexId).getSchema() + .stream().map(s -> generateUniqueSlot(s, indexId == ((OlapTable) table).getBaseIndexId())) + .collect(Collectors.toList()); + } return olapTable.getVisibleIndexIdToMeta().get(indexId).getSchema() .stream() - .map(this::generateUniqueSlot) + .map(s -> generateUniqueSlot(s, indexId == ((OlapTable) table).getBaseIndexId())) .collect(ImmutableList.toImmutableList()); } - private Slot generateUniqueSlot(Column column) { - if (mvNameToSlot.containsKey(column.getName())) { - return mvNameToSlot.get(column.getName()); + private Slot generateUniqueSlot(Column column, boolean isBaseIndex) { + String name = isBaseIndex ? column.getName() + : AbstractSelectMaterializedIndexRule.parseMvColumnToMvName(column.getName(), + column.isAggregated() ? Optional.of(column.getAggregationType().toSql()) : Optional.empty()); + if (cacheSlotWithSlotName.containsKey(name)) { + return cacheSlotWithSlotName.get(name); } - Slot slot = SlotReference.fromColumn(column, qualified()); - mvNameToSlot.put(column.getName(), slot); + Slot slot = SlotReference.fromColumn(column, name, qualified()); + cacheSlotWithSlotName.put(name, slot); return slot; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 165d4685ae..d6195f716e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -170,6 +170,10 @@ public class LogicalRepeat extends LogicalUnary(groupingSets, outputExpressionList, child); } + public LogicalRepeat withAggOutputAndChild(List newOutput, Plan child) { + return new LogicalRepeat<>(groupingSets, newOutput, child); + } + public boolean canBindVirtualSlot() { return bound() && outputExpressions.stream() .noneMatch(output -> output.containsType(VirtualSlotReference.class)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 9d8d502d81..fc484bf6c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -51,14 +52,17 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { private final ImmutableList selectedPartitionIds; private final PreAggStatus preAggStatus; + private final List baseOutputs; + /** * Constructor for PhysicalOlapScan. */ public PhysicalOlapScan(ObjectId id, OlapTable olapTable, List qualifier, long selectedIndexId, List selectedTabletIds, List selectedPartitionIds, DistributionSpec distributionSpec, - PreAggStatus preAggStatus, Optional groupExpression, LogicalProperties logicalProperties) { + PreAggStatus preAggStatus, List baseOutputs, + Optional groupExpression, LogicalProperties logicalProperties) { this(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, - preAggStatus, groupExpression, logicalProperties, null, null); + preAggStatus, baseOutputs, groupExpression, logicalProperties, null, null); } /** @@ -66,7 +70,8 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { */ public PhysicalOlapScan(ObjectId id, OlapTable olapTable, List qualifier, long selectedIndexId, List selectedTabletIds, List selectedPartitionIds, DistributionSpec distributionSpec, - PreAggStatus preAggStatus, Optional groupExpression, LogicalProperties logicalProperties, + PreAggStatus preAggStatus, List baseOutputs, + Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_OLAP_SCAN, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); @@ -76,6 +81,7 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { this.selectedPartitionIds = ImmutableList.copyOf(selectedPartitionIds); this.distributionSpec = distributionSpec; this.preAggStatus = preAggStatus; + this.baseOutputs = ImmutableList.copyOf(baseOutputs); } @Override @@ -105,6 +111,10 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { return preAggStatus; } + public List getBaseOutputs() { + return baseOutputs; + } + @Override public String toString() { return Utils.toSqlString("PhysicalOlapScan[" + id.asInt() + "]" + getGroupIdAsString(), @@ -141,13 +151,14 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { @Override public PhysicalOlapScan withGroupExpression(Optional groupExpression) { return new PhysicalOlapScan(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, - selectedPartitionIds, distributionSpec, preAggStatus, groupExpression, getLogicalProperties()); + selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, + groupExpression, getLogicalProperties()); } @Override public PhysicalOlapScan withLogicalProperties(Optional logicalProperties) { return new PhysicalOlapScan(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, - selectedPartitionIds, distributionSpec, preAggStatus, Optional.empty(), + selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, Optional.empty(), logicalProperties.get()); } @@ -155,7 +166,7 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { public PhysicalOlapScan withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalOlapScan(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, - selectedPartitionIds, distributionSpec, preAggStatus, groupExpression, + selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), physicalProperties, statistics); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index b3e3d62c5b..d59e72e860 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -38,6 +38,7 @@ import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import mockit.Injectable; import org.junit.jupiter.api.Assertions; @@ -65,7 +66,7 @@ public class PhysicalPlanTranslatorTest { LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); PhysicalOlapScan scan = new PhysicalOlapScan(RelationUtil.newRelationId(), t1, qualifier, t1.getBaseIndexId(), Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), - Optional.empty(), t1Properties); + ImmutableList.of(), Optional.empty(), t1Properties); Literal t1FilterRight = new IntegerLiteral(1); Expression t1FilterExpr = new GreaterThan(col1, t1FilterRight); PhysicalFilter filter = diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java index 063f0ca3e6..43b9c8bae1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.PlanConstructor; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import mockit.Injectable; import org.junit.jupiter.api.Assertions; @@ -75,7 +76,7 @@ public class MergeProjectPostProcessTest { t1Output.add(c); LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); PhysicalOlapScan scan = new PhysicalOlapScan(ObjectId.createGenerator().getNextId(), t1, qualifier, 0L, - Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), + Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties); Alias x = new Alias(a, "x"); List projList3 = Lists.newArrayList(x, b, c); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java index 645156b6ce..39e9c69991 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java @@ -41,7 +41,6 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.List; @@ -63,6 +62,8 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP private static final String TEST_TABLE_NAME = "test_tb"; private static final String USER_TAG_MV_NAME = "user_tags_mv"; + private static final String ADVANCE_TABLE_NAME = "advance"; + @Override protected void beforeCreatingConnectContext() throws Exception { FeConstants.default_scheduler_interval_millisecond = 10; @@ -73,6 +74,7 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP protected void runBeforeAll() throws Exception { createDatabase(HR_DB_NAME); useDatabase(HR_DB_NAME); + connectContext.getSessionVariable().enableNereidsTimeout = false; } @BeforeEach @@ -91,6 +93,19 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP + " (time_col date, user_id int, user_name varchar(20), tag_id int) partition by range (time_col) " + " (partition p1 values less than MAXVALUE) " + "distributed by hash(time_col) buckets 3 properties('replication_num' = '1');"); + + createTable("create table " + HR_DB_NAME + "." + ADVANCE_TABLE_NAME + + "( a int, \n" + + " b int, \n" + + " c int\n" + + ")ENGINE=OLAP \n" + + "DISTRIBUTED BY HASH(a) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); } @AfterEach @@ -98,6 +113,7 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP dropTable(EMPS_TABLE_NAME, true); dropTable(DEPTS_TABLE_NAME, true); dropTable(USER_TAG_TABLE_NAME, true); + dropTable(ADVANCE_TABLE_NAME, true); } @Test @@ -152,7 +168,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMvWithTwoTable(union, EMPS_MV_NAME, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggQueryOnAggMV1() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary), " @@ -162,7 +177,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggQueryOnAggMV2() throws Exception { String agg = "select deptno, sum(salary) from " + EMPS_TABLE_NAME + " group by deptno"; @@ -173,7 +187,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggQueryOnAggMV3() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)" @@ -200,7 +213,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * There will be a compensating Project added after matching of the Aggregate. */ - @Disabled //ISSUE #18263 @Test public void testAggQuqeryOnAggMV5() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)" @@ -214,7 +226,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * There will be a compensating Project + Filter added after matching of the Aggregate. */ - @Disabled //ISSUE #18263 @Test public void testAggQuqeryOnAggMV6() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)" @@ -229,7 +240,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP * Aggregation query with groupSets at coarser level of aggregation than * aggregation materialized view. */ - @Disabled //ISSUE #18263 @Test public void testGroupingSetQueryOnAggMV() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " @@ -242,7 +252,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * Aggregation query at coarser level of aggregation than aggregation materialized view. */ - @Disabled //ISSUE #18263 @Test public void testAggQuqeryOnAggMV7() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) " @@ -252,7 +261,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggQueryOnAggMV8() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) " @@ -266,7 +274,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP * Query with cube and arithmetic expr * TODO: enable this when group by cube is supported. */ - @Disabled public void testAggQueryOnAggMV9() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) " + "from " + EMPS_TABLE_NAME + " group by deptno, commission;"; @@ -279,7 +286,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * Query with rollup and arithmetic expr */ - @Disabled //ISSUE #18263 @Test public void testAggQueryOnAggMV10() throws Exception { String createMVSql = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) " @@ -315,7 +321,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMvWithTwoTable(query, EMPS_TABLE_NAME, EMPS_TABLE_NAME); } - @Disabled //ISSUE #18263 @Test public void testJoinOnLeftProjectToJoin() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME @@ -330,7 +335,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, ImmutableMap.of(EMPS_TABLE_NAME, EMPS_MV_NAME, DEPTS_TABLE_NAME, DEPTS_MV_NAME)); } - @Disabled //ISSUE #18263 @Test public void testJoinOnRightProjectToJoin() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary), sum" @@ -345,7 +349,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, ImmutableMap.of(EMPS_TABLE_NAME, EMPS_MV_NAME, DEPTS_TABLE_NAME, DEPTS_MV_NAME)); } - @Disabled //ISSUE #18263 @Test public void testJoinOnProjectsToJoin() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary), sum" @@ -416,7 +419,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when implicit case is fully developed. */ - @Disabled public void testJoinOnCalcToJoin4() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " + EMPS_TABLE_NAME + ";"; @@ -433,7 +435,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when order by column not in project is supported. */ - @Disabled public void testOrderByQueryOnProjectView() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from " + EMPS_TABLE_NAME + ";"; @@ -447,7 +448,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when order by column not in select is supported. */ - @Disabled public void testOrderByQueryOnOrderByView() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from " + EMPS_TABLE_NAME + " order by deptno;"; @@ -456,7 +456,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVAggregateFuncs1() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " @@ -466,7 +465,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVAggregateFuncs2() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " @@ -476,7 +474,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVAggregateFuncs3() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " @@ -486,7 +483,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVAggregateFuncs4() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " @@ -496,7 +492,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVAggregateFuncs5() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) " @@ -506,7 +501,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVCalcGroupByQuery1() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) " @@ -517,7 +511,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVCalcGroupByQuery2() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) " @@ -528,7 +521,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVCalcGroupByQuery3() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) " @@ -552,7 +544,7 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when estimate stats bug fixed. */ - @Disabled + @Test public void testSubQuery() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid " + "from " + EMPS_TABLE_NAME + ";"; @@ -565,7 +557,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when sum(distinct xxx) is supported. */ - @Disabled public void testDistinctQuery() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) " + "from " + EMPS_TABLE_NAME + " group by deptno;"; @@ -587,7 +578,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, ImmutableMap.of(EMPS_TABLE_NAME, EMPS_MV_NAME)); } - @Disabled //ISSUE #18263 @Test public void testMultiMVMultiUsage() throws Exception { String createEmpsMVSql01 = "create materialized view emp_mv_01 as select deptno, empid, salary " @@ -612,7 +602,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, ImmutableMap.of(EMPS_TABLE_NAME, EMPS_MV_NAME, DEPTS_TABLE_NAME, DEPTS_TABLE_NAME)); } - @Disabled //ISSUE #18263 @Test public void testAggregateMVOnCountDistinctQuery1() throws Exception { String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " @@ -703,7 +692,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when order by aggregate function is supported. */ - @Disabled public void testAggFunctionInOrder() throws Exception { String duplicateTable = "CREATE TABLE " + TEST_TABLE_NAME + " ( k1 int(11) NOT NULL , k2 int(11) NOT NULL ," + "v1 varchar(4096) NOT NULL, v2 float NOT NULL , v3 decimal(20, 7) NOT NULL ) ENGINE=OLAP " @@ -750,7 +738,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * bitmap_union_count(to_bitmap()) -> bitmap_union_count without having */ - @Disabled //ISSUE #18263 @Test public void testBitmapUnionRewrite() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME @@ -762,10 +749,23 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP singleTableTest(query, USER_TAG_MV_NAME, true); } + /** + * bitmap_union_count(bitmap_hash()) -> bitmap_union_count without having + */ + @Test + public void testBitmapUnionBitmapHashRewrite() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + + " as select user_id, bitmap_union(bitmap_hash(tag_id)) from " + + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select user_id, bitmap_union_count(bitmap_hash(tag_id)) a from " + USER_TAG_TABLE_NAME + + " group by user_id"; + singleTableTest(query, USER_TAG_MV_NAME, true); + } + /** * bitmap_union_count(to_bitmap()) -> bitmap_union_count with having */ - @Disabled //ISSUE #18263 @Test public void testBitmapUnionInQuery() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME @@ -777,7 +777,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP singleTableTest(query, USER_TAG_MV_NAME, true); } - @Disabled //ISSUE #18263 @Test public void testBitmapUnionInSubquery() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -817,7 +816,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, ImmutableMap.of("user_tags", "user_tags")); } - @Disabled //ISSUE #18263 @Test public void testTwoTupleInQuery() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -830,8 +828,8 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP .analyze(query) .rewrite() .matches(logicalJoin( - logicalAggregate( - logicalProject( + logicalProject( + logicalAggregate( logicalOlapScan().when(scan -> "user_tags_mv".equals( scan.getSelectedMaterializedIndexName().get())))), logicalAggregate( @@ -889,7 +887,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * count distinct to bitmap_union_count in mv */ - @Disabled //ISSUE #18263 @Test public void testCountDistinctToBitmap() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -922,7 +919,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, USER_TAG_TABLE_NAME); } - @Disabled //ISSUE #18263 @Test public void testNDVToHll() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -936,7 +932,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP /** * TODO: enable this when hll is supported. */ - @Disabled public void testApproxCountDistinctToHll() throws Exception { // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " // + "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME @@ -946,7 +941,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP // dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg"); } - @Disabled //ISSUE #18263 @Test public void testHLLUnionFamilyRewrite() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -995,7 +989,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, EMPS_TABLE_NAME); } - @Disabled //ISSUE #18263 @Test public void testCountFieldInQuery() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -1012,7 +1005,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, USER_TAG_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void testCreateMVBaseBitmapAggTable() throws Exception { String createTableSQL = "create table " + HR_DB_NAME + ".agg_table " @@ -1031,7 +1023,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP dropTable("agg_table", true); } - @Disabled //ISSUE #18263 @Test public void testSelectMVWithTableAlias() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " @@ -1048,7 +1039,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP testMv(query, USER_TAG_MV_NAME); } - @Disabled //ISSUE #18263 @Test public void selectBitmapMvWithProjectTest1() throws Exception { createTable("create table t(\n" @@ -1070,7 +1060,6 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP dropTable("t", true); } - @Disabled //ISSUE #18263 @Test public void selectBitmapMvWithProjectTest2() throws Exception { createTable("create table t(\n" @@ -1092,10 +1081,9 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP dropTable("t", true); } - @Disabled //ISSUE #18263 @Test public void selectBitmapMvWithProjectMultiMv() throws Exception { - createTable("create table t(\n" + createTable("create table selectBitmapMvWithProjectMultiMv(\n" + " a int, \n" + " b int, \n" + " c int\n" @@ -1108,14 +1096,60 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP + "\"disable_auto_compaction\" = \"false\"\n" + ");"); createMv("create materialized view mv as" - + " select a, bitmap_union(to_bitmap(b)) from t group by a;"); + + " select a, bitmap_union(to_bitmap(b)) from selectBitmapMvWithProjectMultiMv group by a;"); createMv("create materialized view mv1 as" - + " select c, bitmap_union(to_bitmap(b)) from t group by c;"); + + " select c, bitmap_union(to_bitmap(b)) from selectBitmapMvWithProjectMultiMv group by c;"); createMv("create materialized view mv2 as" - + " select a, c, bitmap_union(to_bitmap(b)) from t group by a, c;"); + + " select a, c, bitmap_union(to_bitmap(b)) from selectBitmapMvWithProjectMultiMv group by a, c;"); - testMv("select a, bitmap_union_count(to_bitmap(b)) as cnt from t group by a", "mv"); - dropTable("t", true); + testMv("select a, bitmap_union_count(to_bitmap(b)) as cnt from selectBitmapMvWithProjectMultiMv group by a", "mv"); + dropTable("selectBitmapMvWithProjectMultiMv", true); + } + + @Test + public void advanceMvAggWithExprTest() throws Exception { + createMv("create materialized view mv1 as" + + " select abs(a)+1 tmp, sum(abs(b+2)) from " + ADVANCE_TABLE_NAME + " group by tmp;"); + + testMv("select abs(a)+1 tmp, sum(abs(b+2)) from " + ADVANCE_TABLE_NAME + " group by tmp", "mv1"); + } + + @Test + public void advanceMvDupColTest() throws Exception { + createMv("create materialized view mv2 as" + + " select a, sum(b), max(b) from " + ADVANCE_TABLE_NAME + " group by a;"); + + testMv("select a, sum(b), max(b) as cnt from " + ADVANCE_TABLE_NAME + " group by a", "mv2"); + testMv("select a, sum(b) as cnt from " + ADVANCE_TABLE_NAME + " group by a", "mv2"); + testMv("select a, max(b) as cnt from " + ADVANCE_TABLE_NAME + " group by a", "mv2"); + testMv("select unix_timestamp(a) tmp, max(b) as cnt from " + ADVANCE_TABLE_NAME + " group by tmp", "mv2"); + } + + @Test + public void advanceMvDupColTest1() throws Exception { + createMv("create materialized view mv2 as" + + " select b, sum(a), max(a) from " + ADVANCE_TABLE_NAME + " group by b;"); + + testMv("select b, sum(a), max(a) as cnt from " + ADVANCE_TABLE_NAME + " group by b", "mv2"); + testMv("select b, sum(a) as cnt from " + ADVANCE_TABLE_NAME + " group by b", "mv2"); + testMv("select b, max(a) as cnt from " + ADVANCE_TABLE_NAME + " group by b", "mv2"); + testMv("select unix_timestamp(b) tmp, max(a) as cnt from " + ADVANCE_TABLE_NAME + " group by tmp", "mv2"); + } + + @Test + public void advanceMvMultiSlotTest() throws Exception { + createMv("create materialized view mv3 as" + + " select abs(a)+b+1,abs(b+2)+c+3 from " + ADVANCE_TABLE_NAME); + + testMv("select abs(a)+b+1,abs(b+2)+c+3 from " + ADVANCE_TABLE_NAME, "mv3"); + } + + @Test + public void advanceMvMultiSlotWithAggTest() throws Exception { + createMv("create materialized view mv4 as" + + " select abs(a)+b+1 tmp, sum(abs(b+2)+c+3) from " + ADVANCE_TABLE_NAME + " group by tmp"); + + testMv("select abs(a)+b+1 tmp, sum(abs(b+2)+c+3) from " + ADVANCE_TABLE_NAME + " group by tmp", "mv4"); } private void testMv(String sql, Map tableToIndex) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java index db0cdfc966..b207f7834a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java @@ -98,7 +98,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M + ");"); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testAggMatching() { singleTableTest("select k2, sum(v1) from t group by k2", "r1", true); @@ -133,7 +133,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M PlanChecker.from(connectContext).checkPlannerResult("select k2, sum(v1) from t group by k2"); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testTranslateWhenPreAggIsOff() { singleTableTest("select k2, min(v1) from t group by k2", scan -> { @@ -171,11 +171,11 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M @Test public void testWithFilter() { PlanChecker.from(connectContext) - .analyze("select k2, sum(v1) from t where k2>3 group by k3") + .analyze("select k2, sum(v1) from t where k2>3 group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); - Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get()); + Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get()); return true; })); } @@ -240,7 +240,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOff()); - Assertions.assertEquals("Input of aggregate function sum((v1 + 1)) should be slot or cast on slot.", + Assertions.assertEquals("Slot((v1 + 1)) in sum((v1 + 1)) is neither key column nor value column.", preAgg.getOffReason()); return true; })); @@ -315,7 +315,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M })); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testKeysOnlyQuery() throws Exception { singleTableTest("select k1 from t1", "r3", false); @@ -329,7 +329,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M /** * Rollup with all the keys should be used. */ - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testRollupWithAllTheKeys() throws Exception { createTable(" CREATE TABLE `t4` (\n" @@ -355,19 +355,19 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M singleTableTest("select k1, sum(v1) from t4 group by k1", "r1", true); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testComplexGroupingExpr() throws Exception { singleTableTest("select k2 + 1, sum(v1) from t group by k2 + 1", "r1", true); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testCountDistinctKeyColumn() { singleTableTest("select k2, count(distinct k3) from t group by k2", "r4", true); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testCountDistinctValueColumn() { singleTableTest("select k1, count(distinct v1) from t group by k1", scan -> { @@ -378,7 +378,7 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements M }); } - @Disabled //ISSUE #18263 + //@Disabled //ISSUE #18263 @Test public void testOnlyValueColumn1() throws Exception { singleTableTest("select sum(v1) from t", "r1", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java index fe1b495fd0..ad9d1df91a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java @@ -259,16 +259,16 @@ public class PlanEqualsTest { PhysicalOlapScan actual = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("a"), olapTable.getBaseIndexId(), selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, - PreAggStatus.on(), Optional.empty(), logicalProperties); + PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties); PhysicalOlapScan expected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("a"), olapTable.getBaseIndexId(), selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, - PreAggStatus.on(), Optional.empty(), logicalProperties); + PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties); Assertions.assertEquals(expected, actual); PhysicalOlapScan unexpected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("b"), olapTable.getBaseIndexId(), selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, - PreAggStatus.on(), Optional.empty(), logicalProperties); + PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties); Assertions.assertNotEquals(unexpected, actual); } diff --git a/regression-test/data/nereids_syntax_p0/advance_mv.out b/regression-test/data/nereids_syntax_p0/advance_mv.out new file mode 100644 index 0000000000..a5e0a11f84 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/advance_mv.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +1 +2 +3 + +-- !select_star -- +11 15 +13 17 +9 13 + +-- !select_star -- +11 15 +13 17 +9 13 + +-- !select_star -- +40 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/aggHaveDupBase.out b/regression-test/data/nereids_syntax_p0/mv/newMv/aggHaveDupBase.out new file mode 100644 index 0000000000..c59641f5cd --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/aggHaveDupBase.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 -4 -4 +1 1 1 +2 2 2 +3 -3 -3 + +-- !select_mv -- +-4 -4 +1 1 +2 2 +3 -3 + +-- !select_mv -- +-4 -4 +1 1 +2 2 +3 -3 + +-- !select_mv -- +\N -4 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/case_ignore.out b/regression-test/data/nereids_syntax_p0/mv/newMv/case_ignore.out new file mode 100644 index 0000000000..46dbfd4eeb --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/case_ignore.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 4 +1 1 +2 2 +3 3 + +-- !select_mv -- +-4 4 +1 1 +2 2 +3 3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.out new file mode 100644 index 0000000000..6a9964f200 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 4 +1 1 +2 2 +3 3 + +-- !select_mv_sub -- +1 +2 +3 +4 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.out new file mode 100644 index 0000000000..fac09ae8ec --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 -3 +1 2 +2 3 +3 -2 + +-- !select_mv_sub -- +-2 +-3 +2 +3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_abs.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_abs.out new file mode 100644 index 0000000000..b965805ffd --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_abs.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 4 +1 1 +2 2 +3 3 + +-- !select_mv_sub -- +1 +2 +3 +4 + +-- !select_mv_sub_add -- +2 +3 +4 +5 + +-- !select_group_mv -- +1 +2 +3 +4 + +-- !select_group_mv_add -- +2 +3 +4 +5 + +-- !select_group_mv_not -- +1 +2 +3 +4 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_bin.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_bin.out new file mode 100644 index 0000000000..3274089ffb --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_bin.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 1111111111111111111111111111111111111111111111111111111111111100 +1 1 +2 10 +3 1111111111111111111111111111111111111111111111111111111111111101 + +-- !select_mv_sub -- +1 +10 +1111111111111111111111111111111111111111111111111111111111111100 +1111111111111111111111111111111111111111111111111111111111111101 + +-- !select_mv_sub_add -- +10a +1111111111111111111111111111111111111111111111111111111111111100a +1111111111111111111111111111111111111111111111111111111111111101a +1a + +-- !select_group_mv -- +1 +10 +1111111111111111111111111111111111111111111111111111111111111100 +1111111111111111111111111111111111111111111111111111111111111101 + +-- !select_group_mv_add -- +10a +1111111111111111111111111111111111111111111111111111111111111100a +1111111111111111111111111111111111111111111111111111111111111101a +1a + +-- !select_group_mv_not -- +1 +10 +1111111111111111111111111111111111111111111111111111111111111100 +1111111111111111111111111111111111111111111111111111111111111101 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.out new file mode 100644 index 0000000000..70f2bed42a --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +1 +1 +1 + +-- !select_k1 -- +1 +2 +2 +3 +3 + +-- !select_star -- +1 1 a +2 2 b +2 2 bb +3 3 c +3 3 c + +-- !select_mv_sub -- +1 1 +2 2 +3 1 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_plus.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_plus.out new file mode 100644 index 0000000000..df92fe4126 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_plus.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 -3 +1 2 +2 3 +3 -2 + +-- !select_mv_sub -- +-2 +-3 +2 +3 + +-- !select_group_mv -- +-2 +-3 +2 +3 + +-- !select_group_mv -- +-4 +1 +2 +3 + +-- !select_group_mv_not -- +-3 +-4 +1 +2 + +-- !select_mv -- +-4 -3 +1 2 +2 3 +3 -2 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_year.out b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_year.out new file mode 100644 index 0000000000..ed40dd09da --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/dup_mv_year.out @@ -0,0 +1,18 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +1 2003 +2 2013 +3 2023 + +-- !select_star -- +1 2003-12-31 2003-12-31T01:02:03 +2 2013-12-31 2013-12-31T01:02:03 +3 2023-12-31 2023-12-31T01:02:03 +4 2033-12-31 2033-12-31T01:02:03 + +-- !select_mv_sub -- +2003 +2013 +2023 +2033 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot1.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot1.out new file mode 100644 index 0000000000..dc239c0f83 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot1.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +1 \N +1 1 +3 7 +5 9 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot2.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot2.out new file mode 100644 index 0000000000..a3c0a07a99 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot2.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +1 1 +3 7 +5 9 + +-- !select_base -- +1 1 +3 7 +5 9 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot3.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot3.out new file mode 100644 index 0000000000..8397a2ecf0 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot3.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-3 1 +2 7 +3 9 +4 \N + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out new file mode 100644 index 0000000000..8397a2ecf0 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-3 1 +2 7 +3 9 +4 \N + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot5.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot5.out new file mode 100644 index 0000000000..9c70e504cf --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot5.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c +3 -3 \N c + +-- !select_mv -- +-4 -8 +1 2 +2 4 +3 \N +3 \N + +-- !select_mv -- +-4 -4 +1 1 +2 2 +3 -3 +3 -3 +3 -3 +3 -3 + +-- !select_mv -- +-4 5.7.99 +1 5.7.99 +2 5.7.99 +3 5.7.99 +3 5.7.99 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot6.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot6.out new file mode 100644 index 0000000000..a63f4cef8e --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot6.out @@ -0,0 +1,18 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +1 1 +3 7 +5 9 + +-- !select_mv -- +1 \N +1 1 +3 7 +5 9 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/mv_with_view.out b/regression-test/data/nereids_syntax_p0/mv/newMv/mv_with_view.out new file mode 100644 index 0000000000..e73b852bfa --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/mv_with_view.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +1 1 1 + +-- !select_mv -- +1 1 a + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/rollback1.out b/regression-test/data/nereids_syntax_p0/mv/newMv/rollback1.out new file mode 100644 index 0000000000..2971ecda2d --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/rollback1.out @@ -0,0 +1,51 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 +1 +2 +3 + +-- !select_mv -- +-4 +1 +2 + +-- !select_mv -- +-4 +1 +2 + +-- !select_mv -- +-4 +1 +2 +3 + +-- !select_mv -- +-4 +-3 +1 +2 + +-- !select_mv -- +-4 +1 +2 + +-- !select_mv -- +-4 +1 +2 + +-- !select_mv -- +-4 +-3 +1 +2 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/single_slot.out b/regression-test/data/nereids_syntax_p0/mv/newMv/single_slot.out new file mode 100644 index 0000000000..16fe69c3ff --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/single_slot.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c + +-- !select_mv -- +2 2 +3 3 +4 2 +5 3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out b/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out new file mode 100644 index 0000000000..67df77e0cc --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 d +1 1 1 a +2 2 2 b +3 -3 \N c +3 2 \N c + +-- !select_mv -- +-4 d -4.0 +1 a 1.0 +2 b 2.0 +3 c -0.5 + +-- !select_mv -- +-4 -4.0 +1 1.0 +2 2.0 +3 -0.5 + +-- !select_mv -- +a 1.0 +b 2.0 +c -0.5 +d -4.0 + +-- !select_mv -- +-0.4 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/MVMultiUsage.out b/regression-test/data/nereids_syntax_p0/mv/ut/MVMultiUsage.out new file mode 100644 index 0000000000..b10e432dad --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/MVMultiUsage.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/MVWithAs.out b/regression-test/data/nereids_syntax_p0/mv/ut/MVWithAs.out new file mode 100644 index 0000000000..30b1f06fdf --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/MVWithAs.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 +2020-01-01 1 a 1 +2020-01-02 2 b 2 + +-- !select_mv -- +3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggCDInBitmap.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggCDInBitmap.out new file mode 100644 index 0000000000..bc9c8312eb --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggCDInBitmap.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +1 \N +2 \N +3 \N + +-- !select_mv -- +1 1 +2 1 +3 1 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.out new file mode 100644 index 0000000000..b10e432dad --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out new file mode 100644 index 0000000000..6db240393b --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- +2 1 +2 2 +3 3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out new file mode 100644 index 0000000000..83bfded26f --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- +\N \N 8 +1 \N 3 +1 1 3 +2 \N 3 +2 2 3 +3 \N 4 +3 3 4 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out new file mode 100644 index 0000000000..d0b1871ed2 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- +1 4 +2 2 +3 2 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out new file mode 100644 index 0000000000..94559632cb --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_emps_mv -- +2 9 +3 3 + +-- !select_star -- +2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 7 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- +2 9 +3 3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out new file mode 100644 index 0000000000..279e3ee410 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 10 +2020-01-04 4 d 21 4 4 + +-- !select_mv -- +4 4 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out new file mode 100644 index 0000000000..b10e432dad --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out new file mode 100644 index 0000000000..b10e432dad --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out new file mode 100644 index 0000000000..b10e432dad --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/bitmapUnionIn.out b/regression-test/data/nereids_syntax_p0/mv/ut/bitmapUnionIn.out new file mode 100644 index 0000000000..88913c5b65 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/bitmapUnionIn.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 +2020-01-01 1 a 2 +2020-01-02 2 b 2 + +-- !select_mv -- +1 2 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/incMVReInSub.out b/regression-test/data/nereids_syntax_p0/mv/ut/incMVReInSub.out new file mode 100644 index 0000000000..ab95a5b36f --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/incMVReInSub.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 +2020-01-01 1 a 2 +2020-01-02 2 b 2 + +-- !select_mv -- +1 \N + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/incRewriteCD.out b/regression-test/data/nereids_syntax_p0/mv/ut/incRewriteCD.out new file mode 100644 index 0000000000..dd06cd3927 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/incRewriteCD.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 +2020-01-01 1 a 2 +2020-01-02 2 b 2 + +-- !select_mv -- +a 2 +b 1 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.out b/regression-test/data/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.out new file mode 100644 index 0000000000..cf5140bb02 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +2 9 2 2 +3 3 3 3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/onStar.out b/regression-test/data/nereids_syntax_p0/mv/ut/onStar.out new file mode 100644 index 0000000000..3440fe7517 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/onStar.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 + +-- !select_mv -- +1 +1 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/orderByOnPView.out b/regression-test/data/nereids_syntax_p0/mv/ut/orderByOnPView.out new file mode 100644 index 0000000000..beefefba56 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/orderByOnPView.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- +1 +1 +2 +3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/projectMV1.out b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV1.out new file mode 100644 index 0000000000..ba455b7e85 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV1.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +1 1 +1 1 +2 2 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/projectMV2.out b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV2.out new file mode 100644 index 0000000000..934d3208d4 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV2.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +2 +2 + +-- !select_base -- +a +a + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/projectMV3.out b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV3.out new file mode 100644 index 0000000000..9537036f41 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV3.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +2 a +2 a + +-- !select_mv2 -- +a +a + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/projectMV4.out b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV4.out new file mode 100644 index 0000000000..45a962ed59 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/projectMV4.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +b + +-- !select_base -- +2 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/subQuery.out b/regression-test/data/nereids_syntax_p0/mv/ut/subQuery.out new file mode 100644 index 0000000000..158f3ca5f0 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/subQuery.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/unionDis.out b/regression-test/data/nereids_syntax_p0/mv/ut/unionDis.out new file mode 100644 index 0000000000..cd7fbba9aa --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/mv/ut/unionDis.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 + +-- !select_mv -- +2 2 +3 3 + diff --git a/regression-test/data/nereids_syntax_p0/rollup/agg.out b/regression-test/data/nereids_syntax_p0/rollup/agg.out new file mode 100644 index 0000000000..a79101110e --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/rollup/agg.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +test_rollup_agg1 AGG_KEYS siteid INT INT No true \N true + citycode SMALLINT SMALLINT No true \N true + username VARCHAR(32) VARCHAR(32) No true \N true + pv BIGINT BIGINT No false 0 SUM true + uv BIGINT BIGINT No false 0 SUM true + vv BIGINT BIGINT Yes false 0 SUM true + +rollup_city AGG_KEYS citycode SMALLINT SMALLINT No true \N true + pv BIGINT BIGINT No false 0 SUM true + vv BIGINT BIGINT Yes false 0 SUM true + +-- !sql -- +1 200 + diff --git a/regression-test/data/nereids_syntax_p0/rollup/agg_date.out b/regression-test/data/nereids_syntax_p0/rollup/agg_date.out new file mode 100644 index 0000000000..a64b546d21 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/rollup/agg_date.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +test_rollup_agg_date1 AGG_KEYS datek1 DATE DATEV2 Yes true \N true + datetimek1 DATETIME DATETIMEV2(0) Yes true \N true + datetimek2 DATETIME DATETIMEV2(3) Yes true \N true + datetimek3 DATETIME DATETIMEV2(6) Yes true \N true + datev1 DATE DATEV2 No false \N MAX true + datetimev1 DATETIME DATETIMEV2(0) No false \N MAX true + datetimev2 DATETIME DATETIMEV2(3) No false \N MAX true + datetimev3 DATETIME DATETIMEV2(6) No false \N MAX true + datetimev4 DATETIME DATETIMEV2(3) Yes false \N MAX true + +rollup_date AGG_KEYS datek1 DATE DATEV2 Yes true \N true + datetimek2 DATETIME DATETIMEV2(3) Yes true \N true + datetimek1 DATETIME DATETIMEV2(0) Yes true \N true + datetimek3 DATETIME DATETIMEV2(6) Yes true \N true + datev1 DATE DATEV2 No false \N MAX true + datetimev1 DATETIME DATETIMEV2(0) No false \N MAX true + datetimev2 DATETIME DATETIMEV2(3) No false \N MAX true + datetimev3 DATETIME DATETIMEV2(6) No false \N MAX true + +-- !sql -- +2022-08-23 2022-08-23T11:11:11 2022-08-23T11:11:11.111 2022-08-23T11:11:11.111111 2022-08-23 2022-08-23T11:11:11 2022-08-23T11:11:11.111 2022-08-23T11:11:11.111111 +2022-08-22 2022-08-22T11:11:11 2022-08-22T11:11:11.111 2022-08-22T11:11:11.111111 2022-08-22 2022-08-22T11:11:11 2022-08-22T11:11:11.111 2022-08-22T11:11:11.111111 + diff --git a/regression-test/data/nereids_syntax_p0/rollup/date.out b/regression-test/data/nereids_syntax_p0/rollup/date.out new file mode 100644 index 0000000000..4e80fba29f --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/rollup/date.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 2020-05-30 + +-- !sql -- +1 2020-05-30T11:11:11 + +-- !sql -- +1 2020-05-30T11:11:11.111 + +-- !sql -- +1 2020-05-30T11:11:11.111111 + diff --git a/regression-test/data/nereids_syntax_p0/rollup/hll.out b/regression-test/data/nereids_syntax_p0/rollup/hll.out new file mode 100644 index 0000000000..0aabbfaca3 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/rollup/hll.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +test_materialized_view_hll1 DUP_KEYS record_id INT INT Yes true \N true + seller_id INT INT Yes true \N true + store_id INT INT Yes true \N true + sale_date DATE DATEV2 Yes false \N NONE true + sale_amt BIGINT BIGINT Yes false \N NONE true + +amt_count AGG_KEYS mv_store_id INT INT Yes true \N true `store_id` + mva_HLL_UNION__hll_hash(`sale_amt`) HLL HLL No false \N HLL_UNION true hll_hash(`sale_amt`) + +-- !sql -- +1 1 + diff --git a/regression-test/data/nereids_syntax_p0/rollup/hll_with_light_sc.out b/regression-test/data/nereids_syntax_p0/rollup/hll_with_light_sc.out new file mode 100644 index 0000000000..985689d249 --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/rollup/hll_with_light_sc.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +test_materialized_view_hll_with_light_sc1 DUP_KEYS record_id INT INT Yes true \N true + seller_id INT INT Yes true \N true + store_id INT INT Yes true \N true + sale_date DATE DATEV2 Yes false \N NONE true + sale_amt BIGINT BIGINT Yes false \N NONE true + +amt_count1 AGG_KEYS mv_store_id INT INT Yes true \N true `store_id` + mva_HLL_UNION__hll_hash(`sale_amt`) HLL HLL No false \N HLL_UNION true hll_hash(`sale_amt`) + +-- !sql -- +1 1 + diff --git a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy index 6d260810e9..87d6a67486 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy @@ -39,7 +39,7 @@ suite ("testIncorrectMVRewriteInSubquery") { sql("select * from user_tags order by time_col;") contains "(user_tags)" } - qt_select_star "select * from user_tags order by time_col,tag_id;" + qt_select_star "select * from user_tags order by time_col, tag_id;" explain { sql("select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;") diff --git a/regression-test/suites/nereids_syntax_p0/advance_mv.groovy b/regression-test/suites/nereids_syntax_p0/advance_mv.groovy new file mode 100644 index 0000000000..ef3119cde7 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/advance_mv.groovy @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("advance_mv") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + def tbName1 = "test_advance_mv_agg_table" + def tbName2 = "test_advance_mv_dup_table" + def tbName3 = "schema_change_dup_mv_regression_test" + + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + sql "DROP TABLE IF EXISTS ${tbName1} FORCE" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName1}( + k1 int, + k2 int, + k3 int, + v1 varchar(10) replace, + v2 bigint sum + ) + AGGREGATE KEY(k1, k2, k3) + DISTRIBUTED BY HASH(k1) buckets 1 properties("replication_num" = "1"); + """ + + sql "DROP TABLE IF EXISTS ${tbName2} FORCE" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName2}( + k1 int, + k2 int, + k3 int, + k4 varchar(10) + ) + DISTRIBUTED BY HASH(k1) buckets 1 properties("replication_num" = "1"); + """ + sql "DROP TABLE IF EXISTS ${tbName3} FORCE" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName3} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATEV2 NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 1 + PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" ); + """ + + sql """INSERT INTO ${tbName3} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20); + """ + sql"""INSERT INTO ${tbName3} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19); + """ + sql """INSERT INTO ${tbName3} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21); + """ + sql """INSERT INTO ${tbName3} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20); + """ + + sql """insert into ${tbName1} values (1,1,1,'a',10);""" + sql """insert into ${tbName1} values (2,2,2,'b',10);""" + sql """insert into ${tbName1} values (3,3,3,'c',10);""" + + sql """insert into ${tbName2} values (4,4,4,'d');""" + sql """insert into ${tbName2} values (5,5,5,'e');""" + sql """insert into ${tbName2} values (6,6,6,'f');""" + // + sql "CREATE materialized VIEW mv1 AS SELECT k1, sum(k2) FROM ${tbName1} GROUP BY k1;" + int max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + explain { + sql("select k1, sum(k2) from ${tbName1} group by k1 order by k1;") + contains "(mv1)" + } + order_qt_select_star "select k1 from ${tbName1} order by k1;" + + sql "CREATE materialized VIEW mv2 AS SELECT abs(k1)+k2+1 tmp, sum(abs(k2+2)+k3+3) FROM ${tbName2} GROUP BY tmp;" + int max_try_secs1 = 60 + while (max_try_secs1--) { + String res = getJobState(tbName2) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs1 < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + explain { + sql("SELECT abs(k1)+k2+1 tmp, sum(abs(k2+2)+k3+3) from FROM ${tbName2} GROUP BY tmp;") + contains "(mv2)" + } + order_qt_select_star "SELECT abs(k1)+k2+1 tmp, sum(abs(k2+2)+k3+3) from FROM ${tbName2} GROUP BY tmp;" + + sql "CREATE materialized VIEW mv3 AS SELECT abs(k1)+k2+1 tmp, abs(k2+2)+k3+3 FROM ${tbName2};" + int max_try_secs2 = 60 + while (max_try_secs2--) { + String res = getJobState(tbName2) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs2 < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + explain { + sql("SELECT abs(k1)+k2+1 tmp, abs(k2+2)+k3+3 FROM ${tbName2};") + contains "(mv3)" + } + order_qt_select_star "SELECT abs(k1)+k2+1 tmp, abs(k2+2)+k3+3 FROM ${tbName2};" + + + sql "create materialized view mv4 as select date, user_id, city, sum(age) from ${tbName3} group by date, user_id, city;" + int max_try_secs3 = 60 + while (max_try_secs3--) { + String res = getJobState(tbName3) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs2 < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + explain { + sql("select sum(age) from ${tbName3};") + contains "(mv4)" + } + order_qt_select_star "select sum(age) from ${tbName3};" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy new file mode 100644 index 0000000000..4373e0b707 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("agg_have_dup_base") { + + def tbName1 = "agg_have_dup_base" + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + sql """ DROP TABLE IF EXISTS agg_have_dup_base; """ + sql """ + create table agg_have_dup_base( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into agg_have_dup_base select 1,1,1,'a';" + sql "insert into agg_have_dup_base select 2,2,2,'b';" + sql "insert into agg_have_dup_base select 3,-3,null,'c';" + + createMV( "create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;") + + sleep(3000) + + + sql "insert into agg_have_dup_base select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from agg_have_dup_base order by k1;" + + explain { + sql("select k1,sum(k2),max(k2) from agg_have_dup_base group by k1;") + contains "(k12s3m)" + } + order_qt_select_mv "select k1,sum(k2),max(k2) from agg_have_dup_base group by k1 order by k1;" + + explain { + sql("select k1,sum(k2) from agg_have_dup_base group by k1;") + contains "(k12s3m)" + } + order_qt_select_mv "select k1,sum(k2) from agg_have_dup_base group by k1 order by k1;" + + explain { + sql("select k1,max(k2) from agg_have_dup_base group by k1;") + contains "(k12s3m)" + } + order_qt_select_mv "select k1,max(k2) from agg_have_dup_base group by k1 order by k1;" + + explain { + sql("select unix_timestamp(k1) tmp,sum(k2) from agg_have_dup_base group by tmp;") + contains "(k12s3m)" + } + order_qt_select_mv "select unix_timestamp(k1) tmp,sum(k2) from agg_have_dup_base group by tmp order by tmp;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy new file mode 100644 index 0000000000..9abf6b5c6d --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("case_ignore") { + sql """ DROP TABLE IF EXISTS case_ignore; """ + + sql """ + create table case_ignore( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into case_ignore select 1,1,1,'a';" + sql "insert into case_ignore select 2,2,2,'b';" + sql "insert into case_ignore select 3,-3,null,'c';" + + createMV ("create materialized view k12a as select K1,abs(K2) from case_ignore;") + sleep(3000) + + sql "insert into case_ignore select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + qt_select_star "select * from case_ignore order by k1;" + + explain { + sql("select k1,abs(k2) from case_ignore order by k1;") + contains "(k12a)" + } + order_qt_select_mv "select k1,abs(k2) from case_ignore order by k1;" + + explain { + sql("select K1,abs(K2) from case_ignore order by K1;") + contains "(k12a)" + } + order_qt_select_mv "select K1,abs(K2) from case_ignore order by K1;" + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy new file mode 100644 index 0000000000..543b2fdfa8 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_gb_mv_abs") { + sql """ DROP TABLE IF EXISTS dup_gb_mv_abs; """ + + sql """ + create table dup_gb_mv_abs( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_gb_mv_abs select 1,1,1,'a';" + sql "insert into dup_gb_mv_abs select 2,2,2,'b';" + sql "insert into dup_gb_mv_abs select 3,-3,null,'c';" + + createMV ("create materialized view k12sa as select k1,sum(abs(k2)) from dup_gb_mv_abs group by k1;") + sleep(3000) + + sql "insert into dup_gb_mv_abs select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from dup_gb_mv_abs order by k1;" + + explain { + sql("select k1,sum(abs(k2)) from dup_gb_mv_abs group by k1;") + contains "(k12sa)" + } + order_qt_select_mv "select k1,sum(abs(k2)) from dup_gb_mv_abs group by k1 order by k1;" + + explain { + sql("select sum(abs(k2)) from dup_gb_mv_abs group by k1;") + contains "(k12sa)" + } + order_qt_select_mv_sub "select sum(abs(k2)) from dup_gb_mv_abs group by k1 order by k1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy new file mode 100644 index 0000000000..6a4e4c1c81 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_gb_mv_plus") { + sql """ DROP TABLE IF EXISTS dup_gb_mv_plus; """ + + sql """ + create table dup_gb_mv_plus( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_gb_mv_plus select 1,1,1,'a';" + sql "insert into dup_gb_mv_plus select 2,2,2,'b';" + sql "insert into dup_gb_mv_plus select 3,-3,null,'c';" + + createMV( "create materialized view k12sp as select k1,sum(k2+1) from dup_gb_mv_plus group by k1;") + sleep(3000) + + sql "insert into dup_gb_mv_plus select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from dup_gb_mv_plus order by k1;" + + explain { + sql("select k1,sum(k2+1) from dup_gb_mv_plus group by k1;") + contains "(k12sp)" + } + order_qt_select_mv "select k1,sum(k2+1) from dup_gb_mv_plus group by k1 order by k1;" + + explain { + sql("select sum(k2+1) from dup_gb_mv_plus group by k1;") + contains "(k12sp)" + } + order_qt_select_mv_sub "select sum(k2+1) from dup_gb_mv_plus group by k1 order by k1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy new file mode 100644 index 0000000000..35b0e55633 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_mv_abs") { + sql """ DROP TABLE IF EXISTS dup_mv_abs; """ + + sql """ + create table dup_mv_abs( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_mv_abs select 1,1,1,'a';" + sql "insert into dup_mv_abs select 2,2,2,'b';" + sql "insert into dup_mv_abs select 3,-3,null,'c';" + + createMV ("create materialized view k12a as select k1,abs(k2) from dup_mv_abs;") + sleep(3000) + + sql "insert into dup_mv_abs select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from dup_mv_abs order by k1;" + + explain { + sql("select k1,abs(k2) from dup_mv_abs order by k1;") + contains "(k12a)" + } + order_qt_select_mv "select k1,abs(k2) from dup_mv_abs order by k1;" + + explain { + sql("select abs(k2) from dup_mv_abs order by k1;") + contains "(k12a)" + } + order_qt_select_mv_sub "select abs(k2) from dup_mv_abs order by k1;" + + explain { + sql("select abs(k2)+1 from dup_mv_abs order by k1;") + contains "(k12a)" + } + order_qt_select_mv_sub_add "select abs(k2)+1 from dup_mv_abs order by k1;" + + explain { + sql("select sum(abs(k2)) from dup_mv_abs group by k1 order by k1;") + contains "(k12a)" + } + order_qt_select_group_mv "select sum(abs(k2)) from dup_mv_abs group by k1 order by k1;" + + explain { + sql("select sum(abs(k2)+1) from dup_mv_abs group by k1 order by k1;") + contains "(k12a)" + } + order_qt_select_group_mv_add "select sum(abs(k2)+1) from dup_mv_abs group by k1 order by k1;" + + explain { + sql("select sum(abs(k2)) from dup_mv_abs group by k3;") + contains "(dup_mv_abs)" + } + order_qt_select_group_mv_not "select sum(abs(k2)) from dup_mv_abs group by k3 order by k3;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy new file mode 100644 index 0000000000..3d06ee478f --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_mv_bin") { + sql """ DROP TABLE IF EXISTS dup_mv_bin; """ + + sql """ + create table dup_mv_bin( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_mv_bin select 1,1,1,'a';" + sql "insert into dup_mv_bin select 2,2,2,'b';" + sql "insert into dup_mv_bin select 3,-3,null,'c';" + + createMV( "create materialized view k12b as select k1,bin(k2) from dup_mv_bin;") + sleep(3000) + + sql "insert into dup_mv_bin select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from dup_mv_bin order by k1;" + + explain { + sql("select k1,bin(k2) from dup_mv_bin order by k1;") + contains "(k12b)" + } + order_qt_select_mv "select k1,bin(k2) from dup_mv_bin order by k1;" + + explain { + sql("select bin(k2) from dup_mv_bin order by k1;") + contains "(k12b)" + } + order_qt_select_mv_sub "select bin(k2) from dup_mv_bin order by k1;" + + explain { + sql("select bin(k2)+1 from dup_mv_bin order by k1;") + contains "(k12b)" + } + order_qt_select_mv_sub_add "select concat(bin(k2),'a') from dup_mv_bin order by k1;" + + explain { + sql("select group_concat(bin(k2)) from dup_mv_bin group by k1 order by k1;") + contains "(k12b)" + } + order_qt_select_group_mv "select group_concat(bin(k2)) from dup_mv_bin group by k1 order by k1;" + + explain { + sql("select group_concat(concat(bin(k2),'a')) from dup_mv_bin group by k1 order by k1;") + contains "(k12b)" + } + order_qt_select_group_mv_add "select group_concat(concat(bin(k2),'a')) from dup_mv_bin group by k1 order by k1;" + + explain { + sql("select group_concat(bin(k2)) from dup_mv_bin group by k3;") + contains "(dup_mv_bin)" + } + order_qt_select_group_mv_not "select group_concat(bin(k2)) from dup_mv_bin group by k3 order by k3;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy new file mode 100644 index 0000000000..a8a81589ca --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_mv_bm_hash") { + sql """ DROP TABLE IF EXISTS dup_mv_bm_hash; """ + + sql """ + create table dup_mv_bm_hash( + k1 int null, + k2 int null, + k3 varchar(100) null + ) + duplicate key (k1) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_mv_bm_hash select 1,1,'a';" + sql "insert into dup_mv_bm_hash select 2,2,'b';" + sql "insert into dup_mv_bm_hash select 3,3,'c';" + + createMV( "create materialized view dup_mv_bm_hash_mv1 as select k1,bitmap_union(to_bitmap(k2)) from dup_mv_bm_hash group by k1;") + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + explain { + sql("select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;") + contains "(dup_mv_bm_hash_mv1)" + } + order_qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;" + + result = "null" + sql "create materialized view dup_mv_bm_hash_mv2 as select k1,bitmap_union(bitmap_hash(k3)) from dup_mv_bm_hash group by k1;" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='dup_mv_bm_hash' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + return + } + Thread.sleep(1000) + } + + sql "SET experimental_enable_nereids_planner=false" + + sql "insert into dup_mv_bm_hash select 2,2,'bb';" + sql "insert into dup_mv_bm_hash select 3,3,'c';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_k1 "select k1 from dup_mv_bm_hash order by k1;" + + order_qt_select_star "select * from dup_mv_bm_hash order by k1,k2,k3;" + + explain { + sql("select k1,bitmap_union_count(bitmap_hash(k3)) from dup_mv_bm_hash group by k1;") + contains "(dup_mv_bm_hash_mv2)" + } + order_qt_select_mv_sub "select k1,bitmap_union_count(bitmap_hash(k3)) from dup_mv_bm_hash group by k1 order by k1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy new file mode 100644 index 0000000000..0e14a8a9cb --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_mv_plus") { + sql """ DROP TABLE IF EXISTS dup_mv_plus; """ + + sql """ + create table dup_mv_plus( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_mv_plus select 1,1,1,'a';" + sql "insert into dup_mv_plus select 2,2,2,'b';" + sql "insert into dup_mv_plus select 3,-3,null,'c';" + + createMV ("create materialized view k12p as select k1,k2+1 from dup_mv_plus;") + sleep(3000) + + sql "insert into dup_mv_plus select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from dup_mv_plus order by k1;" + + explain { + sql("select k1,k2+1 from dup_mv_plus order by k1;") + contains "(k12p)" + } + order_qt_select_mv "select k1,k2+1 from dup_mv_plus order by k1;" + + explain { + sql("select k2+1 from dup_mv_plus order by k1;") + contains "(k12p)" + } + order_qt_select_mv_sub "select k2+1 from dup_mv_plus order by k1;" + + /* + TODO: The selection of the current materialized view is after the constant folding, + so if the rewriting of the constant folding occurs, the corresponding materialized view cannot be selected. + explain { + sql("select k2+1-1 from dup_mv_plus order by k1;") + contains "(k12p)" + } + qt_select_mv_sub_add "select k2+1-1 from dup_mv_plus order by k1;" + */ + + explain { + sql("select sum(k2+1) from dup_mv_plus group by k1 order by k1;") + contains "(k12p)" + } + order_qt_select_group_mv "select sum(k2+1) from dup_mv_plus group by k1 order by k1;" + + explain { + sql("select sum(k1) from dup_mv_plus group by k2+1 order by k2+1;") + contains "(k12p)" + } + order_qt_select_group_mv "select sum(k1) from dup_mv_plus group by k2+1 order by k2+1;" + + /* + explain { + sql("select sum(k2+1-1) from dup_mv_plus group by k1 order by k1;") + contains "(k12p)" + } + qt_select_group_mv_add "select sum(k2+1-1) from dup_mv_plus group by k1 order by k1;" + */ + + explain { + sql("select sum(k2) from dup_mv_plus group by k3;") + contains "(dup_mv_plus)" + } + order_qt_select_group_mv_not "select sum(k2) from dup_mv_plus group by k3 order by k3;" + + explain { + sql("select k1,k2+1 from dup_mv_plus order by k2;") + contains "(dup_mv_plus)" + } + order_qt_select_mv "select k1,k2+1 from dup_mv_plus order by k2;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy new file mode 100644 index 0000000000..0ab0b50072 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("dup_mv_year") { + sql """ DROP TABLE IF EXISTS dup_mv_year; """ + + sql """ + create table dup_mv_year( + k1 int null, + k2 dateV2 null, + k3 datetime null + ) + duplicate key (k1) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into dup_mv_year select 1,'2003-12-31','2003-12-31 01:02:03';" + sql "insert into dup_mv_year select 2,'2013-12-31','2013-12-31 01:02:03';" + sql "insert into dup_mv_year select 3,'2023-12-31','2023-12-31 01:02:03';" + + createMV "create materialized view k12y as select k1,year(k2) from dup_mv_year;" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + explain { + sql("select k1,year(k2) from dup_mv_year order by k1;") + contains "(k12y)" + } + order_qt_select_mv "select k1,year(k2) from dup_mv_year order by k1;" + + result = "null" + sql "create materialized view k13y as select k1,year(k3) from dup_mv_year;" + while (!result.contains("FINISHED")){ + result = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='dup_mv_year' ORDER BY CreateTime DESC LIMIT 1;" + result = result.toString() + logger.info("result: ${result}") + if(result.contains("CANCELLED")){ + return + } + Thread.sleep(1000) + } + sql "SET experimental_enable_nereids_planner=false" + + + sql "insert into dup_mv_year select 4,'2033-12-31','2033-12-31 01:02:03';" + Thread.sleep(1000) + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + order_qt_select_star "select * from dup_mv_year order by k1;" + + explain { + sql("select year(k3) from dup_mv_year order by k1;") + contains "(k13y)" + } + order_qt_select_mv_sub "select year(k3) from dup_mv_year order by k1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy new file mode 100644 index 0000000000..4c0acc8801 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("multi_slot1") { + sql """ DROP TABLE IF EXISTS multi_slot1; """ + + sql """ + create table multi_slot1( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into multi_slot1 select 1,1,1,'a';" + sql "insert into multi_slot1 select 2,2,2,'b';" + sql "insert into multi_slot1 select 3,-3,null,'c';" + + createMV ("create materialized view k1a2p2ap3p as select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1;") + sleep(3000) + + sql "insert into multi_slot1 select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + order_qt_select_star "select * from multi_slot1 order by k1;" + + explain { + sql("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3") + contains "(k1a2p2ap3p)" + } + order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy new file mode 100644 index 0000000000..b86e471051 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("multi_slot2") { + sql """ DROP TABLE IF EXISTS multi_slot2; """ + + sql """ + create table multi_slot2( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into multi_slot2 select 1,1,1,'a';" + sql "insert into multi_slot2 select 2,2,2,'b';" + sql "insert into multi_slot2 select 3,-3,null,'c';" + + boolean createFail = false; + try { + sql "create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2;" + } catch (Exception e) { + createFail = true; + } + assertTrue(createFail); + + createMV ("create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2+1;") + + sleep(3000) + sql "insert into multi_slot2 select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from multi_slot2 order by k1;" + + explain { + sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2+1 order by abs(k1)+k2+1") + contains "(k1a2p2ap3ps)" + } + order_qt_select_mv "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2+1 order by abs(k1)+k2+1;" + + explain { + sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2 order by abs(k1)+k2") + contains "(multi_slot2)" + } + order_qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2 order by abs(k1)+k2;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy new file mode 100644 index 0000000000..e68f560d53 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("multi_slot3") { + sql """ DROP TABLE IF EXISTS multi_slot3; """ + + sql """ + create table multi_slot3( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into multi_slot3 select 1,1,1,'a';" + sql "insert into multi_slot3 select 2,2,2,'b';" + sql "insert into multi_slot3 select 3,-3,null,'c';" + + createMV ("create materialized view k1p2ap3p as select k1+1,abs(k2+2)+k3+3 from multi_slot3;") + + sleep(3000) + + sql "insert into multi_slot3 select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + order_qt_select_star "select * from multi_slot3 order by k1;" + + explain { + sql("select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;") + contains "(k1p2ap3p)" + } + order_qt_select_mv "select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy new file mode 100644 index 0000000000..2ef296dce4 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("multi_slot4") { + sql """ DROP TABLE IF EXISTS multi_slot4; """ + + sql """ + create table multi_slot4( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 2,2,2,'b';" + sql "insert into multi_slot4 select 3,-3,null,'c';" + + createMV ("create materialized view k1p2ap3ps as select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1;") + + sleep(3000) + + sql "insert into multi_slot4 select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from multi_slot4 order by k1;" + + explain { + sql("select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;") + contains "(k1p2ap3ps)" + } + order_qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy new file mode 100644 index 0000000000..bca4183033 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("multi_slot5") { + sql """ DROP TABLE IF EXISTS multi_slot5; """ + + sql """ + create table multi_slot5( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into multi_slot5 select 1,1,1,'a';" + sql "insert into multi_slot5 select 2,2,2,'b';" + sql "insert into multi_slot5 select 3,-3,null,'c';" + + createMV ("create materialized view k123p as select k1,k2+k3 from multi_slot5;") + + sleep(3000) + + sql "insert into multi_slot5 select -4,-4,-4,'d';" + sql "insert into multi_slot5 select 3,-3,null,'c';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + order_qt_select_star "select * from multi_slot5 order by k1,k4;" + + explain { + sql("select k1,k2+k3 from multi_slot5 order by k1;") + contains "(k123p)" + } + order_qt_select_mv "select k1,k2+k3 from multi_slot5 order by k1;" + + explain { + sql("select lhs.k1,rhs.k2 from multi_slot5 as lhs right join multi_slot5 as rhs on lhs.k1=rhs.k1;") + contains "(k123p)" + contains "(multi_slot5)" + } + order_qt_select_mv "select lhs.k1,rhs.k2 from multi_slot5 as lhs right join multi_slot5 as rhs on lhs.k1=rhs.k1 order by lhs.k1;" + + explain { + sql("select k1,version() from multi_slot5;") + contains "(k123p)" + } + order_qt_select_mv "select k1,version() from multi_slot5 order by k1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy new file mode 100644 index 0000000000..72f65007d7 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite ("multi_slot6") { + sql """ DROP TABLE IF EXISTS multi_slot6; """ + + sql """ + create table multi_slot6( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into multi_slot6 select 1,1,1,'a';" + sql "insert into multi_slot6 select 2,2,2,'b';" + sql "insert into multi_slot6 select 3,-3,null,'c';" + + createMV ("create materialized view k1a2p2ap3p as select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6;") + + createMV("create materialized view k1a2p2ap3ps as select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot6 group by abs(k1)+k2+1;") + + sql "insert into multi_slot6 select -4,-4,-4,'d';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from multi_slot6 order by k1;" + + def retry_times = 60 + for (def i = 0; i < retry_times; ++i) { + boolean is_k1a2p2ap3p = false + boolean is_k1a2p2ap3ps = false + boolean is_d_table = false + explain { + sql("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot6 group by abs(k1)+k2+1 order by abs(k1)+k2+1") + check { explainStr, ex, startTime, endTime -> + if (ex != null) { + throw ex; + } + logger.info("explain result: ${explainStr}".toString()) + is_k1a2p2ap3p = explainStr.contains"(k1a2p2ap3p)" + is_k1a2p2ap3ps = explainStr.contains("(k1a2p2ap3ps)") + is_d_table = explainStr.contains("(multi_slot6)") + assert is_k1a2p2ap3p || is_k1a2p2ap3ps || is_d_table + } + } + // FIXME: the mv selector maybe select base table forever when exist multi mv, + // so this pr just treat as success if select base table. + // we should remove is_d_table in the future + if (is_d_table || is_k1a2p2ap3p || is_k1a2p2ap3ps) { + break + } + if (i + 1 == retry_times) { + throw new IllegalStateException("retry and failed too much") + } + sleep(1000) + } + order_qt_select_mv "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot6 group by abs(k1)+k2+1 order by abs(k1)+k2+1;" + + explain { + sql("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3") + contains "(k1a2p2ap3p)" + } + order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3;" + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy new file mode 100644 index 0000000000..09a82de7e4 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/mv_with_view.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("mv_with_view") { + sql """ DROP TABLE IF EXISTS mv_with_view; """ + + sql """ + create table mv_with_view ( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql """insert into mv_with_view select 1,1,1,'a';""" + sql """insert into mv_with_view select 2,2,2,'b';""" + + createMV("create materialized view k132 as select k1,k3,k2 from mv_with_view;") + + sleep(3000) + + sql """insert into mv_with_view select 3,-3,null,'c';""" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + explain { + sql("select * from mv_with_view order by k1;") + contains "(mv_with_view)" + } + order_qt_select_star "select * from mv_with_view order by k1;" + + sql """ + drop view if exists v_k132; + """ + + sql """ + create view v_k132 as select k1,k3,k2 from mv_with_view where k1 = 1; + """ + explain { + sql("select * from v_k132 order by k1;") + contains "(k132)" + } + order_qt_select_mv "select * from v_k132 order by k1;" + + sql """ + drop view if exists v_k124; + """ + + sql """ + create view v_k124 as select k1,k2,k4 from mv_with_view where k1 = 1; + """ + explain { + sql("select * from v_k124 order by k1;") + contains "(mv_with_view)" + } + order_qt_select_mv "select * from v_k124 order by k1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/rollback1.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/rollback1.groovy new file mode 100644 index 0000000000..12d7c541c2 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/rollback1.groovy @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("rollback1") { + sql """ DROP TABLE IF EXISTS rollback1; """ + + sql """ + create table rollback1( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into rollback1 select 1,1,1,'a';" + sql "insert into rollback1 select 2,2,2,'b';" + sql "insert into rollback1 select 3,-3,null,'c';" + + createMV("create materialized view k123p as select k1,k2+k3 from rollback1;") + + sleep(3000) + + sql "insert into rollback1 select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from rollback1 order by k1;" + + qt_select_mv "select k1 p from rollback1 as t where t.k1 in (select k1 from rollback1) order by p;" + qt_select_mv "select k1 p from rollback1 as t where t.k1 in (select k2 from rollback1) order by p;" + qt_select_mv "select k1 p from rollback1 as t where t.k2 in (select k1 from rollback1) order by p;" + qt_select_mv "select k1 p from rollback1 as t where t.k2 in (select k2 from rollback1) order by p;" + qt_select_mv "select k2 p from rollback1 as t where t.k1 in (select k1 from rollback1) order by p;" + qt_select_mv "select k2 p from rollback1 as t where t.k1 in (select k2 from rollback1) order by p;" + qt_select_mv "select k2 p from rollback1 as t where t.k2 in (select k1 from rollback1) order by p;" + qt_select_mv "select k2 p from rollback1 as t where t.k2 in (select k2 from rollback1) order by p;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy new file mode 100644 index 0000000000..9bbba651e9 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("single_slot") { + sql """ DROP TABLE IF EXISTS single_slot; """ + + sql """ + create table single_slot( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into single_slot select 1,1,1,'a';" + sql "insert into single_slot select 2,2,2,'b';" + sql "insert into single_slot select 3,-3,null,'c';" + + createMV("create materialized view k1ap2spa as select abs(k1)+1,sum(abs(k2+1)) from single_slot group by abs(k1)+1;") + + sleep(3000) + + sql "insert into single_slot select -4,-4,-4,'d';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + + order_qt_select_star "select * from single_slot order by k1;" + + explain { + sql("select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;") + contains "(k1ap2spa)" + } + order_qt_select_mv "select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy new file mode 100644 index 0000000000..17a08abc11 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("sum_devide_count") { + sql """ DROP TABLE IF EXISTS sum_devide_count; """ + + sql """ + create table sum_devide_count( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into sum_devide_count select 1,1,1,'a';" + sql "insert into sum_devide_count select 2,2,2,'b';" + sql "insert into sum_devide_count select 3,-3,null,'c';" + + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + test { + sql "create materialized view kavg as select k1,k4,avg(k2) from sum_devide_count group by k1,k4;" + exception "errCode = 2," + } + + createMV ("create materialized view kavg as select k1,k4,sum(k2),count(k2) from sum_devide_count group by k1,k4;") + + sleep(3000) + + sql "SET experimental_enable_nereids_planner=false" + + sql "insert into sum_devide_count select -4,-4,-4,'d';" + sql "insert into sum_devide_count select 3,2,null,'c';" + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + qt_select_star "select * from sum_devide_count order by k1,k2,k3,k4;" + + explain { + sql("select k1,k4,sum(k2)/count(k2) from sum_devide_count group by k1,k4 order by k1,k4;") + contains "(kavg)" + } + order_qt_select_mv "select k1,k4,sum(k2)/count(k2) from sum_devide_count group by k1,k4 order by k1,k4;" + + explain { + sql("select k1,sum(k2)/count(k2) from sum_devide_count group by k1 order by k1;") + contains "(kavg)" + } + order_qt_select_mv "select k1,sum(k2)/count(k2) from sum_devide_count group by k1 order by k1;" + + explain { + sql("select k4,sum(k2)/count(k2) from sum_devide_count group by k4 order by k4;") + contains "(kavg)" + } + order_qt_select_mv "select k4,sum(k2)/count(k2) from sum_devide_count group by k4 order by k4;" + + explain { + sql("select sum(k2)/count(k2) from sum_devide_count;") + contains "(kavg)" + } + order_qt_select_mv "select sum(k2)/count(k2) from sum_devide_count;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy new file mode 100644 index 0000000000..63ec12c6e2 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("MVMultiUsage") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS MVMultiUsage; """ + + sql """ + create table MVMultiUsage ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into MVMultiUsage values("2020-01-01",1,"a",1,1,1);""" + sql """insert into MVMultiUsage values("2020-01-02",2,"b",2,2,2);""" + sql """insert into MVMultiUsage values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view MVMultiUsage_mv as select deptno, empid, salary from MVMultiUsage order by deptno;") + + sleep(3000) + + sql """insert into MVMultiUsage values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from MVMultiUsage order by empid;") + contains "(MVMultiUsage)" + } + order_qt_select_star "select * from MVMultiUsage order by empid;" + + + explain { + sql("select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno);") + contains "(MVMultiUsage_mv)" + notContains "(MVMultiUsage)" + } + order_qt_select_mv "select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno) order by 1;" + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy new file mode 100644 index 0000000000..8d8f4fe121 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("MVWithAs") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS MVWithAs; """ + + sql """ create table MVWithAs ( + time_col dateV2, + user_id int, + user_name varchar(20), + tag_id int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" + sql """insert into MVWithAs values("2020-01-02",2,"b",2);""" + + createMV("create materialized view MVWithAs_mv as select user_id, count(tag_id) from MVWithAs group by user_id;") + + sleep(3000) + + sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" + + explain { + sql("select * from MVWithAs order by time_col;") + contains "(MVWithAs)" + } + order_qt_select_star "select * from MVWithAs order by time_col;" + + explain { + sql("select count(tag_id) from MVWithAs t;") + contains "(MVWithAs_mv)" + } + order_qt_select_mv "select count(tag_id) from MVWithAs t;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy new file mode 100644 index 0000000000..9500c6ff19 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +// testAggTableCountDistinctInBitmapType +suite ("aggCDInBitmap") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggCDInBitmap; """ + + sql """ + CREATE TABLE aggCDInBitmap (k1 int, v1 bitmap bitmap_union) Aggregate KEY (k1) DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1'); + """ + + sql """insert into aggCDInBitmap values(1,to_bitmap(1));""" + sql """insert into aggCDInBitmap values(2,to_bitmap(2));""" + sql """insert into aggCDInBitmap values(3,to_bitmap(3));""" + + + order_qt_select_star "select * from aggCDInBitmap order by 1;" + + + explain { + sql("select k1, count(distinct v1) from aggCDInBitmap group by k1;") + contains "bitmap_union_count" + } + order_qt_select_mv "select k1, count(distinct v1) from aggCDInBitmap group by k1 order by k1;" + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy new file mode 100644 index 0000000000..db05f4409e --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +// testAggregateMVCalcAggFunctionQuery +suite ("aggMVCalcAggFun") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggMVCalcAggFun; """ + + sql """ + create table aggMVCalcAggFun ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggMVCalcAggFun values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggMVCalcAggFun values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggMVCalcAggFun values("2020-01-03",3,"c",3,3,3);""" + + + createMV("create materialized view aggMVCalcAggFunMv as select deptno, empid, sum(salary) from aggMVCalcAggFun group by empid, deptno;") + + sleep(3000) + + sql """insert into aggMVCalcAggFun values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggMVCalcAggFun order by empid;") + contains "(aggMVCalcAggFun)" + } + order_qt_select_star "select * from aggMVCalcAggFun order by empid;" + + + explain { + sql("select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno;") + notContains "(aggMVCalcAggFunMv)" + } + order_qt_select_mv "select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno order by deptno;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy new file mode 100644 index 0000000000..bec69e6b77 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV1") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV1; """ + + sql """ + create table aggOnAggMV1 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV1 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV1 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV1 values("2020-01-03",3,"c",3,3,3);""" + + + createMV("create materialized view aggOnAggMV1_mv as select deptno, sum(salary), max(commission) from aggOnAggMV1 group by deptno ;") + + sleep(3000) + + sql """insert into aggOnAggMV1 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggOnAggMV1 order by empid;") + contains "(aggOnAggMV1)" + } + order_qt_select_star "select * from aggOnAggMV1 order by empid;" + + + explain { + sql("select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;") + contains "(aggOnAggMV1_mv)" + } + order_qt_select_mv "select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;" + + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy new file mode 100644 index 0000000000..035662c915 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV10") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV10; """ + + sql """ + create table aggOnAggMV10 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV10 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV10 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV10 values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view aggOnAggMV10_mv as select deptno, commission, sum(salary) from aggOnAggMV10 group by deptno, commission;") + + sleep(3000) + + sql """insert into aggOnAggMV10 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggOnAggMV10 order by empid;") + contains "(aggOnAggMV10)" + } + order_qt_select_star "select * from aggOnAggMV10 order by empid;" + + explain { + sql("select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission);") + contains "(aggOnAggMV10_mv)" + } + order_qt_select_mv "select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission) order by 1,2;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy new file mode 100644 index 0000000000..07d6004ebb --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV11") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV11; """ + + sql """ + create table aggOnAggMV11 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV11 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV11 values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view aggOnAggMV11_mv as select deptno, count(salary) from aggOnAggMV11 group by deptno;") + + sleep(3000) + + sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggOnAggMV11 order by empid;") + contains "(aggOnAggMV11)" + } + order_qt_select_star "select * from aggOnAggMV11 order by empid;" + + explain { + sql("select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno;") + contains "(aggOnAggMV11)" + } + order_qt_select_mv "select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno order by 1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy new file mode 100644 index 0000000000..ab2bc88f2f --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite ("aggOnAggMV2") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV2; """ + sql """ + create table aggOnAggMV2 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + + sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV2 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,7,2);""" + + explain { + sql("select deptno, sum(salary) from aggOnAggMV2 group by deptno order by deptno;") + contains "(aggOnAggMV2)" + } + order_qt_select_emps_mv "select deptno, sum(salary) from aggOnAggMV2 group by deptno order by deptno;" + + createMV("create materialized view aggOnAggMV2_mv as select deptno, sum(salary) from aggOnAggMV2 group by deptno ;") + + sleep(3000) + + + explain { + sql("select * from aggOnAggMV2 order by empid;") + contains "(aggOnAggMV2)" + } + order_qt_select_star "select * from aggOnAggMV2 order by empid, salary;" + + explain { + sql("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;") + contains "(aggOnAggMV2_mv)" + } + order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;" + + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy new file mode 100644 index 0000000000..cba82744b5 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV3") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV3; """ + + sql """ + create table aggOnAggMV3 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV3 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV3 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV3 values("2020-01-03",3,"c",3,3,10);""" + sql """insert into aggOnAggMV3 values("2020-01-04",4,"d",21,4,4);""" + + + + createMV("create materialized view aggOnAggMV3_mv as select deptno, commission, sum(salary) from aggOnAggMV3 group by deptno, commission ;") + + sleep(3000) + + explain { + sql("select * from aggOnAggMV3 order by empid;") + contains "(aggOnAggMV3)" + } + order_qt_select_star "select * from aggOnAggMV3 order by empid;" + + + explain { + sql("select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;") + contains "(aggOnAggMV3_mv)" + } + order_qt_select_mv "select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;" + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy new file mode 100644 index 0000000000..7d1ab3508e --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV5") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV5; """ + + sql """ + create table aggOnAggMV5 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV5 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV5 values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view aggOnAggMV5_mv as select deptno, commission, sum(salary) from aggOnAggMV5 group by deptno, commission;") + + sleep(3000) + + sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggOnAggMV5 order by empid;") + contains "(aggOnAggMV5)" + } + order_qt_select_star "select * from aggOnAggMV5 order by empid;" + + explain { + sql("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV5 group by deptno) a where sum_salary>10;") + contains "(aggOnAggMV5_mv)" + } + order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV5 group by deptno) a where sum_salary>10 order by 1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy new file mode 100644 index 0000000000..87c9ebffb2 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV6") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV6; """ + + sql """ + create table aggOnAggMV6 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV6 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV6 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV6 values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view aggOnAggMV6_mv as select deptno, commission, sum(salary) from aggOnAggMV6 group by deptno, commission;") + + sleep(3000) + + sql """insert into aggOnAggMV6 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggOnAggMV6 order by empid;") + contains "(aggOnAggMV6)" + } + order_qt_select_star "select * from aggOnAggMV6 order by empid;" + + explain { + sql("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10;") + contains "(aggOnAggMV6_mv)" + } + order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10 order by 1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy new file mode 100644 index 0000000000..b2803ccc66 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("aggOnAggMV7") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS aggOnAggMV7; """ + + sql """ + create table aggOnAggMV7 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into aggOnAggMV7 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV7 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV7 values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view aggOnAggMV7_mv as select deptno, commission, sum(salary) from aggOnAggMV7 group by deptno, commission;") + + sleep(3000) + + sql """insert into aggOnAggMV7 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from aggOnAggMV7 order by empid;") + contains "(aggOnAggMV7)" + } + order_qt_select_star "select * from aggOnAggMV7 order by empid;" + + explain { + sql("select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno;") + contains "(aggOnAggMV7_mv)" + } + order_qt_select_mv "select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno order by 1;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy new file mode 100644 index 0000000000..41cdf5ba48 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("bitmapUnionIn") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS bitmapUnionIn; """ + + sql """ create table bitmapUnionIn ( + time_col dateV2, + user_id int, + user_name varchar(20), + tag_id int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into bitmapUnionIn values("2020-01-01",1,"a",1);""" + sql """insert into bitmapUnionIn values("2020-01-02",2,"b",2);""" + + createMV("create materialized view bitmapUnionIn_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from bitmapUnionIn group by user_id;") + + sleep(3000) + + sql """insert into bitmapUnionIn values("2020-01-01",1,"a",2);""" + + explain { + sql("select * from bitmapUnionIn order by time_col;") + contains "(bitmapUnionIn)" + } + order_qt_select_star "select * from bitmapUnionIn order by time_col,tag_id;" + + explain { + sql("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;") + contains "(bitmapUnionIn_mv)" + } + order_qt_select_mv "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy new file mode 100644 index 0000000000..e854dacf48 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +// nereids_testIncorrectMVRewriteInSubquery +suite ("incMVReInSub") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS incMVReInSub; """ + + sql """ create table incMVReInSub ( + time_col dateV2, + user_id int, + user_name varchar(20), + tag_id int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into incMVReInSub values("2020-01-01",1,"a",1);""" + sql """insert into incMVReInSub values("2020-01-02",2,"b",2);""" + + createMV("create materialized view incMVReInSub_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub group by user_id;") + + sleep(3000) + + sql """insert into incMVReInSub values("2020-01-01",1,"a",2);""" + + explain { + sql("select * from incMVReInSub order by time_col;") + contains "(incMVReInSub)" + } + order_qt_select_star "select * from incMVReInSub order by time_col, user_id, user_name, tag_id;" + + explain { + sql("select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;") + contains "(incMVReInSub)" + } + order_qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy new file mode 100644 index 0000000000..58078a746c --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +// nereids_testIncorrectRewriteCountDistinct +suite ("incRewriteCD") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS incRewriteCD; """ + + sql """ create table incRewriteCD ( + time_col dateV2, + user_id int, + user_name varchar(20), + tag_id int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into incRewriteCD values("2020-01-01",1,"a",1);""" + sql """insert into incRewriteCD values("2020-01-02",2,"b",2);""" + + createMV("create materialized view incRewriteCD_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from incRewriteCD group by user_id;") + + sleep(3000) + + sql """insert into incRewriteCD values("2020-01-01",1,"a",2);""" + + explain { + sql("select * from incRewriteCD order by time_col;") + contains "(incRewriteCD)" + } + order_qt_select_star "select * from incRewriteCD order by time_col,tag_id;" + + explain { + sql("select user_name, count(distinct tag_id) from incRewriteCD group by user_name;") + contains "(incRewriteCD)" + } + order_qt_select_mv "select user_name, count(distinct tag_id) from incRewriteCD group by user_name order by user_name;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy new file mode 100644 index 0000000000..9200918751 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// nereids_testJoinOnLeftProjectToJoin +suite ("joinOnLeftPToJoin") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS joinOnLeftPToJoin; """ + sql """ + create table joinOnLeftPToJoin ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,2,2);""" + sql """insert into joinOnLeftPToJoin values("2020-01-03",3,"c",3,3,3);""" + sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,7,2);""" + + sql """ DROP TABLE IF EXISTS joinOnLeftPToJoin_1; """ + sql """ + create table joinOnLeftPToJoin_1 ( + time_col dateV2, + deptno int, + name varchar, + cost int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",2);""" + sql """insert into joinOnLeftPToJoin_1 values("2020-01-03",3,"c",3);""" + sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",1);""" + + createMV("create materialized view joinOnLeftPToJoin_mv as select deptno, sum(salary), sum(commission) from joinOnLeftPToJoin group by deptno;") + sleep(3000) + createMV("create materialized view joinOnLeftPToJoin_1_mv as select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno;") + sleep(3000) + + explain { + sql("select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno;") + contains "(joinOnLeftPToJoin_mv)" + contains "(joinOnLeftPToJoin_1_mv)" + } + order_qt_select_mv "select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno order by A.deptno;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/onStar.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/onStar.groovy new file mode 100644 index 0000000000..cc94cce896 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/onStar.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("onStar") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS onStar; """ + + sql """ + create table onStar ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into onStar values("2020-01-01",1,"a",1,1,1);""" + sql """insert into onStar values("2020-01-02",2,"b",2,2,2);""" + + createMV("create materialized view onStar_mv as select time_col, deptno,empid, name, salary, commission from onStar order by time_col, deptno, empid;") + + sleep(3000) + + sql """insert into onStar values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from onStar order by empid;") + contains "(onStar_mv)" + } + order_qt_select_star "select * from onStar order by empid;" + + explain { + sql("select * from onStar where deptno = 1;") + contains "(onStar_mv)" + } + order_qt_select_mv "select * from onStar where deptno = 1 order by empid;" + + sql """ DROP TABLE IF EXISTS onStar_tpch; """ + sql """ + CREATE TABLE IF NOT EXISTS onStar_tpch ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) + ) + DUPLICATE KEY(r_regionkey) + DISTRIBUTED BY HASH(r_regionkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + sql """insert into onStar_tpch values(1,'a','a');""" + + explain { + sql("select ref_1.`empid` as c0 from onStar_tpch as ref_0 left join onStar as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;") + contains "(onStar_mv)" + } + order_qt_select_mv "select ref_1.`empid` as c0 from onStar_tpch as ref_0 left join onStar as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy new file mode 100644 index 0000000000..48e22afda2 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +// nereids_testOrderByQueryOnProjectView +suite ("orderByOnPView") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS orderByOnPView; """ + + sql """ + create table orderByOnPView ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into orderByOnPView values("2020-01-01",1,"a",1,1,1);""" + sql """insert into orderByOnPView values("2020-01-02",2,"b",2,2,2);""" + sql """insert into orderByOnPView values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view orderByOnPView_mv as select deptno, empid from orderByOnPView;") + + sleep(3000) + + sql """insert into orderByOnPView values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from orderByOnPView order by empid;") + contains "(orderByOnPView)" + } + order_qt_select_star "select * from orderByOnPView order by empid;" + + + explain { + sql("select empid from orderByOnPView order by deptno;") + contains "(orderByOnPView_mv)" + } + order_qt_select_mv "select empid from orderByOnPView order by deptno;" + + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy new file mode 100644 index 0000000000..cc45984008 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("projectMV1") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS projectMV1; """ + + sql """ + create table projectMV1 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into projectMV1 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into projectMV1 values("2020-01-02",2,"b",2,2,2);""" + + createMV("create materialized view projectMV1_mv as select deptno, empid from projectMV1 order by deptno;") + + sleep(3000) + + sql """insert into projectMV1 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from projectMV1 order by empid;") + contains "(projectMV1)" + } + order_qt_select_star "select * from projectMV1 order by empid;" + + + explain { + sql("select empid, deptno from projectMV1 order by empid;") + contains "(projectMV1_mv)" + } + order_qt_select_mv "select empid, deptno from projectMV1 order by empid;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy new file mode 100644 index 0000000000..a5fa27fd58 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("projectMV2") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS projectMV2; """ + + sql """ + create table projectMV2 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into projectMV2 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into projectMV2 values("2020-01-02",2,"b",2,2,2);""" + + createMV("create materialized view projectMV2_mv as select deptno, empid from projectMV2 order by deptno;") + + sleep(3000) + + sql """insert into projectMV2 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from projectMV2 order by empid;") + contains "(projectMV2)" + } + order_qt_select_star "select * from projectMV2 order by empid;" + + + explain { + sql("select empid + 1 from projectMV2 where deptno = 1 order by empid;") + contains "(projectMV2_mv)" + } + order_qt_select_mv "select empid + 1 from projectMV2 where deptno = 1 order by empid;" + + explain { + sql("select name from projectMV2 where deptno -1 = 0 order by empid;") + contains "(projectMV2)" + } + order_qt_select_base "select name from projectMV2 where deptno -1 = 0 order by empid;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy new file mode 100644 index 0000000000..f986bb26b3 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("projectMV3") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS projectMV3; """ + + sql """ + create table projectMV3 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into projectMV3 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into projectMV3 values("2020-01-02",2,"b",2,2,2);""" + + def result = "null" + + createMV("create materialized view projectMV3_mv as select deptno, empid, name from projectMV3 order by deptno;") + + sleep(3000) + + sql """insert into projectMV3 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from projectMV3 order by empid;") + contains "(projectMV3)" + } + order_qt_select_star "select * from projectMV3 order by empid;" + + + explain { + sql("select empid + 1, name from projectMV3 where deptno = 1 order by empid;") + contains "(projectMV3_mv)" + } + order_qt_select_mv "select empid + 1, name from projectMV3 where deptno = 1 order by empid;" + + explain { + sql("select name from projectMV3 where deptno -1 = 0 order by empid;") + contains "(projectMV3_mv)" + } + order_qt_select_mv2 "select name from projectMV3 where deptno -1 = 0 order by empid;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy new file mode 100644 index 0000000000..f93a50e845 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("projectMV4") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS projectMV4; """ + + sql """ + create table projectMV4 ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into projectMV4 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into projectMV4 values("2020-01-02",2,"b",2,2,2);""" + + def result = "null" + + createMV("create materialized view projectMV4_mv as select name, deptno, salary from projectMV4;") + + sleep(3000) + + sql """insert into projectMV4 values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from projectMV4 order by empid;") + contains "(projectMV4)" + } + order_qt_select_star "select * from projectMV4 order by empid;" + + + explain { + sql("select name from projectMV4 where deptno > 1 and salary > 1 order by name;") + contains "(projectMV4_mv)" + } + order_qt_select_mv "select name from projectMV4 where deptno > 1 and salary > 1 order by name;" + + explain { + sql("select empid from projectMV4 where deptno > 1 and empid > 1 order by empid;") + contains "(projectMV4)" + } + order_qt_select_base "select empid from projectMV4 where deptno > 1 and empid > 1 order by empid;" +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy new file mode 100644 index 0000000000..1ae61477d3 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("subQuery") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS subQuery; """ + + sql """ + create table subQuery ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into subQuery values("2020-01-01",1,"a",1,1,1);""" + sql """insert into subQuery values("2020-01-02",2,"b",2,2,2);""" + sql """insert into subQuery values("2020-01-03",3,"c",3,3,3);""" + + + createMV("create materialized view subQuery_mv as select deptno, empid from subQuery;") + + sleep(3000) + + sql """insert into subQuery values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from subQuery order by empid;") + contains "(subQuery)" + } + order_qt_select_star "select * from subQuery order by empid;" + + /* + explain { + sql("select empid, deptno, salary from subQuery e1 where empid = (select max(empid) from subQuery where deptno = e1.deptno);") + contains "(subQuery_mv)" + contains "(subQuery)" + } + qt_select_mv "select empid, deptno, salary from subQuery e1 where empid = (select max(empid) from subQuery where deptno = e1.deptno) order by deptno;" + */ +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy new file mode 100644 index 0000000000..be10ce1f8e --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("unionDis") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS unionDis; """ + + sql """ + create table unionDis ( + time_col dateV2, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into unionDis values("2020-01-01",1,"a",1,1,1);""" + sql """insert into unionDis values("2020-01-02",2,"b",2,2,2);""" + sql """insert into unionDis values("2020-01-03",3,"c",3,3,3);""" + + createMV("create materialized view unionDis_mv as select empid, deptno from unionDis order by empid, deptno;") + + sleep(3000) + + sql """insert into unionDis values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from unionDis order by empid;") + contains "(unionDis)" + } + order_qt_select_star "select * from unionDis order by empid;" + + + explain { + sql("select empid, deptno from unionDis where empid >1 union select empid, deptno from unionDis where empid <0 order by empid;") + contains "(unionDis_mv)" + notContains "(unionDis)" + } + order_qt_select_mv "select * from (select empid, deptno from unionDis where empid >1 union select empid, deptno from unionDis where empid <0) t order by 1;" + +} diff --git a/regression-test/suites/nereids_syntax_p0/mv/where/k123/k123.groovy b/regression-test/suites/nereids_syntax_p0/mv/where/k123/k123.groovy new file mode 100644 index 0000000000..2bb4ea8627 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/mv/where/k123/k123.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +/* +// Open after implementing the where in mv function +suite ("nereids_k123p") { + sql "SET experimental_enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """ DROP TABLE IF EXISTS d_table; """ + + sql """ + create table d_table( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 2,2,2,'b';" + sql "insert into d_table select 3,-3,null,'c';" + + createMV ("""create materialized view k123p1w as select k1,k2+k3 from d_table where k1 = 1;""") + createMV ("""create materialized view k123p4w as select k1,k2+k3 from d_table where k4 = "b";""") + + sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 2,2,2,'b';" + sql "insert into d_table select 3,-3,null,'c';" + + qt_select_star "select * from d_table order by k1;" + + explain { + sql("select k1,k2+k3 from d_table order by k1;") + contains "(d_table)" + } + qt_select_mv "select k1,k2+k3 from d_table order by k1;" + + explain { + sql("select k1,k2+k3 from d_table where k1 = 1 order by k1;") + contains "(k123p1w)" + } + qt_select_mv "select k1,k2+k3 from d_table where k1 = 1 order by k1;" + + explain { + sql("select k1,k2+k3 from d_table where k1 = 2 order by k1;") + contains "(d_table)" + } + qt_select_mv "select k1,k2+k3 from d_table where k1 = 2 order by k1;" + + explain { + sql("select k1,k2+k3 from d_table where k1 = '1' order by k1;") + contains "(d_table)" + } + qt_select_mv "select k1,k2+k3 from d_table where k1 = '1' order by k1;" + + explain { + sql("select k1,k2+k3 from d_table where k4 = 'b' order by k1;") + contains "(k123p4w)" + } + qt_select_mv "select k1,k2+k3 from d_table where k4 = 'b' order by k1;" + + explain { + sql("select k1,k2+k3 from d_table where k4 = 'a' order by k1;") + contains "(d_table)" + } + qt_select_mv "select k1,k2+k3 from d_table where k4 = 'a' order by k1;" + + explain { + sql("""select k1,k2+k3 from d_table where k1 = 2 and k4 = "b";""") + contains "(k123p4w)" + } + qt_select_mv """select k1,k2+k3 from d_table where k1 = 2 and k4 = "b" order by k1;""" + + qt_select_mv_constant """select bitmap_empty() from d_table where true;""" +} +*/ diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy new file mode 100644 index 0000000000..0808173ae9 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("agg") { + + sql """set enable_nereids_planner=true""" + sql "SET enable_fallback_to_original_planner=false" + + def tbName = "test_rollup_agg1" + + def getJobRollupState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + def getJobColumnState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][9] + } + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName}( + siteid INT(11) NOT NULL, + citycode SMALLINT(6) NOT NULL, + username VARCHAR(32) NOT NULL, + pv BIGINT(20) SUM NOT NULL DEFAULT '0', + uv BIGINT(20) SUM NOT NULL DEFAULT '0' + ) + AGGREGATE KEY (siteid,citycode,username) + DISTRIBUTED BY HASH(siteid) BUCKETS 5 properties("replication_num" = "1"); + """ + sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_city(citycode, pv);""" + int max_try_secs = 60 + while (max_try_secs--) { + String res = getJobRollupState(tbName) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + Thread.sleep(2000) + sql "ALTER TABLE ${tbName} ADD COLUMN vv BIGINT SUM NULL DEFAULT '0' TO rollup_city;" + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobColumnState(tbName) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + sql "SHOW ALTER TABLE ROLLUP WHERE TableName='${tbName}';" + qt_sql "DESC ${tbName} ALL;" + sql "insert into ${tbName} values(1, 1, 'test1', 100,100,100);" + sql "insert into ${tbName} values(2, 1, 'test2', 100,100,100);" + explain { + sql("SELECT citycode,SUM(pv) FROM ${tbName} GROUP BY citycode") + contains("(rollup_city)") + } + qt_sql "SELECT citycode,SUM(pv) FROM ${tbName} GROUP BY citycode" + sql "ALTER TABLE ${tbName} DROP ROLLUP rollup_city;" + sql "DROP TABLE ${tbName} FORCE;" +} diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy new file mode 100644 index 0000000000..4cb9f3821e --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("agg_date", "rollup") { + + sql """set enable_nereids_planner=true""" + sql "SET enable_fallback_to_original_planner=false" + + def tbName = "test_rollup_agg_date1" + + def getJobRollupState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + def getJobColumnState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][9] + } + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName}( + datek1 datev2, + datetimek1 datetimev2, + datetimek2 datetimev2(3), + datetimek3 datetimev2(6), + datev1 datev2 MAX NOT NULL, + datetimev1 datetimev2 MAX NOT NULL, + datetimev2 datetimev2(3) MAX NOT NULL, + datetimev3 datetimev2(6) MAX NOT NULL + ) + AGGREGATE KEY (datek1, datetimek1, datetimek2, datetimek3) + DISTRIBUTED BY HASH(datek1) BUCKETS 5 properties("replication_num" = "1"); + """ + sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_date(datek1,datetimek2,datetimek1,datetimek3,datev1,datetimev1,datetimev2,datetimev3);""" + int max_try_secs = 60 + while (max_try_secs--) { + String res = getJobRollupState(tbName) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + Thread.sleep(2000) + sql "ALTER TABLE ${tbName} ADD COLUMN datetimev4 datetimev2(3) MAX NULL;" + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobColumnState(tbName) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + sql "SHOW ALTER TABLE ROLLUP WHERE TableName='${tbName}';" + qt_sql "DESC ${tbName} ALL;" + sql "insert into ${tbName} values('2022-08-22', '2022-08-22 11:11:11.111111', '2022-08-22 11:11:11.111111', '2022-08-22 11:11:11.111111', '2022-08-22', '2022-08-22 11:11:11.111111', '2022-08-22 11:11:11.111111', '2022-08-22 11:11:11.111111', '2022-08-22 11:11:11.111111');" + sql "insert into ${tbName} values('2022-08-23', '2022-08-23 11:11:11.111111', '2022-08-23 11:11:11.111111', '2022-08-23 11:11:11.111111', '2022-08-23', '2022-08-23 11:11:11.111111', '2022-08-23 11:11:11.111111', '2022-08-23 11:11:11.111111', '2022-08-23 11:11:11.111111');" + explain { + sql("SELECT datek1,datetimek1,datetimek2,datetimek3,max(datev1),max(datetimev1),max(datetimev2),max(datetimev3) FROM ${tbName} GROUP BY datek1,datetimek1,datetimek2,datetimek3") + contains("(rollup_date)") + } + qt_sql """ SELECT datek1,datetimek1,datetimek2,datetimek3,max(datev1),max(datetimev1),max(datetimev2),max(datetimev3) FROM ${tbName} GROUP BY datek1,datetimek1,datetimek2,datetimek3 order by datek1 desc; """ + sql "ALTER TABLE ${tbName} DROP ROLLUP rollup_date;" + sql "DROP TABLE ${tbName} FORCE;" +} diff --git a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy new file mode 100644 index 0000000000..ca2aec6cf0 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("bitmap", "rollup") { + + sql """set enable_nereids_planner=true""" + sql "SET enable_fallback_to_original_planner=false" + + def tbName1 = "test_materialized_view_bitmap1" + + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + sql "DROP TABLE IF EXISTS ${tbName1}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName1}( + k1 BOOLEAN NOT NULL, + k2 TINYINT NOT NULL, + k3 SMALLINT NOT NULL + ) + DISTRIBUTED BY HASH(k1) properties("replication_num" = "1"); + """ + + sql "CREATE MATERIALIZED VIEW test_neg as select k1,bitmap_union(to_bitmap(k2)), bitmap_union(to_bitmap(k3)) FROM ${tbName1} GROUP BY k1;" + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + + explain { + sql "insert into ${tbName1} values(1,1,1);" + contains "to_bitmap_with_check" + } + sql "insert into ${tbName1} values(1,1,1);" + sql "insert into ${tbName1} values(0,1,1);" + + test { + sql "insert into ${tbName1} values(1,-1,-1);" + // check exception message contains + exception "The input: -1 is not valid, to_bitmap only support bigint value from 0 to 18446744073709551615 currently" + } + + sql "DROP TABLE ${tbName1} FORCE;" +} diff --git a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy new file mode 100644 index 0000000000..e035a1c634 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("date", "rollup") { + + sql """set enable_nereids_planner=true""" + sql "SET enable_fallback_to_original_planner=false" + + def tbName1 = "test_materialized_view_date1" + + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + sql "DROP TABLE IF EXISTS ${tbName1}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName1}( + record_id int, + seller_id int, + store_id int, + sale_date date, + sale_date1 datev2, + sale_datetime1 datetimev2, + sale_datetime2 datetimev2(3), + sale_datetime3 datetimev2(6), + sale_amt bigint + ) + DISTRIBUTED BY HASH(record_id) properties("replication_num" = "1"); + """ + + int max_try_secs = 120 + sql "CREATE materialized VIEW amt_max1 AS SELECT store_id, max(sale_date1) FROM ${tbName1} GROUP BY store_id;" + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + Thread.sleep(2000) + max_try_secs = 120 + sql "CREATE materialized VIEW amt_max2 AS SELECT store_id, max(sale_datetime1) FROM ${tbName1} GROUP BY store_id;" + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + Thread.sleep(2000) + max_try_secs = 120 + sql "CREATE materialized VIEW amt_max3 AS SELECT store_id, max(sale_datetime2) FROM ${tbName1} GROUP BY store_id;" + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + Thread.sleep(2000) + max_try_secs = 120 + sql "CREATE materialized VIEW amt_max4 AS SELECT store_id, max(sale_datetime3) FROM ${tbName1} GROUP BY store_id;" + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + + + sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName1}';" + sql "insert into ${tbName1} values(1, 1, 1, '2020-05-30', '2020-05-30', '2020-05-30 11:11:11.111111', '2020-05-30 11:11:11.111111', '2020-05-30 11:11:11.111111',100);" + sql "insert into ${tbName1} values(2, 1, 1, '2020-05-30', '2020-05-30', '2020-04-30 11:11:11.111111', '2020-04-30 11:11:11.111111', '2020-04-30 11:11:11.111111',100);" + Thread.sleep(2000) + explain{ + sql("SELECT store_id, max(sale_date1) FROM ${tbName1} GROUP BY store_id") + contains("(amt_max1)") + } + explain{ + sql("SELECT store_id, max(sale_datetime1) FROM ${tbName1} GROUP BY store_id") + contains("(amt_max2)") + } + explain{ + sql("SELECT store_id, max(sale_datetime2) FROM ${tbName1} GROUP BY store_id") + contains("(amt_max3)") + } + explain{ + sql("SELECT store_id, max(sale_datetime3) FROM ${tbName1} GROUP BY store_id") + contains("(amt_max4)") + } + qt_sql """ SELECT store_id, max(sale_date1) FROM ${tbName1} GROUP BY store_id """ + qt_sql """ SELECT store_id, max(sale_datetime1) FROM ${tbName1} GROUP BY store_id """ + qt_sql """ SELECT store_id, max(sale_datetime2) FROM ${tbName1} GROUP BY store_id """ + qt_sql """ SELECT store_id, max(sale_datetime3) FROM ${tbName1} GROUP BY store_id """ + + sql "DROP TABLE ${tbName1} FORCE;" +} diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll.groovy new file mode 100644 index 0000000000..48c25bcbe4 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/rollup/hll.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("hll", "rollup") { + + sql """set enable_nereids_planner=true""" + sql "SET enable_fallback_to_original_planner=false" + + def tbName1 = "test_materialized_view_hll1" + + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + sql "DROP TABLE IF EXISTS ${tbName1}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName1}( + record_id int, + seller_id int, + store_id int, + sale_date date, + sale_amt bigint + ) + DISTRIBUTED BY HASH(record_id) properties("replication_num" = "1"); + """ + + sql "CREATE materialized VIEW amt_count AS SELECT store_id, hll_union(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;" + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + + qt_sql "DESC ${tbName1} ALL;" + + sql "insert into ${tbName1} values(1, 1, 1, '2020-05-30',100);" + sql "insert into ${tbName1} values(2, 1, 1, '2020-05-30',100);" + qt_sql "SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;" + + explain { + sql("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;") + contains "(amt_count)" + } + + sql "DROP TABLE ${tbName1} FORCE;" +} diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc.groovy new file mode 100644 index 0000000000..7c216d1a99 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("hll_with_light_sc", "rollup") { + + sql """set enable_nereids_planner=true""" + + def tbName1 = "test_materialized_view_hll_with_light_sc1" + + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1; """ + return jobStateResult[0][8] + } + sql "DROP TABLE IF EXISTS ${tbName1}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName1}( + record_id int, + seller_id int, + store_id int, + sale_date date, + sale_amt bigint + ) + DISTRIBUTED BY HASH(record_id) properties("replication_num" = "1", "light_schema_change" = "true"); + """ + + sql "CREATE materialized VIEW amt_count1 AS SELECT store_id, hll_union(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;" + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tbName1) + if (res == "FINISHED") { + sleep(3000) + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + + qt_sql "DESC ${tbName1} ALL;" + + sql "insert into ${tbName1} values(1, 1, 1, '2020-05-30',100);" + sql "insert into ${tbName1} values(2, 1, 1, '2020-05-30',100);" + qt_sql "SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;" + + explain { + sql("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM ${tbName1} GROUP BY store_id;") + contains "(amt_count1)" + } + + sql "DROP TABLE ${tbName1} FORCE;" +}