From 62214cd1f4762a754d4e51b181bcab6b1a2b60c0 Mon Sep 17 00:00:00 2001
From: minghong <englefly@gmail.com>
Date: Fri, 14 Jul 2023 12:54:04 +0800
Subject: [PATCH] [feature](nereids) adjust min/max of column stats for cast
 function (#21772)

cast(A as date), where A is a string column. the min/max of result column stats should be calc like this:
convert A.minExpr to a date dateA, and then get double value from dateA.

add "explain memo plan select ..." to print memo from mysql client

dump column stats for FileScanNode, used in datalake.
---
 .../org/apache/doris/nereids/DorisLexer.g4    |  1 +
 .../org/apache/doris/nereids/DorisParser.g4   |  1 +
 .../apache/doris/nereids/NereidsPlanner.java  |  4 +++
 .../org/apache/doris/nereids/memo/Memo.java   | 11 ++++---
 .../nereids/parser/LogicalPlanBuilder.java    |  3 ++
 .../nereids/stats/ExpressionEstimation.java   | 31 ++++++++++++++++++-
 .../trees/plans/commands/ExplainCommand.java  |  1 +
 .../doris/statistics/ColumnStatistic.java     |  4 +--
 8 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 78071820ff..2f23376d0a 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -251,6 +251,7 @@ MATCH_ANY: 'MATCH_ANY';
 MATCH_ALL: 'MATCH_ALL';
 MATCH_PHRASE: 'MATCH_PHRASE';
 MATCHED: 'MATCHED';
+MEMO:'MEMO';
 MERGE: 'MERGE';
 MINUTE: 'MINUTE';
 MONTH: 'MONTH';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 3fd8e8b1af..c32a09f3f6 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -78,6 +78,7 @@ planType
     | REWRITTEN | LOGICAL  // same type
     | OPTIMIZED | PHYSICAL   // same type
     | SHAPE
+    | MEMO
     | ALL // default type
     ;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index bf74eceee3..25064c1c68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -408,6 +408,10 @@ public class NereidsPlanner extends Planner {
                 return "cost = " + cost + "\n" + optimizedPlan.treeString();
             case SHAPE_PLAN:
                 return optimizedPlan.shape("");
+            case MEMO_PLAN:
+                return cascadesContext.getMemo().toString()
+                    + "\n\n========== OPTIMIZED PLAN ==========\n"
+                    + optimizedPlan.treeString();
             case ALL_PLAN:
                 return "========== PARSED PLAN ==========\n"
                         + parsedPlan.treeString() + "\n\n"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 42acda2511..643ffe819c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.plans.GroupPlan;
 import org.apache.doris.nereids.trees.plans.LeafPlan;
 import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -737,10 +738,12 @@ public class Memo {
             builder.append("\n\n").append(group);
             builder.append("  stats=").append(group.getStatistics()).append("\n");
             Statistics stats = group.getStatistics();
-            if (stats != null && !group.getLogicalExpressions().isEmpty()
-                    && group.getLogicalExpressions().get(0).getPlan() instanceof LogicalOlapScan) {
-                for (Entry e : stats.columnStatistics().entrySet()) {
-                    builder.append("    ").append(e.getKey()).append(":").append(e.getValue()).append("\n");
+            if (stats != null && !group.getLogicalExpressions().isEmpty()) {
+                Plan plan = group.getLogicalExpressions().get(0).getPlan();
+                if (plan instanceof LogicalOlapScan || plan instanceof LogicalFileScan) {
+                    for (Entry e : stats.columnStatistics().entrySet()) {
+                        builder.append("    ").append(e.getKey()).append(":").append(e.getValue()).append("\n");
+                    }
                 }
             }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index dffc10d7df..d4c044ad73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -1901,6 +1901,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
         if (planTypeContext.SHAPE() != null) {
             return ExplainLevel.SHAPE_PLAN;
         }
+        if (planTypeContext.MEMO() != null) {
+            return ExplainLevel.MEMO_PLAN;
+        }
         return ExplainLevel.ALL_PLAN;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
index eca7511684..9a3c54f739 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -18,6 +18,8 @@
 package org.apache.doris.nereids.stats;
 
 import org.apache.doris.analysis.ArithmeticExpr.Operator;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Add;
 import org.apache.doris.nereids.trees.expressions.AggregateExpression;
 import org.apache.doris.nereids.trees.expressions.Alias;
@@ -84,6 +86,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Year;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub;
+import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DataType;
@@ -140,12 +143,38 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
         return columnStat.build();
     }
 
+    @Override
     public ColumnStatistic visitCast(Cast cast, Statistics context) {
         ColumnStatistic stats = context.findColumnStatistics(cast);
         if (stats != null) {
             return stats;
         }
-        return cast.child().accept(this, context);
+        ColumnStatistic childColStats = cast.child().accept(this, context);
+
+        return castMinMax(childColStats, cast.getDataType());
+    }
+
+    private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType) {
+        if (colStats.minExpr instanceof StringLiteral && targetType.isDateLikeType()) {
+            ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats);
+            if (colStats.minExpr != null && colStats.maxExpr != null) {
+                String strMin = colStats.minExpr.getStringValue();
+                try {
+                    DateLiteral dateMinLiteral = new DateLiteral(strMin);
+                    long min = dateMinLiteral.getValue();
+                    builder.setMinValue(min);
+
+                    String strMax = colStats.maxExpr.getStringValue();
+                    DateLiteral dateMaxLiteral = new DateLiteral(strMax);
+                    long max = dateMaxLiteral.getValue();
+                    builder.setMaxValue(max);
+                } catch (AnalysisException e) {
+                    // ignore exception. do not convert min max
+                }
+            }
+            return builder.build();
+        }
+        return colStats;
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index 820e3861d8..bfb0e82a17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -46,6 +46,7 @@ public class ExplainCommand extends Command implements NoForward {
         REWRITTEN_PLAN(true),
         OPTIMIZED_PLAN(true),
         SHAPE_PLAN(true),
+        MEMO_PLAN(true),
         ALL_PLAN(true)
         ;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 48277b3c51..d791ee1e0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -314,8 +314,8 @@ public class ColumnStatistic {
 
     @Override
     public String toString() {
-        return isUnKnown ? "unKnown" : String.format("ndv=%.4f, min=%f, max=%f, sel=%f, count=%.4f",
-                ndv, minValue, maxValue, selectivity, count);
+        return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s), max=%f(%s), count=%.4f",
+                ndv, minValue, minExpr, maxValue, maxExpr, count);
     }
 
     public JSONObject toJson() {