planner: prune duplicate expr in sort (#20325)

planner/core/logical_plan_test.go

@@ -553,6 +553,31 @@ func (s *testPlanSuite) TestColumnPruning(c *C) {
 	}
 }
 
+func (s *testPlanSuite) TestSortByItemsPruning(c *C) {
+	defer testleak.AfterTest(c)()
+	var (
+		input  []string
+		output [][]string
+	)
+	s.testData.GetTestCases(c, &input, &output)
+	s.testData.OnRecord(func() {
+		output = make([][]string, len(input))
+	})
+
+	ctx := context.Background()
+	for i, tt := range input {
+		comment := Commentf("for %s", tt)
+		stmt, err := s.ParseOneStmt(tt, "", "")
+		c.Assert(err, IsNil, comment)
+
+		p, _, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is)
+		c.Assert(err, IsNil)
+		lp, err := logicalOptimize(ctx, flagEliminateProjection|flagPredicatePushDown|flagPrunColumns|flagPrunColumnsAgain, p.(LogicalPlan))
+		c.Assert(err, IsNil)
+		s.checkOrderByItems(lp, c, &output[i], comment)
+	}
+}
+
 func (s *testPlanSuite) TestProjectionEliminator(c *C) {
 	defer testleak.AfterTest(c)()
 	tests := []struct {
@@ -620,6 +645,27 @@ func (s *testPlanSuite) checkDataSourceCols(p LogicalPlan, c *C, ans map[int][]s
 	}
 }
 
+func (s *testPlanSuite) checkOrderByItems(p LogicalPlan, c *C, colList *[]string, comment CommentInterface) {
+	switch p := p.(type) {
+	case *LogicalSort:
+		s.testData.OnRecord(func() {
+			*colList = make([]string, len(p.ByItems))
+		})
+		for i, col := range p.ByItems {
+			s.testData.OnRecord(func() {
+				(*colList)[i] = col.String()
+			})
+			s := col.String()
+			c.Assert(s, Equals, (*colList)[i], comment)
+		}
+	}
+	children := p.Children()
+	c.Assert(len(children), LessEqual, 1, Commentf("For %v Expected <= 1 Child", comment))
+	for _, child := range children {
+		s.checkOrderByItems(child, c, colList, comment)
+	}
+}
+
 func (s *testPlanSuite) TestValidate(c *C) {
 	defer testleak.AfterTest(c)()
 	tests := []struct {

planner/core/rule_column_pruning.go

@@ -137,9 +137,15 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column)
 
 func pruneByItems(old []*util.ByItems) (new []*util.ByItems, parentUsedCols []*expression.Column) {
 	new = make([]*util.ByItems, 0, len(old))
+	seen := make(map[string]struct{}, len(old))
 	for _, byItem := range old {
+		hash := string(byItem.Expr.HashCode(nil))
+		_, hashMatch := seen[hash]
+		seen[hash] = struct{}{}
 		cols := expression.ExtractColumns(byItem.Expr)
-		if len(cols) == 0 {
+		if hashMatch {
+			// do nothing, should be filtered
+		} else if len(cols) == 0 {
 			if !expression.IsRuntimeConstExpr(byItem.Expr) {
 				new = append(new, byItem)
 			}

planner/core/testdata/plan_suite_unexported_in.json

@@ -402,6 +402,14 @@
       "select count(1) from (select count(b) as cnt from t group by c) t1"
     ]
   },
+  {
+    "name": "TestSortByItemsPruning",
+    "cases": [
+      "select * from t where a > 1 order by a asc, a asc limit 10",
+      "select * from t where a > 1 order by a asc, b asc, a asc, c asc limit 10",
+      "select * from t where a > 1 order by pow(a, 2) asc, b asc, pow(a, 2) asc, c asc limit 10"
+    ]
+  },
   {
     "name": "TestDeriveNotNullConds",
     "cases": [

planner/core/testdata/plan_suite_unexported_out.json

@@ -739,6 +739,24 @@
       }
     ]
   },
+  {
+    "Name": "TestSortByItemsPruning",
+    "Cases": [
+      [
+        "test.t.a"
+      ],
+      [
+        "test.t.a",
+        "test.t.b",
+        "test.t.c"
+      ],
+      [
+        "pow(cast(test.t.a, double BINARY), 2)",
+        "test.t.b",
+        "test.t.c"
+      ]
+    ]
+  },
   {
     "Name": "TestDeriveNotNullConds",
     "Cases": [