database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

planner: derive index filters for mv index paths (#54877)

Browse Source 
close pingcap/tidb#54876
This commit is contained in:
Zhou Kunqin
2024-07-27 01:26:44 +08:00
committed by GitHub
parent 197476a41d
commit 56710aeaef
3 changed files with 261 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
pkg/planner/core/indexmerge_path.go
View File

@ -981,7 +981,29 @@ func (ds *DataSource) generateIndexMerge4ComposedIndex(normalPathCnt int, indexM
remainedCNFs = append(remainedCNFs, CNFItem)
}
}
mvp := ds.buildPartialPathUp4MVIndex(combinedPartialPaths, true, remainedCNFs, ds.TableStats.HistColl)
condInIdxFilter := make(map[string]struct{}, len(remainedCNFs))
// try to derive index filters for each path
for _, path := range combinedPartialPaths {
idxFilters, _ := ds.splitIndexFilterConditions(remainedCNFs, path.FullIdxCols, path.FullIdxColLens)
idxFilters = util.CloneExprs(idxFilters)
path.IndexFilters = append(path.IndexFilters, idxFilters...)
for _, idxFilter := range idxFilters {
condInIdxFilter[string(idxFilter.HashCode())] = struct{}{}
}
}
// Collect the table filters.
// Since it's the intersection type index merge here, as long as a filter appears in one path, we don't need it in
// the table filters.
var tableFilters []expression.Expression
for _, CNFItem := range remainedCNFs {
if _, ok := condInIdxFilter[string(CNFItem.HashCode())]; !ok {
tableFilters = append(tableFilters, CNFItem)
}
}
mvp := ds.buildPartialPathUp4MVIndex(combinedPartialPaths, true, tableFilters, ds.TableStats.HistColl)
ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, mvp)
return nil
@ -1046,10 +1068,27 @@ func (ds *DataSource) generateIndexMerge4MVIndex(normalPathCnt int, filters []ex
continue
}
// Here, all partial paths are built from the same MV index, so we can directly use the first one to get the
// metadata.
// And according to buildPartialPaths4MVIndex, there must be at least one partial path if it returns ok.
firstPath := partialPaths[0]
idxFilters, tableFilters := ds.splitIndexFilterConditions(
remainingFilters,
firstPath.FullIdxCols,
firstPath.FullIdxColLens,
)
// Add the index filters to every partial path.
// For union type index merge, this is necessary for correctness.
for _, path := range partialPaths {
clonedIdxFilters := util.CloneExprs(idxFilters)
path.IndexFilters = append(path.IndexFilters, clonedIdxFilters...)
}
ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, ds.buildPartialPathUp4MVIndex(
partialPaths,
isIntersection,
remainingFilters,
tableFilters,
ds.TableStats.HistColl,
),
)
@ -1214,10 +1253,16 @@ func buildPartialPath4MVIndex(
partialPath := &util.AccessPath{Index: mvIndex}
partialPath.Ranges = ranger.FullRange()
for i := 0; i < len(idxCols); i++ {
length := mvIndex.Columns[i].Length
// For full length prefix index, we consider it as non prefix index.
// This behavior is the same as in IndexInfo2Cols(), which is used for non mv index.
if length == idxCols[i].RetType.GetFlen() {
length = types.UnspecifiedLength
}
partialPath.IdxCols = append(partialPath.IdxCols, idxCols[i])
partialPath.IdxColLens = append(partialPath.IdxColLens, mvIndex.Columns[i].Length)
partialPath.IdxColLens = append(partialPath.IdxColLens, length)
partialPath.FullIdxCols = append(partialPath.FullIdxCols, idxCols[i])
partialPath.FullIdxColLens = append(partialPath.FullIdxColLens, mvIndex.Columns[i].Length)
partialPath.FullIdxColLens = append(partialPath.FullIdxColLens, length)
}
if err := detachCondAndBuildRangeForPath(sctx, partialPath, accessFilters, histColl); err != nil {
return nil, false, err

124
tests/integrationtest/r/planner/core/indexmerge_path.result
View File

@ -1081,3 +1081,127 @@ IndexMerge 0.05 root type: union
└─Selection(Probe) 0.05 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1)
└─TableRowIDScan 49.94 cop[tikv] table:t keep order:false, stats:pseudo
SET @@tidb_opt_fix_control = default;
drop table if exists t, t1;
create table t (a int, b varchar(30), c float, j json, pk int primary key,
key mvi1(c, (cast(j->'$.a' as unsigned array)), b),
key mvi2(a, (cast(j->'$.c' as unsigned array))),
key mvi3((cast(j->'$.d' as unsigned array)), c),
key idx(b, c)
);
insert into t values (1, 'test', 1, '{"a":[3,4,5], "c":[7,8,9], "d":[10,11,12]}', 1);
insert into t values (2, 'text', 1, '{"a":[4,5,6], "c":[10,11,12], "d":[13,14,15]}', 2);
insert into t values (1, 'abcd', 1, '{"a":[7,8,9], "c":[13,14,15], "d":[16,17,18]}', 3);
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';
a b c j pk
2 text 1 {"a": [4, 5, 6], "c": [10, 11, 12], "d": [13, 14, 15]} 2
EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';
id estRows task access object operator info
Selection 0.24 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY))
└─IndexMerge 0.30 root type: union
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 6,1 6], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.30 cop[tikv] table:t keep order:false, stats:pseudo
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';
a b c j pk
2 text 1 {"a": [4, 5, 6], "c": [10, 11, 12], "d": [13, 14, 15]} 2
EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';
id estRows task access object operator info
IndexMerge 0.00 root type: intersection
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo
SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');
a b c j pk
EXPLAIN format=brief SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');
id estRows task access object operator info
IndexMerge 0.00 root type: intersection
├─Selection(Build) 2.67 cop[tikv] eq(planner__core__indexmerge_path.t.c, 10), not(like(planner__core__indexmerge_path.t.b, "%test%", 92)), not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(b, c) range:("abc",+inf], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 3,10 3], keep order:false, stats:pseudo
├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo
create table t1 (
a int,
b varchar(30),
c float,
d tinytext,
j json,
key mvi1(b(3), (cast(j as unsigned array))),
key mvi2((cast(j as unsigned array)), b),
key mvi3((cast(j as unsigned array)), d(30)),
key mvi4((cast(j as unsigned array)), d(255))
);
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi1) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg';
id estRows task access object operator info
TableReader 0.00 root data:Selection
└─Selection 0.00 cop[tikv] eq(planner__core__indexmerge_path.t1.b, "abcdefg"), eq(planner__core__indexmerge_path.t1.c, 1), json_contains(planner__core__indexmerge_path.t1.j, cast("[4,5]", json BINARY))
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi2) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg' and
b like '%test%';
id estRows task access object operator info
IndexMerge 0.00 root type: intersection
├─Selection(Build) 0.00 cop[tikv] like(planner__core__indexmerge_path.t1.b, "%test%", 92)
│ └─IndexRangeScan 0.10 cop[tikv] table:t1, index:mvi2(cast(`j` as unsigned array), b) range:[4 "abcdefg",4 "abcdefg"], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] like(planner__core__indexmerge_path.t1.b, "%test%", 92)
│ └─IndexRangeScan 0.10 cop[tikv] table:t1, index:mvi2(cast(`j` as unsigned array), b) range:[5 "abcdefg",5 "abcdefg"], keep order:false, stats:pseudo
└─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1)
└─TableRowIDScan 0.00 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi3) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';
id estRows task access object operator info
IndexMerge 0.01 root type: intersection
├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi3(cast(`j` as unsigned array), d) range:[4,4], keep order:false, stats:pseudo
├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi3(cast(`j` as unsigned array), d) range:[5,5], keep order:false, stats:pseudo
└─Selection(Probe) 0.01 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1), not(like(planner__core__indexmerge_path.t1.d, "%test%", 92))
└─TableRowIDScan 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi4) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';
id estRows task access object operator info
IndexMerge 0.01 root type: intersection
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t1.d, "%test%", 92))
│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:mvi4(cast(`j` as unsigned array), d) range:[4,4], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t1.d, "%test%", 92))
│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:mvi4(cast(`j` as unsigned array), d) range:[5,5], keep order:false, stats:pseudo
└─Selection(Probe) 0.01 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1)
└─TableRowIDScan 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo

88
tests/integrationtest/t/planner/core/indexmerge_path.test
View File

@ -425,3 +425,91 @@ EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b =
EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13);
EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11');
SET @@tidb_opt_fix_control = default;
# Test deriving index filters for mv index paths
drop table if exists t, t1;
create table t (a int, b varchar(30), c float, j json, pk int primary key,
key mvi1(c, (cast(j->'$.a' as unsigned array)), b),
key mvi2(a, (cast(j->'$.c' as unsigned array))),
key mvi3((cast(j->'$.d' as unsigned array)), c),
key idx(b, c)
);
insert into t values (1, 'test', 1, '{"a":[3,4,5], "c":[7,8,9], "d":[10,11,12]}', 1);
insert into t values (2, 'text', 1, '{"a":[4,5,6], "c":[10,11,12], "d":[13,14,15]}', 2);
insert into t values (1, 'abcd', 1, '{"a":[7,8,9], "c":[13,14,15], "d":[16,17,18]}', 3);
# case 1: union type index merge on single mv index from single condition
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';
EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';
# case 2: intersection type index merge on single mv index from single condition
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';
EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';
# case 3: intersection type index merge on multiple indexes from different conditions
SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');
EXPLAIN format=brief SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');
# case 4: test prefix indexes
create table t1 (
a int,
b varchar(30),
c float,
d tinytext,
j json,
key mvi1(b(3), (cast(j as unsigned array))),
key mvi2((cast(j as unsigned array)), b),
key mvi3((cast(j as unsigned array)), d(30)),
key mvi4((cast(j as unsigned array)), d(255))
);
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi1) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg';
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi2) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg' and
b like '%test%';
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi3) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi4) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';