database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](materialized-view) adjust limit for create materialized view on uniq/agg table (#21580)

Browse Source 
adjust limit for create materialized view on uniq/agg table
This commit is contained in:
Pxl
2023-07-10 10:04:17 +08:00
committed by GitHub
parent ee9822fa7e
commit 77336bff44
15 changed files with 175 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -21,11 +21,13 @@ import org.apache.doris.analysis.AddRollupClause;
import org.apache.doris.analysis.AlterClause;
import org.apache.doris.analysis.CancelAlterTableStmt;
import org.apache.doris.analysis.CancelStmt;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt;
import org.apache.doris.analysis.DropMaterializedViewStmt;
import org.apache.doris.analysis.DropRollupClause;
import org.apache.doris.analysis.MVColumnItem;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
@ -33,6 +35,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
@ -463,109 +466,117 @@ public class MaterializedViewHandler extends AlterHandler {
throw new DdlException("MergeOnWrite table can't create materialized view.");
}
// check if mv columns are valid
// a. Aggregate or Unique table:
// 1. For aggregate table, mv columns with aggregate function should be same as base schema
// 2. For aggregate table, the column which is the key of base table should be the key of mv as well.
// b. Duplicate table:
// 1. Columns resolved by semantics are legal
// a. Aggregate table:
// 1. all slot's aggregationType must same with value mv column
// 2. all slot's isKey must same with mv column
// 3. value column'define expr must be slot (except all slot belong replace family)
// b. Unique table:
// 1. mv must not contain group expr
// 2. all slot's isKey same with mv column
// c. Duplicate table:
// 1. Columns resolved by semantics are legal
// update mv columns
List<MVColumnItem> mvColumnItemList = addMVClause.getMVColumnItemList();
List<Column> newMVColumns = Lists.newArrayList();
int numOfKeys = 0;
if (olapTable.getKeysType().isAggregationFamily()) {
if (addMVClause.getMVKeysType() != KeysType.AGG_KEYS) {
throw new DdlException("The materialized view of aggregation"
+ " or unique table must has grouping columns");
if (!addMVClause.isReplay() && olapTable.getKeysType() == KeysType.AGG_KEYS
&& addMVClause.getMVKeysType() != KeysType.AGG_KEYS) {
throw new DdlException("The materialized view of aggregation table must has grouping columns");
}
if (!addMVClause.isReplay() && olapTable.getKeysType() == KeysType.UNIQUE_KEYS
&& addMVClause.getMVKeysType() == KeysType.AGG_KEYS) {
// check b.1
throw new DdlException("The materialized view of unique table must not has grouping columns");
}
addMVClause.setMVKeysType(olapTable.getKeysType());
for (MVColumnItem mvColumnItem : mvColumnItemList) {
if (mvColumnItem.getBaseColumnNames().size() != 1) {
throw new DdlException(
"mvColumnItem.getBaseColumnNames().size() != 1, mvColumnItem.getBaseColumnNames().size() = "
+ mvColumnItem.getBaseColumnNames().size());
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && !mvColumnItem.isKey()) {
mvColumnItem.setAggregationType(AggregateType.REPLACE, true);
}
String mvColumnName = mvColumnItem.getBaseColumnNames().iterator().next();
Column mvColumn = mvColumnItem.toMVColumn(olapTable);
if (mvColumnItem.isKey()) {
++numOfKeys;
}
AggregateType baseAggregationType = mvColumn.getAggregationType();
AggregateType mvAggregationType = mvColumnItem.getAggregationType();
if (mvColumn.isKey() && !mvColumnItem.isKey()) {
throw new DdlException("The column[" + mvColumnName + "] must be the key of materialized view");
}
if (baseAggregationType != mvAggregationType) {
throw new DdlException(
"The aggregation type of column[" + mvColumnName + "] must be same as the aggregate "
+ "type of base column in aggregate table");
}
if (baseAggregationType != null && baseAggregationType.isReplaceFamily() && olapTable
.getKeysNum() != numOfKeys) {
throw new DdlException(
"The materialized view should contain all keys of base table if there is a" + " REPLACE "
+ "value");
}
newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
}
// check useless rollup of same key columns and same order with base table
if (numOfKeys == olapTable.getBaseSchemaKeyColumns().size() && !addMVClause.isReplay()) {
boolean allKeysMatch = true;
for (int i = 0; i < numOfKeys; i++) {
if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName())
.equalsIgnoreCase(olapTable.getBaseSchemaKeyColumns().get(i).getName())) {
allKeysMatch = false;
break;
if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS) {
for (String slotName : mvColumnItem.getBaseColumnNames()) {
if (!addMVClause.isReplay()
&& olapTable
.getColumn(MaterializedIndexMeta
.normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName)))
.isKey()) {
mvColumnItem.setIsKey(true);
}
}
}
if (allKeysMatch) {
throw new DdlException("MV contains all keys in base table with same order for "
+ "aggregation or unique table is useless.");
// check a.2 and b.2
for (String slotName : mvColumnItem.getBaseColumnNames()) {
if (!addMVClause.isReplay() && olapTable
.getColumn(MaterializedIndexMeta
.normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName)))
.isKey() != mvColumnItem.isKey()) {
throw new DdlException("The mvItem[" + mvColumnItem.getName()
+ "]'s isKey must same with all slot, mvItem.isKey="
+ (mvColumnItem.isKey() ? "true" : "false"));
}
}
if (!addMVClause.isReplay() && !mvColumnItem.isKey() && olapTable.getKeysType() == KeysType.AGG_KEYS) {
// check a.1
for (String slotName : mvColumnItem.getBaseColumnNames()) {
if (olapTable
.getColumn(MaterializedIndexMeta
.normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName)))
.getAggregationType() != mvColumnItem.getAggregationType()) {
throw new DdlException("The mvItem[" + mvColumnItem.getName()
+ "]'s AggregationType must same with all slot");
}
}
// check a.3
if (!mvColumnItem.getAggregationType().isReplaceFamily()
&& !(mvColumnItem.getDefineExpr() instanceof SlotRef)
&& !((mvColumnItem.getDefineExpr() instanceof CastExpr)
&& mvColumnItem.getDefineExpr().getChild(0) instanceof SlotRef)) {
throw new DdlException(
"The mvItem[" + mvColumnItem.getName() + "] require slot because it is value column");
}
}
newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
}
} else {
Set<String> partitionOrDistributedColumnName = olapTable.getPartitionColumnNames();
partitionOrDistributedColumnName.addAll(olapTable.getDistributionColumnNames());
boolean hasNewColumn = false;
for (MVColumnItem mvColumnItem : mvColumnItemList) {
Set<String> names = mvColumnItem.getBaseColumnNames();
if (names == null) {
throw new DdlException("Base columns is null");
}
for (String str : names) {
if (partitionOrDistributedColumnName.contains(str)
&& mvColumnItem.getAggregationType() != null) {
throw new DdlException("The partition and distributed columns " + str
+ " must be key column in mv");
if (partitionOrDistributedColumnName.contains(str) && mvColumnItem.getAggregationType() != null) {
throw new DdlException(
"The partition and distributed columns " + str + " must be key column in mv");
}
}
newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
if (mvColumnItem.isKey()) {
++numOfKeys;
}
if (olapTable
.getBaseColumn(CreateMaterializedViewStmt.mvColumnBreaker(mvColumnItem.getName())) == null) {
hasNewColumn = true;
}
}
// check useless rollup of same key columns and same order with base table
if (!addMVClause.isReplay() && addMVClause.getMVKeysType() == KeysType.DUP_KEYS && !hasNewColumn) {
boolean allKeysMatch = true;
for (int i = 0; i < numOfKeys; i++) {
if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName())
.equalsIgnoreCase(olapTable.getBaseSchema().get(i).getName())
&& olapTable.getBaseSchema().get(i).isKey()) {
allKeysMatch = false;
break;
}
}
if (allKeysMatch && !olapTable.isDuplicateWithoutKey()) {
throw new DdlException("MV contain the columns of the base table in prefix order for "
+ "duplicate table is useless.");
}
}
}
if (newMVColumns.size() == olapTable.getBaseSchema().size() && !addMVClause.isReplay()) {
boolean allKeysMatch = true;
for (int i = 0; i < newMVColumns.size(); i++) {
if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName())
.equalsIgnoreCase(olapTable.getBaseSchema().get(i).getName())) {
allKeysMatch = false;
break;
}
}
if (allKeysMatch) {
throw new DdlException("MV same with base table is useless.");
}
}
if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasDeleteSign()) {
newMVColumns.add(new Column(olapTable.getDeleteSignColumn()));
}

4
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -139,6 +139,10 @@ public class CreateMaterializedViewStmt extends DdlStmt {
return dbName;
}
public void setMVKeysType(KeysType type) {
mvKeysType = type;
}
public KeysType getMVKeysType() {
return mvKeysType;
}

4
fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java
View File

@ -179,10 +179,6 @@ public class MVColumnItem {
throw new DdlException("base column's type is null, column=" + result.getName());
}
result.setIsKey(isKey);
// If the mv column type is inconsistent with the base column type, the daily
// test will core.
// So, I comment this line firstly.
// result.setType(type);
} else {
if (type == null) {
throw new DdlException("MVColumnItem type is null");

3
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java
View File

@ -753,8 +753,7 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP
String uniqueTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, k2 int, v1 int) UNIQUE KEY (k1, k2) "
+ "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1','enable_unique_key_merge_on_write' = 'false');";
createTable(uniqueTable);
String createK1MV = "create materialized view only_k1 as select k2 from " + TEST_TABLE_NAME + " group by "
+ "k2;";
String createK1MV = "create materialized view only_k1 as select k2 from " + TEST_TABLE_NAME;
createMv(createK1MV);
String query = "select * from " + TEST_TABLE_NAME + ";";
singleTableTest(query, TEST_TABLE_NAME, false);

5
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java
View File

@ -86,7 +86,6 @@ public class ExplainInsertCommandTest extends TestWithFeService {
+ "properties(\"replication_num\" = \"1\")");
createMv("create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1");
createMv("create materialized view mv2 as select k1, k2, k3 from agg_have_dup_base group by k1, k2, k3");
createMv("create materialized view mv3 as select k1, k2 + k3 from agg_have_dup_base group by k1, k2 + k3");
}
@ -119,9 +118,9 @@ public class ExplainInsertCommandTest extends TestWithFeService {
@Test
public void testWithMV() throws Exception {
String sql = "explain insert into agg_have_dup_base select -4, -4, -4, 'd'";
Assertions.assertEquals(10, getOutputFragment(sql).getOutputExprs().size());
Assertions.assertEquals(8, getOutputFragment(sql).getOutputExprs().size());
String sql1 = "explain insert into agg_have_dup_base select -4, k2, -4, 'd' from agg_have_dup_base";
Assertions.assertEquals(10, getOutputFragment(sql1).getOutputExprs().size());
Assertions.assertEquals(8, getOutputFragment(sql1).getOutputExprs().size());
}
private PlanFragment getOutputFragment(String sql) throws Exception {

3
fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java
View File

@ -666,8 +666,7 @@ public class MaterializedViewFunctionTest {
String uniqueTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, k2 int, v1 int) UNIQUE KEY (k1, k2) "
+ "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'false');";
dorisAssert.withTable(uniqueTable);
String createK1K2MV = "create materialized view only_k1 as select k2, k1 from " + TEST_TABLE_NAME + " group by "
+ "k2, k1;";
String createK1K2MV = "create materialized view only_k1 as select k2, k1 from " + TEST_TABLE_NAME;
String query = "select * from " + TEST_TABLE_NAME + ";";
dorisAssert.withMaterializedView(createK1K2MV).query(query).explainContains(TEST_TABLE_NAME);
dorisAssert.dropTable(TEST_TABLE_NAME, true);