database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[MaterializedView] Implement new materialized view selector (#2821)

Browse Source 
This commit mainly implements the new materialized view selector which supports SPJ<->SPJG.
Two parameters are currently used to regulate this function.
1. test_materialized_view: When this parameter is set to true, the user can create a materialized view for the duplicate table by using 'CREATE MATERIALIZED VIEW' command.
At the same time, if the result of the new materialized views is different from the old version during the query, an error will be reported. This parameter is false by default, which means that the new version of the materialized view function cannot be enabled.
2. use_old_mv_selector: When this parameter is set to true, the result of the old version selector will be selected. If set to false, the result of the new version selector will be selected. This parameter is true by default, which means that the old selector is used.
If the default values of the above two parameters do not change, there will be no behavior changes in the current version.

The main steps for the new selector are as follows:
1. Predicates stage: This stage will mainly filter out all materialized views that do not meet the current query requirements.
2. Priorities stage: This stage will sort the results of the first stage and choose the best materialized view.

The predicates phase is divided into 6 steps:
1. Calculate the predicate gap between the current query and view.
2. Whether the columns in the view can meet the needs of the compensating predicates.
3. Determine whether the group by columns of view match the group by columns of query.
4. Determine whether the aggregate columns of view match the aggregate columns of query.
5. Determine whether the output columns of view match the output columns of query.
6. Add partial materialized views

The priorities phase is divided into two steps:
1. Find the materialized view that matches the best prefix index
2. Find the materialized view with the least amount of data

The biggest difference between the current materialized view selector and the previous one is that it supports SPJ <-> SPJG.
This commit is contained in:
EmmyMiao87
2020-02-27 09:14:32 +08:00
committed by GitHub
parent 7b39d604c3
commit a3e588f39c
32 changed files with 2374 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
View File

@ -18,11 +18,15 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
@ -38,6 +42,15 @@ public class CreateMaterializedViewStmtTest {
private Analyzer analyzer;
@Mocked
private ExprSubstitutionMap exprSubstitutionMap;
@Mocked
private ConnectContext connectContext;
@Mocked
private Config config;
@Before
public void initTest() {
Deencapsulation.setField(Config.class, "enable_materialized_view", true);
}
@Test
public void testFunctionColumnInSelectClause(@Injectable ArithmeticExpr arithmeticExpr) {
@ -178,6 +191,8 @@ public class CreateMaterializedViewStmtTest {
result = havingClause;
slotRef.getColumnName();
result = "k1";
selectStmt.getGroupByClause();
result = null;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
@ -222,6 +237,8 @@ public class CreateMaterializedViewStmtTest {
result = "k1";
slotRef2.getColumnName();
result = "k2";
selectStmt.getGroupByClause();
result = null;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
@ -238,7 +255,8 @@ public class CreateMaterializedViewStmtTest {
@Injectable SlotRef slotRef2,
@Injectable FunctionCallExpr functionCallExpr,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt) throws UserException {
@Injectable SelectStmt selectStmt,
@Injectable GroupByClause groupByClause) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@ -247,6 +265,8 @@ public class CreateMaterializedViewStmtTest {
OrderByElement orderByElement1 = new OrderByElement(functionCallExpr, false, false);
OrderByElement orderByElement2 = new OrderByElement(slotRef1, false, false);
ArrayList<OrderByElement> orderByElementList = Lists.newArrayList(orderByElement1, orderByElement2);
List<Expr> groupByList = Lists.newArrayList();
groupByList.add(slotRef1);
new Expectations() {
{
@ -273,6 +293,10 @@ public class CreateMaterializedViewStmtTest {
result = Lists.newArrayList(slotRef2);
functionCallExpr.getChild(0);
result = slotRef2;
selectStmt.getGroupByClause();
result = groupByClause;
groupByClause.getGroupingExprs();
result = groupByList;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
@ -326,7 +350,8 @@ public class CreateMaterializedViewStmtTest {
@Injectable FunctionCallExpr functionCallExpr,
@Injectable SlotRef functionChild0,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt) throws UserException {
@Injectable SelectStmt selectStmt,
@Injectable GroupByClause groupByClause) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@ -337,6 +362,11 @@ public class CreateMaterializedViewStmtTest {
OrderByElement orderByElement1 = new OrderByElement(slotRef1, false, false);
ArrayList<OrderByElement> orderByElementList = Lists.newArrayList(orderByElement1);
List<Expr> groupByList = Lists.newArrayList();
groupByList.add(slotRef1);
groupByList.add(slotRef2);
new Expectations() {
{
analyzer.getClusterName();
@ -364,6 +394,10 @@ public class CreateMaterializedViewStmtTest {
result = Lists.newArrayList(slotRef1);
functionCallExpr.getChild(0);
result = functionChild0;
selectStmt.getGroupByClause();
result = groupByClause;
groupByClause.getGroupingExprs();
result = groupByList;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
@ -383,7 +417,8 @@ public class CreateMaterializedViewStmtTest {
@Injectable FunctionCallExpr functionCallExpr,
@Injectable SlotRef functionChild0,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt) throws UserException {
@Injectable SelectStmt selectStmt,
@Injectable GroupByClause groupByClause) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@ -401,6 +436,11 @@ public class CreateMaterializedViewStmtTest {
final String columnName3 = "k3";
final String columnName4 = "v1";
final String columnName5 = "sum_v2";
List<Expr> groupByList = Lists.newArrayList();
groupByList.add(slotRef1);
groupByList.add(slotRef2);
groupByList.add(slotRef3);
new Expectations() {
{
analyzer.getClusterName();
@ -434,6 +474,10 @@ public class CreateMaterializedViewStmtTest {
result = columnName4;
functionChild0.getColumnName();
result = columnName5;
selectStmt.getGroupByClause();
result = groupByClause;
groupByClause.getGroupingExprs();
result = groupByList;
}
};
@ -475,11 +519,9 @@ public class CreateMaterializedViewStmtTest {
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregation(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2,
@Injectable SlotRef slotRef3,
@Injectable SlotRef slotRef4,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt) throws UserException {
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt,
@Injectable GroupByClause groupByClause) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@ -494,6 +536,11 @@ public class CreateMaterializedViewStmtTest {
final String columnName2 = "k2";
final String columnName3 = "k3";
final String columnName4 = "v1";
List<Expr> groupByList = Lists.newArrayList();
groupByList.add(slotRef1);
groupByList.add(slotRef2);
groupByList.add(slotRef3);
new Expectations() {
{
analyzer.getClusterName();
@ -529,6 +576,10 @@ public class CreateMaterializedViewStmtTest {
result = 3;
slotRef4.getType().getStorageLayoutBytes();
result = 4;
selectStmt.getGroupByClause();
result = groupByClause;
groupByClause.getGroupingExprs();
result = groupByList;
}
};
@ -565,11 +616,12 @@ public class CreateMaterializedViewStmtTest {
@Test
public void testMVColumns(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2,
@Injectable FunctionCallExpr functionCallExpr,
@Injectable SlotRef functionChild0,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt) throws UserException {
@Injectable SlotRef slotRef2,
@Injectable FunctionCallExpr functionCallExpr,
@Injectable SlotRef functionChild0,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable GroupByClause groupByClause) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@ -583,6 +635,9 @@ public class CreateMaterializedViewStmtTest {
final String columnName1 = "k1";
final String columnName2 = "v1";
final String columnName3 = "sum_v2";
List<Expr> groupByList = Lists.newArrayList();
groupByList.add(slotRef1);
groupByList.add(slotRef2);
new Expectations() {
{
analyzer.getClusterName();
@ -612,6 +667,10 @@ public class CreateMaterializedViewStmtTest {
result = columnName2;
functionChild0.getColumnName();
result = columnName3;
selectStmt.getGroupByClause();
result = groupByClause;
groupByClause.getGroupingExprs();
result = groupByList;
}
};
@ -639,6 +698,46 @@ public class CreateMaterializedViewStmtTest {
Assert.fail(e.getMessage());
}
}
@Test
public void testDeduplicateMV(@Injectable SlotRef slotRef1,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable GroupByClause groupByClause) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
final String columnName1 = "k1";
List<Expr> groupByList = Lists.newArrayList();
groupByList.add(slotRef1);
new Expectations() {
{
analyzer.getClusterName();
result = "default";
selectStmt.getSelectList();
result = selectList;
selectStmt.analyze(analyzer);
selectStmt.getTableRefs();
result = Lists.newArrayList(tableRef);
selectStmt.getWhereClause();
result = null;
slotRef1.getColumnName();
result = columnName1;
selectStmt.getGroupByClause();
result = groupByClause;
groupByClause.getGroupingExprs();
result = groupByList;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
try {
createMaterializedViewStmt.analyze(analyzer);
Assert.fail();
} catch (UserException e) {
System.out.print(e.getMessage());
}
}
}

89
fe/src/test/java/org/apache/doris/analysis/ExprTest.java Normal file
View File

@ -0,0 +1,89 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.jmockit.Deencapsulation;
import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Test;
import java.util.Map;
import java.util.Set;
import mockit.Expectations;
import mockit.Injectable;
import mockit.Mocked;
public class ExprTest {
@Test
public void testGetTableNameToColumnNames(@Mocked Analyzer analyzer,
@Injectable SlotDescriptor slotDesc1,
@Injectable SlotDescriptor slotDesc2,
@Injectable TupleDescriptor tupleDescriptor1,
@Injectable TupleDescriptor tupleDescriptor2,
@Injectable Table tableA,
@Injectable Table tableB) throws AnalysisException {
TableName tableAName = new TableName("test", "tableA");
TableName tableBName = new TableName("test", "tableB");
SlotRef tableAColumn1 = new SlotRef(tableAName, "c1");
SlotRef tableBColumn1 = new SlotRef(tableBName, "c1");
Expr whereExpr = new BinaryPredicate(BinaryPredicate.Operator.EQ, tableAColumn1, tableBColumn1);
Deencapsulation.setField(tableAColumn1, "desc", slotDesc1);
Deencapsulation.setField(tableBColumn1, "desc", slotDesc2);
new Expectations() {
{
slotDesc1.isMaterialized();
result = true;
slotDesc2.isMaterialized();
result = true;
slotDesc1.getColumn().getName();
result = "c1";
slotDesc2.getColumn().getName();
result = "c1";
slotDesc1.getParent();
result = tupleDescriptor1;
slotDesc2.getParent();
result = tupleDescriptor2;
tupleDescriptor1.getTable();
result = tableA;
tupleDescriptor2.getTable();
result = tableB;
tableA.getName();
result = "tableA";
tableB.getName();
result = "tableB";
}
};
Map<String, Set<String>> tableNameToColumnNames = Maps.newHashMap();
whereExpr.getTableNameToColumnNames(tableNameToColumnNames);
Assert.assertEquals(tableNameToColumnNames.size(), 2);
Set<String> tableAColumns = tableNameToColumnNames.get("tableA");
Assert.assertNotEquals(tableAColumns, null);
Assert.assertTrue(tableAColumns.contains("c1"));
Set<String> tableBColumns = tableNameToColumnNames.get("tableB");
Assert.assertNotEquals(tableBColumns, null);
Assert.assertTrue(tableBColumns.contains("c1"));
}
}

2
fe/src/test/java/org/apache/doris/backup/CatalogMocker.java
View File

@ -241,6 +241,7 @@ public class CatalogMocker {
partitionInfo.setDataProperty(TEST_SINGLE_PARTITION_ID, dataProperty);
OlapTable olapTable = new OlapTable(TEST_TBL_ID, TEST_TBL_NAME, TEST_TBL_BASE_SCHEMA,
KeysType.AGG_KEYS, partitionInfo, distributionInfo);
Deencapsulation.setField(olapTable, "baseIndexId", TEST_TBL_ID);
Tablet tablet0 = new Tablet(TEST_TABLET0_ID);
TabletMeta tabletMeta = new TabletMeta(TEST_DB_ID, TEST_TBL_ID, TEST_SINGLE_PARTITION_ID,
@ -312,6 +313,7 @@ public class CatalogMocker {
OlapTable olapTable2 = new OlapTable(TEST_TBL2_ID, TEST_TBL2_NAME, TEST_TBL_BASE_SCHEMA,
KeysType.AGG_KEYS, rangePartitionInfo, distributionInfo2);
Deencapsulation.setField(olapTable2, "baseIndexId", TEST_TBL2_ID);
Tablet baseTabletP1 = new Tablet(TEST_BASE_TABLET_P1_ID);
TabletMeta tabletMetaBaseTabletP1 = new TabletMeta(TEST_DB_ID, TEST_TBL2_ID, TEST_PARTITION1_ID,

18
fe/src/test/java/org/apache/doris/catalog/DatabaseTest.java
View File

@ -17,12 +17,15 @@
package org.apache.doris.catalog;
import mockit.Expectations;
import mockit.Mocked;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.persist.CreateTableInfo;
import org.apache.doris.persist.EditLog;
import org.apache.doris.thrift.TStorageType;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Before;
@ -36,8 +39,12 @@ import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import mockit.Expectations;
import mockit.Mocked;
public class DatabaseTest {
private Database db;
@ -171,6 +178,13 @@ public class DatabaseTest {
Partition partition = new Partition(20000L, "table", index, new RandomDistributionInfo(10));
OlapTable table = new OlapTable(1000, "table", columns, KeysType.AGG_KEYS,
new SinglePartitionInfo(), new RandomDistributionInfo(10));
List<Column> column = Lists.newArrayList();
short schemaHash = 1;
table.setIndexSchemaInfo(new Long(1), "test", column, 1, 1, schemaHash);
Deencapsulation.setField(table, "baseIndexId", 1);
Map<Long, TStorageType> indexIdToStorageType = Maps.newHashMap();
indexIdToStorageType.put(new Long(1), TStorageType.COLUMN);
Deencapsulation.setField(table, "indexIdToStorageType", indexIdToStorageType);
table.addPartition(partition);
db2.createTable(table);
db2.write(dos);

18
fe/src/test/java/org/apache/doris/catalog/TableTest.java
View File

@ -26,8 +26,15 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.thrift.TStorageType;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -71,8 +78,15 @@ public class TableTest {
columns.add(new Column("column10", ScalarType.createType(PrimitiveType.DATE), true, null, "", ""));
columns.add(new Column("column11", ScalarType.createType(PrimitiveType.DATETIME), true, null, "", ""));
Table table1 = new OlapTable(1000L, "group1", columns, KeysType.AGG_KEYS,
new SinglePartitionInfo(), new RandomDistributionInfo(10));
OlapTable table1 = new OlapTable(1000L, "group1", columns, KeysType.AGG_KEYS,
new SinglePartitionInfo(), new RandomDistributionInfo(10));
List<Column> column = Lists.newArrayList();
short schemaHash = 1;
table1.setIndexSchemaInfo(new Long(1), "test", column, 1, 1, schemaHash);
Deencapsulation.setField(table1, "baseIndexId", 1);
Map<Long, TStorageType> indexIdToStorageType = Maps.newHashMap();
indexIdToStorageType.put(new Long(1), TStorageType.COLUMN);
Deencapsulation.setField(table1, "indexIdToStorageType", indexIdToStorageType);
table1.write(dos);
dos.flush();
dos.close();

2
fe/src/test/java/org/apache/doris/common/util/UnitTestUtil.java
View File

@ -37,6 +37,7 @@ import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.Config;
import org.apache.doris.common.LoadException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.load.DppConfig;
import org.apache.doris.load.Load;
import org.apache.doris.system.Backend;
@ -112,6 +113,7 @@ public class UnitTestUtil {
partitionInfo.setIsInMemory(partitionId, false);
OlapTable table = new OlapTable(tableId, TABLE_NAME, columns,
KeysType.AGG_KEYS, partitionInfo, distributionInfo);
Deencapsulation.setField(table, "baseIndexId", indexId);
table.addPartition(partition);
table.setIndexSchemaInfo(indexId, TABLE_NAME, columns, 0, SCHEMA_HASH, (short) 1);
table.setStorageTypeToIndex(indexId, TStorageType.COLUMN);

12
fe/src/test/java/org/apache/doris/persist/CreateTableInfoTest.java
View File

@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.doris.catalog.FakeCatalog;
import org.apache.doris.catalog.ScalarType;
@ -44,6 +45,10 @@ import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.SinglePartitionInfo;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.common.FeConstants;
import org.apache.doris.thrift.TStorageType;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
public class CreateTableInfoTest {
private Catalog catalog;
@ -89,6 +94,13 @@ public class CreateTableInfoTest {
Partition partition = new Partition(20000L, "table", index, distributionInfo);
OlapTable table = new OlapTable(1000L, "table", columns, KeysType.AGG_KEYS,
new SinglePartitionInfo(), distributionInfo);
List<Column> column = Lists.newArrayList();
short schemaHash = 1;
table.setIndexSchemaInfo(new Long(1), "test", column, 1, 1, schemaHash);
Deencapsulation.setField(table, "baseIndexId", 1);
Map<Long, TStorageType> indexIdToStorageType = Maps.newHashMap();
indexIdToStorageType.put(new Long(1), TStorageType.COLUMN);
Deencapsulation.setField(table, "indexIdToStorageType", indexIdToStorageType);
table.addPartition(partition);
CreateTableInfo info = new CreateTableInfo("db1", table);
info.write(dos);

571
fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java Normal file
View File

@ -0,0 +1,571 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
import org.apache.doris.utframe.DorisAssert;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
public class MaterializedViewFunctionTest {
private static String baseDir = "fe";
private static String runningDir = baseDir + "/mocked/MaterializedViewFunctionTest/"
+ UUID.randomUUID().toString() + "/";
private static final String EMPS_TABLE_NAME = "emps";
private static final String EMPS_MV_NAME = "emps_mv";
private static final String HR_DB_NAME = "db1";
private static final String QUERY_USE_EMPS_MV = "rollup: " + EMPS_MV_NAME;
private static final String QUERY_USE_EMPS = "rollup: " + EMPS_TABLE_NAME;
private static final String DEPTS_TABLE_NAME = "depts";
private static final String DEPTS_MV_NAME = "depts_mv";
private static final String QUERY_USE_DEPTS_MV = "rollup: " + DEPTS_MV_NAME;
private static final String QUERY_USE_DEPTS = "rollup: " + DEPTS_TABLE_NAME;
private static DorisAssert dorisAssert;
@BeforeClass
public static void beforeClass() throws Exception {
UtFrameUtils.createMinDorisCluster(runningDir);
dorisAssert = new DorisAssert();
dorisAssert.withEnableMV().withDatabase(HR_DB_NAME).useDatabase(HR_DB_NAME);
}
@Before
public void beforeMethod() throws Exception {
String createTableSQL = "create table " + HR_DB_NAME + "." + EMPS_TABLE_NAME + " (empid int, name varchar, "
+ "deptno int, salary int, commission int) "
+ "distributed by hash(empid) buckets 3 properties('replication_num' = '1');";
dorisAssert.withTable(createTableSQL);
createTableSQL = "create table " + HR_DB_NAME + "." + DEPTS_TABLE_NAME
+ " (deptno int, name varchar, cost int) "
+ "distributed by hash(deptno) buckets 3 properties('replication_num' = '1');";
dorisAssert.withTable(createTableSQL);
}
@After
public void afterMethod() throws Exception {
dorisAssert.dropTable(EMPS_TABLE_NAME);
dorisAssert.dropTable(DEPTS_TABLE_NAME);
}
@AfterClass
public static void afterClass() throws Exception {
UtFrameUtils.cleanDorisFeDir(baseDir);
}
@Test
public void testProjectionMV1() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from "
+ EMPS_TABLE_NAME + " order by deptno;";
String query = "select empid, deptno from " + EMPS_TABLE_NAME + ";";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testProjectionMV2() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from "
+ EMPS_TABLE_NAME + " order by deptno;";
String query1 = "select empid + 1 from " + EMPS_TABLE_NAME + " where deptno = 10;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query1).explainContains(QUERY_USE_EMPS_MV);
String query2 = "select name from " + EMPS_TABLE_NAME + " where deptno -10 = 0;";
dorisAssert.query(query2).explainWithout(QUERY_USE_EMPS_MV);
}
@Test
public void testProjectionMV3() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, name from "
+ EMPS_TABLE_NAME + " order by deptno;";
String query1 = "select empid +1, name from " + EMPS_TABLE_NAME + " where deptno = 10;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query1).explainContains(QUERY_USE_EMPS_MV);
String query2 = "select name from " + EMPS_TABLE_NAME + " where deptno - 10 = 0;";
dorisAssert.query(query2).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testProjectionMV4() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select name, deptno, salary from "
+ EMPS_TABLE_NAME + ";";
String query1 = "select name from " + EMPS_TABLE_NAME + " where deptno > 30 and salary > 3000;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query1).explainContains(QUERY_USE_EMPS_MV);
String query2 = "select empid from " + EMPS_TABLE_NAME + " where deptno > 30 and empid > 10;";
dorisAssert.query(query2).explainWithout(QUERY_USE_EMPS_MV);
}
@Test
public void testUnionQueryOnProjectionMV() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from " +
EMPS_TABLE_NAME + " order by deptno;";
String union = "select empid from " + EMPS_TABLE_NAME + " where deptno > 300" + " union all select empid from"
+ " " + EMPS_TABLE_NAME + " where deptno < 200";
dorisAssert.withMaterializedView(createMVSQL).query(union).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggQueryOnAggMV1() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary), max" + ""
+ "(salary) from " + EMPS_TABLE_NAME + " group by deptno;";
String query = "select sum(salary), deptno from " + EMPS_TABLE_NAME + " group by deptno;";
dorisAssert.withMaterializedView(createMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggQueryOnAggMV2() throws Exception {
String agg = "select deptno, sum(salary) from " + EMPS_TABLE_NAME + " group by deptno";
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as " + agg + ";";
String query = "select * from (select deptno, sum(salary) as sum_salary from " + EMPS_TABLE_NAME + " group "
+ "by" + " deptno) a where (sum_salary * 2) > 3;";
dorisAssert.withMaterializedView(createMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
/*
TODO
The deduplicate materialized view is not yet supported
@Test
public void testAggQueryOnDeduplicatedMV() throws Exception {
String deduplicateSQL = "select deptno, empid, name, salary, commission from " + EMPS_TABLE_NAME + " group "
+ "by" + " deptno, empid, name, salary, commission";
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as " + deduplicateSQL + ";";
String query1 = "select deptno, sum(salary) from (" + deduplicateSQL + ") A group by deptno;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query1).explainContains(QUERY_USE_EMPS_MV);
String query2 = "select deptno, empid from " + EMPS_TABLE_NAME + ";";
dorisAssert.query(query2).explainWithout(QUERY_USE_EMPS_MV);
}
*/
@Test
public void testAggQueryOnAggMV3() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)"
+ " from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select commission, sum(salary) from " + EMPS_TABLE_NAME + " where commission * (deptno + "
+ "commission) = 100 group by commission;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
/**
* Matching failed because the filtering condition under Aggregate
* references columns for aggregation.
*/
@Test
public void testAggQueryOnAggMV4() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)"
+ " from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select deptno, sum(salary) from " + EMPS_TABLE_NAME + " where salary>1000 group by deptno;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainWithout(QUERY_USE_EMPS_MV);
}
/**
* There will be a compensating Project added after matching of the Aggregate.
*/
@Test
public void testAggQuqeryOnAggMV5() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)"
+ " from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select * from (select deptno, sum(salary) as sum_salary from " + EMPS_TABLE_NAME
+ " group by deptno) a where sum_salary>10;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
/**
* There will be a compensating Project + Filter added after matching of the Aggregate.
*
* @throws Exception
*/
@Test
public void testAggQuqeryOnAggMV6() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary)"
+ " from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select * from (select deptno, sum(salary) as sum_salary from " + EMPS_TABLE_NAME
+ " where deptno>=20 group by deptno) a where sum_salary>10;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
/**
* Aggregation query with groupSets at coarser level of aggregation than
* aggregation materialized view.
*/
@Test
public void testGroupingSetQueryOnAggMV() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) " +
"from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select sum(salary), empid, deptno from " + EMPS_TABLE_NAME + " group by rollup(empid,deptno);";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
/**
* Aggregation query at coarser level of aggregation than aggregation materialized view.
*/
@Test
public void testAggQuqeryOnAggMV7() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " " + "group by deptno, commission;";
String query = "select deptno, sum(salary) from " + EMPS_TABLE_NAME + " where deptno>=20 group by deptno;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggQueryOnAggMV8() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno;";
String query = "select deptno, sum(salary) + 1 from " + EMPS_TABLE_NAME + " group by deptno;";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
/**
* Query with cube and arithmetic expr
*/
@Test
public void testAggQueryOnAggMV9() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select deptno, commission, sum(salary) + 1 from " + EMPS_TABLE_NAME
+ " group by cube(deptno,commission);";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
/**
* Query with rollup and arithmetic expr
*/
@Test
public void testAggQueryOnAggMV10() throws Exception {
String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, commission, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno, commission;";
String query = "select deptno, commission, sum(salary) + 1 from " + EMPS_TABLE_NAME
+ " group by rollup (deptno, commission);";
dorisAssert.withMaterializedView(createMVSQL);
dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testJoinOnLeftProjectToJoin() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME
+ " as select deptno, sum(salary), sum(commission) from " + EMPS_TABLE_NAME + " group by deptno;";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno, max(cost) from "
+ DEPTS_TABLE_NAME + " group by deptno;";
String query = "select * from (select deptno , sum(salary) from " + EMPS_TABLE_NAME + " group by deptno) A "
+ "join (select deptno, max(cost) from " + DEPTS_TABLE_NAME + " group by deptno ) B on A.deptno = B"
+ ".deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnRightProjectToJoin() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary), sum" +
"(commission) from " + EMPS_TABLE_NAME + " group by deptno;";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno, max(cost) from "
+ DEPTS_TABLE_NAME + " group by deptno;";
String query = "select * from (select deptno , sum(salary), sum(commission) from " + EMPS_TABLE_NAME
+ " group by deptno) A join (select deptno from " + DEPTS_TABLE_NAME + " group by deptno ) B on A"
+ ".deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnProjectsToJoin() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary), sum" +
"(commission) from " + EMPS_TABLE_NAME + " group by deptno;";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno, max(cost) from "
+ DEPTS_TABLE_NAME + " group by deptno;";
String query = "select * from (select deptno , sum(salary) from " + EMPS_TABLE_NAME + " group by deptno) A "
+ "join (select deptno from " + DEPTS_TABLE_NAME + " group by deptno ) B on A.deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnCalcToJoin0() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + ";";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno from " +
DEPTS_TABLE_NAME + ";";
String query = "select * from (select empid, deptno from " + EMPS_TABLE_NAME + " where deptno > 10 ) A " +
"join (select deptno from " + DEPTS_TABLE_NAME + " ) B on A.deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnCalcToJoin1() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + ";";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno from " +
DEPTS_TABLE_NAME + ";";
String query = "select * from (select empid, deptno from " + EMPS_TABLE_NAME + " ) A join (select " +
"deptno from " + DEPTS_TABLE_NAME + " where deptno > 10 ) B on A.deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnCalcToJoin2() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + ";";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno from " +
DEPTS_TABLE_NAME + ";";
String query = "select * from (select empid, deptno from " + EMPS_TABLE_NAME + " where empid >10 ) A " +
"join (select deptno from " + DEPTS_TABLE_NAME + " where deptno > 10 ) B on A.deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnCalcToJoin3() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + ";";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno from " +
DEPTS_TABLE_NAME + ";";
String query = "select * from (select empid, deptno + 1 deptno from " + EMPS_TABLE_NAME + " where empid >10 )"
+ " A join (select deptno from " + DEPTS_TABLE_NAME
+ " where deptno > 10 ) B on A.deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testJoinOnCalcToJoin4() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + ";";
String createDeptsMVSQL = "create materialized view " + DEPTS_MV_NAME + " as select deptno from " +
DEPTS_TABLE_NAME + ";";
String query = "select * from (select empid, deptno + 1 deptno from " + EMPS_TABLE_NAME
+ " where empid is not null ) A full join (select deptno from " + DEPTS_TABLE_NAME
+ " where deptno is not null ) B on A.deptno = B.deptno;";
dorisAssert.withMaterializedView(createDeptsMVSQL).withMaterializedView(createEmpsMVSQL).query(query)
.explainContains(QUERY_USE_EMPS_MV, QUERY_USE_DEPTS_MV);
}
@Test
public void testOrderByQueryOnProjectView() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from " +
EMPS_TABLE_NAME + ";";
String query = "select empid from " + EMPS_TABLE_NAME + " order by deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testOrderByQueryOnOrderByView() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid from " +
EMPS_TABLE_NAME + " order by deptno;";
String query = "select empid from " + EMPS_TABLE_NAME + " order by deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testQueryOnStar() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, name, " +
"salary, commission from " + EMPS_TABLE_NAME + " order by deptno, empid;";
String query = "select * from " + EMPS_TABLE_NAME + " where deptno = 1";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testQueryOnStarAndJoin() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, name, " +
"salary, commission from " + EMPS_TABLE_NAME + " order by deptno, empid;";
String query = "select * from " + EMPS_TABLE_NAME + " join depts on " + EMPS_TABLE_NAME + ".deptno = " +
DEPTS_TABLE_NAME + ".deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVAggregateFuncs1() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno from " + EMPS_TABLE_NAME + " group by deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVAggregateFuncs2() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno, sum(salary) from " + EMPS_TABLE_NAME + " group by deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVAggregateFuncs3() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno, empid, sum(salary) from " + EMPS_TABLE_NAME + " group by deptno, empid";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVAggregateFuncs4() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno, sum(salary) from " + EMPS_TABLE_NAME + " where deptno > 10 group by deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVAggregateFuncs5() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno, sum(salary) + 1 from " + EMPS_TABLE_NAME + " where deptno > 10 group by deptno";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVCalcGroupByQuery1() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno+1, sum(salary) + 1 from " + EMPS_TABLE_NAME + " where deptno > 10 "
+ "group by deptno+1;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVCalcGroupByQuery2() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno * empid, sum(salary) + 1 from " + EMPS_TABLE_NAME + " where deptno > 10 " +
"group by deptno * empid;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVCalcGroupByQuery3() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select empid, deptno * empid, sum(salary) + 1 from " + EMPS_TABLE_NAME + " where deptno > 10 "
+ "group by empid, deptno * empid;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testAggregateMVCalcAggFunctionQuery() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno, sum(salary + 1) from " + EMPS_TABLE_NAME + " where deptno > 10 "
+ "group by deptno;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainWithout(QUERY_USE_EMPS_MV);
}
@Test
public void testSubQuery() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid "
+ "from " + EMPS_TABLE_NAME + ";";
String query = "select empid, deptno, salary from " + EMPS_TABLE_NAME + " e1 where empid = (select max(empid)"
+ " from " + EMPS_TABLE_NAME + " where deptno = e1.deptno);";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV,
QUERY_USE_EMPS);
}
@Test
public void testDistinctQuery() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, sum(salary) " +
"from " + EMPS_TABLE_NAME + " group by deptno;";
String query1 = "select distinct deptno from " + EMPS_TABLE_NAME + ";";
dorisAssert.withMaterializedView(createEmpsMVSQL);
dorisAssert.query(query1).explainContains(QUERY_USE_EMPS_MV);
String query2 = "select deptno, sum(distinct salary) from " + EMPS_TABLE_NAME + " group by deptno;";
dorisAssert.query(query2).explainWithout(QUERY_USE_EMPS_MV);
}
@Test
public void testSingleMVMultiUsage() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select deptno, empid, salary " +
"from " + EMPS_TABLE_NAME + " order by deptno;";
String query = "select * from (select deptno, empid from " + EMPS_TABLE_NAME + " where deptno>100) A join " +
"(select deptno, empid from " + EMPS_TABLE_NAME + " where deptno >200) B using (deptno);";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV, 2);
}
@Test
public void testMultiMVMultiUsage() throws Exception {
String createEmpsMVSQL01 = "create materialized view emp_mv_01 as select deptno, empid, salary "
+ "from " + EMPS_TABLE_NAME + " order by deptno;";
String createEmpsMVSQL02 = "create materialized view emp_mv_02 as select deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by deptno;";
String query = "select * from (select deptno, empid from " + EMPS_TABLE_NAME + " where deptno>100) A join " +
"(select deptno, sum(salary) from " + EMPS_TABLE_NAME + " where deptno >200 group by deptno) B "
+ "using (deptno);";
dorisAssert.withMaterializedView(createEmpsMVSQL01).withMaterializedView(createEmpsMVSQL02).query(query)
.explainContains("rollup: emp_mv_01", "rollup: emp_mv_02");
}
@Test
public void testMVOnJoinQuery() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select salary, empid, deptno from " +
EMPS_TABLE_NAME + " order by salary;";
String query = "select empid, salary from " + EMPS_TABLE_NAME + " join " + DEPTS_TABLE_NAME
+ " using (deptno) where salary > 300;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV,
QUERY_USE_DEPTS);
}
// TODO: should be support
@Test
public void testAggregateMVOnCountDistinctQuery1() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno, sum(salary) "
+ "from " + EMPS_TABLE_NAME + " group by empid, deptno;";
String query = "select deptno, count(distinct empid) from " + EMPS_TABLE_NAME + " group by deptno;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testQueryAfterTrimingOfUnusedFields() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + " order by empid, deptno;";
String query = "select empid, deptno from (select empid, deptno, salary from " + EMPS_TABLE_NAME + ") A;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV);
}
@Test
public void testUnionAll() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + " order by empid, deptno;";
String query = "select empid, deptno from " + EMPS_TABLE_NAME + " where empid >1 union all select empid,"
+ " deptno from " + EMPS_TABLE_NAME + " where empid <0;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV, 2);
}
@Test
public void testUnionDistinct() throws Exception {
String createEmpsMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " +
EMPS_TABLE_NAME + " order by empid, deptno;";
String query = "select empid, deptno from " + EMPS_TABLE_NAME + " where empid >1 union select empid," +
" deptno from " + EMPS_TABLE_NAME + " where empid <0;";
dorisAssert.withMaterializedView(createEmpsMVSQL).query(query).explainContains(QUERY_USE_EMPS_MV, 2);
}
}

407
fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java Normal file
View File

@ -0,0 +1,407 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
import org.apache.doris.analysis.AggregateInfo;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SelectStmt;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.jmockit.Deencapsulation;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
import java.util.Map;
import java.util.Set;
import mockit.Expectations;
import mockit.Injectable;
public class MaterializedViewSelectorTest {
@Test
public void initTest(@Injectable SelectStmt selectStmt,
@Injectable SlotDescriptor tableAColumn1Desc,
@Injectable AggregateInfo aggregateInfo,
@Injectable Table tableA,
@Injectable TupleDescriptor tableADesc,
@Injectable SlotDescriptor tableAColumn2Desc,
@Injectable SlotDescriptor tableBColumn1Desc,
@Injectable TupleDescriptor tableBDesc,
@Injectable Table tableB,
@Injectable Analyzer analyzer) {
TableName tableAName = new TableName("test", "tableA");
TableName tableBName = new TableName("test", "tableB");
SlotRef tableAColumn1 = new SlotRef(tableAName, "c1");
Deencapsulation.setField(tableAColumn1, "isAnalyzed", true);
SlotRef tableAColumn2 = new SlotRef(tableAName, "c2");
Deencapsulation.setField(tableAColumn2, "isAnalyzed", true);
SlotRef tableBColumn1 = new SlotRef(tableBName, "c1");
Deencapsulation.setField(tableBColumn1, "isAnalyzed", true);
Deencapsulation.setField(tableAColumn1, "desc", tableAColumn1Desc);
Deencapsulation.setField(tableAColumn2, "desc", tableAColumn2Desc);
Deencapsulation.setField(tableBColumn1, "desc", tableBColumn1Desc);
FunctionCallExpr tableAColumn2Sum = new FunctionCallExpr("SUM", Lists.newArrayList(tableAColumn2));
FunctionCallExpr tableBColumn1Max = new FunctionCallExpr("MAX", Lists.newArrayList(tableBColumn1));
new Expectations() {
{
selectStmt.getAggInfo();
result = aggregateInfo;
aggregateInfo.getGroupingExprs();
result = Lists.newArrayList(tableAColumn1);
tableAColumn1Desc.isMaterialized();
result = true;
tableAColumn1Desc.getColumn().getName();
result = "c1";
tableAColumn1Desc.getParent();
result = tableADesc;
tableADesc.getTable();
result = tableA;
tableA.getName();
result = "tableA";
aggregateInfo.getAggregateExprs();
result = Lists.newArrayList(tableAColumn2Sum, tableBColumn1Max);
tableAColumn2Sum.getChildren();
result = Lists.newArrayList(tableAColumn2);
tableBColumn1Max.getChildren();
result = Lists.newArrayList(tableBColumn1);
tableAColumn2.getColumnName();
result = "c2";
tableBColumn1.getColumnName();
result = "c1";
tableAColumn2.getTableName().getTbl();
result = "tableA";
tableBColumn1.getTableName().getTbl();
result = "tableB";
selectStmt.getResultExprs();
result = Lists.newArrayList(tableAColumn1, tableAColumn2Sum, tableBColumn1Max);
tableAColumn2Desc.isMaterialized();
result = true;
tableAColumn2Desc.getColumn().getName();
result = "c2";
tableAColumn2Desc.getParent();
result = tableADesc;
tableBColumn1Desc.isMaterialized();
result = true;
tableBColumn1Desc.getColumn().getName();
result = "c1";
tableBColumn1Desc.getParent();
result = tableBDesc;
tableBDesc.getTable();
result = tableB;
tableB.getName();
result = "tableB";
}
};
MaterializedViewSelector materializedViewSelector = new MaterializedViewSelector(selectStmt, analyzer);
Map<String, Set<String>> columnNamesInPredicates =
Deencapsulation.getField(materializedViewSelector, "columnNamesInPredicates");
Assert.assertEquals(0, columnNamesInPredicates.size());
Assert.assertFalse(Deencapsulation.getField(materializedViewSelector, "isSPJQuery"));
Map<String, Set<String>> columnNamesInGrouping =
Deencapsulation.getField(materializedViewSelector, "columnNamesInGrouping");
Assert.assertEquals(1, columnNamesInGrouping.size());
Set<String> tableAColumnNamesInGrouping = columnNamesInGrouping.get("tableA");
Assert.assertNotEquals(tableAColumnNamesInGrouping, null);
Assert.assertEquals(1, tableAColumnNamesInGrouping.size());
Assert.assertTrue(tableAColumnNamesInGrouping.contains("c1"));
Map<String, Set<MaterializedViewSelector.AggregatedColumn>> aggregateColumnsInQuery =
Deencapsulation.getField(materializedViewSelector, "aggregateColumnsInQuery");
Assert.assertEquals(2, aggregateColumnsInQuery.size());
Set<MaterializedViewSelector.AggregatedColumn> tableAAgggregatedColumns = aggregateColumnsInQuery.get("tableA");
Assert.assertEquals(1, tableAAgggregatedColumns.size());
MaterializedViewSelector.AggregatedColumn aggregatedColumn1 = tableAAgggregatedColumns.iterator().next();
Assert.assertEquals("c2", Deencapsulation.getField(aggregatedColumn1, "columnName"));
Assert.assertTrue("SUM".equalsIgnoreCase(Deencapsulation.getField(aggregatedColumn1, "aggFunctionName")));
Set<MaterializedViewSelector.AggregatedColumn> tableBAgggregatedColumns = aggregateColumnsInQuery.get("tableB");
Assert.assertEquals(1, tableBAgggregatedColumns.size());
MaterializedViewSelector.AggregatedColumn aggregatedColumn2 = tableBAgggregatedColumns.iterator().next();
Assert.assertEquals("c1", Deencapsulation.getField(aggregatedColumn2, "columnName"));
Assert.assertTrue("MAX".equalsIgnoreCase(Deencapsulation.getField(aggregatedColumn2, "aggFunctionName")));
Map<String, Set<String>> columnNamesInQueryOutput =
Deencapsulation.getField(materializedViewSelector, "columnNamesInQueryOutput");
Assert.assertEquals(2, columnNamesInQueryOutput.size());
Set<String> tableAColumnNamesInQueryOutput = columnNamesInQueryOutput.get("tableA");
Assert.assertEquals(2, tableAColumnNamesInQueryOutput.size());
Assert.assertTrue(tableAColumnNamesInQueryOutput.contains("c1"));
Assert.assertTrue(tableAColumnNamesInQueryOutput.contains("c2"));
Set<String> tableBColumnNamesInQueryOutput = columnNamesInQueryOutput.get("tableB");
Assert.assertEquals(1, tableBColumnNamesInQueryOutput.size());
Assert.assertTrue(tableBColumnNamesInQueryOutput.contains("c1"));
}
@Test
public void testCheckCompensatingPredicates(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer) {
Set<String> tableAColumnNames = Sets.newHashSet();
tableAColumnNames.add("C1");
Map<Long, List<Column>> candidateIndexIdToSchema = Maps.newHashMap();
List<Column> index1Columns = Lists.newArrayList();
Column index1Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index1Columns.add(index1Column1);
candidateIndexIdToSchema.put(new Long(1), index1Columns);
List<Column> index2Columns = Lists.newArrayList();
Column index2Column1 = new Column("c1", Type.INT, false, AggregateType.NONE, true, "", "");
index2Columns.add(index2Column1);
candidateIndexIdToSchema.put(new Long(2), index2Columns);
List<Column> index3Columns = Lists.newArrayList();
Column index3Column1 = new Column("c1", Type.INT, false, AggregateType.SUM, true, "", "");
index3Columns.add(index3Column1);
candidateIndexIdToSchema.put(new Long(3), index3Columns);
List<Column> index4Columns = Lists.newArrayList();
Column index4Column2 = new Column("c2", Type.INT, true, null, true, "", "");
index4Columns.add(index4Column2);
candidateIndexIdToSchema.put(new Long(4), index4Columns);
new Expectations() {
{
selectStmt.getAggInfo();
result = null;
selectStmt.getResultExprs();
result = Lists.newArrayList();
}
};
MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer);
Deencapsulation.invoke(selector, "checkCompensatingPredicates", tableAColumnNames, candidateIndexIdToSchema);
Assert.assertEquals(2, candidateIndexIdToSchema.size());
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(1)));
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(2)));
}
@Test
public void testCheckGrouping(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer) {
Set<String> tableAColumnNames = Sets.newHashSet();
tableAColumnNames.add("C1");
Map<Long, List<Column>> candidateIndexIdToSchema = Maps.newHashMap();
List<Column> index1Columns = Lists.newArrayList();
Column index1Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index1Columns.add(index1Column1);
candidateIndexIdToSchema.put(new Long(1), index1Columns);
List<Column> index2Columns = Lists.newArrayList();
Column index2Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index2Columns.add(index2Column1);
Column index2Column2 = new Column("c2", Type.INT, false, AggregateType.SUM, true, "", "");
index2Columns.add(index2Column2);
candidateIndexIdToSchema.put(new Long(2), index2Columns);
List<Column> index3Columns = Lists.newArrayList();
Column index3Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index3Columns.add(index3Column1);
Column index3Column2 = new Column("c1", Type.INT, false, AggregateType.SUM, true, "", "");
index3Columns.add(index3Column2);
candidateIndexIdToSchema.put(new Long(3), index3Columns);
new Expectations() {
{
selectStmt.getAggInfo();
result = null;
selectStmt.getResultExprs();
result = Lists.newArrayList();
}
};
MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer);
Deencapsulation.setField(selector, "isSPJQuery", false);
Deencapsulation.invoke(selector, "checkGrouping", tableAColumnNames, candidateIndexIdToSchema);
Assert.assertEquals(2, candidateIndexIdToSchema.size());
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(1)));
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(2)));
}
@Test
public void testCheckAggregationFunction(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer) {
Map<Long, List<Column>> candidateIndexIdToSchema = Maps.newHashMap();
List<Column> index1Columns = Lists.newArrayList();
Column index1Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index1Columns.add(index1Column1);
candidateIndexIdToSchema.put(new Long(1), index1Columns);
List<Column> index2Columns = Lists.newArrayList();
Column index2Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index2Columns.add(index2Column1);
Column index2Column2 = new Column("c2", Type.INT, false, AggregateType.SUM, true, "", "");
index2Columns.add(index2Column2);
candidateIndexIdToSchema.put(new Long(2), index2Columns);
List<Column> index3Columns = Lists.newArrayList();
Column index3Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index3Columns.add(index3Column1);
Column index3Column2 = new Column("c1", Type.INT, false, AggregateType.SUM, true, "", "");
index3Columns.add(index3Column2);
candidateIndexIdToSchema.put(new Long(3), index3Columns);
new Expectations() {
{
selectStmt.getAggInfo();
result = null;
selectStmt.getResultExprs();
result = Lists.newArrayList();
}
};
MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer);
MaterializedViewSelector.AggregatedColumn aggregatedColumn = Deencapsulation.newInnerInstance
(MaterializedViewSelector.AggregatedColumn.class, selector, "C1", "sum");
Set<MaterializedViewSelector.AggregatedColumn> aggregatedColumnsInQueryOutput = Sets.newHashSet();
aggregatedColumnsInQueryOutput.add(aggregatedColumn);
Deencapsulation.setField(selector, "isSPJQuery", false);
Deencapsulation.invoke(selector, "checkAggregationFunction", aggregatedColumnsInQueryOutput,
candidateIndexIdToSchema);
Assert.assertEquals(2, candidateIndexIdToSchema.size());
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(1)));
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(3)));
}
@Test
public void testCheckOutputColumns(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer) {
Map<Long, List<Column>> candidateIndexIdToSchema = Maps.newHashMap();
List<Column> index1Columns = Lists.newArrayList();
Column index1Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index1Columns.add(index1Column1);
candidateIndexIdToSchema.put(new Long(1), index1Columns);
List<Column> index2Columns = Lists.newArrayList();
Column index2Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index2Columns.add(index2Column1);
Column index2Column2 = new Column("c2", Type.INT, false, AggregateType.NONE, true, "", "");
index2Columns.add(index2Column2);
candidateIndexIdToSchema.put(new Long(2), index2Columns);
List<Column> index3Columns = Lists.newArrayList();
Column index3Column1 = new Column("C2", Type.INT, true, null, true, "", "");
index3Columns.add(index3Column1);
Column index3Column2 = new Column("c1", Type.INT, false, AggregateType.SUM, true, "", "");
index3Columns.add(index3Column2);
candidateIndexIdToSchema.put(new Long(3), index3Columns);
new Expectations() {
{
selectStmt.getAggInfo();
result = null;
selectStmt.getResultExprs();
result = Lists.newArrayList();
}
};
MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer);
Set<String> columnNamesInQueryOutput = Sets.newHashSet();
columnNamesInQueryOutput.add("c1");
columnNamesInQueryOutput.add("c2");
Deencapsulation.invoke(selector, "checkOutputColumns", columnNamesInQueryOutput,
candidateIndexIdToSchema);
Assert.assertEquals(2, candidateIndexIdToSchema.size());
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(2)));
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(3)));
}
@Test
public void testCompensateIndex(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer) {
Map<Long, List<Column>> candidateIndexIdToSchema = Maps.newHashMap();
Map<Long, List<Column>> allVisibleIndexes = Maps.newHashMap();
List<Column> index1Columns = Lists.newArrayList();
Column index1Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index1Columns.add(index1Column1);
allVisibleIndexes.put(new Long(1), index1Columns);
List<Column> index2Columns = Lists.newArrayList();
Column index2Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index2Columns.add(index2Column1);
Column index2Column2 = new Column("c2", Type.INT, false, AggregateType.NONE, true, "", "");
index2Columns.add(index2Column2);
allVisibleIndexes.put(new Long(2), index2Columns);
List<Column> index3Columns = Lists.newArrayList();
Column index3Column1 = new Column("C2", Type.INT, true, null, true, "", "");
index3Columns.add(index3Column1);
Column index3Column2 = new Column("c1", Type.INT, false, AggregateType.SUM, true, "", "");
index3Columns.add(index3Column2);
allVisibleIndexes.put(new Long(3), index3Columns);
new Expectations() {
{
selectStmt.getAggInfo();
result = null;
selectStmt.getResultExprs();
result = Lists.newArrayList();
}
};
MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer);
Deencapsulation.invoke(selector, "compensateIndex", candidateIndexIdToSchema,
allVisibleIndexes, 2);
Assert.assertEquals(2, candidateIndexIdToSchema.size());
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(2)));
Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(3)));
}
@Test
public void testSelectBestRowCountIndex(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer) {
Map<Long, List<Column>> candidateIndexIdToSchema = Maps.newHashMap();
List<Column> index1Columns = Lists.newArrayList();
Column index1Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index1Columns.add(index1Column1);
Column index1Column2 = new Column("c2", Type.INT, false, AggregateType.NONE, true, "", "");
index1Columns.add(index1Column2);
Column index1Column3 = new Column("c3", Type.INT, false, AggregateType.NONE, true, "", "");
index1Columns.add(index1Column3);
candidateIndexIdToSchema.put(new Long(1), index1Columns);
List<Column> index2Columns = Lists.newArrayList();
Column index2Column1 = new Column("c2", Type.INT, true, null, true, "", "");
index2Columns.add(index2Column1);
Column index2Column2 = new Column("c1", Type.INT, false, AggregateType.NONE, true, "", "");
index2Columns.add(index2Column2);
Column index2Column3 = new Column("c3", Type.INT, false, AggregateType.NONE, true, "", "");
index2Columns.add(index2Column3);
candidateIndexIdToSchema.put(new Long(2), index2Columns);
List<Column> index3Columns = Lists.newArrayList();
Column index3Column1 = new Column("c1", Type.INT, true, null, true, "", "");
index3Columns.add(index3Column1);
Column index3Column2 = new Column("c3", Type.INT, false, AggregateType.NONE, true, "", "");
index3Columns.add(index3Column2);
Column index3Column3 = new Column("c2", Type.INT, false, AggregateType.NONE, true, "", "");
index3Columns.add(index3Column3);
candidateIndexIdToSchema.put(new Long(3), index3Columns);
new Expectations() {
{
selectStmt.getAggInfo();
result = null;
selectStmt.getResultExprs();
result = Lists.newArrayList();
}
};
Set<String> equivalenceColumns = Sets.newHashSet();
equivalenceColumns.add("c1");
equivalenceColumns.add("c2");
Set<String> unequivalenceColumns = Sets.newHashSet();
unequivalenceColumns.add("c3");
MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer);
Set<Long> result = Deencapsulation.invoke(selector, "matchBestPrefixIndex", candidateIndexIdToSchema,
equivalenceColumns, unequivalenceColumns);
Assert.assertEquals(2, result.size());
Assert.assertTrue(result.contains(new Long(1)));
Assert.assertTrue(result.contains(new Long(2)));
}
}

5
fe/src/test/java/org/apache/doris/utframe/AnotherDemoTest.java
View File

@ -117,10 +117,7 @@ public class AnotherDemoTest {
@AfterClass
public static void TearDown() {
try {
FileUtils.deleteDirectory(new File(runningDirBase));
} catch (IOException e) {
}
UtFrameUtils.cleanDorisFeDir(runningDirBase);
}
// generate all port from valid ports

73
fe/src/test/java/org/apache/doris/utframe/DemoTest.java
View File

@ -26,24 +26,15 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.utframe.MockedBackendFactory.DefaultBeThriftServiceImpl;
import org.apache.doris.utframe.MockedBackendFactory.DefaultHeartbeatServiceImpl;
import org.apache.doris.utframe.MockedBackendFactory.DefaultPBackendServiceImpl;
import org.apache.doris.utframe.MockedFrontend.EnvVarNotSetException;
import org.apache.doris.utframe.MockedFrontend.FeStartException;
import org.apache.doris.utframe.MockedFrontend.NotInitException;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -51,7 +42,6 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -66,15 +56,6 @@ import java.util.UUID;
*/
public class DemoTest {
private static int fe_http_port;
private static int fe_rpc_port;
private static int fe_query_port;
private static int fe_edit_log_port;
private static int be_heartbeat_port;
private static int be_thrift_port;
private static int be_brpc_port;
private static int be_http_port;
// use a unique dir so that it won't be conflict with other unit test which
// may also start a Mocked Frontend
private static String runningDirBase = "fe";
@ -83,63 +64,13 @@ public class DemoTest {
@BeforeClass
public static void beforeClass() throws EnvVarNotSetException, IOException,
FeStartException, NotInitException, DdlException, InterruptedException {
// get DORIS_HOME
String dorisHome = System.getenv("DORIS_HOME");
if (Strings.isNullOrEmpty(dorisHome)) {
dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString();
}
getPorts();
// start fe in "DORIS_HOME/fe/mocked/"
MockedFrontend frontend = MockedFrontend.getInstance();
Map<String, String> feConfMap = Maps.newHashMap();
// set additional fe config
feConfMap.put("http_port", String.valueOf(fe_http_port));
feConfMap.put("rpc_port", String.valueOf(fe_rpc_port));
feConfMap.put("query_port", String.valueOf(fe_query_port));
feConfMap.put("edit_log_port", String.valueOf(fe_edit_log_port));
feConfMap.put("tablet_create_timeout_second", "10");
frontend.init(dorisHome + "/" + runningDir, feConfMap);
frontend.start(new String[0]);
// start be
MockedBackend backend = MockedBackendFactory.createBackend("127.0.0.1",
be_heartbeat_port, be_thrift_port, be_brpc_port, be_http_port,
new DefaultHeartbeatServiceImpl(be_thrift_port, be_http_port, be_brpc_port),
new DefaultBeThriftServiceImpl(), new DefaultPBackendServiceImpl());
backend.setFeAddress(new TNetworkAddress("127.0.0.1", frontend.getRpcPort()));
backend.start();
// add be
List<Pair<String, Integer>> bes = Lists.newArrayList();
bes.add(Pair.create(backend.getHost(), backend.getHeartbeatPort()));
Catalog.getCurrentSystemInfo().addBackends(bes, false, "default_cluster");
// sleep to wait first heartbeat
Thread.sleep(6000);
UtFrameUtils.createMinDorisCluster(runningDir);
}
@AfterClass
public static void TearDown() {
try {
FileUtils.deleteDirectory(new File(runningDirBase));
} catch (IOException e) {
}
}
// generate all port from valid ports
private static void getPorts() {
fe_http_port = UtFrameUtils.findValidPort();
fe_rpc_port = UtFrameUtils.findValidPort();
fe_query_port = UtFrameUtils.findValidPort();
fe_edit_log_port = UtFrameUtils.findValidPort();
be_heartbeat_port = UtFrameUtils.findValidPort();
be_thrift_port = UtFrameUtils.findValidPort();
be_brpc_port = UtFrameUtils.findValidPort();
be_http_port = UtFrameUtils.findValidPort();
UtFrameUtils.cleanDorisFeDir(runningDirBase);
}
@Test

134
fe/src/test/java/org/apache/doris/utframe/DorisAssert.java Normal file
View File

@ -0,0 +1,134 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.utframe;
import org.apache.doris.alter.AlterJobV2;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
import java.io.IOException;
import java.util.Map;
import java.util.stream.Stream;
public class DorisAssert {
private ConnectContext ctx;
public DorisAssert() throws IOException {
this.ctx = UtFrameUtils.createDefaultCtx();
}
public DorisAssert withEnableMV() {
ctx.getSessionVariable().setTestMaterializedView(true);
Config.enable_materialized_view = true;
return this;
}
public DorisAssert withDatabase(String dbName) throws Exception {
CreateDbStmt createDbStmt =
(CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt("create database " + dbName + ";", ctx);
Catalog.getCurrentCatalog().createDb(createDbStmt);
return this;
}
public DorisAssert useDatabase(String dbName) {
ctx.setDatabase(ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, dbName));
return this;
}
public DorisAssert withTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
return this;
}
public DorisAssert dropTable(String tableName) throws Exception {
DropTableStmt dropTableStmt =
(DropTableStmt) UtFrameUtils.parseAndAnalyzeStmt("drop table " + tableName + ";", ctx);
Catalog.getCurrentCatalog().dropTable(dropTableStmt);
return this;
}
// Add materialized view to the schema
public DorisAssert withMaterializedView(String sql) throws Exception {
CreateMaterializedViewStmt createMaterializedViewStmt =
(CreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
Catalog.getCurrentCatalog().createMaterializedView(createMaterializedViewStmt);
// check alter job
Map<Long, AlterJobV2> alterJobs = Catalog.getCurrentCatalog().getRollupHandler().getAlterJobsV2();
for (AlterJobV2 alterJobV2 : alterJobs.values()) {
while (!alterJobV2.getJobState().isFinalState()) {
System.out.println("alter job " + alterJobV2.getDbId() + " is running. state: " + alterJobV2.getJobState());
Thread.sleep(5000);
}
System.out.println("alter job " + alterJobV2.getDbId() + " is done. state: " + alterJobV2.getJobState());
Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState());
}
// waiting table state to normal
Thread.sleep(5000);
return this;
}
public QueryAssert query(String sql) {
return new QueryAssert(ctx, sql);
}
public class QueryAssert {
private ConnectContext connectContext;
private String sql;
public QueryAssert(ConnectContext connectContext, String sql) {
this.connectContext = connectContext;
this.sql = sql;
}
public void explainContains(String... keywords) throws Exception {
Assert.assertTrue(Stream.of(keywords).allMatch(explainQuery()::contains));
}
public void explainContains(String keywords, int count) throws Exception {
Assert.assertTrue(StringUtils.countMatches(explainQuery(), keywords) == count);
}
public void explainWithout(String s) throws Exception {
Assert.assertFalse(explainQuery().contains(s));
}
private String explainQuery() throws Exception {
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, "explain " + sql);
stmtExecutor.execute();
Planner planner = stmtExecutor.planner();
String explainString = planner.getExplainString(planner.getFragments(), TExplainLevel.VERBOSE);
System.out.println(explainString);
return explainString;
}
}
}

2
fe/src/test/java/org/apache/doris/utframe/MockedFrontend.java
View File

@ -195,7 +195,7 @@ public class MockedFrontend {
Thread feThread = new Thread(new FERunnable(this, args), FE_PROCESS);
feThread.start();
// wait the catalog to be ready until timeout (30 seconds)
waitForCatalogReady(30 * 1000);
waitForCatalogReady(120 * 1000);
System.out.println("Fe process is started");
}

12
fe/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
View File

@ -44,6 +44,9 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.net.ServerSocket;
@ -52,7 +55,9 @@ import java.nio.file.Files;
import java.util.List;
import java.util.Map;
public class UtFrameUtils {
// Help to create a mocked ConnectContext.
public static ConnectContext createDefaultCtx() throws IOException {
SocketChannel channel = SocketChannel.open();
@ -131,6 +136,13 @@ public class UtFrameUtils {
Thread.sleep(6000);
}
public static void cleanDorisFeDir(String baseDir) {
try {
FileUtils.deleteDirectory(new File(baseDir));
} catch (IOException e) {
}
}
public static int findValidPort() {
ServerSocket socket = null;
try {