database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Forbidden float column in short key (#3812)

Browse Source 
* Forbidden float column in short key

When the user does not specify the short key column, doris will automatically supplement the short key column.
However, doris does not support float or double as the short key column, so when adding the short key column, doris should avoid setting those column as the key column.
The short key columns must be less then 3 columns and less then 36 bytes.

The CreateMaterailizedView, AddRollup and CreateDuplicateTable need to forbidden float column in short key.
If the float column is directly encountered during the supplement process, the subsequent columns are all value columns.

Also the float and double could not be the short key column. At the same time, Doris must be at least one short key column.
So the type of first column could not be float or double.
If the varchar is the short key column, it can only be the least one short key column.

Fixed #3811

For duplicate table without order by columns, the order by columns are same as short key columns.
If the order by columns have been designated, the count of short key columns must be <= the count of order by columns.
This commit is contained in:
EmmyMiao87
2020-06-17 14:16:48 +08:00
committed by GitHub
parent e9f7576b9d
commit a62cebfccf
10 changed files with 507 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
@ -481,7 +482,7 @@ public class MaterializedViewHandler extends AlterHandler {
public List<Column> checkAndPrepareMaterializedView(AddRollupClause addRollupClause, OlapTable olapTable,
long baseIndexId, boolean changeStorageFormat)
throws DdlException {
throws DdlException{
String rollupIndexName = addRollupClause.getRollupName();
List<String> rollupColumnNames = addRollupClause.getColumnNames();
if (changeStorageFormat) {
@ -554,28 +555,52 @@ public class MaterializedViewHandler extends AlterHandler {
} else if (KeysType.DUP_KEYS == keysType) {
// supplement the duplicate key
if (addRollupClause.getDupKeys() == null || addRollupClause.getDupKeys().isEmpty()) {
int keyStorageLayoutBytes = 0;
// check the column meta
for (int i = 0; i < rollupColumnNames.size(); i++) {
String columnName = rollupColumnNames.get(i);
Column baseColumn = baseColumnNameToColumn.get(columnName);
if (baseColumn == null) {
throw new DdlException("Column[" + columnName + "] does not exist in base index");
}
keyStorageLayoutBytes += baseColumn.getType().getStorageLayoutBytes();
Column rollupColumn = new Column(baseColumn);
if(changeStorageFormat) {
rollupColumn.setIsKey(baseColumn.isKey());
rollupColumn.setAggregationType(baseColumn.getAggregationType(), true);
} else if ((i + 1) <= FeConstants.shortkey_max_column_count
|| keyStorageLayoutBytes < FeConstants.shortkey_maxsize_bytes) {
rollupColumn.setIsKey(true);
rollupColumn.setAggregationType(null, false);
} else {
rollupColumn.setIsKey(false);
rollupColumn.setAggregationType(AggregateType.NONE, true);
}
rollupSchema.add(rollupColumn);
}
if (changeStorageFormat) {
return rollupSchema;
}
// Supplement key of MV columns
int theBeginIndexOfValue = 0;
int keySizeByte = 0;
for (; theBeginIndexOfValue < rollupSchema.size(); theBeginIndexOfValue++) {
Column column = rollupSchema.get(theBeginIndexOfValue);
keySizeByte += column.getType().getIndexSize();
if (theBeginIndexOfValue + 1 > FeConstants.shortkey_max_column_count
|| keySizeByte > FeConstants.shortkey_maxsize_bytes) {
if (theBeginIndexOfValue == 0 && column.getType().getPrimitiveType().isCharFamily()) {
column.setIsKey(true);
theBeginIndexOfValue++;
}
break;
}
if (column.getType().isFloatingPointType()) {
break;
}
if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
column.setIsKey(true);
theBeginIndexOfValue++;
break;
}
column.setIsKey(true);
}
if (theBeginIndexOfValue == 0) {
throw new DdlException("The first column could not be float or double");
}
// Supplement value of MV columns
for (; theBeginIndexOfValue < rollupSchema.size(); theBeginIndexOfValue++) {
Column rollupColumn = rollupSchema.get(theBeginIndexOfValue);
rollupColumn.setIsKey(false);
rollupColumn.setAggregationType(AggregateType.NONE, true);
}
} else {
/*
* eg.

129
fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
@ -69,8 +70,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
private String dbName;
private KeysType mvKeysType = KeysType.DUP_KEYS;
public CreateMaterializedViewStmt(String mvName, SelectStmt selectStmt,
Map<String, String> properties) {
public CreateMaterializedViewStmt(String mvName, SelectStmt selectStmt, Map<String, String> properties) {
this.mvName = mvName;
this.selectStmt = selectStmt;
this.properties = properties;
@ -116,16 +116,16 @@ public class CreateMaterializedViewStmt extends DdlStmt {
analyzeFromClause();
if (selectStmt.getWhereClause() != null) {
throw new AnalysisException("The where clause is not supported in add materialized view clause, expr:"
+ selectStmt.getWhereClause().toSql());
+ selectStmt.getWhereClause().toSql());
}
if (selectStmt.getHavingPred() != null) {
throw new AnalysisException("The having clause is not supported in add materialized view clause, expr:"
+ selectStmt.getHavingPred().toSql());
+ selectStmt.getHavingPred().toSql());
}
analyzeOrderByClause();
if (selectStmt.getLimit() != -1) {
throw new AnalysisException("The limit clause is not supported in add materialized view clause, expr:"
+ " limit " + selectStmt.getLimit());
+ " limit " + selectStmt.getLimit());
}
}
@ -151,7 +151,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr selectListItemExpr = selectListItem.getExpr();
if (!(selectListItemExpr instanceof SlotRef) && !(selectListItemExpr instanceof FunctionCallExpr)) {
throw new AnalysisException("The materialized view only support the single column or function expr. "
+ "Error column: " + selectListItemExpr.toSql());
+ "Error column: " + selectListItemExpr.toSql());
}
if (selectListItem.getExpr() instanceof SlotRef) {
if (meetAggregate) {
@ -164,6 +164,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnName);
}
MVColumnItem mvColumnItem = new MVColumnItem(columnName);
mvColumnItem.setType(slotRef.getType());
mvColumnItemList.add(mvColumnItem);
} else if (selectListItem.getExpr() instanceof FunctionCallExpr) {
FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItem.getExpr();
@ -174,7 +175,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
&& !functionName.equalsIgnoreCase("min")
&& !functionName.equalsIgnoreCase("max")) {
throw new AnalysisException("The materialized view only support the sum, min and max aggregate "
+ "function. Error function: " + functionCallExpr.toSqlImpl());
+ "function. Error function: " + functionCallExpr.toSqlImpl());
}
Preconditions.checkState(functionCallExpr.getChildren().size() == 1);
@ -182,13 +183,11 @@ public class CreateMaterializedViewStmt extends DdlStmt {
SlotRef slotRef;
if (functionChild0 instanceof SlotRef) {
slotRef = (SlotRef) functionChild0;
}
else if (functionChild0 instanceof CastExpr
&& (functionChild0.getChild(0) instanceof SlotRef)) {
} else if (functionChild0 instanceof CastExpr && (functionChild0.getChild(0) instanceof SlotRef)) {
slotRef = (SlotRef) functionChild0.getChild(0);
} else {
throw new AnalysisException("The children of aggregate function only support one original column. "
+ "Error function: " + functionCallExpr.toSqlImpl());
+ "Error function: " + functionCallExpr.toSqlImpl());
}
meetAggregate = true;
// check duplicate column
@ -226,49 +225,14 @@ public class CreateMaterializedViewStmt extends DdlStmt {
private void analyzeOrderByClause() throws AnalysisException {
if (selectStmt.getOrderByElements() == null) {
/**
* The keys type of Materialized view is aggregation.
* All of group by columns are keys of materialized view.
*/
if (mvKeysType == KeysType.AGG_KEYS) {
for (MVColumnItem mvColumnItem : mvColumnItemList) {
if (mvColumnItem.getAggregationType() != null) {
break;
}
mvColumnItem.setIsKey(true);
}
return;
}
/**
* There is no aggregation function in materialized view.
* Supplement key of MV columns
* For example: select k1, k2 ... kn from t1
* The default key columns are first 36 bytes of the columns in define order.
* If the number of columns in the first 36 is less than 3, the first 3 columns will be used.
* column: k1, k2, k3... km. The key is true.
* Supplement non-key of MV columns
* column: km... kn. The key is false, aggregation type is none, isAggregationTypeImplicit is true.
*/
int keyStorageLayoutBytes = 0;
for (int i = 0; i < selectStmt.getResultExprs().size(); i++) {
MVColumnItem mvColumnItem = mvColumnItemList.get(i);
Expr resultColumn = selectStmt.getResultExprs().get(i);
keyStorageLayoutBytes += resultColumn.getType().getStorageLayoutBytes();
if ((i + 1) <= FeConstants.shortkey_max_column_count
|| keyStorageLayoutBytes < FeConstants.shortkey_maxsize_bytes) {
mvColumnItem.setIsKey(true);
} else {
mvColumnItem.setAggregationType(AggregateType.NONE, true);
}
}
supplyOrderColumn();
return;
}
List<OrderByElement> orderByElements = selectStmt.getOrderByElements();
if (orderByElements.size() > mvColumnItemList.size()) {
throw new AnalysisException("The number of columns in order clause must be less then "
+ "the number of columns in select clause");
throw new AnalysisException("The number of columns in order clause must be less then " + "the number of "
+ "columns in select clause");
}
if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) {
throw new AnalysisException("The key of columns in mv must be all of group by columns");
@ -277,13 +241,13 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr orderByElement = orderByElements.get(i).getExpr();
if (!(orderByElement instanceof SlotRef)) {
throw new AnalysisException("The column in order clause must be original column without calculation. "
+ "Error column: " + orderByElement.toSql());
+ "Error column: " + orderByElement.toSql());
}
MVColumnItem mvColumnItem = mvColumnItemList.get(i);
SlotRef slotRef = (SlotRef) orderByElement;
if (!mvColumnItem.getName().equalsIgnoreCase(slotRef.getColumnName())) {
throw new AnalysisException("The order of columns in order by clause must be same as "
+ "the order of columns in select list");
+ "the order of columns in select list");
}
Preconditions.checkState(mvColumnItem.getAggregationType() == null);
mvColumnItem.setIsKey(true);
@ -301,6 +265,69 @@ public class CreateMaterializedViewStmt extends DdlStmt {
}
}
/*
This function is used to supply order by columns and calculate short key count
*/
private void supplyOrderColumn() throws AnalysisException {
/**
* The keys type of Materialized view is aggregation.
* All of group by columns are keys of materialized view.
*/
if (mvKeysType == KeysType.AGG_KEYS) {
for (MVColumnItem mvColumnItem : mvColumnItemList) {
if (mvColumnItem.getAggregationType() != null) {
break;
}
mvColumnItem.setIsKey(true);
}
} else if (mvKeysType == KeysType.DUP_KEYS) {
/**
* There is no aggregation function in materialized view.
* Supplement key of MV columns
* The key is same as the short key in duplicate table
* For example: select k1, k2 ... kn from t1
* The default key columns are first 36 bytes of the columns in define order.
* If the number of columns in the first 36 is more than 3, the first 3 columns will be used.
* column: k1, k2, k3. The key is true.
* Supplement non-key of MV columns
* column: k4... kn. The key is false, aggregation type is none, isAggregationTypeImplicit is true.
*/
int theBeginIndexOfValue = 0;
// supply key
int keySizeByte = 0;
for (; theBeginIndexOfValue < mvColumnItemList.size(); theBeginIndexOfValue++) {
MVColumnItem column = mvColumnItemList.get(theBeginIndexOfValue);
keySizeByte += column.getType().getIndexSize();
if (theBeginIndexOfValue + 1 > FeConstants.shortkey_max_column_count
|| keySizeByte > FeConstants.shortkey_maxsize_bytes) {
if (theBeginIndexOfValue == 0 && column.getType().getPrimitiveType().isCharFamily()) {
column.setIsKey(true);
theBeginIndexOfValue++;
}
break;
}
if (column.getType().isFloatingPointType()) {
break;
}
if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
column.setIsKey(true);
theBeginIndexOfValue++;
break;
}
column.setIsKey(true);
}
if (theBeginIndexOfValue == 0) {
throw new AnalysisException("The first column could not be float or double type, use decimal instead");
}
// supply value
for (; theBeginIndexOfValue < mvColumnItemList.size(); theBeginIndexOfValue++) {
MVColumnItem mvColumnItem = mvColumnItemList.get(theBeginIndexOfValue);
mvColumnItem.setAggregationType(AggregateType.NONE, true);
}
}
}
@Override
public String toSql() {
return null;

27
fe/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
@ -283,11 +284,29 @@ public class CreateTableStmt extends DdlStmt {
keysDesc = new KeysDesc(KeysType.AGG_KEYS, keysColumnNames);
} else {
for (ColumnDef columnDef : columnDefs) {
keyLength += columnDef.getType().getStorageLayoutBytes();
if (keysColumnNames.size() < FeConstants.shortkey_max_column_count
|| keyLength < FeConstants.shortkey_maxsize_bytes) {
keysColumnNames.add(columnDef.getName());
keyLength += columnDef.getType().getIndexSize();
if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes) {
if (keysColumnNames.size() == 0
&& columnDef.getType().getPrimitiveType().isCharFamily()) {
keysColumnNames.add(columnDef.getName());
}
break;
}
if (columnDef.getType().isFloatingPointType()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
keysColumnNames.add(columnDef.getName());
break;
}
keysColumnNames.add(columnDef.getName());
}
// The OLAP table must has at least one short key and the float and double should not be short key.
// So the float and double could not be the first column in OLAP table.
if (keysColumnNames.isEmpty()) {
throw new AnalysisException("The first column could not be float or double,"
+ " use decimal instead.");
}
keysDesc = new KeysDesc(KeysType.DUP_KEYS, keysColumnNames);
}

11
fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Type;
/**
* This is a result of semantic analysis for AddMaterializedViewClause.
@ -27,6 +28,8 @@ import org.apache.doris.catalog.AggregateType;
*/
public class MVColumnItem {
private String name;
// the origin type of slot ref
private Type type;
private boolean isKey;
private AggregateType aggregationType;
private boolean isAggregationTypeImplicit;
@ -40,6 +43,14 @@ public class MVColumnItem {
return name;
}
public Type getType() {
return type;
}
public void setType(Type type) {
this.type = type;
}
public void setIsKey(boolean key) {
isKey = key;
}

37
fe/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -4879,29 +4879,30 @@ public class Catalog {
* contains at most one VARCHAR column. And if contains, it should
* be at the last position of the short key list.
*/
shortKeyColumnCount = 1;
shortKeyColumnCount = 0;
int shortKeySizeByte = 0;
Column firstColumn = indexColumns.get(0);
if (firstColumn.getDataType() != PrimitiveType.VARCHAR) {
shortKeySizeByte = firstColumn.getOlapColumnIndexSize();
int maxShortKeyColumnCount = Math.min(indexColumns.size(), FeConstants.shortkey_max_column_count);
for (int i = 1; i < maxShortKeyColumnCount; i++) {
Column column = indexColumns.get(i);
shortKeySizeByte += column.getOlapColumnIndexSize();
if (shortKeySizeByte > FeConstants.shortkey_maxsize_bytes) {
break;
}
if (column.getDataType() == PrimitiveType.VARCHAR) {
int maxShortKeyColumnCount = Math.min(indexColumns.size(), FeConstants.shortkey_max_column_count);
for (int i = 0; i < maxShortKeyColumnCount; i++) {
Column column = indexColumns.get(i);
shortKeySizeByte += column.getOlapColumnIndexSize();
if (shortKeySizeByte > FeConstants.shortkey_maxsize_bytes) {
if (column.getDataType().isCharFamily()) {
++shortKeyColumnCount;
break;
}
++shortKeyColumnCount;
break;
}
if (column.getType().isFloatingPointType()) {
break;
}
if (column.getDataType() == PrimitiveType.VARCHAR) {
++shortKeyColumnCount;
break;
}
++shortKeyColumnCount;
}
if (shortKeyColumnCount == 0) {
throw new DdlException("The first column could not be float or double type, use decimal instead");
}
// else
// first column type is VARCHAR
// use only first column as shortKey
// do nothing here
} // end calc shortKeyColumnCount

8
fe/src/main/java/org/apache/doris/catalog/PrimitiveType.java
View File

@ -17,15 +17,15 @@
package org.apache.doris.catalog;
import java.util.List;
import org.apache.doris.mysql.MysqlColType;
import org.apache.doris.thrift.TPrimitiveType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
public enum PrimitiveType {
INVALID_TYPE("INVALID_TYPE", -1, TPrimitiveType.INVALID_TYPE),
@ -671,6 +671,10 @@ public enum PrimitiveType {
return (this == VARCHAR || this == CHAR || this == HLL);
}
public boolean isCharFamily() {
return (this == VARCHAR || this == CHAR);
}
public boolean isIntegerType() {
return (this == TINYINT || this == SMALLINT
|| this == INT || this == BIGINT);

8
fe/src/main/java/org/apache/doris/catalog/Type.java
View File

@ -1030,4 +1030,12 @@ public abstract class Type {
public int getStorageLayoutBytes() {
return 0;
}
public int getIndexSize() {
if (this.getPrimitiveType() == PrimitiveType.CHAR) {
return ((ScalarType) this).getLength();
} else {
return this.getPrimitiveType().getOlapColumnIndexSize();
}
}
}

323
fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
@ -584,15 +585,19 @@ public class CreateMaterializedViewStmtTest {
result = columnName3;
slotRef4.getColumnName();
result = columnName4;
selectStmt.getResultExprs();
result = Lists.newArrayList(slotRef1, slotRef2, slotRef3, slotRef4);
slotRef1.getType().getStorageLayoutBytes();
result = 35;
slotRef2.getType().getStorageLayoutBytes();
result = 2;
slotRef3.getType().getStorageLayoutBytes();
result = 3;
slotRef4.getType().getStorageLayoutBytes();
slotRef1.getType().getIndexSize();
result = 34;
slotRef1.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef2.getType().getIndexSize();
result = 1;
slotRef2.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef3.getType().getIndexSize();
result = 1;
slotRef3.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef4.getType().getIndexSize();
result = 4;
selectStmt.getAggInfo(); // return null, so that the mv can be a duplicate mv
result = null;
@ -631,6 +636,306 @@ public class CreateMaterializedViewStmtTest {
}
}
/*
ISSUE: #3811
*/
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregationWithFloat(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
SelectListItem selectListItem2 = new SelectListItem(slotRef2, null);
selectList.addItem(selectListItem2);
SelectListItem selectListItem3 = new SelectListItem(slotRef3, null);
selectList.addItem(selectListItem3);
SelectListItem selectListItem4 = new SelectListItem(slotRef4, null);
selectList.addItem(selectListItem4);
final String columnName1 = "k1";
final String columnName2 = "k2";
final String columnName3 = "v1";
final String columnName4 = "v2";
new Expectations() {
{
analyzer.getClusterName();
result = "default";
selectStmt.getAggInfo();
result = null;
selectStmt.getSelectList();
result = selectList;
selectStmt.getTableRefs();
result = Lists.newArrayList(tableRef);
selectStmt.getWhereClause();
result = null;
selectStmt.getHavingPred();
result = null;
selectStmt.getOrderByElements();
result = null;
selectStmt.getLimit();
result = -1;
selectStmt.analyze(analyzer);
slotRef1.getColumnName();
result = columnName1;
slotRef2.getColumnName();
result = columnName2;
slotRef3.getColumnName();
result = columnName3;
slotRef4.getColumnName();
result = columnName4;
slotRef1.getType().getIndexSize();
result = 1;
slotRef1.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef2.getType().getIndexSize();
result = 2;
slotRef2.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef3.getType().getIndexSize();
result = 3;
slotRef3.getType().isFloatingPointType();
result = true;
selectStmt.getAggInfo(); // return null, so that the mv can be a duplicate mv
result = null;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
try {
createMaterializedViewStmt.analyze(analyzer);
Assert.assertEquals(KeysType.DUP_KEYS, createMaterializedViewStmt.getMVKeysType());
List<MVColumnItem> mvColumns = createMaterializedViewStmt.getMVColumnItemList();
Assert.assertEquals(4, mvColumns.size());
MVColumnItem mvColumn0 = mvColumns.get(0);
Assert.assertTrue(mvColumn0.isKey());
Assert.assertFalse(mvColumn0.isAggregationTypeImplicit());
Assert.assertEquals(columnName1, mvColumn0.getName());
Assert.assertEquals(null, mvColumn0.getAggregationType());
MVColumnItem mvColumn1 = mvColumns.get(1);
Assert.assertTrue(mvColumn1.isKey());
Assert.assertFalse(mvColumn1.isAggregationTypeImplicit());
Assert.assertEquals(columnName2, mvColumn1.getName());
Assert.assertEquals(null, mvColumn1.getAggregationType());
MVColumnItem mvColumn2 = mvColumns.get(2);
Assert.assertFalse(mvColumn2.isKey());
Assert.assertTrue(mvColumn2.isAggregationTypeImplicit());
Assert.assertEquals(columnName3, mvColumn2.getName());
Assert.assertEquals(AggregateType.NONE, mvColumn2.getAggregationType());
MVColumnItem mvColumn3 = mvColumns.get(3);
Assert.assertFalse(mvColumn3.isKey());
Assert.assertTrue(mvColumn3.isAggregationTypeImplicit());
Assert.assertEquals(columnName4, mvColumn3.getName());
Assert.assertEquals(AggregateType.NONE, mvColumn3.getAggregationType());
} catch (UserException e) {
Assert.fail(e.getMessage());
}
}
/*
ISSUE: #3811
*/
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregationWithVarchar(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
SelectListItem selectListItem2 = new SelectListItem(slotRef2, null);
selectList.addItem(selectListItem2);
SelectListItem selectListItem3 = new SelectListItem(slotRef3, null);
selectList.addItem(selectListItem3);
SelectListItem selectListItem4 = new SelectListItem(slotRef4, null);
selectList.addItem(selectListItem4);
final String columnName1 = "k1";
final String columnName2 = "k2";
final String columnName3 = "v1";
final String columnName4 = "v2";
new Expectations() {
{
analyzer.getClusterName();
result = "default";
selectStmt.getAggInfo();
result = null;
selectStmt.getSelectList();
result = selectList;
selectStmt.getTableRefs();
result = Lists.newArrayList(tableRef);
selectStmt.getWhereClause();
result = null;
selectStmt.getHavingPred();
result = null;
selectStmt.getOrderByElements();
result = null;
selectStmt.getLimit();
result = -1;
selectStmt.analyze(analyzer);
slotRef1.getColumnName();
result = columnName1;
slotRef2.getColumnName();
result = columnName2;
slotRef3.getColumnName();
result = columnName3;
slotRef4.getColumnName();
result = columnName4;
slotRef1.getType().getIndexSize();
result = 1;
slotRef1.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef2.getType().getIndexSize();
result = 2;
slotRef2.getType().getPrimitiveType();
result = PrimitiveType.INT;
slotRef3.getType().getIndexSize();
result = 3;
slotRef3.getType().getPrimitiveType();
result = PrimitiveType.VARCHAR;
selectStmt.getAggInfo(); // return null, so that the mv can be a duplicate mv
result = null;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
try {
createMaterializedViewStmt.analyze(analyzer);
Assert.assertEquals(KeysType.DUP_KEYS, createMaterializedViewStmt.getMVKeysType());
List<MVColumnItem> mvColumns = createMaterializedViewStmt.getMVColumnItemList();
Assert.assertEquals(4, mvColumns.size());
MVColumnItem mvColumn0 = mvColumns.get(0);
Assert.assertTrue(mvColumn0.isKey());
Assert.assertFalse(mvColumn0.isAggregationTypeImplicit());
Assert.assertEquals(columnName1, mvColumn0.getName());
Assert.assertEquals(null, mvColumn0.getAggregationType());
MVColumnItem mvColumn1 = mvColumns.get(1);
Assert.assertTrue(mvColumn1.isKey());
Assert.assertFalse(mvColumn1.isAggregationTypeImplicit());
Assert.assertEquals(columnName2, mvColumn1.getName());
Assert.assertEquals(null, mvColumn1.getAggregationType());
MVColumnItem mvColumn2 = mvColumns.get(2);
Assert.assertTrue(mvColumn2.isKey());
Assert.assertFalse(mvColumn2.isAggregationTypeImplicit());
Assert.assertEquals(columnName3, mvColumn2.getName());
Assert.assertEquals(null, mvColumn2.getAggregationType());
MVColumnItem mvColumn3 = mvColumns.get(3);
Assert.assertFalse(mvColumn3.isKey());
Assert.assertTrue(mvColumn3.isAggregationTypeImplicit());
Assert.assertEquals(columnName4, mvColumn3.getName());
Assert.assertEquals(AggregateType.NONE, mvColumn3.getAggregationType());
} catch (UserException e) {
Assert.fail(e.getMessage());
}
}
/*
ISSUE: #3811
*/
@Test
public void testMVColumnsWithFirstFloat(@Injectable SlotRef slotRef1,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
final String columnName1 = "k1";
new Expectations() {
{
analyzer.getClusterName();
result = "default";
selectStmt.getAggInfo();
result = null;
selectStmt.getSelectList();
result = selectList;
selectStmt.getTableRefs();
result = Lists.newArrayList(tableRef);
selectStmt.getWhereClause();
result = null;
selectStmt.getHavingPred();
result = null;
selectStmt.getOrderByElements();
result = null;
selectStmt.analyze(analyzer);
slotRef1.getColumnName();
result = columnName1;
slotRef1.getType().isFloatingPointType();
result = true;
selectStmt.getAggInfo(); // return null, so that the mv can be a duplicate mv
result = null;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
try {
createMaterializedViewStmt.analyze(analyzer);
Assert.fail("The first column could not be float or double, use decimal instead");
} catch (UserException e) {
System.out.print(e.getMessage());
}
}
/*
ISSUE: #3811
*/
@Test
public void testMVColumnsWithFirstVarchar(@Injectable SlotRef slotRef1,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
final String columnName1 = "k1";
new Expectations() {
{
analyzer.getClusterName();
result = "default";
selectStmt.getAggInfo();
result = null;
selectStmt.getSelectList();
result = selectList;
selectStmt.getTableRefs();
result = Lists.newArrayList(tableRef);
selectStmt.getWhereClause();
result = null;
selectStmt.getHavingPred();
result = null;
selectStmt.getOrderByElements();
result = null;
selectStmt.getLimit();
result = -1;
selectStmt.analyze(analyzer);
slotRef1.getColumnName();
result = columnName1;
slotRef1.getType().getPrimitiveType();
result = PrimitiveType.VARCHAR;
}
};
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
try {
createMaterializedViewStmt.analyze(analyzer);
Assert.assertEquals(KeysType.DUP_KEYS, createMaterializedViewStmt.getMVKeysType());
List<MVColumnItem> mvColumns = createMaterializedViewStmt.getMVColumnItemList();
Assert.assertEquals(1, mvColumns.size());
MVColumnItem mvColumn0 = mvColumns.get(0);
Assert.assertTrue(mvColumn0.isKey());
Assert.assertFalse(mvColumn0.isAggregationTypeImplicit());
Assert.assertEquals(columnName1, mvColumn0.getName());
Assert.assertEquals(null, mvColumn0.getAggregationType());
} catch (UserException e) {
Assert.fail(e.getMessage());
}
}
@Test
public void testMVColumns(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2,

6
fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -69,9 +69,9 @@ public class QueryPlanTest {
Catalog.getCurrentCatalog().createDb(createDbStmt);
createTable("create table test.test1\n" +
"(\n" +
" query_id varchar(48) comment \"Unique query id\",\n" +
" time datetime not null comment \"Query start time\",\n" +
"(\n" +
" time datetime not null comment \"Query start time\",\n" +
" query_id varchar(48) comment \"Unique query id\",\n" +
" client_ip varchar(32) comment \"Client IP\",\n" +
" user varchar(64) comment \"User name\",\n" +
" db varchar(96) comment \"Database of this query\",\n" +

8
fe/src/test/java/org/apache/doris/utframe/DorisAssert.java
View File

@ -141,11 +141,15 @@ public class DorisAssert {
}
public String explainQuery() throws Exception {
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, "explain " + sql);
return internalExecute("explain " + sql);
}
private String internalExecute(String sql) throws Exception {
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
stmtExecutor.execute();
QueryState queryState = connectContext.getState();
if (queryState.getStateType() == QueryState.MysqlStateType.ERR) {
switch (queryState.getErrType()){
switch (queryState.getErrType()) {
case ANALYSIS_ERR:
throw new AnalysisException(queryState.getErrorMessage());
case OTHER_ERR: