database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[SQL][Bug]fix semi/anti join error when table has delete sign column (#4498)

Browse Source 
It is possible to report "Illegal column/field reference'table2.DORIS_DELETE_SIGN' of semi-/anti-join"
when executing a semi/anti join statement on a table with hidden columns.
This is because the filter conditions of semi/anti join cannot added in the where statement.

Now we add delete flag related where predicate in OlapScanNode level.
This commit is contained in:
Zhengguo Yang
2020-09-03 17:15:15 +08:00
committed by GitHub
parent 1a30bcbf36
commit d0d394ad7e
4 changed files with 53 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -655,7 +655,7 @@ public class Analyzer {
// find table all name
for (TupleDescriptor desc : tupleByAlias.get(tblName.toString())) {
//result = desc;
if (!isVisible(desc.getId())) {
if (!colName.equalsIgnoreCase(Column.DELETE_SIGN) && !isVisible(desc.getId())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_ILLEGAL_COLUMN_REFERENCE_ERROR,
Joiner.on(".").join(tblName.getTbl(),colName));
}

17
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Type;
@ -340,7 +339,6 @@ public class SelectStmt extends QueryStmt {
return;
}
super.analyze(analyzer);
fromClause_.setNeedToSql(needToSql);
fromClause_.analyze(analyzer);
@ -533,21 +531,6 @@ public class SelectStmt extends QueryStmt {
whereClause = new BoolLiteral(true);
}
}
// filter deleted data by and DELETE_SIGN column = 0, DELETE_SIGN is a hidden column
// that indicates whether the row is delete
for (TableRef tableRef : fromClause_.getTableRefs()) {
if (!isForbiddenMVRewrite() && tableRef instanceof BaseTableRef && tableRef.getTable() instanceof OlapTable
&& ((OlapTable) tableRef.getTable()).getKeysType() == KeysType.UNIQUE_KEYS
&& ((OlapTable) tableRef.getTable()).hasDeleteSign()) {
BinaryPredicate filterDeleteExpr = new BinaryPredicate(BinaryPredicate.Operator.EQ,
new SlotRef(tableRef.getName(), Column.DELETE_SIGN), new IntLiteral(0));
if (whereClause == null) {
whereClause = filterDeleteExpr;
} else {
whereClause = new CompoundPredicate(CompoundPredicate.Operator.AND, filterDeleteExpr, whereClause);
}
}
}
Expr deDuplicatedWhere = deduplicateOrs(whereClause);
if (deDuplicatedWhere != null) {
whereClause = deDuplicatedWhere;

9
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.analysis.GroupByClause;
import org.apache.doris.analysis.GroupingInfo;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.InlineViewRef;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.JoinOperator;
import org.apache.doris.analysis.LiteralExpr;
@ -52,6 +53,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
@ -1361,6 +1363,13 @@ public class SingleNodePlanner {
switch (tblRef.getTable().getType()) {
case OLAP:
OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
if (((OlapTable) tblRef.getTable()).hasDeleteSign()) {
Expr conjunct = new BinaryPredicate(BinaryPredicate.Operator.EQ,
new SlotRef(tblRef.getAliasAsName(), Column.DELETE_SIGN), new IntLiteral(0));
conjunct.analyze(analyzer);
analyzer.registerConjunct(conjunct, tblRef.getDesc().getId());
}
olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened());
scanNode = olapNode;
break;

44
fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java
View File

@ -64,11 +64,42 @@ public class SelectStmtTest {
"\"storage_type\" = \"COLUMN\",\n" +
"\"replication_num\" = \"1\"\n" +
");";
String tbl1 = "CREATE TABLE db1.table1 (\n" +
" `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" +
" `citycode` smallint(6) NULL COMMENT \"\",\n" +
" `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" +
" `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"UNIQUE KEY(`siteid`, `citycode`, `username`)\n" +
"COMMENT \"OLAP\"\n" +
"DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\",\n" +
"\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\"\n" +
")";
String tbl2 = "CREATE TABLE db1.table2 (\n" +
" `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" +
" `citycode` smallint(6) NULL COMMENT \"\",\n" +
" `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" +
" `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"UNIQUE KEY(`siteid`, `citycode`, `username`)\n" +
"COMMENT \"OLAP\"\n" +
"DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\",\n" +
"\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\"\n" +
")";
dorisAssert = new DorisAssert();
dorisAssert.withDatabase("db1").useDatabase("db1");
dorisAssert.withTable(createTblStmtStr)
.withTable(createBaseAllStmtStr)
.withTable(createPratitionTableStr);
.withTable(createPratitionTableStr)
.withTable(tbl1)
.withTable(tbl2);
}
@Test
@ -411,4 +442,15 @@ public class SelectStmtTest {
.explainQuery()
.contains("`datekey` = 20200730"));
}
@Test
public void testDeleteSign() throws Exception {
String sql1 = "SELECT * FROM db1.table1 LEFT ANTI JOIN db1.table2 ON db1.table1.siteid = db1.table2.siteid;";
Assert.assertTrue(dorisAssert.query(sql1).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0"));
String sql2 = "SELECT * FROM db1.table1 JOIN db1.table2 ON db1.table1.siteid = db1.table2.siteid;";
Assert.assertTrue(dorisAssert.query(sql2).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0"));
String sql3 = "SELECT * FROM table1";
Assert.assertTrue(dorisAssert.query(sql3).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0"));
String sql4 = " SELECT * FROM table1 table2";
Assert.assertTrue(dorisAssert.query(sql4).explainQuery().contains("`table2`.`__DORIS_DELETE_SIGN__` = 0"));
}
}