database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](cache) Some view stmt cannot be obtained when view in the subquery and add cache key UT (#7375)

Browse Source 
1. Fix bug that some view stmt cannot be obtained when view in the subquery
2. Add cache key UT
This commit is contained in:
Zeno Yang
2021-12-21 10:21:28 +08:00
committed by GitHub
parent 2d72c039ad
commit 30db2cdd19
5 changed files with 195 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View File

@ -429,6 +429,10 @@ public class InlineViewRef extends TableRef {
return view;
}
public QueryStmt getQueryStmt() {
return queryStmt;
}
@Override
public String tableRefToSql() {
// Enclose the alias in quotes if Hive cannot parse it without quotes.

3
fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java vendored
View File

@ -217,7 +217,7 @@ public class CacheAnalyzer {
latestTable.Debug();
addAllViewStmt(selectStmt);
String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, ",");
String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, "|");
if (now == 0) {
now = nowtime();
@ -459,6 +459,7 @@ public class CacheAnalyzer {
addAllViewStmt(inlineViewRef.getViewStmt());
allViewStmtSet.add(inlineViewRef.getView().getInlineViewDef());
}
addAllViewStmt(inlineViewRef.getQueryStmt());
}
}
}

4
fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java vendored
View File

@ -62,8 +62,8 @@ public class PartitionCache extends Cache {
return nokeyStmt;
}
private String getSqlWithViewStmt() {
return nokeyStmt.toSql() + allViewExpandStmtListStr;
public String getSqlWithViewStmt() {
return nokeyStmt.toSql() + "|" + allViewExpandStmtListStr;
}
public PartitionCache(TUniqueId queryId, SelectStmt selectStmt) {

4
fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java vendored
View File

@ -40,8 +40,8 @@ public class SqlCache extends Cache {
this.allViewExpandStmtListStr = allViewExpandStmtListStr;
}
private String getSqlWithViewStmt() {
return selectStmt.toSql() + allViewExpandStmtListStr;
public String getSqlWithViewStmt() {
return selectStmt.toSql() + "|" + allViewExpandStmtListStr;
}
public InternalService.PFetchCacheResult getCacheData(Status status) {

186
fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.RangePartitionInfo;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
@ -60,6 +61,7 @@ import org.apache.doris.qe.cache.CacheCoordinator;
import org.apache.doris.qe.cache.PartitionCache;
import org.apache.doris.qe.cache.PartitionRange;
import org.apache.doris.qe.cache.RowBatchBuilder;
import org.apache.doris.qe.cache.SqlCache;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
@ -131,7 +133,7 @@ public class PartitionCacheTest {
}
@Before
public void setUp() {
public void setUp() throws Exception {
MockedAuth.mockedAuth(auth);
MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1");
new MockUp<Util>() {
@ -155,6 +157,15 @@ public class PartitionCacheTest {
db.createTable(tbl2);
db.createTable(tbl3);
View view1 = createEventView1();
View view2 = createEventView2();
View view3 = createEventView3();
View view4 = createEventNestedView();
db.createTable(view1);
db.createTable(view2);
db.createTable(view3);
db.createTable(view4);
new Expectations(catalog) {
{
catalog.getAuth();
@ -395,6 +406,37 @@ public class PartitionCacheTest {
return table;
}
private View createEventView1() {
String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " +
"eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate";
View view = new View(30000L, "view1", null);
view.setInlineViewDefWithSqlMode(originStmt, 0L);
return view;
}
private View createEventView2() {
String originStmt = "select eventdate, userid FROM appevent";
View view = new View(30001L, "view2", null);
view.setInlineViewDefWithSqlMode(originStmt, 0L);
return view;
}
private View createEventView3() {
String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " +
"eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate";
View view = new View(30002L, "view3", null);
view.setInlineViewDefWithSqlMode(originStmt, 0L);
return view;
}
private View createEventNestedView() {
String originStmt = "select eventdate, COUNT(userid) FROM view2 WHERE " +
"eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate";
View view = new View(30003L, "view4", null);
view.setInlineViewDefWithSqlMode(originStmt, 0L);
return view;
}
private ScanNode createEventScanNode(Collection<Long> selectedPartitionIds){
OlapTable table = createEventTable();
TupleDescriptor desc = new TupleDescriptor(new TupleId(30002));
@ -913,5 +955,147 @@ public class PartitionCacheTest {
Assert.fail(e.getMessage());
}
}
@Test
public void testSqlCacheKey() {
Catalog.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"
);
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT <slot 2> `eventdate` AS `eventdate`, <slot 3> count(`userid`) " +
"AS `count(``userid``)` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` " +
">= '2020-01-12 00:00:00' AND `eventdate` <= '2020-01-14 00:00:00' GROUP BY `eventdate`|");
}
@Test
public void testSqlCacheKeyWithView() {
Catalog.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("SELECT * from testDb.view1");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `testDb`.`view1`.`eventdate` AS `eventdate`, `testDb`.`view1`." +
"`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view1`|select eventdate, COUNT(userid) " +
"FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate");
}
@Test
public void testSqlCacheKeyWithSubSelectView() {
Catalog.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"select origin.eventdate as eventdate, origin.userid as userid\n" +
"from (\n" +
" select view2.eventdate as eventdate, view2.userid as userid \n" +
" from testDb.view2 view2 \n" +
" where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" +
") origin"
);
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`userid` AS " +
"`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM " +
"`testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12 00:00:00' AND `view2`.`eventdate`" +
" <= '2020-01-14 00:00:00') origin|select eventdate, userid FROM appevent");
}
@Test
public void testPartitionCacheKeyWithView() {
Catalog.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("SELECT * from testDb.view3");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L);
List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
CacheAnalyzer ca = new CacheAnalyzer(context,parseStmt, scanNodes);
ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first
try {
PartitionCache cache = (PartitionCache) ca.getCache();
cache.rewriteSelectStmt(null);
Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null);
Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testDb`.`view3`.`eventdate` AS " +
"`eventdate`, `testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM " +
"`testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>=" +
"\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate");
} catch (Exception e) {
LOG.warn("ex={}", e);
Assert.fail(e.getMessage());
}
}
@Test
public void testPartitionCacheKeyWithSubSelectView() {
Catalog.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"select origin.eventdate as eventdate, origin.cnt as cnt\n" +
"from (\n" +
" SELECT eventdate, COUNT(userid) as cnt \n" +
" FROM view2 \n" +
" WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate\n" +
") origin"
);
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L);
List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
CacheAnalyzer ca = new CacheAnalyzer(context,parseStmt, scanNodes);
ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first
try {
PartitionCache cache = (PartitionCache) ca.getCache();
cache.rewriteSelectStmt(null);
Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null);
Assert.assertEquals(cache.getSqlWithViewStmt(),
"SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` FROM (SELECT " +
"<slot 4> `eventdate` AS `eventdate`, <slot 5> count(`userid`) AS `cnt` FROM " +
"`testDb`.`view2` GROUP BY `eventdate`) origin|select eventdate, userid FROM appevent");
} catch (Exception e) {
LOG.warn("ex={}", e);
Assert.fail(e.getMessage());
}
}
@Test
public void testSqlCacheKeyWithNestedView() {
Catalog.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("SELECT * from testDb.view4");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `testDb`.`view4`.`eventdate` AS `eventdate`, " +
"`testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view4`|select " +
"eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and " +
"eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent");
}
}