From 30db2cdd192f04e153ef8eefdc1b63510f293f65 Mon Sep 17 00:00:00 2001 From: Zeno Yang Date: Tue, 21 Dec 2021 10:21:28 +0800 Subject: [PATCH] [fix](cache) Some view stmt cannot be obtained when view in the subquery and add cache key UT (#7375) 1. Fix bug that some view stmt cannot be obtained when view in the subquery 2. Add cache key UT --- .../apache/doris/analysis/InlineViewRef.java | 4 + .../apache/doris/qe/cache/CacheAnalyzer.java | 3 +- .../apache/doris/qe/cache/PartitionCache.java | 4 +- .../org/apache/doris/qe/cache/SqlCache.java | 4 +- .../apache/doris/qe/PartitionCacheTest.java | 186 +++++++++++++++++- 5 files changed, 195 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java index 82f9ff438a..750fcfea9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java @@ -429,6 +429,10 @@ public class InlineViewRef extends TableRef { return view; } + public QueryStmt getQueryStmt() { + return queryStmt; + } + @Override public String tableRefToSql() { // Enclose the alias in quotes if Hive cannot parse it without quotes. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index b00f9aa6dc..b490d9081f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -217,7 +217,7 @@ public class CacheAnalyzer { latestTable.Debug(); addAllViewStmt(selectStmt); - String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, ","); + String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet, "|"); if (now == 0) { now = nowtime(); @@ -459,6 +459,7 @@ public class CacheAnalyzer { addAllViewStmt(inlineViewRef.getViewStmt()); allViewStmtSet.add(inlineViewRef.getView().getInlineViewDef()); } + addAllViewStmt(inlineViewRef.getQueryStmt()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java index 5c379f2061..78a4061d4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java @@ -62,8 +62,8 @@ public class PartitionCache extends Cache { return nokeyStmt; } - private String getSqlWithViewStmt() { - return nokeyStmt.toSql() + allViewExpandStmtListStr; + public String getSqlWithViewStmt() { + return nokeyStmt.toSql() + "|" + allViewExpandStmtListStr; } public PartitionCache(TUniqueId queryId, SelectStmt selectStmt) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java index 7d197a2d38..a9ba2912a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java @@ -40,8 +40,8 @@ public class SqlCache extends Cache { this.allViewExpandStmtListStr = allViewExpandStmtListStr; } - private String getSqlWithViewStmt() { - return selectStmt.toSql() + allViewExpandStmtListStr; + public String getSqlWithViewStmt() { + return selectStmt.toSql() + "|" + allViewExpandStmtListStr; } public InternalService.PFetchCacheResult getCacheData(Status status) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java index 415920d650..6329fad876 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java @@ -38,6 +38,7 @@ import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.View; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; @@ -60,6 +61,7 @@ import org.apache.doris.qe.cache.CacheCoordinator; import org.apache.doris.qe.cache.PartitionCache; import org.apache.doris.qe.cache.PartitionRange; import org.apache.doris.qe.cache.RowBatchBuilder; +import org.apache.doris.qe.cache.SqlCache; import org.apache.doris.service.FrontendOptions; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; @@ -131,7 +133,7 @@ public class PartitionCacheTest { } @Before - public void setUp() { + public void setUp() throws Exception { MockedAuth.mockedAuth(auth); MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); new MockUp() { @@ -155,6 +157,15 @@ public class PartitionCacheTest { db.createTable(tbl2); db.createTable(tbl3); + View view1 = createEventView1(); + View view2 = createEventView2(); + View view3 = createEventView3(); + View view4 = createEventNestedView(); + db.createTable(view1); + db.createTable(view2); + db.createTable(view3); + db.createTable(view4); + new Expectations(catalog) { { catalog.getAuth(); @@ -395,6 +406,37 @@ public class PartitionCacheTest { return table; } + private View createEventView1() { + String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " + + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; + View view = new View(30000L, "view1", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private View createEventView2() { + String originStmt = "select eventdate, userid FROM appevent"; + View view = new View(30001L, "view2", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private View createEventView3() { + String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " + + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"; + View view = new View(30002L, "view3", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private View createEventNestedView() { + String originStmt = "select eventdate, COUNT(userid) FROM view2 WHERE " + + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; + View view = new View(30003L, "view4", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + private ScanNode createEventScanNode(Collection selectedPartitionIds){ OlapTable table = createEventTable(); TupleDescriptor desc = new TupleDescriptor(new TupleId(30002)); @@ -913,5 +955,147 @@ public class PartitionCacheTest { Assert.fail(e.getMessage()); } } + + @Test + public void testSqlCacheKey() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `eventdate` AS `eventdate`, count(`userid`) " + + "AS `count(``userid``)` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` " + + ">= '2020-01-12 00:00:00' AND `eventdate` <= '2020-01-14 00:00:00' GROUP BY `eventdate`|"); + } + + @Test + public void testSqlCacheKeyWithView() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("SELECT * from testDb.view1"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `testDb`.`view1`.`eventdate` AS `eventdate`, `testDb`.`view1`." + + "`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view1`|select eventdate, COUNT(userid) " + + "FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"); + } + + @Test + public void testSqlCacheKeyWithSubSelectView() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "select origin.eventdate as eventdate, origin.userid as userid\n" + + "from (\n" + + " select view2.eventdate as eventdate, view2.userid as userid \n" + + " from testDb.view2 view2 \n" + + " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" + + ") origin" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`userid` AS " + + "`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM " + + "`testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12 00:00:00' AND `view2`.`eventdate`" + + " <= '2020-01-14 00:00:00') origin|select eventdate, userid FROM appevent"); + } + + @Test + public void testPartitionCacheKeyWithView() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("SELECT * from testDb.view3"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context,parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testDb`.`view3`.`eventdate` AS " + + "`eventdate`, `testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM " + + "`testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>=" + + "\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testPartitionCacheKeyWithSubSelectView() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "select origin.eventdate as eventdate, origin.cnt as cnt\n" + + "from (\n" + + " SELECT eventdate, COUNT(userid) as cnt \n" + + " FROM view2 \n" + + " WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate\n" + + ") origin" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context,parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + Assert.assertEquals(cache.getSqlWithViewStmt(), + "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` FROM (SELECT " + + " `eventdate` AS `eventdate`, count(`userid`) AS `cnt` FROM " + + "`testDb`.`view2` GROUP BY `eventdate`) origin|select eventdate, userid FROM appevent"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testSqlCacheKeyWithNestedView() { + Catalog.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("SELECT * from testDb.view4"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `testDb`.`view4`.`eventdate` AS `eventdate`, " + + "`testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view4`|select " + + "eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent"); + } }