database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](fe ut) Fix OlapQueryCacheTest failed (#27305)

Browse Source 
1. 
```
java.lang.NullPointerException: null
        at org.apache.doris.catalog.Env.getCurrentSystemInfo(Env.java:793) ~[classes/:?]
        at org.apache.doris.qe.SimpleScheduler$UpdateBlacklistThread.run(SimpleScheduler.java:206) ~[classes/:?]
        at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_382]

java.lang.NullPointerException
        at org.apache.doris.qe.OlapQueryCacheTest.setUp(OlapQueryCacheTest.java:226)
```

2.
```
[ERROR] testSqlCacheKeyWithNestedViewForNereids  Time elapsed: 1.962 s  <<< FAILURE!
java.lang.AssertionError: SELECT command denied to user 'testCluster:testUser'@'192.168.1.1' for table 'internal: testCluster:testDb: appevent'
	at org.apache.doris.qe.OlapQueryCacheTest.parseSqlByNereids(OlapQueryCacheTest.java:579)
	at org.apache.doris.qe.OlapQueryCacheTest.testSqlCacheKeyWithNestedViewForNereids(OlapQueryCacheTest.java:1338)
```

3.
```
[ERROR] Tests run: 28, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 113.63 s <<< FAILURE! - in org.apache.doris.qe.OlapQueryCacheTest
[ERROR] testCacheModeTable  Time elapsed: 1.657 s  <<< ERROR!
java.lang.IllegalArgumentException: Value of type org.apache.doris.qe.QueryState incompatible with return type org.apache.doris.system.SystemInfoService of org.apache.doris.catalog.Env#getCurrentSystemInfo()
        at org.apache.doris.qe.OlapQueryCacheTest.setUp(OlapQueryCacheTest.java:156)
```
This commit is contained in:
Xinyi Zou
2023-11-21 16:47:57 +08:00
committed by GitHub
parent 721282242b
commit 7e707f5d64
4 changed files with 53 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/DorisFE.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.journal.bdbje.BDBTool;
import org.apache.doris.journal.bdbje.BDBToolOptions;
import org.apache.doris.persist.meta.MetaReader;
import org.apache.doris.qe.QeService;
import org.apache.doris.qe.SimpleScheduler;
import org.apache.doris.service.ExecuteEnv;
import org.apache.doris.service.FeServer;
import org.apache.doris.service.FrontendOptions;
@ -194,6 +195,8 @@ public class DorisFE {
Env.getCurrentEnv().setHttpReady(true);
}
SimpleScheduler.init();
if (options.enableQeService) {
QeService qeService = new QeService(Config.query_port, Config.arrow_flight_sql_port,
ExecuteEnv.getInstance().getScheduler());

2
fe/fe-core/src/main/java/org/apache/doris/qe/SimpleScheduler.java
View File

@ -53,7 +53,7 @@ public class SimpleScheduler {
private static Map<Long, Pair<Integer, String>> blacklistBackends = Maps.newConcurrentMap();
private static UpdateBlacklistThread updateBlacklistThread;
static {
public static void init() {
updateBlacklistThread = new UpdateBlacklistThread();
updateBlacklistThread.start();
}

169
fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.qe;
import org.apache.doris.analysis.AccessTestUtil;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.CreateViewStmt;
import org.apache.doris.analysis.PartitionValue;
@ -26,6 +27,7 @@ import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
@ -50,13 +52,10 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.mysql.MysqlChannel;
import org.apache.doris.mysql.MysqlSerializer;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.MockedAuth;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
@ -77,7 +76,6 @@ import org.apache.doris.qe.cache.RowBatchBuilder;
import org.apache.doris.qe.cache.SqlCache;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TUniqueId;
@ -96,17 +94,15 @@ import org.junit.Test;
import java.io.StringReader;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Function;
public class OlapQueryCacheTest {
private static final Logger LOG = LogManager.getLogger(OlapQueryCacheTest.class);
public static String clusterName = "testCluster";
public static String dbName = "testDb";
public static String fullDbName = "testCluster:testDb";
public static String tableName = "testTbl";
public static String userName = "testUser";
private static ConnectContext context;
@ -115,21 +111,12 @@ public class OlapQueryCacheTest {
private Cache.HitRange hitRange;
private Analyzer analyzer;
private Database db;
@Mocked
private AccessControllerManager accessManager;
@Mocked
private SystemInfoService service;
@Mocked
private Env env;
@Mocked
private InternalCatalog catalog;
@Mocked
private ConnectContext ctx;
private QueryState state;
private ConnectScheduler scheduler;
@Mocked
MysqlChannel channel;
@Mocked
ConnectScheduler scheduler;
private MysqlChannel channel = null;
@BeforeClass
public static void start() {
@ -150,8 +137,15 @@ public class OlapQueryCacheTest {
@Before
public void setUp() throws Exception {
MockedAuth.mockedAccess(accessManager);
MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1");
state = new QueryState();
scheduler = new ConnectScheduler(10);
ctx = new ConnectContext();
SessionVariable sessionVariable = new SessionVariable();
Deencapsulation.setField(sessionVariable, "beNumberForTest", 1);
MysqlSerializer serializer = MysqlSerializer.newInstance();
env = AccessTestUtil.fetchAdminCatalog();
new MockUp<Util>() {
@Mock
public boolean showHiddenColumns() {
@ -160,82 +154,28 @@ public class OlapQueryCacheTest {
};
new MockUp<Env>() {
@Mock
public SystemInfoService getCurrentSystemInfo() {
return service;
}
};
db = new Database(1L, fullDbName);
new Expectations(catalog) {
{
catalog.getDbNullable(fullDbName);
minTimes = 0;
result = db;
catalog.getDbNullable(dbName);
minTimes = 0;
result = db;
catalog.getDbNullable(db.getId());
minTimes = 0;
result = db;
catalog.getDbNames();
minTimes = 0;
result = Lists.newArrayList(fullDbName);
Env getCurrentEnv() {
return env;
}
};
CatalogMgr dsMgr = new CatalogMgr();
new Expectations(dsMgr) {
{
dsMgr.getCatalog((String) any);
minTimes = 0;
result = catalog;
dsMgr.getCatalogOrException((String) any, (Function) any);
minTimes = 0;
result = catalog;
dsMgr.getCatalogOrAnalysisException((String) any);
minTimes = 0;
result = catalog;
}
};
new Expectations(env) {
{
env.getAccessManager();
minTimes = 0;
result = accessManager;
env.getCurrentCatalog();
minTimes = 0;
result = catalog;
env.getInternalCatalog();
minTimes = 0;
result = catalog;
env.getCatalogMgr();
minTimes = 0;
result = dsMgr;
}
};
FunctionSet fs = new FunctionSet();
fs.init();
Deencapsulation.setField(env, "functionSet", fs);
QueryState state = new QueryState();
channel.reset();
SessionVariable sessionVariable = new SessionVariable();
Deencapsulation.setField(sessionVariable, "beNumberForTest", 1);
channel.reset();
new Expectations(channel) {
{
channel.sendOnePacket((ByteBuffer) any);
minTimes = 0;
channel.reset();
minTimes = 0;
channel.getSerializer();
minTimes = 0;
result = MysqlSerializer.newInstance();
result = serializer;
}
};
@ -290,7 +230,7 @@ public class OlapQueryCacheTest {
ctx.getDatabase();
minTimes = 0;
result = dbName;
result = fullDbName;
ctx.getSessionVariable();
minTimes = 0;
@ -305,17 +245,32 @@ public class OlapQueryCacheTest {
ctx.getCurrentCatalog();
minTimes = 0;
result = catalog;
result = env.getCurrentCatalog();
ctx.getCatalog(anyString);
minTimes = 0;
result = catalog;
result = env.getCurrentCatalog();
ConnectContext.get();
minTimes = 0;
result = ctx;
ctx.getRemoteIP();
minTimes = 0;
result = "192.168.1.1";
ctx.getCurrentUserIdentity();
minTimes = 0;
UserIdentity userIdentity = new UserIdentity(userName, "192.168.1.1");
userIdentity.setIsAnalyzed();
result = userIdentity;
}
};
analyzer = new Analyzer(env, ctx);
newRangeList = Lists.newArrayList();
db = ((InternalCatalog) env.getCurrentCatalog()).getDbNullable(fullDbName);
// table and view init use analyzer, should init after analyzer build
OlapTable tbl1 = createOrderTable();
db.createTable(tbl1);
@ -600,7 +555,6 @@ public class OlapQueryCacheTest {
@Test
public void testCacheNode() throws Exception {
Env.getCurrentSystemInfo();
CacheCoordinator cp = CacheCoordinator.getInstance();
cp.debugModel = true;
Backend bd1 = new Backend(1, "", 1000);
@ -626,7 +580,6 @@ public class OlapQueryCacheTest {
@Test
public void testCacheModeNone() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("select @@version_comment limit 1");
List<ScanNode> scanNodes = Lists.newArrayList();
CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
@ -636,7 +589,6 @@ public class OlapQueryCacheTest {
@Test
public void testCacheModeTable() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT country, COUNT(userid) FROM userprofile GROUP BY country"
);
@ -650,7 +602,6 @@ public class OlapQueryCacheTest {
@Test
public void testWithinMinTime() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT country, COUNT(userid) FROM userprofile GROUP BY country"
);
@ -664,7 +615,6 @@ public class OlapQueryCacheTest {
@Test
public void testPartitionModel() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(DISTINCT userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-15\" GROUP BY eventdate"
@ -680,7 +630,6 @@ public class OlapQueryCacheTest {
@Test
public void testParseByte() throws Exception {
Env.getCurrentSystemInfo();
RowBatchBuilder sb = new RowBatchBuilder(CacheMode.Partition);
byte[] buffer = new byte[]{10, 50, 48, 50, 48, 45, 48, 51, 45, 49, 48, 1, 51, 2, 67, 78};
PartitionRange.PartitionKeyType key1 = sb.getKeyFromRow(buffer, 0, Type.DATE);
@ -693,7 +642,6 @@ public class OlapQueryCacheTest {
@Test
public void testPartitionIntTypeSql() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200112 and `date`<=20200115 GROUP BY date"
);
@ -737,7 +685,6 @@ public class OlapQueryCacheTest {
@Test
public void testSimpleCacheSql() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-15\" GROUP BY eventdate"
@ -780,7 +727,6 @@ public class OlapQueryCacheTest {
@Test
public void testHitSqlCache() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate"
@ -795,7 +741,6 @@ public class OlapQueryCacheTest {
@Test
public void testHitPartPartition() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate"
@ -841,7 +786,6 @@ public class OlapQueryCacheTest {
@Test
public void testNoUpdatePartition() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate"
@ -883,7 +827,6 @@ public class OlapQueryCacheTest {
@Test
public void testUpdatePartition() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-15\" GROUP BY eventdate"
@ -932,7 +875,6 @@ public class OlapQueryCacheTest {
@Test
public void testRewriteMultiPredicate1() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>\"2020-01-11\" and "
+ "eventdate<\"2020-01-16\""
@ -976,7 +918,6 @@ public class OlapQueryCacheTest {
@Test
public void testRewriteJoin() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT appevent.eventdate, country, COUNT(appevent.userid) FROM appevent"
+ " INNER JOIN userprofile ON appevent.userid = userprofile.userid"
@ -1021,7 +962,6 @@ public class OlapQueryCacheTest {
@Test
public void testSubSelect() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, sum(pv) FROM (SELECT eventdate, COUNT(userid) AS pv FROM appevent WHERE "
+ "eventdate>\"2020-01-11\" AND eventdate<\"2020-01-16\""
@ -1074,7 +1014,6 @@ public class OlapQueryCacheTest {
@Test
public void testNotHitPartition() throws Exception {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate"
@ -1103,7 +1042,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKey() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate"
@ -1125,7 +1063,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithChineseChar() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" and city=\"北京\" GROUP BY eventdate"
@ -1145,7 +1082,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithView() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("SELECT * from testDb.view1");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
@ -1166,7 +1102,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithViewForNereids() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSqlByNereids("SELECT * from testDb.view1");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
@ -1185,7 +1120,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithSubSelectView() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"select origin.eventdate as eventdate, origin.userid as userid\n"
+ "from (\n"
@ -1214,7 +1148,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithSubSelectViewForNereids() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSqlByNereids(
"select origin.eventdate as eventdate, origin.userid as userid\n"
+ "from (\n"
@ -1243,7 +1176,6 @@ public class OlapQueryCacheTest {
@Test
public void testPartitionCacheKeyWithView() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("SELECT * from testDb.view3");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L);
@ -1270,7 +1202,6 @@ public class OlapQueryCacheTest {
@Test
public void testPartitionCacheKeyWithSubSelectView() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"select origin.eventdate as eventdate, origin.cnt as cnt\n"
+ "from (\n"
@ -1294,8 +1225,8 @@ public class OlapQueryCacheTest {
Assert.assertEquals(cache.getSqlWithViewStmt(),
"SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` "
+ "FROM (SELECT <slot 4> `eventdate` AS `eventdate`, <slot 5> count(`userid`) AS `cnt` "
+ "FROM `testDb`.`view2` GROUP BY `eventdate`) origin|SELECT `eventdate` AS `eventdate`, "
+ "`userid` AS `userid` FROM `testCluster:testDb`.`appevent`");
+ "FROM `testCluster:testDb`.`view2` GROUP BY `eventdate`) origin|SELECT `eventdate` "
+ "AS `eventdate`, `userid` AS `userid` FROM `testCluster:testDb`.`appevent`");
} catch (Exception e) {
LOG.warn("ex={}", e);
Assert.fail(e.getMessage());
@ -1304,7 +1235,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithNestedView() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql("SELECT * from testDb.view4");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
@ -1317,7 +1247,7 @@ public class OlapQueryCacheTest {
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view4`.`eventdate` AS `eventdate`, "
+ "`testCluster:testDb`.`view4`.`__count_1` AS `__count_1` FROM `testCluster:testDb`.`view4`|"
+ "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `__count_1` FROM `testDb`.`view2` "
+ "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `__count_1` FROM `testCluster:testDb`.`view2` "
+ "WHERE `eventdate` >= '2020-01-12' AND `eventdate` <= '2020-01-14' GROUP BY `eventdate`|"
+ "SELECT `eventdate` AS `eventdate`, `userid` AS `userid` FROM `testCluster:testDb`.`appevent`");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
@ -1325,7 +1255,6 @@ public class OlapQueryCacheTest {
@Test
public void testSqlCacheKeyWithNestedViewForNereids() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSqlByNereids("SELECT * from testDb.view4");
ArrayList<Long> selectedPartitionIds
= Lists.newArrayList(20200112L, 20200113L, 20200114L);
@ -1337,15 +1266,14 @@ public class OlapQueryCacheTest {
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT * from testDb.view4|SELECT `eventdate` AS `eventdate`, "
+ "count(`userid`) AS `__count_1` FROM `testDb`.`view2` WHERE `eventdate` >= '2020-01-12' AND "
+ "`eventdate` <= '2020-01-14' GROUP BY `eventdate`|SELECT `eventdate` AS `eventdate`, "
+ "count(`userid`) AS `__count_1` FROM `testCluster:testDb`.`view2` WHERE `eventdate` >= '2020-01-12' "
+ "AND `eventdate` <= '2020-01-14' GROUP BY `eventdate`|SELECT `eventdate` AS `eventdate`, "
+ "`userid` AS `userid` FROM `testCluster:testDb`.`appevent`");
Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum());
}
@Test
public void testCacheLocalViewMultiOperand() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT COUNT(userid)\n"
+ "FROM (\n"
@ -1369,7 +1297,6 @@ public class OlapQueryCacheTest {
@Test
// test that some partitions do not exist in the table
public void testNotExistPartitionSql() {
Env.getCurrentSystemInfo();
StatementBase parseStmt = parseSql(
"SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200110 and `date`<=20200115 GROUP BY date"
);

1
fe/fe-core/src/test/java/org/apache/doris/qe/SimpleSchedulerTest.java
View File

@ -46,6 +46,7 @@ public class SimpleSchedulerTest {
@BeforeClass
public static void setUp() {
SimpleScheduler.init();
FeConstants.heartbeat_interval_second = 2;
be1 = new Backend(1000L, "192.168.100.0", 9050);
be2 = new Backend(1001L, "192.168.100.1", 9050);