database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](test) Refactor FE unit test framework that starts a FE server. (#9388)

Browse Source 
Currently, we use `UtFrameUtils` to start a FE server in the FE unit test. 
Each test class has to do some initialization and clean up stuff with the JUnit4
`@BeforeClass` and `@AfterClass` annotation. It's redundant and boring.
Besides, almost all the APIs in `UtFrameUtils` has a `ConnectContext` parameter, which is not easy to use.

This PR proposes to use an inherit-manner, i.e., wrap all the common logic in base class `TestWithFeService`,
leveraging the 
JUnit5 `@BeforeAll` and `@AfterAll` annotation to narrow down the setup and cleanup lifecycle to each test class instance.
At the same time, the derived concrete test class could directly use utility methods inherited from the base class,
without calling a util class and passing a `ConnectContext` argument.

`UtFrameUtils` and `DorisAssert`  are marked as deprecated. We could remove these two classes
if this refactor works well for a time.
This commit is contained in:
Shuo Wang
2022-05-07 21:28:42 +08:00
committed by GitHub
parent fd11a6b493
commit 1746f61388
18 changed files with 890 additions and 665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fe/fe-core/pom.xml
View File

@ -220,11 +220,25 @@ under the License.
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<!-- https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-engine -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.junit.vintage/junit-vintage-engine -->
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->

51
fe/fe-core/src/test/java/org/apache/doris/analysis/AdminSetConfigStmtTest.java
View File

@ -20,54 +20,37 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.jupiter.api.Test;
import java.util.UUID;
public class AdminSetConfigStmtTest {
private static String runningDir = "fe/mocked/AdminSetConfigStmtTest/" + UUID.randomUUID().toString() + "/";
private static ConnectContext connectContext;
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@BeforeClass
public static void beforeClass() throws Exception {
UtFrameUtils.createDorisCluster(runningDir);
// create connect context
connectContext = UtFrameUtils.createDefaultCtx();
}
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class AdminSetConfigStmtTest extends TestWithFeService {
@Test
public void testNormal() throws Exception {
String stmt = "admin set frontend config(\"alter_table_timeout_second\" = \"60\");";
AdminSetConfigStmt adminSetConfigStmt = (AdminSetConfigStmt) UtFrameUtils.parseAndAnalyzeStmt(stmt, connectContext);
AdminSetConfigStmt adminSetConfigStmt = (AdminSetConfigStmt) parseAndAnalyzeStmt(stmt);
Catalog.getCurrentCatalog().setConfig(adminSetConfigStmt);
}
@Test
public void testUnknownConfig() throws Exception {
String stmt = "admin set frontend config(\"unknown_config\" = \"unknown\");";
AdminSetConfigStmt adminSetConfigStmt = (AdminSetConfigStmt) UtFrameUtils.parseAndAnalyzeStmt(stmt, connectContext);
expectedEx.expect(DdlException.class);
expectedEx.expectMessage("errCode = 2, detailMessage = Config 'unknown_config' does not exist");
Catalog.getCurrentCatalog().setConfig(adminSetConfigStmt);
AdminSetConfigStmt adminSetConfigStmt = (AdminSetConfigStmt) parseAndAnalyzeStmt(stmt);
DdlException exception = assertThrows(DdlException.class,
() -> Catalog.getCurrentCatalog().setConfig(adminSetConfigStmt));
assertEquals("errCode = 2, detailMessage = Config 'unknown_config' does not exist",
exception.getMessage());
}
@Test
public void testEmptyConfig() throws Exception {
String stmt = "admin set frontend config;";
expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage("errCode = 2, detailMessage = config parameter size is not equal to 1");
AdminSetConfigStmt adminSetConfigStmt = (AdminSetConfigStmt) UtFrameUtils.parseAndAnalyzeStmt(stmt, connectContext);
public void testEmptyConfig() {
AnalysisException exception =
assertThrows(AnalysisException.class,
() -> parseAndAnalyzeStmt("admin set frontend config;"));
assertEquals("errCode = 2, detailMessage = config parameter size is not equal to 1",
exception.getMessage());
}
}

68
fe/fe-core/src/test/java/org/apache/doris/analysis/AdminShowReplicaTest.java
View File

@ -25,76 +25,43 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowExecutor;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.StringReader;
import java.lang.reflect.Method;
import java.util.UUID;
public class AdminShowReplicaTest {
// use a unique dir so that it won't be conflict with other unit test which
// may also start a Mocked Frontend
private static String runningDir = "fe/mocked/AdminShowReplicaTest/" + UUID.randomUUID().toString() + "/";
private static ConnectContext connectContext;
@BeforeClass
public static void beforeClass() throws Exception {
UtFrameUtils.createDorisCluster(runningDir);
// create connect context
connectContext = UtFrameUtils.createDefaultCtx();
// create database
String createDbStmtStr = "create database test;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
Catalog.getCurrentCatalog().createDb(createDbStmt);
public class AdminShowReplicaTest extends TestWithFeService {
@Override
protected void runBeforeAll() throws Exception {
createDatabase("test");
createTable("create table test.tbl1\n" +
"(k1 date, k2 int)\n" +
"partition by range(k1)\n" +
"(\n" +
" partition p1 values less than(\"2021-07-01\"),\n" +
" partition p2 values less than(\"2021-08-01\")\n" +
")\n" +
"distributed by hash(k2) buckets 10\n" +
"properties(\"replication_num\" = \"1\");");
}
@AfterClass
public static void tearDown() {
File file = new File(runningDir);
file.delete();
}
private static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().createTable(createTableStmt);
"(k1 date, k2 int)\n" +
"partition by range(k1)\n" +
"(\n" +
" partition p1 values less than(\"2021-07-01\"),\n" +
" partition p2 values less than(\"2021-08-01\")\n" +
")\n" +
"distributed by hash(k2) buckets 10\n" +
"properties(\"replication_num\" = \"1\");");
}
@Test
public void testShowReplicaDistribution() throws Exception {
String stmtStr = "admin show replica distribution from test.tbl1 partition(p1)";
AdminShowReplicaDistributionStmt stmt = (AdminShowReplicaDistributionStmt) UtFrameUtils.parseAndAnalyzeStmt(
stmtStr, connectContext);
AdminShowReplicaDistributionStmt stmt = (AdminShowReplicaDistributionStmt) parseAndAnalyzeStmt(
stmtStr);
ShowExecutor executor = new ShowExecutor(connectContext, stmt);
ShowResultSet resultSet = executor.execute();
Assert.assertEquals(1, resultSet.getResultRows().size());
Assert.assertEquals(7, resultSet.getResultRows().get(0).size());
stmtStr = "show data skew from test.tbl1 partition(p1)";
ShowDataSkewStmt skewStmt = (ShowDataSkewStmt) UtFrameUtils.parseAndAnalyzeStmt(
stmtStr, connectContext);
ShowDataSkewStmt skewStmt = (ShowDataSkewStmt) parseAndAnalyzeStmt(stmtStr);
executor = new ShowExecutor(connectContext, skewStmt);
resultSet = executor.execute();
Assert.assertEquals(10, resultSet.getResultRows().size());
@ -190,5 +157,4 @@ public class AdminShowReplicaTest {
return true;
}
}

29
fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java
View File

@ -21,34 +21,26 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.DorisAssert;
import org.apache.doris.utframe.TestWithFeService;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import java.util.UUID;
public class AggregateTest {
private static String baseDir = "fe";
private static String runningDir = baseDir + "/mocked/AggregateTest/"
+ UUID.randomUUID().toString() + "/";
public class AggregateTest extends TestWithFeService {
private static final String TABLE_NAME = "table1";
private static final String DB_NAME = "db1";
private static DorisAssert dorisAssert;
@BeforeClass
public static void beforeClass() throws Exception{
@Override
protected void runBeforeAll() throws Exception {
FeConstants.runningUnitTest = true;
UtFrameUtils.createDorisCluster(runningDir);
dorisAssert = new DorisAssert();
dorisAssert.withDatabase(DB_NAME).useDatabase(DB_NAME);
String createTableSQL = "create table " + DB_NAME + "." + TABLE_NAME + " (empid int, name varchar, " +
"deptno int, salary int, commission int, time DATETIME) "
+ "distributed by hash(empid) buckets 3 properties('replication_num' = '1');";
dorisAssert.withTable(createTableSQL);
"deptno int, salary int, commission int, time DATETIME) "
+ "distributed by hash(empid) buckets 3 properties('replication_num' = '1');";
createTable(createTableSQL);
}
/**
@ -181,9 +173,4 @@ public class AggregateTest {
Assert.fail("must be AnalysisException.");
} while(false);
}
@AfterClass
public static void afterClass() throws Exception {
UtFrameUtils.cleanDorisFeDir(baseDir);
}
}

34
fe/fe-core/src/test/java/org/apache/doris/analysis/ListPartitionPrunerTest.java
View File

@ -17,29 +17,16 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.FeConstants;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.UUID;
import org.junit.jupiter.api.Test;
public class ListPartitionPrunerTest extends PartitionPruneTestBase {
@BeforeClass
public static void beforeClass() throws Exception {
@Override
protected void runBeforeAll() throws Exception {
FeConstants.runningUnitTest = true;
runningDir = "fe/mocked/ListPartitionPrunerTest/" + UUID.randomUUID().toString() + "/";
UtFrameUtils.createDorisCluster(runningDir);
connectContext = UtFrameUtils.createDefaultCtx();
String createDbStmtStr = "create database test;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
Catalog.getCurrentCatalog().createDb(createDbStmt);
createDatabase("test");
String createSinglePartColWithSinglePartKey =
"create table test.t1\n"
@ -84,15 +71,10 @@ public class ListPartitionPrunerTest extends PartitionPruneTestBase {
+ "distributed by hash(k2) buckets 1\n"
+ "properties('replication_num' = '1');";
createTable(createSinglePartColWithSinglePartKey);
createTable(createSinglePartColWithMultiPartKey);
createTable(createMultiPartColWithSinglePartKey);
createTable(createMultiPartColWithMultiPartKey);
}
@AfterClass
public static void tearDown() throws Exception {
UtFrameUtils.cleanDorisFeDir(runningDir);
createTables(createSinglePartColWithSinglePartKey,
createSinglePartColWithMultiPartKey,
createMultiPartColWithSinglePartKey,
createMultiPartColWithMultiPartKey);
}
private void initTestCases() {

19
fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
View File

@ -17,19 +17,14 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.Assert;
import java.util.ArrayList;
import java.util.List;
public class PartitionPruneTestBase {
protected static String runningDir;
protected static ConnectContext connectContext;
public abstract class PartitionPruneTestBase extends TestWithFeService {
protected List<TestCase> cases = new ArrayList<>();
protected void doTest() throws Exception {
@ -41,16 +36,10 @@ public class PartitionPruneTestBase {
}
}
protected static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().createTable(createTableStmt);
}
private void assertExplainContains(int version, String sql, String subString) throws Exception {
Assert.assertTrue(String.format("version=%d, sql=%s, expectResult=%s",
version, sql, subString),
UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "explain " + sql)
.contains(subString));
version, sql, subString),
getSQLPlanOrErrorMsg("explain " + sql).contains(subString));
}
protected void addCase(String sql, String v1Result, String v2Result) {

34
fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
View File

@ -17,29 +17,16 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.FeConstants;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.UUID;
import org.junit.jupiter.api.Test;
public class RangePartitionPruneTest extends PartitionPruneTestBase {
@BeforeClass
public static void beforeClass() throws Exception {
@Override
protected void runBeforeAll() throws Exception {
FeConstants.runningUnitTest = true;
runningDir = "fe/mocked/RangePartitionPruneTest/" + UUID.randomUUID().toString() + "/";
UtFrameUtils.createDorisCluster(runningDir);
connectContext = UtFrameUtils.createDefaultCtx();
String createDbStmtStr = "create database test;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
Catalog.getCurrentCatalog().createDb(createDbStmt);
createDatabase("test");
String singleColumnPartitionTable =
"CREATE TABLE `test`.`t1` (\n" +
@ -117,15 +104,10 @@ public class RangePartitionPruneTest extends PartitionPruneTestBase {
"DISTRIBUTED BY HASH(`k1`) BUCKETS 10\n" +
"PROPERTIES ('replication_num' = '1');";
createTable(singleColumnPartitionTable);
createTable(notNullSingleColumnPartitionTable);
createTable(multipleColumnsPartitionTable);
createTable(notNullMultipleColumnsPartitionTable);
}
@AfterClass
public static void tearDown() throws Exception {
UtFrameUtils.cleanDorisFeDir(runningDir);
createTables(singleColumnPartitionTable,
notNullSingleColumnPartitionTable,
multipleColumnsPartitionTable,
notNullMultipleColumnsPartitionTable);
}
private void initTestCases() {

63
fe/fe-core/src/test/java/org/apache/doris/catalog/AdminStmtTest.java
View File

@ -18,22 +18,17 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.AdminSetReplicaStatusStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.Replica.ReplicaStatus;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.persist.SetReplicaStatusOperationLog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.Lists;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
@ -42,44 +37,20 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
public class AdminStmtTest {
// use a unique dir so that it won't be conflict with other unit test which
// may also start a Mocked Frontend
private static String runningDir = "fe/mocked/AdminStmtTest/" + UUID.randomUUID().toString() + "/";
private static ConnectContext connectContext;
@BeforeClass
public static void beforeClass() throws Exception {
UtFrameUtils.createDorisCluster(runningDir);
// create connect context
connectContext = UtFrameUtils.createDefaultCtx();
// create database
String createDbStmtStr = "create database test;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
Catalog.getCurrentCatalog().createDb(createDbStmt);
String sql = "CREATE TABLE test.tbl1 (\n" +
" `id` int(11) NULL COMMENT \"\",\n" +
" `id2` bitmap bitmap_union NULL\n" +
") ENGINE=OLAP\n" +
"AGGREGATE KEY(`id`)\n" +
"DISTRIBUTED BY HASH(`id`) BUCKETS 3\n" +
"PROPERTIES (\n" +
" \"replication_num\" = \"1\"\n" +
");";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().createTable(createTableStmt);
}
@AfterClass
public static void tearDown() {
File file = new File(runningDir);
file.delete();
public class AdminStmtTest extends TestWithFeService {
@Override
protected void runBeforeAll() throws Exception {
createDatabase("test");
createTable( "CREATE TABLE test.tbl1 (\n" +
" `id` int(11) NULL COMMENT \"\",\n" +
" `id2` bitmap bitmap_union NULL\n" +
") ENGINE=OLAP\n" +
"AGGREGATE KEY(`id`)\n" +
"DISTRIBUTED BY HASH(`id`) BUCKETS 3\n" +
"PROPERTIES (\n" +
" \"replication_num\" = \"1\"\n" +
");");
}
@Test
@ -108,7 +79,7 @@ public class AdminStmtTest {
// set replica to bad
String adminStmt = "admin set replica status properties ('tablet_id' = '" + tabletId + "', 'backend_id' = '"
+ backendId + "', 'status' = 'bad');";
AdminSetReplicaStatusStmt stmt = (AdminSetReplicaStatusStmt) UtFrameUtils.parseAndAnalyzeStmt(adminStmt, connectContext);
AdminSetReplicaStatusStmt stmt = (AdminSetReplicaStatusStmt) parseAndAnalyzeStmt(adminStmt);
Catalog.getCurrentCatalog().setReplicaStatus(stmt);
replica = Catalog.getCurrentInvertedIndex().getReplica(tabletId, backendId);
Assert.assertTrue(replica.isBad());
@ -116,7 +87,7 @@ public class AdminStmtTest {
// set replica to ok
adminStmt = "admin set replica status properties ('tablet_id' = '" + tabletId + "', 'backend_id' = '"
+ backendId + "', 'status' = 'ok');";
stmt = (AdminSetReplicaStatusStmt) UtFrameUtils.parseAndAnalyzeStmt(adminStmt, connectContext);
stmt = (AdminSetReplicaStatusStmt) parseAndAnalyzeStmt(adminStmt);
Catalog.getCurrentCatalog().setReplicaStatus(stmt);
replica = Catalog.getCurrentInvertedIndex().getReplica(tabletId, backendId);
Assert.assertFalse(replica.isBad());

122
fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
View File

@ -17,63 +17,54 @@
package org.apache.doris.planner;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.Expr;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.doris.utframe.TestWithFeService;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.List;
import java.util.UUID;
public class PlannerTest {
private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/";
private static ConnectContext ctx;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Rule
public ExpectedException expectedEx = ExpectedException.none();
public class PlannerTest extends TestWithFeService {
@After
public void tearDown() throws Exception {
FileUtils.deleteDirectory(new File(runningDir));
}
@Override
protected void runBeforeAll() throws Exception {
// Create database `db1`.
createDatabase("db1");
@BeforeClass
public static void setUp() throws Exception {
UtFrameUtils.createDorisCluster(runningDir);
ctx = UtFrameUtils.createDefaultCtx();
String createDbStmtStr = "create database db1;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, ctx);
Catalog.getCurrentCatalog().createDb(createDbStmt);
// 3. create table tbl1
String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2 varchar(32), k3 varchar(32), k4 int) "
+ "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
// Create tables.
String tbl1 = "create table db1.tbl1(" +
"k1 varchar(32), " +
"k2 varchar(32), " +
"k3 varchar(32), " +
"k4 int) " +
"AGGREGATE KEY(k1, k2,k3,k4) " +
"distributed by hash(k1) buckets 3 " +
"properties('replication_num' = '1');";
createTblStmtStr = "create table db1.tbl2(k1 int, k2 int sum) "
+ "AGGREGATE KEY(k1) partition by range(k1) () distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
String tbl2 = "create table db1.tbl2(" +
"k1 int, " +
"k2 int sum) " +
"AGGREGATE KEY(k1) " +
"partition by range(k1) () " +
"distributed by hash(k1) buckets 3 " +
"properties('replication_num' = '1');";
createTblStmtStr = "create table db1.tbl3 (k1 date, k2 varchar(128) NULL, k3 varchar(5000) NULL) "
+ "DUPLICATE KEY(k1, k2, k3) distributed by hash(k1) buckets 1 properties ('replication_num' = '1');";
createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
String tbl3 = "create table db1.tbl3 (" +
"k1 date, " +
"k2 varchar(128) NULL, " +
"k3 varchar(5000) NULL) " +
"DUPLICATE KEY(k1, k2, k3) " +
"distributed by hash(k1) buckets 1 " +
"properties ('replication_num' = '1');";
createTables(tbl1, tbl2, tbl3);
}
@Test
@ -87,7 +78,7 @@ public class PlannerTest {
+ " db1.tbl1 b\n"
+ " on (a.k1 = b.k1)\n"
+ "where b.k1 = 'a'";
StmtExecutor stmtExecutor1 = new StmtExecutor(ctx, sql1);
StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1);
stmtExecutor1.execute();
Planner planner1 = stmtExecutor1.planner();
List<PlanFragment> fragments1 = planner1.getFragments();
@ -112,7 +103,7 @@ public class PlannerTest {
+ " union all\n"
+ " (select * from db1.tbl1 where k1='b' and k4=5)\n"
+ " order by 3 limit 3)";
StmtExecutor stmtExecutor2 = new StmtExecutor(ctx, sql2);
StmtExecutor stmtExecutor2 = new StmtExecutor(connectContext, sql2);
stmtExecutor2.execute();
Planner planner2 = stmtExecutor2.planner();
List<PlanFragment> fragments2 = planner2.getFragments();
@ -128,7 +119,7 @@ public class PlannerTest {
+ " db1.tbl1 b\n"
+ " on (a.k1 = b.k1)\n"
+ "where b.k1 = 'a'";
StmtExecutor stmtExecutor3 = new StmtExecutor(ctx, sql3);
StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, sql3);
stmtExecutor3.execute();
Planner planner3 = stmtExecutor3.planner();
List<PlanFragment> fragments3 = planner3.getFragments();
@ -154,7 +145,7 @@ public class PlannerTest {
+ " (select * from db1.tbl1 where k1='b' and k4=5)\n"
+ " order by 3 limit 3)";
StmtExecutor stmtExecutor4 = new StmtExecutor(ctx, sql4);
StmtExecutor stmtExecutor4 = new StmtExecutor(connectContext, sql4);
stmtExecutor4.execute();
Planner planner4 = stmtExecutor4.planner();
List<PlanFragment> fragments4 = planner4.getFragments();
@ -170,7 +161,7 @@ public class PlannerTest {
+ " db1.tbl1 b\n"
+ " on (a.k1 = b.k1)\n"
+ "where b.k1 = 'a'";
StmtExecutor stmtExecutor5 = new StmtExecutor(ctx, sql5);
StmtExecutor stmtExecutor5 = new StmtExecutor(connectContext, sql5);
stmtExecutor5.execute();
Planner planner5 = stmtExecutor5.planner();
List<PlanFragment> fragments5 = planner5.getFragments();
@ -185,7 +176,7 @@ public class PlannerTest {
+ "except distinct\n"
+ "(select * from db1.tbl1 where k1='a' and k4=2)\n"
+ "order by 3 limit 3";
StmtExecutor stmtExecutor6 = new StmtExecutor(ctx, sql6);
StmtExecutor stmtExecutor6 = new StmtExecutor(connectContext, sql6);
stmtExecutor6.execute();
Planner planner6 = stmtExecutor6.planner();
List<PlanFragment> fragments6 = planner6.getFragments();
@ -200,7 +191,7 @@ public class PlannerTest {
+ "except\n"
+ "(select * from db1.tbl1 where k1='a' and k4=2)\n"
+ "order by 3 limit 3";
StmtExecutor stmtExecutor7 = new StmtExecutor(ctx, sql7);
StmtExecutor stmtExecutor7 = new StmtExecutor(connectContext, sql7);
stmtExecutor7.execute();
Planner planner7 = stmtExecutor7.planner();
List<PlanFragment> fragments7 = planner7.getFragments();
@ -216,7 +207,7 @@ public class PlannerTest {
+ "intersect\n"
+ "(select * from db1.tbl1 where k1='a' and k4=2)\n"
+ "order by 3 limit 3";
StmtExecutor stmtExecutor8 = new StmtExecutor(ctx, sql8);
StmtExecutor stmtExecutor8 = new StmtExecutor(connectContext, sql8);
stmtExecutor8.execute();
Planner planner8 = stmtExecutor8.planner();
List<PlanFragment> fragments8 = planner8.getFragments();
@ -245,7 +236,7 @@ public class PlannerTest {
+ " (select * from db1.tbl1 where k1='b' and k4=5)\n"
+ " order by 3 limit 3)";
StmtExecutor stmtExecutor9 = new StmtExecutor(ctx, sql9);
StmtExecutor stmtExecutor9 = new StmtExecutor(connectContext, sql9);
stmtExecutor9.execute();
Planner planner9 = stmtExecutor9.planner();
List<PlanFragment> fragments9 = planner9.getFragments();
@ -255,7 +246,7 @@ public class PlannerTest {
Assert.assertEquals(2, StringUtils.countMatches(plan9, "EXCEPT"));
String sql10 = "select 499 union select 670 except select 499";
StmtExecutor stmtExecutor10 = new StmtExecutor(ctx, sql10);
StmtExecutor stmtExecutor10 = new StmtExecutor(connectContext, sql10);
stmtExecutor10.execute();
Planner planner10 = stmtExecutor10.planner();
List<PlanFragment> fragments10 = planner10.getFragments();
@ -268,7 +259,7 @@ public class PlannerTest {
"(SELECT '01' x) a \n" +
"INNER JOIN\n" +
"(SELECT '01' x UNION all SELECT '02') b";
StmtExecutor stmtExecutor11 = new StmtExecutor(ctx, sql11);
StmtExecutor stmtExecutor11 = new StmtExecutor(connectContext, sql11);
stmtExecutor11.execute();
Planner planner11 = stmtExecutor11.planner();
SetOperationNode setNode11 = (SetOperationNode)(planner11.getFragments().get(1).getPlanRoot());
@ -280,7 +271,7 @@ public class PlannerTest {
"(SELECT k1 from db1.tbl1 \n" +
"UNION all \n" +
"SELECT k1 from db1.tbl1) b;";
StmtExecutor stmtExecutor12 = new StmtExecutor(ctx, sql12);
StmtExecutor stmtExecutor12 = new StmtExecutor(connectContext, sql12);
stmtExecutor12.execute();
Planner planner12 = stmtExecutor12.planner();
SetOperationNode setNode12 = (SetOperationNode)(planner12.getFragments().get(1).getPlanRoot());
@ -314,7 +305,7 @@ public class PlannerTest {
" )\n" +
") t\n" +
"WHERE IF(k2 IS NULL, 'ALL', k2) = 'ALL'";
StmtExecutor stmtExecutor1 = new StmtExecutor(ctx, sql1);
StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1);
stmtExecutor1.execute();
Planner planner1 = stmtExecutor1.planner();
List<PlanFragment> fragments1 = planner1.getFragments();
@ -341,7 +332,7 @@ public class PlannerTest {
" GROUP BY k1, k2, k3\n" +
") t\n" +
"WHERE IF(k2 IS NULL, 'ALL', k2) = 'ALL'";
StmtExecutor stmtExecutor2 = new StmtExecutor(ctx, sql2);
StmtExecutor stmtExecutor2 = new StmtExecutor(connectContext, sql2);
stmtExecutor2.execute();
Planner planner2 = stmtExecutor2.planner();
List<PlanFragment> fragments2 = planner2.getFragments();
@ -356,7 +347,7 @@ public class PlannerTest {
"b as ( select '543' as user_id) " +
"select user_id from a union all select user_id from b";
StmtExecutor stmtExecutor1 = new StmtExecutor(ctx, sql1);
StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1);
stmtExecutor1.execute();
Planner planner1 = stmtExecutor1.planner();
List<PlanFragment> fragments1 = planner1.getFragments();
@ -367,7 +358,7 @@ public class PlannerTest {
@Test
public void testAccessingVisibleColumnWithoutPartition() throws Exception {
String sql = "select count(k1) from db1.tbl2";
StmtExecutor stmtExecutor = new StmtExecutor(ctx, sql);
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
stmtExecutor.execute();
Assert.assertNotNull(stmtExecutor.planner());
}
@ -379,7 +370,7 @@ public class PlannerTest {
"LEFT JOIN (SELECT 1 AS line, k1, k2, k3 FROM db1.tbl3) t\n" +
"ON t.k1 = a.k1 AND t.k3 = a.k3\n" +
"GROUP BY a.k1, a.k3";
StmtExecutor stmtExecutor = new StmtExecutor(ctx, sql);
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
stmtExecutor.execute();
Assert.assertNotNull(stmtExecutor.planner());
Planner planner = stmtExecutor.planner();
@ -403,7 +394,7 @@ public class PlannerTest {
@Test
public void testBigintSlotRefCompareDecimalLiteral() {
java.util.function.BiConsumer<String, String> compare = (sql1, sql2) -> {
StmtExecutor stmtExecutor1 = new StmtExecutor(ctx, sql1);
StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, sql1);
try {
stmtExecutor1.execute();
} catch (Exception e) {
@ -413,7 +404,7 @@ public class PlannerTest {
List<PlanFragment> fragments1 = planner1.getFragments();
String plan1 = planner1.getExplainString(fragments1, new ExplainOptions(false, false));
StmtExecutor stmtExecutor2 = new StmtExecutor(ctx, sql2);
StmtExecutor stmtExecutor2 = new StmtExecutor(connectContext, sql2);
try {
stmtExecutor2.execute();
} catch (Exception e) {
@ -441,12 +432,11 @@ public class PlannerTest {
}
@Test
public void testStringType() throws Exception {
public void testStringType() {
String createTbl1 = "create table db1.tbl1(k1 string, k2 varchar(32), k3 varchar(32), k4 int) "
+ "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets 3 properties('replication_num' = '1')";
expectedEx.expect(AnalysisException.class);
expectedEx.expectMessage("String Type should not be used in key column[k1].");
UtFrameUtils.parseAndAnalyzeStmt(createTbl1, ctx);
AnalysisException exception =
assertThrows(AnalysisException.class, () -> parseAndAnalyzeStmt(createTbl1));
assertTrue(exception.getMessage().contains("String Type should not be used in key column[k1]."));
}
}

586
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

File diff suppressed because it is too large Load Diff

77
fe/fe-core/src/test/java/org/apache/doris/utframe/DemoTest.java
View File

@ -18,34 +18,24 @@
package org.apache.doris.utframe;
import org.apache.doris.alter.AlterJobV2;
import org.apache.doris.alter.AlterJobV2.JobState;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.MockedFrontend.EnvVarNotSetException;
import org.apache.doris.utframe.MockedFrontend.FeStartException;
import org.apache.doris.utframe.MockedFrontend.NotInitException;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/*
* This demo shows how to run unit test with mocked FE and BE.
@ -55,88 +45,77 @@ import java.util.UUID;
* 3. Make a schema change to tbl.
* 4. send a query and get query plan
*/
public class DemoTest {
public class DemoTest extends TestWithFeService {
// use a unique dir so that it won't be conflict with other unit test which
// may also start a Mocked Frontend
private static String runningDirBase = "fe";
private static String runningDir = runningDirBase + "/mocked/DemoTest/" + UUID.randomUUID().toString() + "/";
@BeforeClass
public static void beforeClass() throws EnvVarNotSetException, IOException,
FeStartException, NotInitException, DdlException, InterruptedException {
@Override
protected void runBeforeAll() throws Exception {
FeConstants.default_scheduler_interval_millisecond = 10;
UtFrameUtils.createDorisCluster(runningDir);
}
@AfterClass
public static void TearDown() {
UtFrameUtils.cleanDorisFeDir(runningDirBase);
}
@Test
public void testCreateDbAndTable() throws Exception {
// 1. create connect context
ConnectContext ctx = UtFrameUtils.createDefaultCtx();
connectContext = createDefaultCtx();
// 2. create database db1
String createDbStmtStr = "create database db1;";
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, ctx);
Catalog.getCurrentCatalog().createDb(createDbStmt);
createDatabase("db1");
System.out.println(Catalog.getCurrentCatalog().getDbNames());
// 3. create table tbl1
String createTblStmtStr = "create table db1.tbl1(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
createTable("create table db1.tbl1(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');");
// 4. get and test the created db and table
Database db = Catalog.getCurrentCatalog().getDbOrMetaException("default_cluster:db1");
OlapTable tbl = db.getTableOrMetaException("tbl1", Table.TableType.OLAP);
tbl.readLock();
try {
Assert.assertNotNull(tbl);
Assertions.assertNotNull(tbl);
System.out.println(tbl.getName());
Assert.assertEquals("Doris", tbl.getEngine());
Assert.assertEquals(1, tbl.getBaseSchema().size());
Assertions.assertEquals("Doris", tbl.getEngine());
Assertions.assertEquals(1, tbl.getBaseSchema().size());
} finally {
tbl.readUnlock();
}
// 5. process a schema change job
String alterStmtStr = "alter table db1.tbl1 add column k2 int default '1'";
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(alterStmtStr, ctx);
AlterTableStmt alterTableStmt = (AlterTableStmt) parseAndAnalyzeStmt(alterStmtStr);
Catalog.getCurrentCatalog().getAlterInstance().processAlterTable(alterTableStmt);
// 6. check alter job
Map<Long, AlterJobV2> alterJobs = Catalog.getCurrentCatalog().getSchemaChangeHandler().getAlterJobsV2();
Assert.assertEquals(1, alterJobs.size());
Assertions.assertEquals(1, alterJobs.size());
for (AlterJobV2 alterJobV2 : alterJobs.values()) {
while (!alterJobV2.getJobState().isFinalState()) {
System.out.println("alter job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState());
Thread.sleep(1000);
}
System.out.println("alter job " + alterJobV2.getJobId() + " is done. state: " + alterJobV2.getJobState());
Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState());
Assertions.assertEquals(JobState.FINISHED, alterJobV2.getJobState());
}
OlapTable tbl1 = db.getTableOrMetaException("tbl1", Table.TableType.OLAP);
tbl1.readLock();
try {
Assert.assertEquals(2, tbl1.getBaseSchema().size());
Assertions.assertEquals(2, tbl1.getBaseSchema().size());
String baseIndexName = tbl1.getIndexNameById(tbl.getBaseIndexId());
Assert.assertEquals(baseIndexName, tbl1.getName());
Assertions.assertEquals(baseIndexName, tbl1.getName());
MaterializedIndexMeta indexMeta = tbl1.getIndexMetaByIndexId(tbl1.getBaseIndexId());
Assert.assertNotNull(indexMeta);
Assertions.assertNotNull(indexMeta);
} finally {
tbl1.readUnlock();
}
// 7. query
// TODO: we can not process real query for now. So it has to be a explain query
String queryStr = "explain select * from db1.tbl1";
StmtExecutor stmtExecutor = new StmtExecutor(ctx, queryStr);
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, queryStr);
stmtExecutor.execute();
Planner planner = stmtExecutor.planner();
List<PlanFragment> fragments = planner.getFragments();
Assert.assertEquals(1, fragments.size());
Assertions.assertEquals(1, fragments.size());
PlanFragment fragment = fragments.get(0);
Assert.assertTrue(fragment.getPlanRoot() instanceof OlapScanNode);
Assert.assertEquals(0, fragment.getChildren().size());
Assertions.assertTrue(fragment.getPlanRoot() instanceof OlapScanNode);
Assertions.assertEquals(0, fragment.getChildren().size());
}
}

6
fe/fe-core/src/test/java/org/apache/doris/utframe/DorisAssert.java
View File

@ -50,6 +50,12 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
/**
* This class is deprecated.
* If you want to start a FE server in unit test, please let your
* test class extend {@link TestWithFeService}.
*/
@Deprecated
public class DorisAssert {
private ConnectContext ctx;

382
fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java Normal file
View File

@ -0,0 +1,382 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.utframe;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.net.DatagramSocket;
import java.net.ServerSocket;
import java.net.SocketException;
import java.nio.channels.SocketChannel;
import java.nio.file.Files;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.CreateViewStmt;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.utframe.MockedBackendFactory.DefaultBeThriftServiceImpl;
import org.apache.doris.utframe.MockedBackendFactory.DefaultHeartbeatServiceImpl;
import org.apache.doris.utframe.MockedBackendFactory.DefaultPBackendServiceImpl;
import org.apache.doris.utframe.MockedFrontend.EnvVarNotSetException;
import org.apache.doris.utframe.MockedFrontend.FeStartException;
import org.apache.doris.utframe.MockedFrontend.NotInitException;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestInstance;
/**
* This is the base class for unit class that wants to start a FE service.
* <p>
* Concrete test class must be derived class of {@link TestWithFeService}, {@link DemoTest} is
* an example.
* <p>
* This class use {@link TestInstance} in JUnit5 to do initialization and cleanup stuff. Unlike
* deprecated legacy combination-based implementation {@link UtFrameUtils}, we use an inherit-manner,
* thus we could wrap common logic in this base class. It's more easy to use.
* <p>
* Note:
* Unit-test method in derived classes must use the JUnit5 {@link org.junit.jupiter.api.Test}
* annotation, rather than the old JUnit4 {@link org.junit.Test} or others.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public abstract class TestWithFeService {
protected String runningDir =
"fe/mocked/" + getClass().getSimpleName() + "/" + UUID.randomUUID() + "/";
protected ConnectContext connectContext;
@BeforeAll
public final void beforeAll() throws Exception {
connectContext = createDefaultCtx();
createDorisCluster();
runBeforeAll();
}
@AfterAll
public final void afterAll() throws Exception {
runAfterAll();
cleanDorisFeDir(runningDir);
}
protected void runBeforeAll() throws Exception {
}
protected void runAfterAll() throws Exception {
}
// Help to create a mocked ConnectContext.
protected ConnectContext createDefaultCtx() throws IOException {
SocketChannel channel = SocketChannel.open();
ConnectContext ctx = new ConnectContext(channel);
ctx.setCluster(SystemInfoService.DEFAULT_CLUSTER);
ctx.setCurrentUserIdentity(UserIdentity.ROOT);
ctx.setQualifiedUser(PaloAuth.ROOT_USER);
ctx.setRemoteIP("127.0.0.1");
ctx.setCatalog(Catalog.getCurrentCatalog());
ctx.setThreadLocalInfo();
return ctx;
}
// Parse an origin stmt and analyze it. Return a StatementBase instance.
protected StatementBase parseAndAnalyzeStmt(String originStmt)
throws Exception {
System.out.println("begin to parse stmt: " + originStmt);
SqlScanner input =
new SqlScanner(new StringReader(originStmt),
connectContext.getSessionVariable().getSqlMode());
SqlParser parser = new SqlParser(input);
Analyzer analyzer = new Analyzer(connectContext.getCatalog(), connectContext);
StatementBase statementBase = null;
try {
statementBase = SqlParserUtils.getFirstStmt(parser);
} catch (AnalysisException e) {
String errorMessage = parser.getErrorMsg(originStmt);
System.err.println("parse failed: " + errorMessage);
if (errorMessage == null) {
throw e;
} else {
throw new AnalysisException(errorMessage, e);
}
}
statementBase.analyze(analyzer);
return statementBase;
}
// for analyzing multi statements
protected List<StatementBase> parseAndAnalyzeStmts(String originStmt) throws Exception {
System.out.println("begin to parse stmts: " + originStmt);
SqlScanner input = new SqlScanner(new StringReader(originStmt), connectContext.getSessionVariable().getSqlMode());
SqlParser parser = new SqlParser(input);
Analyzer analyzer = new Analyzer(connectContext.getCatalog(), connectContext);
List<StatementBase> statementBases = null;
try {
statementBases = SqlParserUtils.getMultiStmts(parser);
} catch (AnalysisException e) {
String errorMessage = parser.getErrorMsg(originStmt);
System.err.println("parse failed: " + errorMessage);
if (errorMessage == null) {
throw e;
} else {
throw new AnalysisException(errorMessage, e);
}
}
for (StatementBase stmt : statementBases) {
stmt.analyze(analyzer);
}
return statementBases;
}
protected String generateRandomFeRunningDir(Class testSuiteClass) {
return generateRandomFeRunningDir(testSuiteClass.getSimpleName());
}
protected String generateRandomFeRunningDir(String testSuiteName) {
return "fe" + "/mocked/" + testSuiteName + "/" + UUID.randomUUID().toString() + "/";
}
protected int startFEServer(String runningDir) throws EnvVarNotSetException, IOException,
FeStartException, NotInitException, DdlException, InterruptedException {
// get DORIS_HOME
String dorisHome = System.getenv("DORIS_HOME");
if (Strings.isNullOrEmpty(dorisHome)) {
dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString();
}
Config.plugin_dir = dorisHome + "/plugins";
Config.custom_config_dir = dorisHome + "/conf";
File file = new File(Config.custom_config_dir);
if (!file.exists()) {
file.mkdir();
}
int fe_http_port = findValidPort();
int fe_rpc_port = findValidPort();
int fe_query_port = findValidPort();
int fe_edit_log_port = findValidPort();
// start fe in "DORIS_HOME/fe/mocked/"
MockedFrontend frontend = MockedFrontend.getInstance();
Map<String, String> feConfMap = Maps.newHashMap();
// set additional fe config
feConfMap.put("http_port", String.valueOf(fe_http_port));
feConfMap.put("rpc_port", String.valueOf(fe_rpc_port));
feConfMap.put("query_port", String.valueOf(fe_query_port));
feConfMap.put("edit_log_port", String.valueOf(fe_edit_log_port));
feConfMap.put("tablet_create_timeout_second", "10");
frontend.init(dorisHome + "/" + runningDir, feConfMap);
frontend.start(new String[0]);
return fe_rpc_port;
}
protected void createDorisCluster()
throws InterruptedException, NotInitException, IOException, DdlException,
EnvVarNotSetException, FeStartException {
createDorisCluster(runningDir, 1);
}
protected void createDorisCluster(String runningDir, int backendNum)
throws EnvVarNotSetException, IOException, FeStartException,
NotInitException, DdlException, InterruptedException {
int fe_rpc_port = startFEServer(runningDir);
for (int i = 0; i < backendNum; i++) {
createBackend("127.0.0.1", fe_rpc_port);
// sleep to wait first heartbeat
Thread.sleep(6000);
}
}
// Create multi backends with different host for unit test.
// the host of BE will be "127.0.0.1", "127.0.0.2"
protected void createDorisClusterWithMultiTag(String runningDir,
int backendNum)
throws EnvVarNotSetException, IOException, FeStartException, NotInitException,
DdlException, InterruptedException {
// set runningUnitTest to true, so that for ut, the agent task will be send to "127.0.0.1" to make cluster running well.
FeConstants.runningUnitTest = true;
int fe_rpc_port = startFEServer(runningDir);
for (int i = 0; i < backendNum; i++) {
String host = "127.0.0." + (i + 1);
createBackend(host, fe_rpc_port);
}
// sleep to wait first heartbeat
Thread.sleep(6000);
}
protected void createBackend(String beHost, int fe_rpc_port)
throws IOException, InterruptedException {
int be_heartbeat_port = findValidPort();
int be_thrift_port = findValidPort();
int be_brpc_port = findValidPort();
int be_http_port = findValidPort();
// start be
MockedBackend backend = MockedBackendFactory.createBackend(beHost,
be_heartbeat_port, be_thrift_port, be_brpc_port, be_http_port,
new DefaultHeartbeatServiceImpl(be_thrift_port, be_http_port, be_brpc_port),
new DefaultBeThriftServiceImpl(), new DefaultPBackendServiceImpl());
backend.setFeAddress(new TNetworkAddress("127.0.0.1", fe_rpc_port));
backend.start();
// add be
Backend be = new Backend(Catalog.getCurrentCatalog().getNextId(), backend.getHost(), backend.getHeartbeatPort());
Map<String, DiskInfo> disks = Maps.newHashMap();
DiskInfo diskInfo1 = new DiskInfo("/path" + be.getId());
diskInfo1.setTotalCapacityB(1000000);
diskInfo1.setAvailableCapacityB(500000);
diskInfo1.setDataUsedCapacityB(480000);
disks.put(diskInfo1.getRootPath(), diskInfo1);
be.setDisks(ImmutableMap.copyOf(disks));
be.setAlive(true);
be.setOwnerClusterName(SystemInfoService.DEFAULT_CLUSTER);
be.setBePort(be_thrift_port);
be.setHttpPort(be_http_port);
be.setBrpcPort(be_brpc_port);
Catalog.getCurrentSystemInfo().addBackend(be);
}
protected void cleanDorisFeDir(String baseDir) {
try {
FileUtils.deleteDirectory(new File(baseDir));
} catch (IOException e) {
e.printStackTrace();
}
}
protected int findValidPort() {
int port = 0;
while (true) {
try (ServerSocket socket = new ServerSocket(0)) {
socket.setReuseAddress(true);
port = socket.getLocalPort();
try (DatagramSocket datagramSocket = new DatagramSocket(port)) {
datagramSocket.setReuseAddress(true);
break;
} catch (SocketException e) {
System.out.println("The port " + port + " is invalid and try another port.");
}
} catch (IOException e) {
throw new IllegalStateException("Could not find a free TCP/IP port to start HTTP Server on");
}
}
return port;
}
protected String getSQLPlanOrErrorMsg(String sql) throws Exception {
return getSQLPlanOrErrorMsg(sql, false);
}
protected String getSQLPlanOrErrorMsg(String sql, boolean isVerbose) throws Exception {
connectContext.getState().reset();
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
connectContext.setExecutor(stmtExecutor);
ConnectContext.get().setExecutor(stmtExecutor);
stmtExecutor.execute();
if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR) {
Planner planner = stmtExecutor.planner();
return planner.getExplainString(planner.getFragments(), new ExplainOptions(isVerbose, false));
} else {
return connectContext.getState().getErrorMessage();
}
}
protected Planner getSQLPlanner(String queryStr) throws Exception {
connectContext.getState().reset();
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, queryStr);
stmtExecutor.execute();
if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR) {
return stmtExecutor.planner();
} else {
return null;
}
}
protected StmtExecutor getSqlStmtExecutor(String queryStr) throws Exception {
connectContext.getState().reset();
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, queryStr);
stmtExecutor.execute();
if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR) {
return stmtExecutor;
} else {
return null;
}
}
protected void createDatabase(String db) throws Exception {
String createDbStmtStr = "CREATE DATABASE " + db;
CreateDbStmt createDbStmt = (CreateDbStmt) parseAndAnalyzeStmt(createDbStmtStr);
Catalog.getCurrentCatalog().createDb(createDbStmt);
}
protected void createTable(String sql) throws Exception {
createTables(sql);
}
protected void createTables(String... sqls) throws Exception {
for (String sql : sqls) {
CreateTableStmt stmt = (CreateTableStmt) parseAndAnalyzeStmt(sql);
Catalog.getCurrentCatalog().createTable(stmt);
}
}
protected void createView(String sql) throws Exception {
CreateViewStmt createViewStmt = (CreateViewStmt) parseAndAnalyzeStmt(sql);
Catalog.getCurrentCatalog().createView(createViewStmt);
}
protected void assertSQLPlanOrErrorMsgContains(String sql, String expect) throws Exception {
// Note: adding `EXPLAIN` is necessary for non-query SQL, e.g., DDL, DML, etc.
// TODO: Use a graceful way to get explain plan string, rather than modifying the SQL string.
Assertions.assertTrue(getSQLPlanOrErrorMsg("EXPLAIN " + sql).contains(expect));
}
protected void assertSQLPlanOrErrorMsgContains(String sql, String... expects) throws Exception {
String str = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
for (String expect : expects) {
Assertions.assertTrue(str.contains(expect));
}
}
}

7
fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
View File

@ -62,7 +62,12 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
* This class is deprecated.
* If you want to start a FE server in unit test, please let your test
* class extend {@link TestWithFeService}.
*/
@Deprecated
public class UtFrameUtils {
// Help to create a mocked ConnectContext.