database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](http) get create table stmt for a given query (#11979)

Browse Source 
This API can help user to get all create table statement of a given SQL.
So that it will be easier to run the case in other Doris cluster.
See document for details
This commit is contained in:
Mingyu Chen
2022-08-25 15:02:05 +08:00
committed by GitHub
parent 003fdf2b36
commit aec24d4da1
15 changed files with 446 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md Normal file
View File

@ -0,0 +1,108 @@
---
{
"title": "Query Schema Action",
"language": "en"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Query Schema Action
## Request
```
POST /api/query_schema/<ns_name>/<db_name>
```
## Description
The Query Schema Action can return the table creation statement for the given SQL-related table. Can be used to test some query scenarios locally.
The API was released in version 1.2.
## Path parameters
* `<db_name>`
Specify the database name. This database will be considered as the default database for the current session, and will be used if the table name in SQL does not qualify the database name.
## Query parameters
None
## Request body
```
text/plain
sql
```
* "sql" field is the SQL string.
## Response
* Return value
```
CREATE TABLE `tbl1` (
`k1` int(11) NULL,
`k2` int(11) NULL
) ENGINE=OLAP
DUPLICATE KEY(`k1`, `k2`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
CREATE TABLE `tbl2` (
`k1` int(11) NULL,
`k2` int(11) NULL
) ENGINE=OLAP
DUPLICATE KEY(`k1`, `k2`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
```
## Example
1. Write the SQL in local file 1.sql
```
select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1;
```
2. Use curl to get the table creation statement.
```
curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql
```

3
docs/sidebars.json
View File

@ -939,7 +939,8 @@
"admin-manual/http-actions/fe/show-proc-action",
"admin-manual/http-actions/fe/check-decommission-action",
"admin-manual/http-actions/fe/health-action",
"admin-manual/http-actions/fe/check-storage-type-action"
"admin-manual/http-actions/fe/check-storage-type-action",
"admin-manual/http-actions/fe/query-schema-action"
]
},
"admin-manual/http-actions/restore-tablet",

107
docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md Normal file
View File

@ -0,0 +1,107 @@
---
{
"title": "Query Schema Action",
"language": "zh-CN"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Query Schema Action
## Request
```
POST /api/query_schema/<ns_name>/<db_name>
```
## Description
Query Schema Action 可以返回给定的 SQL 有关的表的建表语句。可以用于本地测试一些查询场景。
该 API 在 1.2 版本中发布。
## Path parameters
* `<db_name>`
指定数据库名称。该数据库会被视为当前session的默认数据库，如果在 SQL 中的表名没有限定数据库名称的话，则使用该数据库。
## Query parameters
无
## Request body
```
text/plain
sql
```
* sql 字段为具体的 SQL
## Response
* 返回结果集
```
CREATE TABLE `tbl1` (
`k1` int(11) NULL,
`k2` int(11) NULL
) ENGINE=OLAP
DUPLICATE KEY(`k1`, `k2`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
CREATE TABLE `tbl2` (
`k1` int(11) NULL,
`k2` int(11) NULL
) ENGINE=OLAP
DUPLICATE KEY(`k1`, `k2`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
```
## Example
1. 在本地文件 1.sql 中写入 SQL
```
select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1;
```
2. 使用 curl 命令获取建表语句
```
curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql
```

2
fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
View File

@ -188,7 +188,7 @@ public class InsertStmt extends DdlStmt {
public void getTables(Analyzer analyzer, Map<Long, TableIf> tableMap, Set<String> parentViewNameSet)
throws AnalysisException {
// get dbs of statement
queryStmt.getTables(analyzer, tableMap, parentViewNameSet);
queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet);
tblName.analyze(analyzer);
// disallow external catalog
Util.prohibitExternalCatalog(tblName.getCtl(), this.getClass().getSimpleName());

13
fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
View File

@ -454,18 +454,17 @@ public abstract class QueryStmt extends StatementBase implements Queriable {
}
if (pos > resultExprs.size()) {
throw new AnalysisException(
errorPrefix + ": ordinal exceeds number of items in select list: "
+ expr.toSql());
errorPrefix + ": ordinal exceeds number of items in select list: " + expr.toSql());
}
// Create copy to protect against accidentally shared state.
return resultExprs.get((int) pos - 1).clone();
}
public void getWithClauseTables(Analyzer analyzer, Map<Long, TableIf> tableMap, Set<String> parentViewNameSet)
throws AnalysisException {
public void getWithClauseTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf> tableMap,
Set<String> parentViewNameSet) throws AnalysisException {
if (withClause != null) {
withClause.getTables(analyzer, tableMap, parentViewNameSet);
withClause.getTables(analyzer, expandView, tableMap, parentViewNameSet);
}
}
@ -542,8 +541,8 @@ public abstract class QueryStmt extends StatementBase implements Queriable {
// tmp in child stmt "(select siteid, citycode from tmp)" do not contain with_Clause
// so need to check is view name by parentViewNameSet.
// issue link: https://github.com/apache/doris/issues/4598
public abstract void getTables(Analyzer analyzer, Map<Long, TableIf> tables, Set<String> parentViewNameSet)
throws AnalysisException;
public abstract void getTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf> tables,
Set<String> parentViewNameSet) throws AnalysisException;
// get TableRefs in this query, including physical TableRefs of this statement and
// nested statements of inline views and with_Clause.

29
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -292,18 +292,18 @@ public class SelectStmt extends QueryStmt {
}
@Override
public void getTables(Analyzer analyzer, Map<Long, TableIf> tableMap, Set<String> parentViewNameSet)
throws AnalysisException {
getWithClauseTables(analyzer, tableMap, parentViewNameSet);
public void getTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf> tableMap,
Set<String> parentViewNameSet) throws AnalysisException {
getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet);
for (TableRef tblRef : fromClause) {
if (tblRef instanceof InlineViewRef) {
// Inline view reference
QueryStmt inlineStmt = ((InlineViewRef) tblRef).getViewStmt();
inlineStmt.getTables(analyzer, tableMap, parentViewNameSet);
inlineStmt.getTables(analyzer, expandView, tableMap, parentViewNameSet);
} else if (tblRef instanceof TableValuedFunctionRef) {
TableValuedFunctionRef tblFuncRef = (TableValuedFunctionRef) tblRef;
tableMap.put(tblFuncRef.getTableFunction().getTable().getId(),
tblFuncRef.getTableFunction().getTable());
tblFuncRef.getTableFunction().getTable());
} else {
String dbName = tblRef.getName().getDb();
String tableName = tblRef.getName().getTbl();
@ -320,14 +320,19 @@ public class SelectStmt extends QueryStmt {
.getCatalogOrAnalysisException(tblRef.getName().getCtl()).getDbOrAnalysisException(dbName);
TableIf table = db.getTableOrAnalysisException(tableName);
// check auth
if (!Env.getCurrentEnv().getAuth()
.checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT",
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
dbName + ": " + tableName);
if (expandView && (table instanceof View)) {
View view = (View) table;
view.getQueryStmt().getTables(analyzer, expandView, tableMap, parentViewNameSet);
} else {
// check auth
if (!Env.getCurrentEnv().getAuth()
.checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT",
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
dbName + ": " + tableName);
}
tableMap.put(table.getId(), table);
}
tableMap.put(table.getId(), table);
}
}
}

8
fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
View File

@ -213,11 +213,11 @@ public class SetOperationStmt extends QueryStmt {
}
@Override
public void getTables(Analyzer analyzer, Map<Long, TableIf> tableMap, Set<String> parentViewNameSet)
throws AnalysisException {
getWithClauseTables(analyzer, tableMap, parentViewNameSet);
public void getTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf> tableMap,
Set<String> parentViewNameSet) throws AnalysisException {
getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet);
for (SetOperand op : operands) {
op.getQueryStmt().getTables(analyzer, tableMap, parentViewNameSet);
op.getQueryStmt().getTables(analyzer, expandView, tableMap, parentViewNameSet);
}
}

9
fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java
View File

@ -102,8 +102,7 @@ public class WithClause implements ParseNode {
Preconditions.checkNotNull(other);
views = Lists.newArrayList();
for (View view : other.views) {
views.add(new View(view.getName(), view.getQueryStmt().clone(),
view.getOriginalColLabels()));
views.add(new View(view.getName(), view.getQueryStmt().clone(), view.getOriginalColLabels()));
}
}
@ -113,12 +112,12 @@ public class WithClause implements ParseNode {
}
}
public void getTables(Analyzer analyzer, Map<Long, TableIf> tableMap, Set<String> parentViewNameSet)
throws AnalysisException {
public void getTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf> tableMap,
Set<String> parentViewNameSet) throws AnalysisException {
for (View view : views) {
QueryStmt stmt = view.getQueryStmt();
parentViewNameSet.add(view.getName());
stmt.getTables(analyzer, tableMap, parentViewNameSet);
stmt.getTables(analyzer, expandView, tableMap, parentViewNameSet);
}
}

9
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -2703,8 +2703,7 @@ public class Env {
if (table.getType() == TableType.VIEW) {
View view = (View) table;
sb.append("CREATE VIEW `").append(table.getName()).append("` AS ").append(view.getInlineViewDef());
sb.append(";");
createTableStmt.add(sb.toString());
createTableStmt.add(sb + ";");
return;
}
@ -3045,7 +3044,7 @@ public class Env {
sb.append("\n)");
}
createTableStmt.add(sb.toString());
createTableStmt.add(sb + ";");
// 2. add partition
if (separatePartition && (table instanceof OlapTable) && ((OlapTable) table).getPartitions().size() > 1) {
@ -3077,7 +3076,7 @@ public class Env {
sb.append("(\"version_info\" = \"");
sb.append(partition.getVisibleVersion()).append("\"");
sb.append(");");
addPartitionStmt.add(sb.toString());
addPartitionStmt.add(sb + ";");
}
}
}
@ -3104,7 +3103,7 @@ public class Env {
}
}
sb.append(");");
createRollupStmt.add(sb.toString());
createRollupStmt.add(sb + ";");
}
}
}

135
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java
View File

@ -17,41 +17,56 @@
package org.apache.doris.httpv2.rest;
import org.apache.doris.common.DdlException;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.httpv2.util.ExecutionResultSet;
import org.apache.doris.httpv2.util.StatementSubmitter;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.SystemInfoService;
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.io.StringReader;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* For execute stmt via http
* For execute stmt or get create table stmt via http
*/
@RestController
public class StmtExecutionAction extends RestBaseController {
private static final Logger LOG = LogManager.getLogger(StmtExecutionAction.class);
private static StatementSubmitter stmtSubmitter = new StatementSubmitter();
private static final String PARAM_SYNC = "sync";
private static final String PARAM_LIMIT = "limit";
private static final long DEFAULT_ROW_LIMIT = 1000;
private static final long MAX_ROW_LIMIT = 10000;
@ -59,48 +74,75 @@ public class StmtExecutionAction extends RestBaseController {
* Execute a SQL.
* Request body:
* {
* "stmt" : "select * from tbl1"
* "is_sync": 1, // optional
* "limit" : 1000 // optional
* "stmt" : "select * from tbl1" // required
* }
*/
@RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST})
public Object executeSQL(
@PathVariable(value = NS_KEY) String ns,
@PathVariable(value = DB_KEY) String dbName,
HttpServletRequest request, HttpServletResponse response,
@RequestBody String stmtBody) throws DdlException {
public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName,
HttpServletRequest request, HttpServletResponse response, @RequestBody String body) {
ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false);
if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) {
return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now");
}
boolean isSync = true;
String syncParam = request.getParameter(PARAM_SYNC);
if (!Strings.isNullOrEmpty(syncParam)) {
isSync = syncParam.equals("1");
}
String limitParam = request.getParameter(PARAM_LIMIT);
long limit = DEFAULT_ROW_LIMIT;
if (!Strings.isNullOrEmpty(limitParam)) {
limit = Math.min(Long.valueOf(limitParam), MAX_ROW_LIMIT);
if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) {
ns = InternalCatalog.INTERNAL_CATALOG_NAME;
}
Type type = new TypeToken<StmtRequestBody>() {
}.getType();
StmtRequestBody stmtRequestBody = new Gson().fromJson(stmtBody, type);
StmtRequestBody stmtRequestBody = new Gson().fromJson(body, type);
if (Strings.isNullOrEmpty(stmtRequestBody.stmt)) {
return ResponseEntityBuilder.badRequest("Missing statement request body");
}
LOG.info("stmt: {}", stmtRequestBody.stmt);
LOG.info("stmt: {}, isSync:{}, limit: {}", stmtRequestBody.stmt, stmtRequestBody.is_sync,
stmtRequestBody.limit);
ConnectContext.get().changeDefaultCatalog(ns);
ConnectContext.get().setDatabase(getFullDbName(dbName));
return executeQuery(authInfo, stmtRequestBody.is_sync, stmtRequestBody.limit, stmtRequestBody);
}
// 2. Submit stmt
StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext(
stmtRequestBody.stmt, authInfo.fullUserName, authInfo.password, limit
);
/**
* Get all create table stmt of a SQL
*
* @param ns
* @param dbName
* @param request
* @param response
* @param sql plain text of sql
* @return plain text of create table stmts
*/
@RequestMapping(path = "/api/query_schema/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST})
public String querySchema(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName,
HttpServletRequest request, HttpServletResponse response, @RequestBody String sql) {
checkWithCookie(request, response, false);
if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) {
ns = InternalCatalog.INTERNAL_CATALOG_NAME;
}
LOG.info("sql: {}", sql);
ConnectContext.get().changeDefaultCatalog(ns);
ConnectContext.get().setDatabase(getFullDbName(dbName));
return getSchema(sql);
}
/**
* Execute a query
*
* @param authInfo
* @param isSync
* @param limit
* @param stmtRequestBody
* @return
*/
@NotNull
private ResponseEntity executeQuery(ActionAuthorizationInfo authInfo, boolean isSync, long limit,
StmtRequestBody stmtRequestBody) {
StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext(stmtRequestBody.stmt,
authInfo.fullUserName, authInfo.password, limit);
Future<ExecutionResultSet> future = stmtSubmitter.submit(stmtCtx);
if (isSync) {
@ -119,7 +161,38 @@ public class StmtExecutionAction extends RestBaseController {
}
}
@NotNull
private String getSchema(String sql) {
SqlParser parser = new SqlParser(new SqlScanner(new StringReader(sql)));
StatementBase stmt = null;
try {
stmt = SqlParserUtils.getStmt(parser, 0);
if (!(stmt instanceof QueryStmt)) {
return "Only support query stmt";
}
Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get());
QueryStmt queryStmt = (QueryStmt) stmt;
Map<Long, TableIf> tableMap = Maps.newHashMap();
Set<String> parentViewNameSet = Sets.newHashSet();
queryStmt.getTables(analyzer, true, tableMap, parentViewNameSet);
List<String> createStmts = Lists.newArrayList();
for (TableIf tbl : tableMap.values()) {
List<String> createTableStmts = Lists.newArrayList();
Env.getDdlStmt(tbl, createTableStmts, null, null, false, true);
if (!createTableStmts.isEmpty()) {
createStmts.add(createTableStmts.get(0));
}
}
return Joiner.on("\n\n").join(createStmts);
} catch (Exception e) {
return "Error:" + e.getMessage();
}
}
private static class StmtRequestBody {
public Boolean is_sync = true; // CHECKSTYLE IGNORE THIS LINE
public Long limit = DEFAULT_ROW_LIMIT;
public String stmt;
}
}

4
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -630,11 +630,11 @@ public class StmtExecutor implements ProfileWriter {
Set<String> parentViewNameSet = Sets.newHashSet();
if (parsedStmt instanceof QueryStmt) {
queryStmt = (QueryStmt) parsedStmt;
queryStmt.getTables(analyzer, tableMap, parentViewNameSet);
queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet);
} else if (parsedStmt instanceof CreateTableAsSelectStmt) {
CreateTableAsSelectStmt parsedStmt = (CreateTableAsSelectStmt) this.parsedStmt;
queryStmt = parsedStmt.getQueryStmt();
queryStmt.getTables(analyzer, tableMap, parentViewNameSet);
queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet);
} else if (parsedStmt instanceof InsertStmt) {
InsertStmt insertStmt = (InsertStmt) parsedStmt;
insertStmt.getTables(analyzer, tableMap, parentViewNameSet);

109
fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
View File

@ -17,15 +17,28 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.io.StringReader;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* test for CreateTableAsSelectStmt.
**/
@ -71,11 +84,10 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
createTableAsSelect(selectFromDecimal);
Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `amount_decimal` decimal(10, 2) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n"
+ "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\","
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1));
String selectFromDecimal1 =
"create table `test`.`select_decimal_table_1` PROPERTIES(\"replication_num\" = \"1\") "
@ -88,7 +100,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\","
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
} else {
Assertions.assertEquals(
@ -97,7 +109,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\","
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1));
}
}
@ -119,11 +131,10 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
createTableAsSelect(selectFromVarchar);
ShowResultSet showResultSet = showCreateTableByName("select_varchar");
Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) REPLACE NOT NULL\n"
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n"
+ " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -138,7 +149,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\""
+ ",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ ",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet1.getResultRows().get(0).get(1));
String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") "
@ -153,7 +164,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DUPLICATE KEY(`_col0`, `_col1`, `_col2`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet2.getResultRows().get(0).get(1));
}
@ -167,17 +178,16 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DUPLICATE KEY(`amount`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\","
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1));
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1));
String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") "
+ "as select userId as alias_name, username from `test`.`varchar_table`";
createTableAsSelect(selectAlias2);
ShowResultSet showResultSet2 = showCreateTableByName("select_alias_2");
Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" + " `alias_name` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) REPLACE NOT NULL\n"
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n"
+ " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet2.getResultRows().get(0).get(1));
}
@ -193,7 +203,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") "
+ "as select vt.userId as userId1, jt.userId as userId2, vt.username, jt.status "
@ -202,11 +212,10 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
ShowResultSet showResultSet1 = showCreateTableByName("select_join1");
Assertions.assertEquals("CREATE TABLE `select_join1` (\n" + " `userId1` varchar(255) NOT NULL,\n"
+ " `userId2` varchar(255) NOT NULL,\n" + " `username` varchar(255) REPLACE NOT NULL,\n"
+ " `status` int(11) REPLACE NOT NULL\n"
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n"
+ " `status` int(11) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId1`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet1.getResultRows().get(0).get(1));
}
@ -223,7 +232,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`user`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -237,7 +246,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\""
+ " = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1));
+ " = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1));
}
@Test
@ -251,7 +260,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet.getResultRows().get(0).get(1));
String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")"
+ "as with source_data as (select 1 as id union all select 2 as id) select * from source_data;";
@ -261,7 +270,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "DUPLICATE KEY(`id`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\","
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1));
+ "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1));
}
@Test
@ -272,13 +281,12 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
createTableAsSelect(selectFromPartition);
ShowResultSet showResultSet = showCreateTableByName("selectPartition");
Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) REPLACE NOT NULL\n"
+ ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n"
+ " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "PARTITION BY LIST(`userId`)\n"
+ "(PARTITION p1 VALUES IN (\"CA\",\"GB\",\"US\",\"ZH\"))\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")",
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
showResultSet.getResultRows().get(0).get(1));
}
@ -294,6 +302,55 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
+ "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1));
+ "\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1));
}
@Test
public void testQuerySchema() throws Exception {
connectContext.setDatabase("default_cluster:test");
String create1 = "create table test.qs1 (k1 int, k2 int) distributed by hash(k1) "
+ "buckets 1 properties('replication_num' = '1')";
String create2 = "create table test.qs2 (k1 int, k2 int) distributed by hash(k1) "
+ "buckets 1 properties('replication_num' = '1')";
createTables(create1, create2);
String view1 = "create view test.v1 as select qs1.k1 from qs1 join qs2 on qs1.k1 = qs2.k1";
String view2 = "create view test.v2 as with cte(s1) as (select * from v1) select * from cte";
createView(view1);
createView(view2);
String sql1 = "select * from v1";
org.apache.doris.analysis.SqlParser parser = new SqlParser(
new org.apache.doris.analysis.SqlScanner(new StringReader(sql1)));
QueryStmt stmt = (QueryStmt) SqlParserUtils.getStmt(parser, 0);
Map<Long, TableIf> tableMap = Maps.newHashMap();
Set<String> parentViewNameSet = Sets.newHashSet();
Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get());
stmt.getTables(analyzer, true, tableMap, parentViewNameSet);
List<String> createStmts = Lists.newArrayList();
for (TableIf tbl : tableMap.values()) {
List<String> createTableStmts = Lists.newArrayList();
Env.getDdlStmt(tbl, createTableStmts, null, null, false, true);
createStmts.add(createTableStmts.get(0));
if (tbl.getName().equals("qs1")) {
Assert.assertEquals("CREATE TABLE `qs1` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n"
+ ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");",
createTableStmts.get(0));
} else {
Assert.assertEquals("CREATE TABLE `qs2` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n"
+ ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");",
createTableStmts.get(0));
}
}
Assert.assertEquals(2, createStmts.size());
}
}

2
fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java
View File

@ -197,7 +197,7 @@ public class StmtExecutorTest {
minTimes = 0;
result = false;
queryStmt.getTables((Analyzer) any, (SortedMap) any, Sets.newHashSet());
queryStmt.getTables((Analyzer) any, anyBoolean, (SortedMap) any, Sets.newHashSet());
minTimes = 0;
queryStmt.getRedirectStatus();

2
regression-test/data/query/show/test_array_show_create.out
View File

@ -1,4 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select --
test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array<smallint(6)> NOT NULL,\n `k3` array<int(11)> NOT NULL,\n `k4` array<bigint(20)> NOT NULL,\n `k5` array<char(1)> NOT NULL,\n `k6` array<varchar(20)> NULL,\n `k7` array<date> NOT NULL,\n `k8` array<datetime> NOT NULL,\n `k9` array<float> NOT NULL,\n `k10` array<double> NOT NULL,\n `k11` array<decimal(20, 6)> NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n)
test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array<smallint(6)> NOT NULL,\n `k3` array<int(11)> NOT NULL,\n `k4` array<bigint(20)> NOT NULL,\n `k5` array<char(1)> NOT NULL,\n `k6` array<varchar(20)> NULL,\n `k7` array<date> NOT NULL,\n `k8` array<datetime> NOT NULL,\n `k9` array<float> NOT NULL,\n `k10` array<double> NOT NULL,\n `k11` array<decimal(20, 6)> NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n);

4
regression-test/suites/query/show/test_array_show_create.groovy
View File

@ -42,7 +42,7 @@ suite("test_array_show_create", "query") {
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"storage_format" = "V2"
)
);
"""
// DDL/DML return 1 row and 3 column, the only value is update row count
@ -69,4 +69,4 @@ suite("test_array_show_create", "query") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
}
}