diff --git a/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md b/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md new file mode 100644 index 0000000000..ccf9bdb368 --- /dev/null +++ b/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md @@ -0,0 +1,108 @@ +--- +{ + "title": "Query Schema Action", + "language": "en" +} +--- + + + +# Query Schema Action + + +## Request + +``` +POST /api/query_schema// +``` + +## Description + +The Query Schema Action can return the table creation statement for the given SQL-related table. Can be used to test some query scenarios locally. + +The API was released in version 1.2. + +## Path parameters + +* `` + + Specify the database name. This database will be considered as the default database for the current session, and will be used if the table name in SQL does not qualify the database name. + +## Query parameters + +None + +## Request body + +``` +text/plain + +sql +``` + +* "sql" field is the SQL string. + +## Response + +* Return value + + ``` + CREATE TABLE `tbl1` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + + CREATE TABLE `tbl2` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + ``` + +## Example + +1. Write the SQL in local file 1.sql + + ``` + select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1; + ``` + +2. Use curl to get the table creation statement. + + ``` + curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql + ``` \ No newline at end of file diff --git a/docs/sidebars.json b/docs/sidebars.json index 4263fc94f1..2de4954562 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -939,7 +939,8 @@ "admin-manual/http-actions/fe/show-proc-action", "admin-manual/http-actions/fe/check-decommission-action", "admin-manual/http-actions/fe/health-action", - "admin-manual/http-actions/fe/check-storage-type-action" + "admin-manual/http-actions/fe/check-storage-type-action", + "admin-manual/http-actions/fe/query-schema-action" ] }, "admin-manual/http-actions/restore-tablet", diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md new file mode 100644 index 0000000000..b5855c008d --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md @@ -0,0 +1,107 @@ +--- +{ + "title": "Query Schema Action", + "language": "zh-CN" +} +--- + + + +# Query Schema Action + + +## Request + +``` +POST /api/query_schema// +``` + +## Description + +Query Schema Action 可以返回给定的 SQL 有关的表的建表语句。可以用于本地测试一些查询场景。 +该 API 在 1.2 版本中发布。 + +## Path parameters + +* `` + + 指定数据库名称。该数据库会被视为当前session的默认数据库，如果在 SQL 中的表名没有限定数据库名称的话，则使用该数据库。 + +## Query parameters + +无 + +## Request body + +``` +text/plain + +sql +``` + +* sql 字段为具体的 SQL + +## Response + +* 返回结果集 + + ``` + CREATE TABLE `tbl1` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + + CREATE TABLE `tbl2` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + ``` + +## Example + +1. 在本地文件 1.sql 中写入 SQL + + ``` + select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1; + ``` + +2. 使用 curl 命令获取建表语句 + + ``` + curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql + ``` \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java index a155129067..9a1aac6216 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -188,7 +188,7 @@ public class InsertStmt extends DdlStmt { public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) throws AnalysisException { // get dbs of statement - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); tblName.analyze(analyzer); // disallow external catalog Util.prohibitExternalCatalog(tblName.getCtl(), this.getClass().getSimpleName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java index 7c4079dedd..03ee4048f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -454,18 +454,17 @@ public abstract class QueryStmt extends StatementBase implements Queriable { } if (pos > resultExprs.size()) { throw new AnalysisException( - errorPrefix + ": ordinal exceeds number of items in select list: " - + expr.toSql()); + errorPrefix + ": ordinal exceeds number of items in select list: " + expr.toSql()); } // Create copy to protect against accidentally shared state. return resultExprs.get((int) pos - 1).clone(); } - public void getWithClauseTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { + public void getWithClauseTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { if (withClause != null) { - withClause.getTables(analyzer, tableMap, parentViewNameSet); + withClause.getTables(analyzer, expandView, tableMap, parentViewNameSet); } } @@ -542,8 +541,8 @@ public abstract class QueryStmt extends StatementBase implements Queriable { // tmp in child stmt "(select siteid, citycode from tmp)" do not contain with_Clause // so need to check is view name by parentViewNameSet. // issue link: https://github.com/apache/doris/issues/4598 - public abstract void getTables(Analyzer analyzer, Map tables, Set parentViewNameSet) - throws AnalysisException; + public abstract void getTables(Analyzer analyzer, boolean expandView, Map tables, + Set parentViewNameSet) throws AnalysisException; // get TableRefs in this query, including physical TableRefs of this statement and // nested statements of inline views and with_Clause. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 0fa4d0f9c7..8e4ae4eaa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -292,18 +292,18 @@ public class SelectStmt extends QueryStmt { } @Override - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { - getWithClauseTables(analyzer, tableMap, parentViewNameSet); + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { + getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet); for (TableRef tblRef : fromClause) { if (tblRef instanceof InlineViewRef) { // Inline view reference QueryStmt inlineStmt = ((InlineViewRef) tblRef).getViewStmt(); - inlineStmt.getTables(analyzer, tableMap, parentViewNameSet); + inlineStmt.getTables(analyzer, expandView, tableMap, parentViewNameSet); } else if (tblRef instanceof TableValuedFunctionRef) { TableValuedFunctionRef tblFuncRef = (TableValuedFunctionRef) tblRef; tableMap.put(tblFuncRef.getTableFunction().getTable().getId(), - tblFuncRef.getTableFunction().getTable()); + tblFuncRef.getTableFunction().getTable()); } else { String dbName = tblRef.getName().getDb(); String tableName = tblRef.getName().getTbl(); @@ -320,14 +320,19 @@ public class SelectStmt extends QueryStmt { .getCatalogOrAnalysisException(tblRef.getName().getCtl()).getDbOrAnalysisException(dbName); TableIf table = db.getTableOrAnalysisException(tableName); - // check auth - if (!Env.getCurrentEnv().getAuth() - .checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT", - ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), - dbName + ": " + tableName); + if (expandView && (table instanceof View)) { + View view = (View) table; + view.getQueryStmt().getTables(analyzer, expandView, tableMap, parentViewNameSet); + } else { + // check auth + if (!Env.getCurrentEnv().getAuth() + .checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + dbName + ": " + tableName); + } + tableMap.put(table.getId(), table); } - tableMap.put(table.getId(), table); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java index 5d81c38b59..7fdd67c8d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java @@ -213,11 +213,11 @@ public class SetOperationStmt extends QueryStmt { } @Override - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { - getWithClauseTables(analyzer, tableMap, parentViewNameSet); + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { + getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet); for (SetOperand op : operands) { - op.getQueryStmt().getTables(analyzer, tableMap, parentViewNameSet); + op.getQueryStmt().getTables(analyzer, expandView, tableMap, parentViewNameSet); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java index e9e1e57d10..fb4fdbec6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java @@ -102,8 +102,7 @@ public class WithClause implements ParseNode { Preconditions.checkNotNull(other); views = Lists.newArrayList(); for (View view : other.views) { - views.add(new View(view.getName(), view.getQueryStmt().clone(), - view.getOriginalColLabels())); + views.add(new View(view.getName(), view.getQueryStmt().clone(), view.getOriginalColLabels())); } } @@ -113,12 +112,12 @@ public class WithClause implements ParseNode { } } - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { for (View view : views) { QueryStmt stmt = view.getQueryStmt(); parentViewNameSet.add(view.getName()); - stmt.getTables(analyzer, tableMap, parentViewNameSet); + stmt.getTables(analyzer, expandView, tableMap, parentViewNameSet); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 3cfa5d8502..66dc944aab 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2703,8 +2703,7 @@ public class Env { if (table.getType() == TableType.VIEW) { View view = (View) table; sb.append("CREATE VIEW `").append(table.getName()).append("` AS ").append(view.getInlineViewDef()); - sb.append(";"); - createTableStmt.add(sb.toString()); + createTableStmt.add(sb + ";"); return; } @@ -3045,7 +3044,7 @@ public class Env { sb.append("\n)"); } - createTableStmt.add(sb.toString()); + createTableStmt.add(sb + ";"); // 2. add partition if (separatePartition && (table instanceof OlapTable) && ((OlapTable) table).getPartitions().size() > 1) { @@ -3077,7 +3076,7 @@ public class Env { sb.append("(\"version_info\" = \""); sb.append(partition.getVisibleVersion()).append("\""); sb.append(");"); - addPartitionStmt.add(sb.toString()); + addPartitionStmt.add(sb + ";"); } } } @@ -3104,7 +3103,7 @@ public class Env { } } sb.append(");"); - createRollupStmt.add(sb.toString()); + createRollupStmt.add(sb + ";"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index d4f787132c..34ebae2415 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -17,41 +17,56 @@ package org.apache.doris.httpv2.rest; -import org.apache.doris.common.DdlException; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.QueryStmt; +import org.apache.doris.analysis.SqlParser; +import org.apache.doris.analysis.SqlScanner; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.util.ExecutionResultSet; import org.apache.doris.httpv2.util.StatementSubmitter; import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.SystemInfoService; +import com.google.common.base.Joiner; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.io.StringReader; import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** - * For execute stmt via http + * For execute stmt or get create table stmt via http */ @RestController public class StmtExecutionAction extends RestBaseController { private static final Logger LOG = LogManager.getLogger(StmtExecutionAction.class); private static StatementSubmitter stmtSubmitter = new StatementSubmitter(); - private static final String PARAM_SYNC = "sync"; - private static final String PARAM_LIMIT = "limit"; - private static final long DEFAULT_ROW_LIMIT = 1000; private static final long MAX_ROW_LIMIT = 10000; @@ -59,48 +74,75 @@ public class StmtExecutionAction extends RestBaseController { * Execute a SQL. * Request body: * { - * "stmt" : "select * from tbl1" + * "is_sync": 1, // optional + * "limit" : 1000 // optional + * "stmt" : "select * from tbl1" // required * } */ @RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) - public Object executeSQL( - @PathVariable(value = NS_KEY) String ns, - @PathVariable(value = DB_KEY) String dbName, - HttpServletRequest request, HttpServletResponse response, - @RequestBody String stmtBody) throws DdlException { + public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, @RequestBody String body) { ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); - if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { - return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); - } - - boolean isSync = true; - String syncParam = request.getParameter(PARAM_SYNC); - if (!Strings.isNullOrEmpty(syncParam)) { - isSync = syncParam.equals("1"); - } - - String limitParam = request.getParameter(PARAM_LIMIT); - long limit = DEFAULT_ROW_LIMIT; - if (!Strings.isNullOrEmpty(limitParam)) { - limit = Math.min(Long.valueOf(limitParam), MAX_ROW_LIMIT); + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalCatalog.INTERNAL_CATALOG_NAME; } Type type = new TypeToken() { }.getType(); - StmtRequestBody stmtRequestBody = new Gson().fromJson(stmtBody, type); + StmtRequestBody stmtRequestBody = new Gson().fromJson(body, type); if (Strings.isNullOrEmpty(stmtRequestBody.stmt)) { return ResponseEntityBuilder.badRequest("Missing statement request body"); } - LOG.info("stmt: {}", stmtRequestBody.stmt); + LOG.info("stmt: {}, isSync:{}, limit: {}", stmtRequestBody.stmt, stmtRequestBody.is_sync, + stmtRequestBody.limit); + ConnectContext.get().changeDefaultCatalog(ns); ConnectContext.get().setDatabase(getFullDbName(dbName)); + return executeQuery(authInfo, stmtRequestBody.is_sync, stmtRequestBody.limit, stmtRequestBody); + } - // 2. Submit stmt - StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext( - stmtRequestBody.stmt, authInfo.fullUserName, authInfo.password, limit - ); + + /** + * Get all create table stmt of a SQL + * + * @param ns + * @param dbName + * @param request + * @param response + * @param sql plain text of sql + * @return plain text of create table stmts + */ + @RequestMapping(path = "/api/query_schema/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) + public String querySchema(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, @RequestBody String sql) { + checkWithCookie(request, response, false); + + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalCatalog.INTERNAL_CATALOG_NAME; + } + LOG.info("sql: {}", sql); + + ConnectContext.get().changeDefaultCatalog(ns); + ConnectContext.get().setDatabase(getFullDbName(dbName)); + return getSchema(sql); + } + + /** + * Execute a query + * + * @param authInfo + * @param isSync + * @param limit + * @param stmtRequestBody + * @return + */ + @NotNull + private ResponseEntity executeQuery(ActionAuthorizationInfo authInfo, boolean isSync, long limit, + StmtRequestBody stmtRequestBody) { + StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext(stmtRequestBody.stmt, + authInfo.fullUserName, authInfo.password, limit); Future future = stmtSubmitter.submit(stmtCtx); if (isSync) { @@ -119,7 +161,38 @@ public class StmtExecutionAction extends RestBaseController { } } + @NotNull + private String getSchema(String sql) { + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(sql))); + StatementBase stmt = null; + try { + stmt = SqlParserUtils.getStmt(parser, 0); + if (!(stmt instanceof QueryStmt)) { + return "Only support query stmt"; + } + Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); + QueryStmt queryStmt = (QueryStmt) stmt; + Map tableMap = Maps.newHashMap(); + Set parentViewNameSet = Sets.newHashSet(); + queryStmt.getTables(analyzer, true, tableMap, parentViewNameSet); + + List createStmts = Lists.newArrayList(); + for (TableIf tbl : tableMap.values()) { + List createTableStmts = Lists.newArrayList(); + Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); + if (!createTableStmts.isEmpty()) { + createStmts.add(createTableStmts.get(0)); + } + } + return Joiner.on("\n\n").join(createStmts); + } catch (Exception e) { + return "Error:" + e.getMessage(); + } + } + private static class StmtRequestBody { + public Boolean is_sync = true; // CHECKSTYLE IGNORE THIS LINE + public Long limit = DEFAULT_ROW_LIMIT; public String stmt; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index fd4e28dd52..ac913cb727 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -630,11 +630,11 @@ public class StmtExecutor implements ProfileWriter { Set parentViewNameSet = Sets.newHashSet(); if (parsedStmt instanceof QueryStmt) { queryStmt = (QueryStmt) parsedStmt; - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); } else if (parsedStmt instanceof CreateTableAsSelectStmt) { CreateTableAsSelectStmt parsedStmt = (CreateTableAsSelectStmt) this.parsedStmt; queryStmt = parsedStmt.getQueryStmt(); - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); } else if (parsedStmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) parsedStmt; insertStmt.getTables(analyzer, tableMap, parentViewNameSet); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 28c09dc57b..df6354dff2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -17,15 +17,28 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.io.StringReader; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * test for CreateTableAsSelectStmt. **/ @@ -71,11 +84,10 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { createTableAsSelect(selectFromDecimal); Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" + " `userId` varchar(255) NOT NULL,\n" + " `amount_decimal` decimal(10, 2) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" - + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" - + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1)); String selectFromDecimal1 = "create table `test`.`select_decimal_table_1` PROPERTIES(\"replication_num\" = \"1\") " @@ -88,7 +100,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } else { Assertions.assertEquals( @@ -97,7 +109,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } } @@ -119,11 +131,10 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { createTableAsSelect(selectFromVarchar); ShowResultSet showResultSet = showCreateTableByName("select_varchar"); Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -138,7 +149,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"" - + ",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + ",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") " @@ -153,7 +164,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DUPLICATE KEY(`_col0`, `_col1`, `_col2`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -167,17 +178,16 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DUPLICATE KEY(`amount`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1)); + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") " + "as select userId as alias_name, username from `test`.`varchar_table`"; createTableAsSelect(selectAlias2); ShowResultSet showResultSet2 = showCreateTableByName("select_alias_2"); Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" + " `alias_name` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -193,7 +203,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") " + "as select vt.userId as userId1, jt.userId as userId2, vt.username, jt.status " @@ -202,11 +212,10 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { ShowResultSet showResultSet1 = showCreateTableByName("select_join1"); Assertions.assertEquals("CREATE TABLE `select_join1` (\n" + " `userId1` varchar(255) NOT NULL,\n" + " `userId2` varchar(255) NOT NULL,\n" + " `username` varchar(255) REPLACE NOT NULL,\n" - + " `status` int(11) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n" + + " `status` int(11) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId1`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -223,7 +232,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`user`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -237,7 +246,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\"" - + " = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); + + " = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @Test @@ -251,7 +260,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")" + "as with source_data as (select 1 as id union all select 2 as id) select * from source_data;"; @@ -261,7 +270,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "DUPLICATE KEY(`id`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1)); + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @Test @@ -272,13 +281,12 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { createTableAsSelect(selectFromPartition); ShowResultSet showResultSet = showCreateTableByName("selectPartition"); Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "PARTITION BY LIST(`userId`)\n" + "(PARTITION p1 VALUES IN (\"CA\",\"GB\",\"US\",\"ZH\"))\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -294,6 +302,55 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); + + "\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); + } + + @Test + public void testQuerySchema() throws Exception { + connectContext.setDatabase("default_cluster:test"); + String create1 = "create table test.qs1 (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties('replication_num' = '1')"; + String create2 = "create table test.qs2 (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties('replication_num' = '1')"; + createTables(create1, create2); + + String view1 = "create view test.v1 as select qs1.k1 from qs1 join qs2 on qs1.k1 = qs2.k1"; + String view2 = "create view test.v2 as with cte(s1) as (select * from v1) select * from cte"; + + createView(view1); + createView(view2); + + String sql1 = "select * from v1"; + + org.apache.doris.analysis.SqlParser parser = new SqlParser( + new org.apache.doris.analysis.SqlScanner(new StringReader(sql1))); + QueryStmt stmt = (QueryStmt) SqlParserUtils.getStmt(parser, 0); + Map tableMap = Maps.newHashMap(); + Set parentViewNameSet = Sets.newHashSet(); + Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); + stmt.getTables(analyzer, true, tableMap, parentViewNameSet); + + List createStmts = Lists.newArrayList(); + for (TableIf tbl : tableMap.values()) { + List createTableStmts = Lists.newArrayList(); + Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); + createStmts.add(createTableStmts.get(0)); + if (tbl.getName().equals("qs1")) { + Assert.assertEquals("CREATE TABLE `qs1` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n" + + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");", + createTableStmts.get(0)); + } else { + Assert.assertEquals("CREATE TABLE `qs2` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n" + + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");", + createTableStmts.get(0)); + } + } + Assert.assertEquals(2, createStmts.size()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 4f20dd9866..39603385b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -197,7 +197,7 @@ public class StmtExecutorTest { minTimes = 0; result = false; - queryStmt.getTables((Analyzer) any, (SortedMap) any, Sets.newHashSet()); + queryStmt.getTables((Analyzer) any, anyBoolean, (SortedMap) any, Sets.newHashSet()); minTimes = 0; queryStmt.getRedirectStatus(); diff --git a/regression-test/data/query/show/test_array_show_create.out b/regression-test/data/query/show/test_array_show_create.out index fd1da0cb7a..bda800f37e 100644 --- a/regression-test/data/query/show/test_array_show_create.out +++ b/regression-test/data/query/show/test_array_show_create.out @@ -1,4 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array NOT NULL,\n `k3` array NOT NULL,\n `k4` array NOT NULL,\n `k5` array NOT NULL,\n `k6` array NULL,\n `k7` array NOT NULL,\n `k8` array NOT NULL,\n `k9` array NOT NULL,\n `k10` array NOT NULL,\n `k11` array NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n) +test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array NOT NULL,\n `k3` array NOT NULL,\n `k4` array NOT NULL,\n `k5` array NOT NULL,\n `k6` array NULL,\n `k7` array NOT NULL,\n `k8` array NOT NULL,\n `k9` array NOT NULL,\n `k10` array NOT NULL,\n `k11` array NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n); diff --git a/regression-test/suites/query/show/test_array_show_create.groovy b/regression-test/suites/query/show/test_array_show_create.groovy index b5692663f4..382e2700ae 100644 --- a/regression-test/suites/query/show/test_array_show_create.groovy +++ b/regression-test/suites/query/show/test_array_show_create.groovy @@ -42,7 +42,7 @@ suite("test_array_show_create", "query") { PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2" - ) + ); """ // DDL/DML return 1 row and 3 column, the only value is update row count @@ -69,4 +69,4 @@ suite("test_array_show_create", "query") { try_sql("DROP TABLE IF EXISTS ${testTable}") } -} \ No newline at end of file +}