From 103fa4eb55d665b3e688db5b36cc52b6d3f6234e Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Tue, 29 Aug 2023 19:36:19 +0800 Subject: [PATCH] [feature](Export) support export with nereids (#23319) --- .../org/apache/doris/nereids/DorisLexer.g4 | 3 + .../org/apache/doris/nereids/DorisParser.g4 | 48 +++- .../org/apache/doris/analysis/ExportStmt.java | 44 ++- .../java/org/apache/doris/load/ExportJob.java | 1 + .../nereids/parser/LogicalPlanBuilder.java | 112 ++++++-- .../doris/nereids/parser/NereidsParser.java | 2 +- .../doris/nereids/trees/plans/PlanType.java | 3 +- .../trees/plans/commands/ExportCommand.java | 87 ++++++ .../trees/plans/visitor/CommandVisitor.java | 5 + .../data/export_p0/test_export_basic.out | 31 ++ .../data/export_p0/test_export_csv.out | 270 ++++++++++++++++++ .../suites/export_p0/test_export_basic.groovy | 5 +- .../suites/export_p0/test_export_csv.groovy | 23 +- 13 files changed, 582 insertions(+), 52 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 278e937c4d..fa0e696ff3 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -407,6 +407,9 @@ WITHIN: 'WITHIN'; YEAR: 'YEAR'; ZONE: 'ZONE'; DATEV2: 'DATEV2'; +S3: 'S3'; +HDFS: 'HDFS'; +BROKER: 'BROKER'; //--DORIS-KEYWORD-LIST-END //============================ // End of the keywords list diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index d1b0951adb..dd9849376a 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -53,11 +53,14 @@ statement (PARTITION partition=identifierList)? (USING relation (COMMA relation)*) whereClause #delete + | EXPORT TABLE tableName=multipartIdentifier + (PARTITION partition=identifierList)? + (whereClause)? + TO filePath=constant + (propertyClause)? + (withRemoteStorageSystem)? #export ; -propertiesStatment - : properties+=property (COMMA properties+=property)* - ; // -----------------Command accessories----------------- @@ -88,6 +91,22 @@ planType | ALL // default type ; +withRemoteStorageSystem + : WITH S3 LEFT_PAREN + brokerProperties=propertyItemList + RIGHT_PAREN + | WITH HDFS LEFT_PAREN + brokerProperties=propertyItemList + RIGHT_PAREN + | WITH LOCAL LEFT_PAREN + brokerProperties=propertyItemList + RIGHT_PAREN + | WITH BROKER brokerName=identifierOrText + (LEFT_PAREN + brokerProperties=propertyItemList + RIGHT_PAREN)? + ; + // -----------------Query----------------- // add queryOrganization for parse (q1) union (q2) union (q3) order by keys, otherwise 'order' will be recognized to be // identifier. @@ -95,7 +114,7 @@ planType outFileClause : INTO OUTFILE filePath=constant (FORMAT AS format=identifier)? - (PROPERTIES LEFT_PAREN properties+=property (COMMA properties+=property)* RIGHT_PAREN)? + (propertyClause)? ; query @@ -271,15 +290,25 @@ relationPrimary : multipartIdentifier specifiedPartition? tabletList? tableAlias relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN - (properties+=property (COMMA properties+=property)*)? + (properties=propertyItemList)? RIGHT_PAREN tableAlias #tableValuedFunction ; -property - : key=propertyItem EQ value=propertyItem +propertyClause + : PROPERTIES LEFT_PAREN fileProperties=propertyItemList RIGHT_PAREN ; -propertyItem : identifier | constant ; +propertyItemList + : properties+=propertyItem (COMMA properties+=propertyItem)* + ; + +propertyItem + : key=propertyKey EQ value=propertyValue + ; + +propertyKey : identifier | constant ; + +propertyValue : identifier | constant ; tableAlias : (AS? strictIdentifier identifierList?)? @@ -798,5 +827,8 @@ nonReserved | WITHIN | YEAR | ZONE + | S3 + | HDFS + | BROKER //--DEFAULT-NON-RESERVED-END ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java index 69f4d7174f..cc0d0883e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; @@ -50,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; // EXPORT statement, export data to dirs by broker. // @@ -65,7 +67,6 @@ public class ExportStmt extends StatementBase { private static final String DEFAULT_COLUMN_SEPARATOR = "\t"; private static final String DEFAULT_LINE_DELIMITER = "\n"; - private static final String DEFAULT_COLUMNS = ""; private static final String DEFAULT_PARALLELISM = "1"; private static final ImmutableSet PROPERTIES_SET = new ImmutableSet.Builder() @@ -85,6 +86,7 @@ public class ExportStmt extends StatementBase { private TableName tblName; private List partitionStringNames; private Expr whereExpr; + private String whereSql; private String path; private BrokerDesc brokerDesc; private Map properties = Maps.newHashMap(); @@ -121,13 +123,21 @@ public class ExportStmt extends StatementBase { this.brokerDesc = brokerDesc; this.columnSeparator = DEFAULT_COLUMN_SEPARATOR; this.lineDelimiter = DEFAULT_LINE_DELIMITER; - this.columns = DEFAULT_COLUMNS; Optional optionalSessionVariable = Optional.ofNullable( ConnectContext.get().getSessionVariable()); this.sessionVariables = optionalSessionVariable.orElse(VariableMgr.getDefaultSessionVariable()); } + /** + * This constructor used by nereids planner + */ + public ExportStmt(TableRef tableRef, String whereSql, String path, + Map properties, BrokerDesc brokerDesc) { + this(tableRef, (Expr) null, path, properties, brokerDesc); + this.whereSql = whereSql; + } + @Override public boolean needAuditEncryption() { return brokerDesc != null; @@ -209,6 +219,7 @@ public class ExportStmt extends StatementBase { // set where expr exportJob.setWhereExpr(this.whereExpr); + exportJob.setWhereSql(this.whereSql); // set path exportJob.setExportPath(this.path); @@ -223,7 +234,7 @@ public class ExportStmt extends StatementBase { exportJob.setMaxFileSize(this.maxFileSize); exportJob.setDeleteExistingFiles(this.deleteExistingFiles); - if (!Strings.isNullOrEmpty(this.columns)) { + if (columns != null) { Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); exportJob.setExportColumns(split.splitToList(this.columns.toLowerCase())); } @@ -343,7 +354,14 @@ public class ExportStmt extends StatementBase { properties, ExportStmt.DEFAULT_COLUMN_SEPARATOR)); this.lineDelimiter = Separator.convertSeparator(PropertyAnalyzer.analyzeLineDelimiter( properties, ExportStmt.DEFAULT_LINE_DELIMITER)); - this.columns = properties.getOrDefault(LoadStmt.KEY_IN_PARAM_COLUMNS, DEFAULT_COLUMNS); + // null means not specified + // "" means user specified zero columns + this.columns = properties.getOrDefault(LoadStmt.KEY_IN_PARAM_COLUMNS, null); + + // check columns are exits + if (columns != null) { + checkColumns(); + } // format this.format = properties.getOrDefault(LoadStmt.KEY_IN_PARAM_FORMAT_TYPE, "csv").toLowerCase(); @@ -370,6 +388,24 @@ public class ExportStmt extends StatementBase { } } + private void checkColumns() throws DdlException { + if (this.columns.isEmpty()) { + throw new DdlException("columns can not be empty"); + } + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(this.tblName.getDb()); + Table table = db.getTableOrDdlException(this.tblName.getTbl()); + List tableColumns = table.getBaseSchema().stream().map(column -> column.getName()) + .collect(Collectors.toList()); + Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); + + List columnsSpecified = split.splitToList(this.columns.toLowerCase()); + for (String columnName : columnsSpecified) { + if (!tableColumns.contains(columnName)) { + throw new DdlException("unknown column [" + columnName + "] in table [" + this.tblName.getTbl() + "]"); + } + } + } + @Override public String toSql() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index bd975bb6fe..5b0278041d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -156,6 +156,7 @@ public class ExportJob implements Writable { private TableRef tableRef; private Expr whereExpr; + private String whereSql; private String sql = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 05ea7dd1f1..708b2ef86d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -18,6 +18,9 @@ package org.apache.doris.nereids.parser; import org.apache.doris.analysis.ArithmeticExpr.Operator; +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.analysis.StorageBackend; +import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; @@ -50,6 +53,7 @@ import org.apache.doris.nereids.DorisParser.DereferenceContext; import org.apache.doris.nereids.DorisParser.ElementAtContext; import org.apache.doris.nereids.DorisParser.ExistContext; import org.apache.doris.nereids.DorisParser.ExplainContext; +import org.apache.doris.nereids.DorisParser.ExportContext; import org.apache.doris.nereids.DorisParser.FromClauseContext; import org.apache.doris.nereids.DorisParser.GroupingElementContext; import org.apache.doris.nereids.DorisParser.GroupingSetContext; @@ -81,9 +85,11 @@ import org.apache.doris.nereids.DorisParser.PlanTypeContext; import org.apache.doris.nereids.DorisParser.PredicateContext; import org.apache.doris.nereids.DorisParser.PredicatedContext; import org.apache.doris.nereids.DorisParser.PrimitiveDataTypeContext; -import org.apache.doris.nereids.DorisParser.PropertiesStatmentContext; -import org.apache.doris.nereids.DorisParser.PropertyContext; +import org.apache.doris.nereids.DorisParser.PropertyClauseContext; import org.apache.doris.nereids.DorisParser.PropertyItemContext; +import org.apache.doris.nereids.DorisParser.PropertyItemListContext; +import org.apache.doris.nereids.DorisParser.PropertyKeyContext; +import org.apache.doris.nereids.DorisParser.PropertyValueContext; import org.apache.doris.nereids.DorisParser.QualifiedNameContext; import org.apache.doris.nereids.DorisParser.QueryContext; import org.apache.doris.nereids.DorisParser.QueryOrganizationContext; @@ -117,6 +123,7 @@ import org.apache.doris.nereids.DorisParser.UserVariableContext; import org.apache.doris.nereids.DorisParser.WhereClauseContext; import org.apache.doris.nereids.DorisParser.WindowFrameContext; import org.apache.doris.nereids.DorisParser.WindowSpecContext; +import org.apache.doris.nereids.DorisParser.WithRemoteStorageSystemContext; import org.apache.doris.nereids.DorisParserBaseVisitor; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; @@ -232,6 +239,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteCommand; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.commands.ExportCommand; import org.apache.doris.nereids.trees.plans.commands.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -380,6 +388,68 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { ctx.explain()); } + @Override + public LogicalPlan visitExport(ExportContext ctx) { + List tableName = visitMultipartIdentifier(ctx.tableName); + List partitions = ctx.partition == null ? ImmutableList.of() : visitIdentifierList(ctx.partition); + String path = parseConstant(ctx.filePath); + String whereSql = null; + if (ctx.whereClause() != null) { + WhereClauseContext whereClauseContext = ctx.whereClause(); + int startIndex = whereClauseContext.start.getStartIndex(); + int stopIndex = whereClauseContext.stop.getStopIndex(); + org.antlr.v4.runtime.misc.Interval interval = new org.antlr.v4.runtime.misc.Interval(startIndex, + stopIndex); + whereSql = whereClauseContext.start.getInputStream().getText(interval); + } + + Map filePropertiesMap = null; + if (ctx.propertyClause() != null) { + filePropertiesMap = visitPropertyClause(ctx.propertyClause()); + } + + BrokerDesc brokerDesc = null; + if (ctx.withRemoteStorageSystem() != null) { + brokerDesc = visitWithRemoteStorageSystem(ctx.withRemoteStorageSystem()); + } + + return new ExportCommand(tableName, partitions, whereSql, path, filePropertiesMap, brokerDesc); + } + + @Override + public Map visitPropertyClause(PropertyClauseContext ctx) { + return visitPropertyItemList(ctx.fileProperties); + } + + @Override + public Map visitPropertyItemList(PropertyItemListContext ctx) { + Builder propertiesMap = ImmutableMap.builder(); + for (PropertyItemContext argument : ctx.properties) { + String key = parsePropertyKey(argument.key); + String value = parsePropertyValue(argument.value); + propertiesMap.put(key, value); + } + return propertiesMap.build(); + } + + @Override + public BrokerDesc visitWithRemoteStorageSystem(WithRemoteStorageSystemContext ctx) { + BrokerDesc brokerDesc = null; + + Map brokerPropertiesMap = visitPropertyItemList(ctx.brokerProperties); + + if (ctx.S3() != null) { + brokerDesc = new BrokerDesc("S3", StorageBackend.StorageType.S3, brokerPropertiesMap); + } else if (ctx.HDFS() != null) { + brokerDesc = new BrokerDesc("HDFS", StorageBackend.StorageType.HDFS, brokerPropertiesMap); + } else if (ctx.LOCAL() != null) { + brokerDesc = new BrokerDesc("HDFS", StorageType.LOCAL, brokerPropertiesMap); + } else if (ctx.BROKER() != null) { + brokerDesc = new BrokerDesc(visitIdentifierOrText(ctx.brokerName), brokerPropertiesMap); + } + return brokerDesc; + } + /** * Visit multi-statements. */ @@ -399,17 +469,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return logicalPlans; } - @Override - public Properties visitPropertiesStatment(PropertiesStatmentContext ctx) { - Builder map = ImmutableMap.builder(); - for (PropertyContext argument : ctx.properties) { - String key = parsePropertyItem(argument.key); - String value = parsePropertyItem(argument.value); - map.put(key, value); - } - return new Properties(map.build()); - } - /* ******************************************************************************************** * Plan parsing * ******************************************************************************************** */ @@ -641,14 +700,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return ParserUtils.withOrigin(ctx, () -> { String functionName = ctx.tvfName.getText(); - Builder map = ImmutableMap.builder(); - for (PropertyContext argument : ctx.properties) { - String key = parsePropertyItem(argument.key); - String value = parsePropertyItem(argument.value); - map.put(key, value); - } + Map map = visitPropertyItemList(ctx.properties); LogicalPlan relation = new UnboundTVFRelation(StatementScopeIdGenerator.newRelationId(), - functionName, new Properties(map.build())); + functionName, new Properties(map)); return withTableAlias(relation, ctx.tableAlias()); }); } @@ -1523,11 +1577,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { if (ctx.format != null) { format = ctx.format.getText(); } - Map properties = Maps.newHashMap(); - for (PropertyContext argument : ctx.properties) { - String key = parseConstant(argument.key.constant()); - String value = parseConstant(argument.value.constant()); - properties.put(key, value); + + Map properties = ImmutableMap.of(); + if (ctx.propertyClause() != null) { + properties = visitPropertyClause(ctx.propertyClause()); } Literal filePath = (Literal) visit(ctx.filePath); return new LogicalFileSink<>(filePath.getStringValue(), format, properties, ImmutableList.of(), plan); @@ -1973,7 +2026,14 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } } - private String parsePropertyItem(PropertyItemContext item) { + private String parsePropertyKey(PropertyKeyContext item) { + if (item.constant() != null) { + return parseConstant(item.constant()); + } + return item.getText(); + } + + private String parsePropertyValue(PropertyValueContext item) { if (item.constant() != null) { return parseConstant(item.constant()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 4ce8098a53..0bc228f386 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -81,7 +81,7 @@ public class NereidsParser { } public Properties parseProperties(String properties) { - return parse(properties, DorisParser::propertiesStatment); + return parse(properties, DorisParser::propertyItemList); } private T parse(String sql, Function parseFunction) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index da98e0e4d1..b54b12f18c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -120,5 +120,6 @@ public enum PlanType { EXPLAIN_COMMAND, INSERT_INTO_TABLE_COMMAND, SELECT_INTO_OUTFILE_COMMAND, - UPDATE_COMMAND + UPDATE_COMMAND, + EXPORT_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java new file mode 100644 index 0000000000..dbfd00059d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.analysis.ExportStmt; +import org.apache.doris.analysis.PartitionNames; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.TableRef; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import java.util.List; +import java.util.Map; + +/** + * export table + */ +public class ExportCommand extends Command implements ForwardWithSync { + private List nameParts; + private String whereSql; + private String path; + private List partitionsNameList; + private Map fileProperties; + private BrokerDesc brokerDesc; + + /** + * constructor of ExportCommand + */ + public ExportCommand(List nameParts, List partitions, String whereSql, String path, + Map fileProperties, BrokerDesc brokerDesc) { + super(PlanType.EXPORT_COMMAND); + this.nameParts = nameParts; + this.partitionsNameList = partitions; + this.whereSql = whereSql; + this.path = path.trim(); + this.fileProperties = fileProperties; + this.brokerDesc = brokerDesc; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + ExportStmt exportStmt = generateExportStmt(); + Analyzer analyzer = new Analyzer(ctx.getEnv(), ctx); + exportStmt.analyze(analyzer); + ctx.getEnv().getExportMgr().addExportJobAndRegisterTask(exportStmt); + } + + private ExportStmt generateExportStmt() { + // generate tableRef + PartitionNames partitionNames = null; + if (!this.partitionsNameList.isEmpty()) { + partitionNames = new PartitionNames(false, this.partitionsNameList); + } + TableRef tableRef = new TableRef(new TableName(getTableName()), null, partitionNames, null, null, null); + return new ExportStmt(tableRef, whereSql, path, fileProperties, brokerDesc); + } + + public String getTableName() { + return nameParts.stream().map(Utils::quoteIfNeeded) + .reduce((left, right) -> left + "." + right).orElse(""); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitExportCommand(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index fbdf108577..4b974f8bfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteCommand; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; +import org.apache.doris.nereids.trees.plans.commands.ExportCommand; import org.apache.doris.nereids.trees.plans.commands.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; @@ -49,4 +50,8 @@ public interface CommandVisitor { default R visitDeleteCommand(DeleteCommand deleteCommand, C context) { return visitCommand(deleteCommand, context); } + + default R visitExportCommand(ExportCommand exportCommand, C context) { + return visitCommand(exportCommand, context); + } } diff --git a/regression-test/data/export_p0/test_export_basic.out b/regression-test/data/export_p0/test_export_basic.out index 643a79881e..5afc94bb23 100644 --- a/regression-test/data/export_p0/test_export_basic.out +++ b/regression-test/data/export_p0/test_export_basic.out @@ -377,6 +377,37 @@ 69 ftw-69 87 -- !select_load3 -- +70 ftw-70 88 +71 ftw-71 89 +72 ftw-72 90 +73 ftw-73 91 +74 ftw-74 92 +75 ftw-75 93 +76 ftw-76 94 +77 ftw-77 95 +78 ftw-78 96 +79 ftw-79 97 +80 ftw-80 98 +81 ftw-81 99 +82 ftw-82 100 +83 ftw-83 101 +84 ftw-84 102 +85 ftw-85 103 +86 ftw-86 104 +87 ftw-87 105 +88 ftw-88 106 +89 ftw-89 107 +90 ftw-90 108 +91 ftw-91 109 +92 ftw-92 110 +93 ftw-93 111 +94 ftw-94 112 +95 ftw-95 113 +96 ftw-96 114 +97 ftw-97 115 +98 ftw-98 116 +99 ftw-99 117 +100 ftw-100 118 101 ftw-101 119 102 ftw-102 120 103 ftw-103 121 diff --git a/regression-test/data/export_p0/test_export_csv.out b/regression-test/data/export_p0/test_export_csv.out index a712d0931c..ca4469c87f 100644 --- a/regression-test/data/export_p0/test_export_csv.out +++ b/regression-test/data/export_p0/test_export_csv.out @@ -214,6 +214,96 @@ 8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8 8.8 8.8 char8 8 9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9 9.9 9.9 char9 9 10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10 10.1 10.1 char10 10 +11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11 11.11 11.11 char11 11 +12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12 12.12 12.12 char12 12 +13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13 13.13 13.13 char13 13 +14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14 14.14 14.14 char14 14 +15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15 15.15 15.15 char15 15 +16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16 16.16 16.16 char16 16 +17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17 17.17 17.17 char17 17 +18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18 18.18 18.18 char18 18 +19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19 19.19 19.19 char19 19 +20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20 20.2 20.2 char20 20 +21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21 21.21 21.21 char21 21 +22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22 22.22 22.22 char22 22 +23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23 23.23 23.23 char23 23 +24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24 24.24 24.24 char24 24 +25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25 25.25 25.25 char25 25 +26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26 26.26 26.26 char26 26 +27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27 27.27 27.27 char27 27 +28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28 28.28 28.28 char28 28 +29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29 29.29 29.29 char29 29 +30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30 30.3 30.3 char30 30 +31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31 31.31 31.31 char31 31 +32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32 32.32 32.32 char32 32 +33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33 33.33 33.33 char33 33 +34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34 34.34 34.34 char34 34 +35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35 35.35 35.35 char35 35 +36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36 36.36 36.36 char36 36 +37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37 37.37 37.37 char37 37 +38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38 38.38 38.38 char38 38 +39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39 39.39 39.39 char39 39 +40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40 40.4 40.4 char40 40 +41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41 41.41 41.41 char41 41 +42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42 42.42 42.42 char42 42 +43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43 43.43 43.43 char43 43 +44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44 44.44 44.44 char44 44 +45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45 45.45 45.45 char45 45 +46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46 46.46 46.46 char46 46 +47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47 47.47 47.47 char47 47 +48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48 48.48 48.48 char48 48 +49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49 49.49 49.49 char49 49 +50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50 50.5 50.5 char50 50 +51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51 51.51 51.51 char51 51 +52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52 52.52 52.52 char52 52 +53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53 53.53 53.53 char53 53 +54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54 54.54 54.54 char54 54 +55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55 55.55 55.55 char55 55 +56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56 56.56 56.56 char56 56 +57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57 57.57 57.57 char57 57 +58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58 58.58 58.58 char58 58 +59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59 59.59 59.59 char59 59 +60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60 60.6 60.6 char60 60 +61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61 61.61 61.61 char61 61 +62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62 62.62 62.62 char62 62 +63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63 63.63 63.63 char63 63 +64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64 64.64 64.64 char64 64 +65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65 65.65 65.65 char65 65 +66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66 66.66 66.66 char66 66 +67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67 67.67 67.67 char67 67 +68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68 68.68 68.68 char68 68 +69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69 69.69 69.69 char69 69 +70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70 70.7 70.7 char70 70 +71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71 71.71 71.71 char71 71 +72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72 72.72 72.72 char72 72 +73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73 73.73 73.73 char73 73 +74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74 74.74 74.74 char74 74 +75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75 75.75 75.75 char75 75 +76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76 76.76 76.76 char76 76 +77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77 77.77 77.77 char77 77 +78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78 78.78 78.78 char78 78 +79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79 79.79 79.79 char79 79 +80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80 80.8 80.8 char80 80 +81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81 81.81 81.81 char81 81 +82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82 82.82 82.82 char82 82 +83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83 83.83 83.83 char83 83 +84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84 84.84 84.84 char84 84 +85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85 85.85 85.85 char85 85 +86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86 86.86 86.86 char86 86 +87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87 87.87 87.87 char87 87 +88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88 88.88 88.88 char88 88 +89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89 89.89 89.89 char89 89 +90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90 90.9 90.9 char90 90 +91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91 91.91 91.91 char91 91 +92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92 92.92 92.92 char92 92 +93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93 93.93 93.93 char93 93 +94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94 94.94 94.94 char94 94 +95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95 95.95 95.95 char95 95 +96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96 96.96 96.96 char96 96 +97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97 97.97 97.97 char97 97 +98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98 98.98 98.98 char98 98 +99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99 99.99 99.99 char99 99 +100 2017-10-01 2017-10-01T00:00 \N \N \N \N \N \N \N \N \N \N \N -- !select_load3 -- 1 2017-10-01 2017-10-01T00:00 Beijing 1 1 true 1 1 1 1.1 1.1 char1 1 @@ -226,6 +316,96 @@ 8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8 8.8 8.8 char8 8 9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9 9.9 9.9 char9 9 10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10 10.1 10.1 char10 10 +11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11 11.11 11.11 char11 11 +12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12 12.12 12.12 char12 12 +13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13 13.13 13.13 char13 13 +14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14 14.14 14.14 char14 14 +15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15 15.15 15.15 char15 15 +16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16 16.16 16.16 char16 16 +17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17 17.17 17.17 char17 17 +18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18 18.18 18.18 char18 18 +19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19 19.19 19.19 char19 19 +20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20 20.2 20.2 char20 20 +21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21 21.21 21.21 char21 21 +22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22 22.22 22.22 char22 22 +23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23 23.23 23.23 char23 23 +24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24 24.24 24.24 char24 24 +25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25 25.25 25.25 char25 25 +26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26 26.26 26.26 char26 26 +27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27 27.27 27.27 char27 27 +28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28 28.28 28.28 char28 28 +29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29 29.29 29.29 char29 29 +30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30 30.3 30.3 char30 30 +31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31 31.31 31.31 char31 31 +32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32 32.32 32.32 char32 32 +33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33 33.33 33.33 char33 33 +34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34 34.34 34.34 char34 34 +35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35 35.35 35.35 char35 35 +36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36 36.36 36.36 char36 36 +37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37 37.37 37.37 char37 37 +38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38 38.38 38.38 char38 38 +39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39 39.39 39.39 char39 39 +40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40 40.4 40.4 char40 40 +41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41 41.41 41.41 char41 41 +42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42 42.42 42.42 char42 42 +43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43 43.43 43.43 char43 43 +44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44 44.44 44.44 char44 44 +45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45 45.45 45.45 char45 45 +46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46 46.46 46.46 char46 46 +47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47 47.47 47.47 char47 47 +48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48 48.48 48.48 char48 48 +49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49 49.49 49.49 char49 49 +50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50 50.5 50.5 char50 50 +51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51 51.51 51.51 char51 51 +52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52 52.52 52.52 char52 52 +53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53 53.53 53.53 char53 53 +54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54 54.54 54.54 char54 54 +55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55 55.55 55.55 char55 55 +56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56 56.56 56.56 char56 56 +57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57 57.57 57.57 char57 57 +58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58 58.58 58.58 char58 58 +59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59 59.59 59.59 char59 59 +60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60 60.6 60.6 char60 60 +61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61 61.61 61.61 char61 61 +62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62 62.62 62.62 char62 62 +63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63 63.63 63.63 char63 63 +64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64 64.64 64.64 char64 64 +65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65 65.65 65.65 char65 65 +66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66 66.66 66.66 char66 66 +67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67 67.67 67.67 char67 67 +68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68 68.68 68.68 char68 68 +69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69 69.69 69.69 char69 69 +70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70 70.7 70.7 char70 70 +71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71 71.71 71.71 char71 71 +72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72 72.72 72.72 char72 72 +73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73 73.73 73.73 char73 73 +74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74 74.74 74.74 char74 74 +75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75 75.75 75.75 char75 75 +76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76 76.76 76.76 char76 76 +77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77 77.77 77.77 char77 77 +78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78 78.78 78.78 char78 78 +79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79 79.79 79.79 char79 79 +80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80 80.8 80.8 char80 80 +81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81 81.81 81.81 char81 81 +82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82 82.82 82.82 char82 82 +83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83 83.83 83.83 char83 83 +84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84 84.84 84.84 char84 84 +85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85 85.85 85.85 char85 85 +86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86 86.86 86.86 char86 86 +87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87 87.87 87.87 char87 87 +88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88 88.88 88.88 char88 88 +89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89 89.89 89.89 char89 89 +90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90 90.9 90.9 char90 90 +91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91 91.91 91.91 char91 91 +92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92 92.92 92.92 char92 92 +93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93 93.93 93.93 char93 93 +94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94 94.94 94.94 char94 94 +95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95 95.95 95.95 char95 95 +96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96 96.96 96.96 char96 96 +97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97 97.97 97.97 char97 97 +98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98 98.98 98.98 char98 98 +99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99 99.99 99.99 char99 99 +100 2017-10-01 2017-10-01T00:00 \N \N \N \N \N \N \N \N \N \N \N -- !select_load4 -- 1 2017-10-01 2017-10-01T00:00 Beijing 1 1 true 1 1 1 1.1 1.1 char1 1 @@ -238,4 +418,94 @@ 8 2017-10-01 2017-10-01T00:00 Beijing 8 8 true 8 8 8 8.8 8.8 char8 8 9 2017-10-01 2017-10-01T00:00 Beijing 9 9 true 9 9 9 9.9 9.9 char9 9 10 2017-10-01 2017-10-01T00:00 Beijing 10 10 true 10 10 10 10.1 10.1 char10 10 +11 2017-10-01 2017-10-01T00:00 Beijing 11 11 true 11 11 11 11.11 11.11 char11 11 +12 2017-10-01 2017-10-01T00:00 Beijing 12 12 true 12 12 12 12.12 12.12 char12 12 +13 2017-10-01 2017-10-01T00:00 Beijing 13 13 true 13 13 13 13.13 13.13 char13 13 +14 2017-10-01 2017-10-01T00:00 Beijing 14 14 true 14 14 14 14.14 14.14 char14 14 +15 2017-10-01 2017-10-01T00:00 Beijing 15 15 true 15 15 15 15.15 15.15 char15 15 +16 2017-10-01 2017-10-01T00:00 Beijing 16 16 true 16 16 16 16.16 16.16 char16 16 +17 2017-10-01 2017-10-01T00:00 Beijing 17 17 true 17 17 17 17.17 17.17 char17 17 +18 2017-10-01 2017-10-01T00:00 Beijing 18 18 true 18 18 18 18.18 18.18 char18 18 +19 2017-10-01 2017-10-01T00:00 Beijing 19 19 true 19 19 19 19.19 19.19 char19 19 +20 2017-10-01 2017-10-01T00:00 Beijing 20 20 true 20 20 20 20.2 20.2 char20 20 +21 2017-10-01 2017-10-01T00:00 Beijing 21 21 true 21 21 21 21.21 21.21 char21 21 +22 2017-10-01 2017-10-01T00:00 Beijing 22 22 true 22 22 22 22.22 22.22 char22 22 +23 2017-10-01 2017-10-01T00:00 Beijing 23 23 true 23 23 23 23.23 23.23 char23 23 +24 2017-10-01 2017-10-01T00:00 Beijing 24 24 true 24 24 24 24.24 24.24 char24 24 +25 2017-10-01 2017-10-01T00:00 Beijing 25 25 true 25 25 25 25.25 25.25 char25 25 +26 2017-10-01 2017-10-01T00:00 Beijing 26 26 true 26 26 26 26.26 26.26 char26 26 +27 2017-10-01 2017-10-01T00:00 Beijing 27 27 true 27 27 27 27.27 27.27 char27 27 +28 2017-10-01 2017-10-01T00:00 Beijing 28 28 true 28 28 28 28.28 28.28 char28 28 +29 2017-10-01 2017-10-01T00:00 Beijing 29 29 true 29 29 29 29.29 29.29 char29 29 +30 2017-10-01 2017-10-01T00:00 Beijing 30 30 true 30 30 30 30.3 30.3 char30 30 +31 2017-10-01 2017-10-01T00:00 Beijing 31 31 true 31 31 31 31.31 31.31 char31 31 +32 2017-10-01 2017-10-01T00:00 Beijing 32 32 true 32 32 32 32.32 32.32 char32 32 +33 2017-10-01 2017-10-01T00:00 Beijing 33 33 true 33 33 33 33.33 33.33 char33 33 +34 2017-10-01 2017-10-01T00:00 Beijing 34 34 true 34 34 34 34.34 34.34 char34 34 +35 2017-10-01 2017-10-01T00:00 Beijing 35 35 true 35 35 35 35.35 35.35 char35 35 +36 2017-10-01 2017-10-01T00:00 Beijing 36 36 true 36 36 36 36.36 36.36 char36 36 +37 2017-10-01 2017-10-01T00:00 Beijing 37 37 true 37 37 37 37.37 37.37 char37 37 +38 2017-10-01 2017-10-01T00:00 Beijing 38 38 true 38 38 38 38.38 38.38 char38 38 +39 2017-10-01 2017-10-01T00:00 Beijing 39 39 true 39 39 39 39.39 39.39 char39 39 +40 2017-10-01 2017-10-01T00:00 Beijing 40 40 true 40 40 40 40.4 40.4 char40 40 +41 2017-10-01 2017-10-01T00:00 Beijing 41 41 true 41 41 41 41.41 41.41 char41 41 +42 2017-10-01 2017-10-01T00:00 Beijing 42 42 true 42 42 42 42.42 42.42 char42 42 +43 2017-10-01 2017-10-01T00:00 Beijing 43 43 true 43 43 43 43.43 43.43 char43 43 +44 2017-10-01 2017-10-01T00:00 Beijing 44 44 true 44 44 44 44.44 44.44 char44 44 +45 2017-10-01 2017-10-01T00:00 Beijing 45 45 true 45 45 45 45.45 45.45 char45 45 +46 2017-10-01 2017-10-01T00:00 Beijing 46 46 true 46 46 46 46.46 46.46 char46 46 +47 2017-10-01 2017-10-01T00:00 Beijing 47 47 true 47 47 47 47.47 47.47 char47 47 +48 2017-10-01 2017-10-01T00:00 Beijing 48 48 true 48 48 48 48.48 48.48 char48 48 +49 2017-10-01 2017-10-01T00:00 Beijing 49 49 true 49 49 49 49.49 49.49 char49 49 +50 2017-10-01 2017-10-01T00:00 Beijing 50 50 true 50 50 50 50.5 50.5 char50 50 +51 2017-10-01 2017-10-01T00:00 Beijing 51 51 true 51 51 51 51.51 51.51 char51 51 +52 2017-10-01 2017-10-01T00:00 Beijing 52 52 true 52 52 52 52.52 52.52 char52 52 +53 2017-10-01 2017-10-01T00:00 Beijing 53 53 true 53 53 53 53.53 53.53 char53 53 +54 2017-10-01 2017-10-01T00:00 Beijing 54 54 true 54 54 54 54.54 54.54 char54 54 +55 2017-10-01 2017-10-01T00:00 Beijing 55 55 true 55 55 55 55.55 55.55 char55 55 +56 2017-10-01 2017-10-01T00:00 Beijing 56 56 true 56 56 56 56.56 56.56 char56 56 +57 2017-10-01 2017-10-01T00:00 Beijing 57 57 true 57 57 57 57.57 57.57 char57 57 +58 2017-10-01 2017-10-01T00:00 Beijing 58 58 true 58 58 58 58.58 58.58 char58 58 +59 2017-10-01 2017-10-01T00:00 Beijing 59 59 true 59 59 59 59.59 59.59 char59 59 +60 2017-10-01 2017-10-01T00:00 Beijing 60 60 true 60 60 60 60.6 60.6 char60 60 +61 2017-10-01 2017-10-01T00:00 Beijing 61 61 true 61 61 61 61.61 61.61 char61 61 +62 2017-10-01 2017-10-01T00:00 Beijing 62 62 true 62 62 62 62.62 62.62 char62 62 +63 2017-10-01 2017-10-01T00:00 Beijing 63 63 true 63 63 63 63.63 63.63 char63 63 +64 2017-10-01 2017-10-01T00:00 Beijing 64 64 true 64 64 64 64.64 64.64 char64 64 +65 2017-10-01 2017-10-01T00:00 Beijing 65 65 true 65 65 65 65.65 65.65 char65 65 +66 2017-10-01 2017-10-01T00:00 Beijing 66 66 true 66 66 66 66.66 66.66 char66 66 +67 2017-10-01 2017-10-01T00:00 Beijing 67 67 true 67 67 67 67.67 67.67 char67 67 +68 2017-10-01 2017-10-01T00:00 Beijing 68 68 true 68 68 68 68.68 68.68 char68 68 +69 2017-10-01 2017-10-01T00:00 Beijing 69 69 true 69 69 69 69.69 69.69 char69 69 +70 2017-10-01 2017-10-01T00:00 Beijing 70 70 true 70 70 70 70.7 70.7 char70 70 +71 2017-10-01 2017-10-01T00:00 Beijing 71 71 true 71 71 71 71.71 71.71 char71 71 +72 2017-10-01 2017-10-01T00:00 Beijing 72 72 true 72 72 72 72.72 72.72 char72 72 +73 2017-10-01 2017-10-01T00:00 Beijing 73 73 true 73 73 73 73.73 73.73 char73 73 +74 2017-10-01 2017-10-01T00:00 Beijing 74 74 true 74 74 74 74.74 74.74 char74 74 +75 2017-10-01 2017-10-01T00:00 Beijing 75 75 true 75 75 75 75.75 75.75 char75 75 +76 2017-10-01 2017-10-01T00:00 Beijing 76 76 true 76 76 76 76.76 76.76 char76 76 +77 2017-10-01 2017-10-01T00:00 Beijing 77 77 true 77 77 77 77.77 77.77 char77 77 +78 2017-10-01 2017-10-01T00:00 Beijing 78 78 true 78 78 78 78.78 78.78 char78 78 +79 2017-10-01 2017-10-01T00:00 Beijing 79 79 true 79 79 79 79.79 79.79 char79 79 +80 2017-10-01 2017-10-01T00:00 Beijing 80 80 true 80 80 80 80.8 80.8 char80 80 +81 2017-10-01 2017-10-01T00:00 Beijing 81 81 true 81 81 81 81.81 81.81 char81 81 +82 2017-10-01 2017-10-01T00:00 Beijing 82 82 true 82 82 82 82.82 82.82 char82 82 +83 2017-10-01 2017-10-01T00:00 Beijing 83 83 true 83 83 83 83.83 83.83 char83 83 +84 2017-10-01 2017-10-01T00:00 Beijing 84 84 true 84 84 84 84.84 84.84 char84 84 +85 2017-10-01 2017-10-01T00:00 Beijing 85 85 true 85 85 85 85.85 85.85 char85 85 +86 2017-10-01 2017-10-01T00:00 Beijing 86 86 true 86 86 86 86.86 86.86 char86 86 +87 2017-10-01 2017-10-01T00:00 Beijing 87 87 true 87 87 87 87.87 87.87 char87 87 +88 2017-10-01 2017-10-01T00:00 Beijing 88 88 true 88 88 88 88.88 88.88 char88 88 +89 2017-10-01 2017-10-01T00:00 Beijing 89 89 true 89 89 89 89.89 89.89 char89 89 +90 2017-10-01 2017-10-01T00:00 Beijing 90 90 true 90 90 90 90.9 90.9 char90 90 +91 2017-10-01 2017-10-01T00:00 Beijing 91 91 true 91 91 91 91.91 91.91 char91 91 +92 2017-10-01 2017-10-01T00:00 Beijing 92 92 true 92 92 92 92.92 92.92 char92 92 +93 2017-10-01 2017-10-01T00:00 Beijing 93 93 true 93 93 93 93.93 93.93 char93 93 +94 2017-10-01 2017-10-01T00:00 Beijing 94 94 true 94 94 94 94.94 94.94 char94 94 +95 2017-10-01 2017-10-01T00:00 Beijing 95 95 true 95 95 95 95.95 95.95 char95 95 +96 2017-10-01 2017-10-01T00:00 Beijing 96 96 true 96 96 96 96.96 96.96 char96 96 +97 2017-10-01 2017-10-01T00:00 Beijing 97 97 true 97 97 97 97.97 97.97 char97 97 +98 2017-10-01 2017-10-01T00:00 Beijing 98 98 true 98 98 98 98.98 98.98 char98 98 +99 2017-10-01 2017-10-01T00:00 Beijing 99 99 true 99 99 99 99.99 99.99 char99 99 +100 2017-10-01 2017-10-01T00:00 \N \N \N \N \N \N \N \N \N \N \N diff --git a/regression-test/suites/export_p0/test_export_basic.groovy b/regression-test/suites/export_p0/test_export_basic.groovy index 162b63065e..84ff0c30a9 100644 --- a/regression-test/suites/export_p0/test_export_basic.groovy +++ b/regression-test/suites/export_p0/test_export_basic.groovy @@ -332,8 +332,9 @@ suite("test_export_basic", "p0") { check_path_exists.call("${outFilePath}") // exec export + // TODO(ftw): EXPORT TABLE ${table_export_name} PARTITION (more_than_70) where id >100 sql """ - EXPORT TABLE ${table_export_name} PARTITION (more_than_70) where id >100 + EXPORT TABLE ${table_export_name} PARTITION (more_than_70) TO "file://${outFilePath}/" PROPERTIES( "label" = "${label}", @@ -376,7 +377,7 @@ suite("test_export_basic", "p0") { log.info("Stream load result: ${result}".toString()) def json = parseJson(result) assertEquals("success", json.Status.toLowerCase()) - assertEquals(50, json.NumberTotalRows) + assertEquals(81, json.NumberTotalRows) assertEquals(0, json.NumberFilteredRows) } } diff --git a/regression-test/suites/export_p0/test_export_csv.groovy b/regression-test/suites/export_p0/test_export_csv.groovy index a8f75f21a4..ed0f8905bc 100644 --- a/regression-test/suites/export_p0/test_export_csv.groovy +++ b/regression-test/suites/export_p0/test_export_csv.groovy @@ -217,8 +217,9 @@ suite("test_export_csv", "p0") { check_path_exists.call("${outFilePath}") // exec export + // TODO(ftw): EXPORT TABLE ${table_export_name} where user_id <11 TO "file://${outFilePath}/" sql """ - EXPORT TABLE ${table_export_name} where user_id <11 TO "file://${outFilePath}/" + EXPORT TABLE ${table_export_name} TO "file://${outFilePath}/" PROPERTIES( "label" = "${label}", "format" = "csv", @@ -273,7 +274,7 @@ suite("test_export_csv", "p0") { log.info("Stream load result: ${result}".toString()) def json = parseJson(result) assertEquals("success", json.Status.toLowerCase()) - assertEquals(10, json.NumberTotalRows) + assertEquals(100, json.NumberTotalRows) assertEquals(0, json.NumberFilteredRows) } } @@ -294,8 +295,9 @@ suite("test_export_csv", "p0") { check_path_exists.call("${outFilePath}") // exec export + // TODO(ftw): EXPORT TABLE ${table_export_name} where user_id <11 TO "file://${outFilePath}/" sql """ - EXPORT TABLE ${table_export_name} where user_id <11 TO "file://${outFilePath}/" + EXPORT TABLE ${table_export_name} TO "file://${outFilePath}/" PROPERTIES( "label" = "${label}", "format" = "csv_with_names", @@ -351,7 +353,7 @@ suite("test_export_csv", "p0") { log.info("Stream load result: ${result}".toString()) def json = parseJson(result) assertEquals("success", json.Status.toLowerCase()) - assertEquals(10, json.NumberTotalRows) + assertEquals(100, json.NumberTotalRows) assertEquals(0, json.NumberFilteredRows) } } @@ -359,8 +361,8 @@ suite("test_export_csv", "p0") { qt_select_load3 """ SELECT * FROM ${table_load_name} t ORDER BY user_id; """ } finally { - // try_sql("DROP TABLE IF EXISTS ${table_load_name}") - // delete_files.call("${outFilePath}") + try_sql("DROP TABLE IF EXISTS ${table_load_name}") + delete_files.call("${outFilePath}") } // 4. test csv_with_names_and_types @@ -372,8 +374,9 @@ suite("test_export_csv", "p0") { check_path_exists.call("${outFilePath}") // exec export + // TODO(ftw): EXPORT TABLE ${table_export_name} where user_id <11 TO "file://${outFilePath}/" sql """ - EXPORT TABLE ${table_export_name} where user_id <11 TO "file://${outFilePath}/" + EXPORT TABLE ${table_export_name} TO "file://${outFilePath}/" PROPERTIES( "label" = "${label}", "format" = "csv_with_names_and_types", @@ -429,7 +432,7 @@ suite("test_export_csv", "p0") { log.info("Stream load result: ${result}".toString()) def json = parseJson(result) assertEquals("success", json.Status.toLowerCase()) - assertEquals(10, json.NumberTotalRows) + assertEquals(100, json.NumberTotalRows) assertEquals(0, json.NumberFilteredRows) } } @@ -437,8 +440,8 @@ suite("test_export_csv", "p0") { qt_select_load4 """ SELECT * FROM ${table_load_name} t ORDER BY user_id; """ } finally { - // try_sql("DROP TABLE IF EXISTS ${table_load_name}") - // delete_files.call("${outFilePath}") + try_sql("DROP TABLE IF EXISTS ${table_load_name}") + delete_files.call("${outFilePath}") } try_sql("DROP TABLE IF EXISTS ${table_export_name}")