database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Union stmt support 'OutFileClause' (#7026)

Browse Source 
The union(set operation) stmt also need to analyze 'OutFileClause'.

Whether the fragment is colocate only needs to check the plan node belonging to this fragment.
This commit is contained in:
EmmyMiao87
2021-11-06 13:28:52 +08:00
committed by GitHub
parent 5ca271299a
commit 3cef2fb0a8
7 changed files with 121 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
View File

@ -158,7 +158,7 @@ public class OutFileClause {
return schema;
}
private void analyze(Analyzer analyzer) throws UserException {
public void analyze(Analyzer analyzer, List<Expr> resultExprs) throws UserException {
if (isAnalyzed) {
// If the query stmt is rewritten, the whole stmt will be analyzed again.
// But some of fields in this OutfileClause has been changed,
@ -187,13 +187,9 @@ public class OutFileClause {
throw new AnalysisException("Must specify BROKER properties in OUTFILE clause");
}
isAnalyzed = true;
}
public void analyze(Analyzer analyzer, SelectStmt stmt) throws UserException {
analyze(analyzer);
if (isParquetFormat()) {
analyzeForParquetFormat(stmt.getResultExprs());
analyzeForParquetFormat(resultExprs);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View File

@ -537,7 +537,7 @@ public class SelectStmt extends QueryStmt {
}
}
if (hasOutFileClause()) {
outFileClause.analyze(analyzer, this);
outFileClause.analyze(analyzer, resultExprs);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
View File

@ -258,6 +258,8 @@ public class SetOperationStmt extends QueryStmt {
setOpsResultExprs_ = Expr.cloneList(resultExprs);
if (evaluateOrderBy) createSortTupleInfo(analyzer);
baseTblResultExprs = resultExprs;
if (hasOutFileClause()) outFileClause.analyze(analyzer, resultExprs);
}
/**

1
fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
View File

@ -48,7 +48,6 @@ import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo;
import org.apache.doris.load.loadv2.LoadJobFinalOperation;
import org.apache.doris.load.routineload.RoutineLoadJob;
import org.apache.doris.load.sync.SyncJob;
import org.apache.doris.master.Checkpoint;
import org.apache.doris.mysql.privilege.UserPropertyInfo;
import org.apache.doris.persist.AlterRoutineLoadJobOperationLog;
import org.apache.doris.persist.AlterViewInfo;

15
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -1222,23 +1222,10 @@ public class Coordinator {
}
if (planFragment.hasColocatePlanNode()) {
colocateFragmentIds.add(planFragment.getId().asInt());
return true;
}
if (node instanceof HashJoinNode) {
HashJoinNode joinNode = (HashJoinNode) node;
if (joinNode.isColocate()) {
colocateFragmentIds.add(joinNode.getFragmentId().asInt());
return true;
}
}
for (PlanNode childNode : node.getChildren()) {
if (childNode.getFragmentId().asInt() == planFragment.getId().asInt() && isColocateFragment(planFragment, childNode)) {
return true;
}
}
return false;
}

101
fe/fe-core/src/test/java/org/apache/doris/analysis/OutFileClauseFunctionTest.java Normal file
View File

@ -0,0 +1,101 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.utframe.DorisAssert;
import org.apache.doris.utframe.UtFrameUtils;
import java.io.File;
import java.util.UUID;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
public class OutFileClauseFunctionTest {
private static String baseDir = "fe";
private static String runningDir = baseDir + "/mocked/MaterializedViewFunctionTest/"
+ UUID.randomUUID().toString() + "/";
private static ConnectContext ctx;
private static final String DB_NAME = "db1";
@BeforeClass
public static void beforeClass() throws Exception {
FeConstants.default_scheduler_interval_millisecond = 10;
FeConstants.runningUnitTest = true;
Config.enable_outfile_to_local = true;
ctx = UtFrameUtils.createDefaultCtx();
UtFrameUtils.createDorisCluster(runningDir);
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt("create database db1;", ctx);
Catalog.getCurrentCatalog().createDb(createDbStmt);
String createTableSQL = "create table " + DB_NAME
+ ".test (k1 int, k2 varchar ) "
+ "distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTableSQL, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
}
@AfterClass
public static void tearDown() {
File file = new File(runningDir);
file.delete();
}
@Test
public void testSelectStmtOutFileClause() throws Exception {
String query1 = "select * from db1.test into outfile \"file:///" + runningDir + "/result_\";";
QueryStmt analyzedQueryStmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(query1, ctx);
Assert.assertTrue(analyzedQueryStmt.hasOutFileClause());
OutFileClause outFileClause = analyzedQueryStmt.getOutFileClause();
boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
Assert.assertTrue(isOutFileClauseAnalyzed);
Assert.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_CSV_PLAIN);
}
@Test
public void testSetOperationStmtOutFileClause() throws Exception {
String query1 = "select * from db1.test union select * from db1.test into outfile \"file:///" + runningDir + "/result_\";";
QueryStmt analyzedSetOperationStmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(query1, ctx);
Assert.assertTrue(analyzedSetOperationStmt.hasOutFileClause());
OutFileClause outFileClause = analyzedSetOperationStmt.getOutFileClause();
boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
Assert.assertTrue(isOutFileClauseAnalyzed);
Assert.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_CSV_PLAIN);
}
@Test
public void testParquetFormat() throws Exception {
String query1 = "select * from db1.test union select * from db1.test into outfile \"file:///" + runningDir + "/result_\" FORMAT AS PARQUET;";
QueryStmt analyzedSetOperationStmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(query1, ctx);
Assert.assertTrue(analyzedSetOperationStmt.hasOutFileClause());
OutFileClause outFileClause = analyzedSetOperationStmt.getOutFileClause();
boolean isOutFileClauseAnalyzed = Deencapsulation.getField(outFileClause, "isAnalyzed");
Assert.assertTrue(isOutFileClauseAnalyzed);
Assert.assertEquals(outFileClause.getFileFormatType(), TFileFormatType.FORMAT_PARQUET);
}
}

14
fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java
View File

@ -172,4 +172,18 @@ public class ColocatePlanTest {
Assert.assertEquals(instanceInfo.size(), 2);
}
@Test
public void checkColocatePlanFragment() throws Exception {
String sql = "select a.k1 from db1.test_colocate a, db1.test_colocate b where a.k1=b.k1 and a.k2=b.k2 group by a.k1;";
StmtExecutor executor = UtFrameUtils.getSqlStmtExecutor(ctx, sql);
Planner planner = executor.planner();
Coordinator coordinator = Deencapsulation.getField(executor, "coord");
boolean isColocateFragment0 = Deencapsulation.invoke(coordinator, "isColocateFragment",
planner.getFragments().get(1), planner.getFragments().get(1).getPlanRoot());
Assert.assertFalse(isColocateFragment0);
boolean isColocateFragment1 = Deencapsulation.invoke(coordinator, "isColocateFragment",
planner.getFragments().get(2), planner.getFragments().get(2).getPlanRoot());
Assert.assertTrue(isColocateFragment1);
}
}