database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[ODBC] Support ODBC Sink for insert into data to ODBC external table (#5033)

Browse Source 
issue:#5031

1. Support ODBC Sink for insert into data to ODBC external table.
2. Support Transaction for ODBC sink to make sure insert into data is atomicital.
3. The document about ODBC sink has been modified
This commit is contained in:
HappenLee
2020-12-13 21:53:27 +08:00
committed by GitHub
parent 1267d6bf66
commit 115d4332aa
18 changed files with 574 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java
View File

@ -55,6 +55,7 @@ public class OdbcTable extends Table {
private static final String ODBC_DRIVER = "driver";
private static final String ODBC_TYPE = "odbc_type";
// map now odbc external table Doris support now
private static Map<String, TOdbcTableType> TABLE_TYPE_MAP;
static {
Map<String, TOdbcTableType> tempMap = new HashMap<>();
@ -64,6 +65,19 @@ public class OdbcTable extends Table {
TABLE_TYPE_MAP = Collections.unmodifiableMap(tempMap);
}
// For different databases, special characters need to be escaped
private static String mysqlProperName(String name) {
return "`" + name + "`";
}
public static String databaseProperName(TOdbcTableType tableType, String name) {
switch (tableType) {
case MYSQL:
return mysqlProperName(name);
}
return name;
}
private String odbcCatalogResourceName;
private String host;
private String port;

3
fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.planner;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TDataSink;
@ -54,6 +55,8 @@ public abstract class DataSink {
public static DataSink createDataSink(Table table) throws AnalysisException {
if (table instanceof MysqlTable) {
return new MysqlTableSink((MysqlTable) table);
} else if (table instanceof OdbcTable) {
return new OdbcTableSink((OdbcTable)table);
} else {
throw new AnalysisException("Unknown table type " + table.getType());
}

18
fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java
View File

@ -50,18 +50,6 @@ import java.util.List;
public class OdbcScanNode extends ScanNode {
private static final Logger LOG = LogManager.getLogger(OdbcScanNode.class);
private static String mysqlProperName(String name) {
return "`" + name + "`";
}
private static String databaseProperName(TOdbcTableType tableType, String name) {
switch (tableType) {
case MYSQL:
return mysqlProperName(name);
}
return name;
}
// Now some database have different function call like doris, now doris do not
// push down the function call except MYSQL
private static boolean shouldPushDownConjunct(TOdbcTableType tableType, Expr expr) {
@ -88,7 +76,7 @@ public class OdbcScanNode extends ScanNode {
super(id, desc, "SCAN ODBC");
connectString = tbl.getConnectString();
odbcType = tbl.getOdbcTableType();
tblName = databaseProperName(odbcType, tbl.getOdbcTableName());
tblName = OdbcTable.databaseProperName(odbcType, tbl.getOdbcTableName());
}
@Override
@ -156,7 +144,7 @@ public class OdbcScanNode extends ScanNode {
continue;
}
Column col = slot.getColumn();
columns.add(databaseProperName(odbcType, col.getName()));
columns.add(OdbcTable.databaseProperName(odbcType, col.getName()));
}
// this happens when count(*)
if (0 == columns.size()) {
@ -176,7 +164,7 @@ public class OdbcScanNode extends ScanNode {
for (SlotRef slotRef : slotRefs) {
SlotRef tmpRef = (SlotRef) slotRef.clone();
tmpRef.setTblName(null);
tmpRef.setLabel(databaseProperName(odbcType, tmpRef.getColumnName()));
tmpRef.setLabel(OdbcTable.databaseProperName(odbcType, tmpRef.getColumnName()));
sMap.put(slotRef, tmpRef);
}
ArrayList<Expr> odbcConjuncts = Expr.cloneList(conjuncts, sMap);

74
fe/fe-core/src/main/java/org/apache/doris/planner/OdbcTableSink.java Normal file
View File

@ -0,0 +1,74 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TDataSinkType;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TOdbcTableSink;
import org.apache.doris.thrift.TOdbcTableType;
public class OdbcTableSink extends DataSink {
private final TOdbcTableType odbcType ;
private final String tblName;
private final String originTblName;
private final String connectString;
private final boolean useTransaction;
public OdbcTableSink(OdbcTable odbcTable) {
connectString = odbcTable.getConnectString();
originTblName = odbcTable.getName();
odbcType = odbcTable.getOdbcTableType();
tblName = odbcTable.databaseProperName(odbcType, odbcTable.getOdbcTableName());
useTransaction = ConnectContext.get().getSessionVariable().isEnableOdbcTransaction();
}
@Override
public String getExplainString(String prefix, TExplainLevel explainLevel) {
StringBuilder strBuilder = new StringBuilder();
strBuilder.append(prefix + "ODBC TABLE SINK:\n");
strBuilder.append(prefix + "TABLENAME IN DORIS: ").append(originTblName).append("\n");
strBuilder.append(prefix + "TABLE TYPE: ").append(odbcType.toString()).append("\n");
strBuilder.append(prefix + "TABLENAME OF EXTERNAL TABLE: ").append(tblName).append("\n");
strBuilder.append(prefix + "EnableTransaction: ").append(useTransaction ? "true" : "false").append("\n");
return strBuilder.toString();
}
@Override
protected TDataSink toThrift() {
TDataSink tDataSink = new TDataSink(TDataSinkType.ODBC_TABLE_SINK);
TOdbcTableSink odbcTableSink = new TOdbcTableSink();
odbcTableSink.setConnectString(connectString);
odbcTableSink.setTable(tblName);
odbcTableSink.setUseTransaction(useTransaction);
tDataSink.setOdbcTableSink(odbcTableSink);
return tDataSink;
}
@Override
public PlanNodeId getExchNodeId() {
return null;
}
@Override
public DataPartition getOutputPartition() {
return null;
}
}

10
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -76,7 +76,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_INSERT_STRICT = "enable_insert_strict";
public static final String ENABLE_SPILLING = "enable_spilling";
public static final String PREFER_JOIN_METHOD = "prefer_join_method";
public static final String ENABLE_ODBC_TRANSCATION = "enable_odbc_transcation";
public static final String ENABLE_SQL_CACHE = "enable_sql_cache";
public static final String ENABLE_PARTITION_CACHE = "enable_partition_cache";
@ -235,6 +236,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT)
private boolean enableInsertStrict = false;
@VariableMgr.VarAttr(name = ENABLE_ODBC_TRANSCATION)
private boolean enableOdbcTransaction = false;
@VariableMgr.VarAttr(name = ENABLE_SQL_CACHE)
private boolean enableSqlCache = false;
@ -428,6 +432,10 @@ public class SessionVariable implements Serializable, Writable {
return enableBucketShuffleJoin;
}
public boolean isEnableOdbcTransaction() {
return enableOdbcTransaction;
}
public String getPreferJoinMethod() {return preferJoinMethod; }
public void setPreferJoinMethod(String preferJoinMethod) {this.preferJoinMethod = preferJoinMethod; }

19
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -1254,6 +1254,25 @@ public class QueryPlanTest {
Assert.assertTrue(explainString.contains("LIMIT 10"));
}
@Test
public void testOdbcSink() throws Exception {
connectContext.setDatabase("default_cluster:test");
// insert into odbc_oracle table
String queryStr = "explain insert into odbc_oracle select * from odbc_mysql";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
Assert.assertTrue(explainString.contains("TABLENAME IN DORIS: odbc_oracle"));
Assert.assertTrue(explainString.contains("TABLE TYPE: ORACLE"));
Assert.assertTrue(explainString.contains("TABLENAME OF EXTERNAL TABLE: tbl1"));
// enable transaction of ODBC Sink
Deencapsulation.setField(connectContext.getSessionVariable(), "enableOdbcTransaction", true);
queryStr = "explain insert into odbc_oracle select * from odbc_mysql";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr);
Assert.assertTrue(explainString.contains("EnableTransaction: true"));
}
@Test
public void testPreferBroadcastJoin() throws Exception {
connectContext.setDatabase("default_cluster:test");