database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](jdbc catalog) Add mysql jdbc catalog function to filter push-down identification (#21745)

Browse Source
This commit is contained in:
zy-kkk
2023-07-19 23:48:23 +08:00
committed by GitHub
parent 58f2593ba1
commit 2daad2151d
14 changed files with 99 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
View File

@ -47,8 +47,8 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.planner.DataPartition;
import org.apache.doris.planner.DataSink;
import org.apache.doris.planner.ExportSink;
import org.apache.doris.planner.JdbcTableSink;
import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.external.jdbc.JdbcTableSink;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.service.FrontendOptions;

2
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -137,7 +137,6 @@ import org.apache.doris.planner.ExchangeNode;
import org.apache.doris.planner.HashJoinNode;
import org.apache.doris.planner.HashJoinNode.DistributionMode;
import org.apache.doris.planner.IntersectNode;
import org.apache.doris.planner.JdbcScanNode;
import org.apache.doris.planner.JoinNodeBase;
import org.apache.doris.planner.MultiCastDataSink;
import org.apache.doris.planner.MultiCastPlanFragment;
@ -160,6 +159,7 @@ import org.apache.doris.planner.UnionNode;
import org.apache.doris.planner.external.HiveScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.SystemInfoService;

1
fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.planner.external.odbc.OdbcTableSink;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TExplainLevel;

2
fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
View File

@ -38,6 +38,8 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.odbc.OdbcScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TPartitionType;

2
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -73,6 +73,8 @@ import org.apache.doris.planner.external.HiveScanNode;
import org.apache.doris.planner.external.MaxComputeScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.odbc.OdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException;

49
fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcFunctionPushDownRule.java vendored Normal file
View File

@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner.external.jdbc;
import org.apache.doris.thrift.TOdbcTableType;
import java.util.TreeSet;
public class JdbcFunctionPushDownRule {
private static final TreeSet<String> UNSUPPORTED_MYSQL_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
static {
UNSUPPORTED_MYSQL_FUNCTIONS.add("date_trunc");
UNSUPPORTED_MYSQL_FUNCTIONS.add("money_format");
}
public static boolean isUnsupportedFunctions(TOdbcTableType tableType, String filter) {
if (tableType.equals(TOdbcTableType.MYSQL)) {
return isMySQLUnsupportedFunctions(filter);
} else {
return false;
}
}
private static boolean isMySQLUnsupportedFunctions(String filter) {
for (String func : UNSUPPORTED_MYSQL_FUNCTIONS) {
if (filter.contains(func)) {
return true;
}
}
return false;
}
}

6
fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java → fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java vendored
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
package org.apache.doris.planner.external.jdbc;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.BinaryPredicate;
@ -36,6 +36,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.ExternalScanNode;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
@ -137,6 +138,9 @@ public class JdbcScanNode extends ExternalScanNode {
if (filter.equals("TRUE")) {
filter = "1 = 1";
}
if (JdbcFunctionPushDownRule.isUnsupportedFunctions(jdbcType, filter)) {
continue;
}
filters.add(filter);
conjuncts.remove(p);
}

5
fe/fe-core/src/main/java/org/apache/doris/planner/JdbcTableSink.java → fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcTableSink.java vendored
View File

@ -15,9 +15,12 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
package org.apache.doris.planner.external.jdbc;
import org.apache.doris.catalog.JdbcTable;
import org.apache.doris.planner.DataPartition;
import org.apache.doris.planner.DataSink;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TDataSinkType;

4
fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java → fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java vendored
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
package org.apache.doris.planner.external.odbc;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.Expr;
@ -29,7 +29,9 @@ import org.apache.doris.catalog.JdbcTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.ExternalScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsRecursiveDerive;
import org.apache.doris.statistics.query.StatsDelta;

5
fe/fe-core/src/main/java/org/apache/doris/planner/OdbcTableSink.java → fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcTableSink.java vendored
View File

@ -15,10 +15,13 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
package org.apache.doris.planner.external.odbc;
import org.apache.doris.catalog.JdbcTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.planner.DataPartition;
import org.apache.doris.planner.DataSink;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TDataSinkType;