From 0c98355fffbd346474d9440a4456494feb48633b Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 30 May 2023 16:57:39 +0800 Subject: [PATCH] [fix](catalog) fix create catalog with resource replay issue and kerberos auth issue (#20137) 1. Fix create catalog with resource replay bug. If user create catalog using `create catalog hive with resource xxx`, when replaying edit log, there is a bug that resource may be dropped, causing NPE and FE will fail to start. In this PR, I add a new FE config `disallow_create_catalog_with_resource`, default is true. So that `with resource` will not be allowed, and it will be deprecated later. And also fix the replay bug to avoid NPE. 2. Fix issue when creating 2 hive catalogs to connect with and without kerberos authentication. When user create 2 hive catalogs, one use simple auth, the other use kerberos auth. The query may fail with error like: `Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.` So I add a default property for hive catalog: `"ipc.client.fallback-to-simple-auth-allowed" = "true"`. Which means this property will be added automatically when user creating hive catalog, to avoid such problem. 3. Fix calling `hdfsExists()` issue When calling `hdfsExists()` with non-zero return code, should check if it encounters error or is file not found. 3. Some code refactor Avoid import `org.apache.parquet.Strings` --- be/src/io/fs/hdfs_file_system.cpp | 10 ++ be/src/io/hdfs_builder.cpp | 2 + be/src/vec/exec/scan/scanner_scheduler.cpp | 3 +- be/src/vec/exec/scan/vfile_scanner.cpp | 1 + be/src/vec/exec/scan/vfile_scanner.h | 4 + be/src/vec/exec/scan/vscanner.h | 4 + fe/check/checkstyle/import-control.xml | 1 + .../java/org/apache/doris/common/Config.java | 4 + .../doris/analysis/CreateCatalogStmt.java | 8 ++ .../doris/analysis/DropSqlBlockRuleStmt.java | 4 +- .../doris/analysis/ReplaceTableClause.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 2 +- .../doris/common/util/DocGenerator.java | 2 +- .../org/apache/doris/common/util/S3URI.java | 2 +- .../org/apache/doris/common/util/URI.java | 2 +- .../doris/datasource/CatalogFactory.java | 72 ++++++++------ .../apache/doris/datasource/CatalogIf.java | 11 +++ .../apache/doris/datasource/CatalogMgr.java | 41 ++++---- .../doris/datasource/CatalogProperty.java | 9 +- .../doris/datasource/EsExternalCatalog.java | 4 +- .../doris/datasource/ExternalCatalog.java | 11 ++- .../doris/datasource/HMSExternalCatalog.java | 14 ++- .../doris/datasource/JdbcExternalCatalog.java | 5 +- .../datasource/MaxComputeExternalCatalog.java | 5 +- .../hudi/HudiHMSExternalCatalog.java | 6 +- .../iceberg/IcebergDLFExternalCatalog.java | 5 +- .../iceberg/IcebergExternalCatalog.java | 4 +- .../IcebergExternalCatalogFactory.java | 14 +-- .../iceberg/IcebergGlueExternalCatalog.java | 5 +- .../iceberg/IcebergHMSExternalCatalog.java | 5 +- .../iceberg/IcebergRestExternalCatalog.java | 5 +- .../datasource/test/TestExternalCatalog.java | 6 +- .../doris/fs/remote/dfs/DFSFileSystem.java | 69 +++++++++---- .../httpv2/rest/ExtraBasepathAction.java | 2 +- .../doris/httpv2/rest/manager/HttpUtils.java | 2 +- .../org/apache/doris/ldap/LdapManager.java | 2 +- .../load/routineload/KafkaRoutineLoadJob.java | 2 +- .../org/apache/doris/persist/EditLog.java | 2 +- .../apache/doris/statistics/Histogram.java | 2 +- .../doris/datasource/CatalogMgrTest.java | 26 ++--- .../property/PropertyConverterTest.java | 28 +++--- .../doris/external/hms/HmsCatalogTest.java | 9 +- .../hive/test_catalog_with_resource.out | 31 ------ .../suites/es_p0/test_es_query.groovy | 13 +-- .../suites/es_p0/test_es_query_nereids.groovy | 13 +-- .../es_p0/test_es_query_no_http_url.groovy | 13 +-- .../hive/test_catalog_with_resource.groovy | 98 ------------------- .../hive/test_different_parquet_types.groovy | 6 +- .../hive/test_hive_orc.groovy | 4 +- .../hive/test_hive_other.groovy | 4 +- .../hive/test_hive_parquet.groovy | 4 +- .../hive/test_hive_partitions.groovy | 4 +- .../mysql/test_external_catalog_mysql.groovy | 20 +--- .../test_clickhouse_jdbc_catalog.groovy | 9 +- .../test_mysql_jdbc_catalog.groovy | 24 +---- .../test_mysql_jdbc_catalog_nereids.groovy | 12 +-- .../test_oracle_jdbc_catalog.groovy | 19 +--- .../test_pg_jdbc_catalog.groovy | 24 +---- .../test_sqlserver_jdbc_catalog.groovy | 9 +- .../hive_catalog_orc.groovy | 4 +- .../hive_catalog_parquet.groovy | 4 +- 61 files changed, 298 insertions(+), 434 deletions(-) delete mode 100644 regression-test/data/external_catalog_p0/hive/test_catalog_with_resource.out delete mode 100644 regression-test/suites/external_catalog_p0/hive/test_catalog_with_resource.groovy diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp index c338390af4..93271d86f8 100644 --- a/be/src/io/fs/hdfs_file_system.cpp +++ b/be/src/io/fs/hdfs_file_system.cpp @@ -201,6 +201,16 @@ Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const { CHECK_HDFS_HANDLE(_fs_handle); Path real_path = convert_path(path, _namenode); int is_exists = hdfsExists(_fs_handle->hdfs_fs, real_path.string().c_str()); +#ifdef USE_HADOOP_HDFS + // when calling hdfsExists() and return non-zero code, + // if root_cause is nullptr, which means the file does not exist. + // if root_cause is not nullptr, which means it encounter other error, should return. + // NOTE: not for libhdfs3 since it only runs on MaxOS, don't have to support it. + char* root_cause = hdfsGetLastExceptionRootCause(); + if (root_cause != nullptr) { + return Status::IOError("failed to check path existence {}: {}", path.native(), root_cause); + } +#endif *res = (is_exists == 0); return Status::OK(); } diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index be1281441a..73edc326c3 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -142,6 +142,8 @@ Status createHDFSBuilder(const THdfsParams& hdfsParams, HDFSCommonBuilder* build } } + hdfsBuilderConfSetStr(builder->get(), "ipc.client.fallback-to-simple-auth-allowed", "true"); + if (builder->is_need_kinit()) { RETURN_IF_ERROR(builder->run_kinit()); } diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp index 4fdb4a2caf..a01a9e8b9b 100644 --- a/be/src/vec/exec/scan/scanner_scheduler.cpp +++ b/be/src/vec/exec/scan/scanner_scheduler.cpp @@ -338,8 +338,9 @@ void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler, ScannerContext break; } if (status.is()) { - // The only case in this if branch is external table file delete and fe cache has not been updated yet. + // The only case in this "if" branch is external table file delete and fe cache has not been updated yet. // Set status to OK. + LOG(INFO) << "scan range not found: " << scanner->get_current_scan_range_name(); status = Status::OK(); eos = true; } diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 767106b936..a1a15e61e2 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -581,6 +581,7 @@ Status VFileScanner::_get_next_reader() { } const TFileRangeDesc& range = _ranges[_next_range++]; + _current_range_path = range.path; // create reader for specific format // TODO: add json, avro diff --git a/be/src/vec/exec/scan/vfile_scanner.h b/be/src/vec/exec/scan/vfile_scanner.h index 162be02bda..df4204d5ae 100644 --- a/be/src/vec/exec/scan/vfile_scanner.h +++ b/be/src/vec/exec/scan/vfile_scanner.h @@ -77,6 +77,8 @@ public: std::string get_name() override { return VFileScanner::NAME; } + std::string get_current_scan_range_name() override { return _current_range_path; } + protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; @@ -168,6 +170,8 @@ private: std::unordered_map _slot_id_to_filter_conjuncts; // not single(zero or multi) slot filter conjuncts VExprContextSPtrs _not_single_slot_filter_conjuncts; + // save the path of current scan range + std::string _current_range_path = ""; private: Status _init_expr_ctxes(); diff --git a/be/src/vec/exec/scan/vscanner.h b/be/src/vec/exec/scan/vscanner.h index bba93dec47..7103449940 100644 --- a/be/src/vec/exec/scan/vscanner.h +++ b/be/src/vec/exec/scan/vscanner.h @@ -66,6 +66,10 @@ public: virtual std::string get_name() { return ""; } + // return the readable name of current scan range. + // eg, for file scanner, return the current file path. + virtual std::string get_current_scan_range_name() { return "not implemented"; } + protected: // Subclass should implement this to return data. virtual Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) = 0; diff --git a/fe/check/checkstyle/import-control.xml b/fe/check/checkstyle/import-control.xml index f6ca72d1e9..9ff32f7d51 100644 --- a/fe/check/checkstyle/import-control.xml +++ b/fe/check/checkstyle/import-control.xml @@ -34,6 +34,7 @@ under the License. + diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 7b56b39714..04d0d22e6b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1997,4 +1997,8 @@ public class Config extends ConfigBase { @ConfField public static int analyze_task_timeout_in_minutes = 120; + @ConfField(mutable = true, masterOnly = true, description = { + "是否禁止使用 WITH REOSOURCE 语句创建 Catalog。", + "Whether to disable creating catalog with WITH RESOURCE statement."}) + public static boolean disallow_create_catalog_with_resource = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java index 9d36259c93..7d2bbb2b61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java @@ -19,6 +19,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -92,6 +93,13 @@ public class CreateCatalogStmt extends DdlStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED, analyzer.getQualifiedUser(), catalogName); } + + if (Config.disallow_create_catalog_with_resource && !Strings.isNullOrEmpty(resource)) { + throw new AnalysisException("Create catalog with resource is deprecated and is not allowed." + + " You can set `disallow_create_catalog_with_resource=false` in fe.conf" + + " to enable it temporarily."); + } + String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " "); properties.put(CREATE_TIME_PROP, currentDateTime); PropertyAnalyzer.checkCatalogProperties(properties, false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java index 7265102662..1a90eec2ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java @@ -24,8 +24,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; +import com.google.common.base.Joiner; import lombok.Getter; -import org.apache.parquet.Strings; import java.util.List; @@ -53,7 +53,7 @@ public class DropSqlBlockRuleStmt extends DdlStmt { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("DROP SQL_BLOCK_RULE ").append(Strings.join(ruleNames, ",")); + sb.append("DROP SQL_BLOCK_RULE ").append(Joiner.on(",").join(ruleNames)); return sb.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java index 06f6fa1e4d..20d153ce3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java @@ -21,7 +21,7 @@ import org.apache.doris.alter.AlterOpType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.parquet.Strings; +import com.google.common.base.Strings; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 2084abab2f..abc2cfa0b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -66,6 +66,7 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; @@ -73,7 +74,6 @@ import com.google.common.collect.Sets; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.parquet.Strings; import java.io.DataInput; import java.io.DataOutput; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java index c54486d995..bb6bc59b7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -25,9 +25,9 @@ import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; +import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.commons.io.output.FileWriterWithEncoding; -import org.apache.parquet.Strings; import org.jetbrains.annotations.NotNull; import java.io.BufferedReader; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java index 0c177df51a..faa5890556 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java @@ -19,8 +19,8 @@ package org.apache.doris.common.util; import org.apache.doris.common.UserException; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; -import org.apache.parquet.Strings; import org.apache.parquet.glob.GlobExpander; import java.net.URI; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/URI.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/URI.java index d480e41272..5e064bf91e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/URI.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/URI.java @@ -19,8 +19,8 @@ package org.apache.doris.common.util; import org.apache.doris.common.AnalysisException; +import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; -import org.apache.parquet.Strings; import java.util.TreeMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index a691c88957..f530bcc5f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -30,27 +30,24 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.iceberg.IcebergExternalCatalogFactory; import org.apache.doris.datasource.test.TestExternalCatalog; -import org.apache.parquet.Strings; +import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; -import java.util.Optional; /** * A factory to create catalog instance of log or covert catalog into log. */ public class CatalogFactory { + private static final Logger LOG = LogManager.getLogger(CatalogFactory.class); + /** * Convert the sql statement into catalog log. */ - public static CatalogLog constructorCatalogLog(long catalogId, StatementBase stmt) { + public static CatalogLog createCatalogLog(long catalogId, StatementBase stmt) { CatalogLog log = new CatalogLog(); - if (stmt instanceof CreateCatalogStmt) { - log.setCatalogId(catalogId); - log.setCatalogName(((CreateCatalogStmt) stmt).getCatalogName()); - log.setResource(((CreateCatalogStmt) stmt).getResource()); - log.setComment(((CreateCatalogStmt) stmt).getComment()); - log.setProps(((CreateCatalogStmt) stmt).getProperties()); - } else if (stmt instanceof DropCatalogStmt) { + if (stmt instanceof DropCatalogStmt) { log.setCatalogId(catalogId); } else if (stmt instanceof AlterCatalogPropertyStmt) { log.setCatalogId(catalogId); @@ -74,21 +71,33 @@ public class CatalogFactory { /** * create the catalog instance from catalog log. */ - public static CatalogIf constructorFromLog(CatalogLog log) throws DdlException { - return constructorCatalog(log.getCatalogId(), log.getCatalogName(), log.getResource(), - log.getComment(), log.getProps()); + public static CatalogIf createFromLog(CatalogLog log) throws DdlException { + return createCatalog(log.getCatalogId(), log.getCatalogName(), log.getResource(), + log.getComment(), log.getProps(), true); } - private static CatalogIf constructorCatalog(long catalogId, String name, String resource, String comment, - Map props) throws DdlException { + /** + * create the catalog instance from creating statement. + */ + public static CatalogIf createFromStmt(long catalogId, CreateCatalogStmt stmt) + throws DdlException { + return createCatalog(catalogId, stmt.getCatalogName(), stmt.getResource(), + stmt.getComment(), stmt.getProperties(), false); + } + + private static CatalogIf createCatalog(long catalogId, String name, String resource, String comment, + Map props, boolean isReplay) throws DdlException { // get catalog type from resource or properties String catalogType; if (!Strings.isNullOrEmpty(resource)) { - Resource catalogResource = Optional.ofNullable(Env.getCurrentEnv().getResourceMgr().getResource(resource)) - .orElseThrow(() -> new DdlException("Resource doesn't exist: " + resource)); - catalogType = catalogResource.getType().name().toLowerCase(); - if (props.containsKey(CatalogMgr.CATALOG_TYPE_PROP)) { - throw new DdlException("Can not set 'type' when creating catalog with resource"); + Resource catalogResource = Env.getCurrentEnv().getResourceMgr().getResource(resource); + if (catalogResource == null) { + // This is temp bug fix to continue replaying edit log even if resource doesn't exist. + // In new version, create catalog with resource is not allowed by default. + LOG.warn("Resource doesn't exist: {} when create catalog {}", resource, name); + catalogType = "hms"; + } else { + catalogType = catalogResource.getType().name().toLowerCase(); } } else { String type = props.get(CatalogMgr.CATALOG_TYPE_PROP); @@ -99,38 +108,41 @@ public class CatalogFactory { } // create catalog - CatalogIf catalog; + ExternalCatalog catalog; switch (catalogType) { case "hms": - catalog = new HMSExternalCatalog(catalogId, name, resource, props); + catalog = new HMSExternalCatalog(catalogId, name, resource, props, comment); break; case "es": - catalog = new EsExternalCatalog(catalogId, name, resource, props); + catalog = new EsExternalCatalog(catalogId, name, resource, props, comment); break; case "jdbc": - catalog = new JdbcExternalCatalog(catalogId, name, resource, props); + catalog = new JdbcExternalCatalog(catalogId, name, resource, props, comment); break; case "iceberg": - catalog = IcebergExternalCatalogFactory.createCatalog(catalogId, name, resource, props); + catalog = IcebergExternalCatalogFactory.createCatalog(catalogId, name, resource, props, comment); break; case "max_compute": - catalog = new MaxComputeExternalCatalog(catalogId, name, resource, props); + catalog = new MaxComputeExternalCatalog(catalogId, name, resource, props, comment); break; // case "hudi": - // catalog = new HudiHMSExternalCatalog(catalogId, name, resource, props); + // catalog = new HudiHMSExternalCatalog(catalogId, name, resource, props, comment); // break; case "test": if (!FeConstants.runningUnitTest) { throw new DdlException("test catalog is only for FE unit test"); } - catalog = new TestExternalCatalog(catalogId, name, resource, props); + catalog = new TestExternalCatalog(catalogId, name, resource, props, comment); break; default: throw new DdlException("Unknown catalog type: " + catalogType); } - if (catalog instanceof ExternalCatalog && !Strings.isNullOrEmpty(comment)) { - ((ExternalCatalog) catalog).setComment(comment); + if (!isReplay) { + // set some default properties when creating catalog. + // do not call this method when replaying edit log. Because we need to keey the original properties. + catalog.setDefaultProps(); } return catalog; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index eac17865a2..ce13c9881e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -25,6 +25,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -150,4 +151,14 @@ public interface CatalogIf { } String getComment(); + + default CatalogLog constructEditLog() { + CatalogLog log = new CatalogLog(); + log.setCatalogId(getId()); + log.setCatalogName(getName()); + log.setResource(Strings.nullToEmpty(getResource())); + log.setComment(getComment()); + log.setProps(getProperties()); + return log; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index f389f49f32..11a10c2a78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -50,12 +50,12 @@ import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.parquet.Strings; import java.io.DataInput; import java.io.DataOutput; @@ -114,8 +114,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable { nameToCatalog.put(catalog.getName(), catalog); idToCatalog.put(catalog.getId(), catalog); if (!Strings.isNullOrEmpty(catalog.getResource())) { - Env.getCurrentEnv().getResourceMgr().getResource(catalog.getResource()) - .addReference(catalog.getName(), ReferenceType.CATALOG); + Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(catalog.getResource()); + if (resource != null) { + resource.addReference(catalog.getName(), ReferenceType.CATALOG); + } } } @@ -255,19 +257,19 @@ public class CatalogMgr implements Writable, GsonPostProcessable { * Create and hold the catalog instance and write the meta log. */ public void createCatalog(CreateCatalogStmt stmt) throws UserException { + long id = Env.getCurrentEnv().getNextId(); + CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt); writeLock(); try { - if (nameToCatalog.containsKey(stmt.getCatalogName())) { + if (nameToCatalog.containsKey(catalog.getName())) { if (stmt.isSetIfNotExists()) { LOG.warn("Catalog {} is already exist.", stmt.getCatalogName()); return; } throw new DdlException("Catalog had already exist with name: " + stmt.getCatalogName()); } - long id = Env.getCurrentEnv().getNextId(); - CatalogLog log = CatalogFactory.constructorCatalogLog(id, stmt); - replayCreateCatalog(log, false); - Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_CREATE_CATALOG, log); + createCatalogInternal(catalog, false); + Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_CREATE_CATALOG, catalog.constructEditLog()); } finally { writeUnlock(); } @@ -287,7 +289,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { if (catalog == null) { throw new DdlException("No catalog found with name: " + stmt.getCatalogName()); } - CatalogLog log = CatalogFactory.constructorCatalogLog(catalog.getId(), stmt); + CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt); replayDropCatalog(log); Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_DROP_CATALOG, log); @@ -312,7 +314,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { if (nameToCatalog.get(stmt.getNewCatalogName()) != null) { throw new DdlException("Catalog with name " + stmt.getNewCatalogName() + " already exist"); } - CatalogLog log = CatalogFactory.constructorCatalogLog(catalog.getId(), stmt); + CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt); replayAlterCatalogName(log); Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_NAME, log); @@ -340,7 +342,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { .equalsIgnoreCase(stmt.getNewProperties().get("type"))) { throw new DdlException("Can't modify the type of catalog property with name: " + stmt.getCatalogName()); } - CatalogLog log = CatalogFactory.constructorCatalogLog(catalog.getId(), stmt); + CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt); replayAlterCatalogProps(log); Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log); } finally { @@ -444,7 +446,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { StringBuilder sb = new StringBuilder(); sb.append("\nCREATE CATALOG `").append(ClusterNamespace.getNameFromFullName(showStmt.getCatalog())) .append("`"); - if (!com.google.common.base.Strings.isNullOrEmpty(catalog.getComment())) { + if (!Strings.isNullOrEmpty(catalog.getComment())) { sb.append("\nCOMMENT \"").append(catalog.getComment()).append("\"\n"); } if (catalog.getProperties().size() > 0) { @@ -469,7 +471,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { if (catalog == null) { throw new DdlException("No catalog found with name: " + stmt.getCatalogName()); } - CatalogLog log = CatalogFactory.constructorCatalogLog(catalog.getId(), stmt); + CatalogLog log = CatalogFactory.createCatalogLog(catalog.getId(), stmt); refreshCatalog(log); } @@ -486,23 +488,26 @@ public class CatalogMgr implements Writable, GsonPostProcessable { /** * Reply for create catalog event. */ - public CatalogIf replayCreateCatalog(CatalogLog log, boolean isReplay) throws DdlException { + public void replayCreateCatalog(CatalogLog log) throws DdlException { + CatalogIf catalog = CatalogFactory.createFromLog(log); + createCatalogInternal(catalog, true); + } + + private void createCatalogInternal(CatalogIf catalog, boolean isReplay) throws DdlException { writeLock(); try { - CatalogIf catalog = CatalogFactory.constructorFromLog(log); if (!isReplay && catalog instanceof ExternalCatalog) { ((ExternalCatalog) catalog).checkProperties(); } - Map props = log.getProps(); + Map props = catalog.getProperties(); if (props.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { // need refresh - long catalogId = log.getCatalogId(); + long catalogId = catalog.getId(); Integer metadataRefreshIntervalSec = Integer.valueOf(props.get(METADATA_REFRESH_INTERVAL_SEC)); Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec}; Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec); } addCatalog(catalog); - return catalog; } finally { writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java index 5b9466cbfd..fd1403c72f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java @@ -86,7 +86,10 @@ public class CatalogProperty implements Writable { public Map getProperties() { Map mergedProperties = Maps.newHashMap(); if (!Strings.isNullOrEmpty(resource)) { - mergedProperties = catalogResource().getCopiedProperties(); + Resource res = catalogResource(); + if (res != null) { + mergedProperties = res.getCopiedProperties(); + } } mergedProperties.putAll(properties); return mergedProperties; @@ -102,6 +105,10 @@ public class CatalogProperty implements Writable { return hadoopProperties; } + public void addProperty(String key, String val) { + this.properties.put(key, val); + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java index 10d00fa987..323a31feb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java @@ -47,8 +47,8 @@ public class EsExternalCatalog extends ExternalCatalog { /** * Default constructor for EsExternalCatalog. */ - public EsExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name, InitCatalogLog.Type.ES); + public EsExternalCatalog(long catalogId, String name, String resource, Map props, String comment) { + super(catalogId, name, InitCatalogLog.Type.ES, comment); this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index a3f5584d5f..ae070bf507 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -39,6 +39,7 @@ import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterCatalogExecutor; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -47,7 +48,6 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.parquet.Strings; import org.jetbrains.annotations.Nullable; import java.io.DataInput; @@ -91,15 +91,20 @@ public abstract class ExternalCatalog private ExternalSchemaCache schemaCache; private String comment; - public ExternalCatalog(long catalogId, String name, InitCatalogLog.Type logType) { + public ExternalCatalog(long catalogId, String name, InitCatalogLog.Type logType, String comment) { this.id = catalogId; this.name = name; this.logType = logType; + this.comment = com.google.common.base.Strings.nullToEmpty(comment); } protected List listDatabaseNames() { throw new UnsupportedOperationException("Unsupported operation: " - + "listDatabaseNames from remote client when init catalog with " + logType.name()); + + "listDatabaseNames from remote client when init catalog with " + logType.name()); + } + + public void setDefaultProps() { + // set some default properties when creating catalog } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java index 049ccbaa5a..531c626e3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java @@ -60,6 +60,7 @@ public class HMSExternalCatalog extends ExternalCatalog { private long lastSyncedEventId = -1L; public static final String ENABLE_SELF_SPLITTER = "enable.self.splitter"; public static final String FILE_META_CACHE_TTL_SECOND = "file.meta.cache.ttl-second"; + private static final String PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH = "ipc.client.fallback-to-simple-auth-allowed"; // -1 means file cache no ttl set public static final int FILE_META_CACHE_NO_TTL = -1; @@ -69,8 +70,9 @@ public class HMSExternalCatalog extends ExternalCatalog { /** * Default constructor for HMSExternalCatalog. */ - public HMSExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name, InitCatalogLog.Type.HMS); + public HMSExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, InitCatalogLog.Type.HMS, comment); props = PropertyConverter.convertToMetaProperties(props); catalogProperty = new CatalogProperty(resource, props); } @@ -261,4 +263,12 @@ public class HMSExternalCatalog extends ExternalCatalog { Env.getCurrentEnv().getExtMetaCacheMgr().getMetaStoreCache(this).setNewFileCache(); } } + + @Override + public void setDefaultProps() { + if (catalogProperty.getOrDefault(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty()) { + // always allow fallback to simple auth, so to support both kerberos and simple auth + catalogProperty.addProperty(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true"); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java index 95bc431a24..ca373daba8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java @@ -47,9 +47,10 @@ public class JdbcExternalCatalog extends ExternalCatalog { // or Gson will throw exception with HikariCP private transient JdbcClient jdbcClient; - public JdbcExternalCatalog(long catalogId, String name, String resource, Map props) + public JdbcExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) throws DdlException { - super(catalogId, name, InitCatalogLog.Type.JDBC); + super(catalogId, name, InitCatalogLog.Type.JDBC, comment); this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/MaxComputeExternalCatalog.java index 92a967ae45..d3f77a985f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/MaxComputeExternalCatalog.java @@ -36,8 +36,9 @@ public class MaxComputeExternalCatalog extends ExternalCatalog { private static final String odpsUrlTemplate = "http://service.{}.maxcompute.aliyun.com/api"; private static final String tunnelUrlTemplate = "http://dt.{}.maxcompute.aliyun.com"; - public MaxComputeExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name, InitCatalogLog.Type.MAX_COMPUTE); + public MaxComputeExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, InitCatalogLog.Type.MAX_COMPUTE, comment); catalogProperty = new CatalogProperty(resource, props); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiHMSExternalCatalog.java index 42e9b88519..8006dff472 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiHMSExternalCatalog.java @@ -40,8 +40,9 @@ public class HudiHMSExternalCatalog extends HMSExternalCatalog { * @param resource * @param props */ - public HudiHMSExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name, resource, props); + public HudiHMSExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, resource, props, comment); } @@ -64,3 +65,4 @@ public class HudiHMSExternalCatalog extends HMSExternalCatalog { } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java index 6a74f7abb8..a243a17b31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java @@ -28,8 +28,9 @@ import java.util.Map; public class IcebergDLFExternalCatalog extends IcebergExternalCatalog { - public IcebergDLFExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name); + public IcebergDLFExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, comment); props.put(HMSProperties.HIVE_METASTORE_TYPE, "dlf"); props = PropertyConverter.convertToMetaProperties(props); catalogProperty = new CatalogProperty(resource, props); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 3fc528568f..a5b7f4d7af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -49,8 +49,8 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog { protected Catalog catalog; protected SupportsNamespaces nsCatalog; - public IcebergExternalCatalog(long catalogId, String name) { - super(catalogId, name, InitCatalogLog.Type.ICEBERG); + public IcebergExternalCatalog(long catalogId, String name, String comment) { + super(catalogId, name, InitCatalogLog.Type.ICEBERG, comment); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalogFactory.java index 32b54182a4..6ea5a3a73b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalogFactory.java @@ -18,27 +18,27 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.common.DdlException; -import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalCatalog; import java.util.Map; public class IcebergExternalCatalogFactory { - public static CatalogIf createCatalog(long catalogId, String name, String resource, Map props) - throws DdlException { + public static ExternalCatalog createCatalog(long catalogId, String name, String resource, Map props, + String comment) throws DdlException { String catalogType = props.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE); if (catalogType == null) { throw new DdlException("Missing " + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE + " property"); } switch (catalogType) { case IcebergExternalCatalog.ICEBERG_REST: - return new IcebergRestExternalCatalog(catalogId, name, resource, props); + return new IcebergRestExternalCatalog(catalogId, name, resource, props, comment); case IcebergExternalCatalog.ICEBERG_HMS: - return new IcebergHMSExternalCatalog(catalogId, name, resource, props); + return new IcebergHMSExternalCatalog(catalogId, name, resource, props, comment); case IcebergExternalCatalog.ICEBERG_GLUE: - return new IcebergGlueExternalCatalog(catalogId, name, resource, props); + return new IcebergGlueExternalCatalog(catalogId, name, resource, props, comment); case IcebergExternalCatalog.ICEBERG_DLF: - return new IcebergDLFExternalCatalog(catalogId, name, resource, props); + return new IcebergDLFExternalCatalog(catalogId, name, resource, props, comment); default: throw new DdlException("Unknown " + IcebergExternalCatalog.ICEBERG_CATALOG_TYPE + " value: " + catalogType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java index c3cc47ce6e..4977c84e49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java @@ -36,8 +36,9 @@ public class IcebergGlueExternalCatalog extends IcebergExternalCatalog { // As a default placeholder. The path just use for 'create table', query stmt will not use it. private static final String CHECKED_WAREHOUSE = "s3://doris"; - public IcebergGlueExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name); + public IcebergGlueExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, comment); props = PropertyConverter.convertToMetaProperties(props); catalogProperty = new CatalogProperty(resource, props); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java index f9a15e8ee7..0300477361 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java @@ -29,8 +29,9 @@ import java.util.Map; public class IcebergHMSExternalCatalog extends IcebergExternalCatalog { - public IcebergHMSExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name); + public IcebergHMSExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, comment); props = PropertyConverter.convertToMetaProperties(props); catalogProperty = new CatalogProperty(resource, props); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java index 6f425495f5..b021f84da6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java @@ -31,8 +31,9 @@ import java.util.Map; public class IcebergRestExternalCatalog extends IcebergExternalCatalog { - public IcebergRestExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name); + public IcebergRestExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, comment); props = PropertyConverter.convertToMetaProperties(props); catalogProperty = new CatalogProperty(resource, props); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java index 70193a167b..f680c79d02 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java @@ -41,8 +41,9 @@ public class TestExternalCatalog extends ExternalCatalog { private TestCatalogProvider catalogProvider; - public TestExternalCatalog(long catalogId, String name, String resource, Map props) { - super(catalogId, name, InitCatalogLog.Type.TEST); + public TestExternalCatalog(long catalogId, String name, String resource, Map props, + String comment) { + super(catalogId, name, InitCatalogLog.Type.TEST, comment); this.catalogProperty = new CatalogProperty(resource, props); Class providerClazz = null; try { @@ -108,3 +109,4 @@ public class TestExternalCatalog extends ExternalCatalog { Map>> getMetadata(); } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java index d8df27a5c0..677ca658be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java @@ -50,9 +50,11 @@ import java.nio.ByteBuffer; import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.PrivilegedAction; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; public class DFSFileSystem extends RemoteFileSystem { @@ -74,36 +76,62 @@ public class DFSFileSystem extends RemoteFileSystem { if (dfsFileSystem != null) { return dfsFileSystem; } - String username = properties.get(HdfsResource.HADOOP_USER_NAME); + Configuration conf = new HdfsConfiguration(); - boolean isSecurityEnabled = false; for (Map.Entry propEntry : properties.entrySet()) { conf.set(propEntry.getKey(), propEntry.getValue()); - if (propEntry.getKey().equals(HdfsResource.HADOOP_SECURITY_AUTHENTICATION) - && propEntry.getValue().equals(AuthType.KERBEROS.getDesc())) { - isSecurityEnabled = true; - } } - try { - if (isSecurityEnabled) { - UserGroupInformation.setConfiguration(conf); - UserGroupInformation.loginUserFromKeytab( - properties.get(HdfsResource.HADOOP_KERBEROS_PRINCIPAL), - properties.get(HdfsResource.HADOOP_KERBEROS_KEYTAB)); + + UserGroupInformation ugi = getUgi(conf); + AtomicReference exception = new AtomicReference<>(); + + dfsFileSystem = ugi.doAs((PrivilegedAction) () -> { + try { + String username = properties.get(HdfsResource.HADOOP_USER_NAME); + if (username == null) { + return FileSystem.get(new Path(remotePath).toUri(), conf); + } else { + return FileSystem.get(new Path(remotePath).toUri(), conf, username); + } + } catch (Exception e) { + exception.set(e); + return null; } - if (username == null) { - dfsFileSystem = FileSystem.get(new Path(remotePath).toUri(), conf); - } else { - dfsFileSystem = FileSystem.get(new Path(remotePath).toUri(), conf, username); - } - } catch (Exception e) { - LOG.error("errors while connect to " + remotePath, e); - throw new UserException("errors while connect to " + remotePath, e); + }); + + if (dfsFileSystem == null) { + LOG.error("errors while connect to " + remotePath, exception.get()); + throw new UserException("errors while connect to " + remotePath, exception.get()); } operations = new HDFSFileOperations(dfsFileSystem); return dfsFileSystem; } + private UserGroupInformation getUgi(Configuration conf) throws UserException { + String authentication = conf.get(HdfsResource.HADOOP_SECURITY_AUTHENTICATION, null); + if (AuthType.KERBEROS.getDesc().equals(authentication)) { + conf.set("hadoop.security.authorization", "true"); + UserGroupInformation.setConfiguration(conf); + String principal = conf.get(HdfsResource.HADOOP_KERBEROS_PRINCIPAL); + String keytab = conf.get(HdfsResource.HADOOP_KERBEROS_KEYTAB); + try { + UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); + UserGroupInformation.setLoginUser(ugi); + LOG.info("kerberos authentication successful"); + return ugi; + } catch (IOException e) { + throw new UserException(e); + } + } else { + String hadoopUserName = conf.get(HdfsResource.HADOOP_USER_NAME); + if (hadoopUserName == null) { + hadoopUserName = "hadoop"; + LOG.warn("hadoop.username is unset, use default user: hadoop"); + } + return UserGroupInformation.createRemoteUser(hadoopUserName); + } + } + @Override public Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize) { LOG.debug("download from {} to {}, file size: {}.", remoteFilePath, localFilePath, fileSize); @@ -450,3 +478,4 @@ public class DFSFileSystem extends RemoteFileSystem { return new Status(Status.ErrCode.COMMON_ERROR, "mkdir is not implemented."); } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java index 5a04c8a4ec..aba1273be0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ExtraBasepathAction.java @@ -20,7 +20,7 @@ package org.apache.doris.httpv2.rest; import org.apache.doris.common.Config; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; -import org.apache.parquet.Strings; +import com.google.common.base.Strings; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java index ccaaaf05ca..3a065ee0ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java @@ -24,6 +24,7 @@ import org.apache.doris.httpv2.entity.ResponseBody; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.system.Frontend; +import com.google.common.base.Strings; import com.google.gson.reflect.TypeToken; import org.apache.commons.io.IOUtils; import org.apache.http.client.config.RequestConfig; @@ -34,7 +35,6 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; -import org.apache.parquet.Strings; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapManager.java b/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapManager.java index 65e88da9d8..92bfc4fb0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapManager.java @@ -25,11 +25,11 @@ import org.apache.doris.common.LdapConfig; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.Role; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.parquet.Strings; import java.util.List; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java index 51f27eb107..9409218cfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java @@ -46,6 +46,7 @@ import org.apache.doris.transaction.TransactionStatus; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; @@ -54,7 +55,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.parquet.Strings; import java.io.DataInput; import java.io.DataOutput; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index d72c2f8f6b..55eb7759b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -841,7 +841,7 @@ public class EditLog { } case OperationType.OP_CREATE_CATALOG: { CatalogLog log = (CatalogLog) journal.getData(); - env.getCatalogMgr().replayCreateCatalog(log, true); + env.getCatalogMgr().replayCreateCatalog(log); break; } case OperationType.OP_DROP_CATALOG: { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java index 643c25c82b..b346e3f8fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Histogram.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.statistics.util.InternalQueryResult.ResultRow; import org.apache.doris.statistics.util.StatisticsUtil; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -31,7 +32,6 @@ import com.google.gson.JsonParser; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.parquet.Strings; import java.util.Collections; import java.util.List; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 2db2926a40..ea49f99c25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -20,7 +20,6 @@ package org.apache.doris.datasource; import org.apache.doris.analysis.AlterCatalogNameStmt; import org.apache.doris.analysis.AlterCatalogPropertyStmt; import org.apache.doris.analysis.CreateCatalogStmt; -import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.analysis.CreateRoleStmt; import org.apache.doris.analysis.CreateUserStmt; import org.apache.doris.analysis.DropCatalogStmt; @@ -116,13 +115,9 @@ public class CatalogMgrTest extends TestWithFeService { // TODO: 2023/1/20 zdtodo // Assert.assertTrue(auth.getDbPrivTable().hasPrivsOfCatalog(user1, "testc")); - // create hms catalog by resource - CreateResourceStmt hmsResource = (CreateResourceStmt) parseAndAnalyzeStmt( - "create resource hms_resource properties('type' = 'hms', 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');", - rootCtx); - env.getResourceMgr().createResource(hmsResource); + // create hms catalog CreateCatalogStmt hiveCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( - "create catalog hive with resource hms_resource;", + "create catalog hive properties('type' = 'hms', 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');", rootCtx); env.getCatalogMgr().createCatalog(hiveCatalog); // deprecated: create hms catalog by properties @@ -136,14 +131,9 @@ public class CatalogMgrTest extends TestWithFeService { rootCtx); env.getCatalogMgr().createCatalog(iceBergCatalog); - // create es catalog by resource - CreateResourceStmt esResource = (CreateResourceStmt) parseAndAnalyzeStmt( - "create resource es_resource properties('type' = 'es', 'hosts' = 'http://192.168.0.1', 'user' = 'user1');", - rootCtx); - - env.getResourceMgr().createResource(esResource); + // create es catalog CreateCatalogStmt esCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( - "create catalog es with resource es_resource;", + "create catalog es properties('type' = 'es', 'hosts' = 'http://192.168.0.1', 'user' = 'user1');", rootCtx); env.getCatalogMgr().createCatalog(esCatalog); // deprecated: create es catalog by properties @@ -242,7 +232,7 @@ public class CatalogMgrTest extends TestWithFeService { CatalogIf catalog = env.getCatalogMgr().getCatalog(MY_CATALOG); // type, hive.metastore.uris and create_time - Assert.assertEquals(3, catalog.getProperties().size()); + Assert.assertEquals(4, catalog.getProperties().size()); Assert.assertEquals("thrift://172.16.5.9:9083", catalog.getProperties().get("hive.metastore.uris")); // test add property @@ -252,14 +242,14 @@ public class CatalogMgrTest extends TestWithFeService { AlterCatalogPropertyStmt alterStmt = new AlterCatalogPropertyStmt(MY_CATALOG, alterProps2); mgr.alterCatalogProps(alterStmt); catalog = env.getCatalogMgr().getCatalog(MY_CATALOG); - Assert.assertEquals(5, catalog.getProperties().size()); + Assert.assertEquals(6, catalog.getProperties().size()); Assert.assertEquals("service1", catalog.getProperties().get("dfs.nameservices")); String showDetailCatalog = "SHOW CATALOG my_catalog"; ShowCatalogStmt showDetailStmt = (ShowCatalogStmt) parseAndAnalyzeStmt(showDetailCatalog); showResultSet = mgr.showCatalogs(showDetailStmt); - Assert.assertEquals(5, showResultSet.getResultRows().size()); + Assert.assertEquals(6, showResultSet.getResultRows().size()); for (List row : showResultSet.getResultRows()) { Assertions.assertEquals(2, row.size()); if (row.get(0).equalsIgnoreCase("type")) { @@ -328,7 +318,7 @@ public class CatalogMgrTest extends TestWithFeService { CatalogIf hms = mgr2.getCatalog(MY_CATALOG); properties = hms.getProperties(); - Assert.assertEquals(5, properties.size()); + Assert.assertEquals(6, properties.size()); Assert.assertEquals("hms", properties.get("type")); Assert.assertEquals("thrift://172.16.5.9:9083", properties.get("hive.metastore.uris")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index f6fe3e9250..dcd4b3c3e6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -237,10 +237,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(queryOld); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_s3_old"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 11); + Assertions.assertEquals(properties.size(), 12); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 20); + Assertions.assertEquals(hdProps.size(), 21); } @Test @@ -255,10 +255,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(query); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_s3"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 10); + Assertions.assertEquals(properties.size(), 11); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 19); + Assertions.assertEquals(hdProps.size(), 20); } @Test @@ -275,10 +275,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(queryOld); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_glue_old"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 19); + Assertions.assertEquals(properties.size(), 20); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 28); + Assertions.assertEquals(hdProps.size(), 29); String query = "create catalog hms_glue properties (\n" + " 'type'='hms',\n" @@ -291,10 +291,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmtNew = createStmt(query); HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew, "hms_glue"); Map propertiesNew = catalogNew.getCatalogProperty().getProperties(); - Assertions.assertEquals(propertiesNew.size(), 19); + Assertions.assertEquals(propertiesNew.size(), 20); Map hdPropsNew = catalogNew.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdPropsNew.size(), 28); + Assertions.assertEquals(hdPropsNew.size(), 29); } @Test @@ -309,10 +309,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(query); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_obs"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 10); + Assertions.assertEquals(properties.size(), 11); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 14); + Assertions.assertEquals(hdProps.size(), 15); } @Test @@ -328,10 +328,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(query); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_cos"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 11); + Assertions.assertEquals(properties.size(), 12); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 23); + Assertions.assertEquals(hdProps.size(), 24); } @Test @@ -346,10 +346,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(query); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_oss"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 10); + Assertions.assertEquals(properties.size(), 11); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 22); + Assertions.assertEquals(hdProps.size(), 23); } private static HMSExternalCatalog createAndGetCatalog(CreateCatalogStmt analyzedStmt, String name) diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index e1bcede00d..003f20c694 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -19,7 +19,6 @@ package org.apache.doris.external.hms; import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.CreateDbStmt; -import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -71,13 +70,9 @@ public class HmsCatalogTest extends TestWithFeService { rootCtx.setEnv(env); mgr = env.getCatalogMgr(); - // create hms catalog by resource - CreateResourceStmt hmsResource = (CreateResourceStmt) parseAndAnalyzeStmt( - "create resource hms_resource properties('type' = 'hms', 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');", - rootCtx); - env.getResourceMgr().createResource(hmsResource); + // create hms catalog CreateCatalogStmt hmsCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( - "create catalog hms_ctl with resource hms_resource;", + "create catalog hms_ctl properties('type' = 'hms', 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');", rootCtx); mgr.createCatalog(hmsCatalog); diff --git a/regression-test/data/external_catalog_p0/hive/test_catalog_with_resource.out b/regression-test/data/external_catalog_p0/hive/test_catalog_with_resource.out deleted file mode 100644 index c134f2cee0..0000000000 --- a/regression-test/data/external_catalog_p0/hive/test_catalog_with_resource.out +++ /dev/null @@ -1,31 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql11 -- -jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n); - --- !sql12 -- -jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root2",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n); - --- !sql13 -- -jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root2",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n); - --- !sql14 -- -jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root2",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n); - --- !sql21 -- -hive1_catalog \nCREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"hive.metastore.uris" = "thrift://127.0.0.1:91837",\n"create_time" = "2023-05-15 22:02:06.511"\n); - --- !sql22 -- -hive1_catalog \nCREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"new_config" = "value1",\n"hive.metastore.uris" = "thrift://127.0.0.1:91837",\n"create_time" = "2023-05-15 22:02:06.511"\n); - --- !sql23 -- -hive1_catalog \nCREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"new_config" = "value1",\n"hive.metastore.uris" = "thrift://127.0.0.2:91837",\n"create_time" = "2023-05-15 22:02:06.511"\n); - --- !sql31 -- -es1_catalog \nCREATE CATALOG `es1_catalog` PROPERTIES (\n"type" = "es",\n"nodes_discovery" = "true",\n"hosts" = "http://127.0.0.1:29207",\n"enable_keyword_sniff" = "true",\n"create_time" = "2023-05-15 22:02:06.522"\n); - --- !sql22 -- -es1_catalog \nCREATE CATALOG `es1_catalog` PROPERTIES (\n"type" = "es",\n"nodes_discovery" = "true",\n"hosts" = "http://127.0.0.1:29207",\n"enable_keyword_sniff" = "false",\n"create_time" = "2023-05-15 22:02:06.522"\n); - --- !sql23 -- -es1_catalog \nCREATE CATALOG `es1_catalog` PROPERTIES (\n"type" = "es",\n"nodes_discovery" = "true",\n"hosts" = "http://127.0.0.2:29207",\n"enable_keyword_sniff" = "false",\n"create_time" = "2023-05-15 22:02:06.522"\n); - diff --git a/regression-test/suites/es_p0/test_es_query.groovy b/regression-test/suites/es_p0/test_es_query.groovy index f60a805691..2b0251c1e1 100644 --- a/regression-test/suites/es_p0/test_es_query.groovy +++ b/regression-test/suites/es_p0/test_es_query.groovy @@ -26,9 +26,6 @@ suite("test_es_query", "p0") { sql """drop catalog if exists es6;""" sql """drop catalog if exists es7;""" sql """drop catalog if exists es8;""" - sql """drop resource if exists es6_resource;""" - sql """drop resource if exists es7_resource;""" - sql """drop resource if exists es8_resource;""" sql """drop table if exists test_v1;""" sql """drop table if exists test_v2;""" @@ -44,7 +41,7 @@ suite("test_es_query", "p0") { """ // test new create catalog syntax - sql """create resource if not exists es7_resource properties( + sql """create catalog if not exists es7 properties( "type"="es", "hosts"="http://127.0.0.1:$es_7_port", "nodes_discovery"="false", @@ -52,7 +49,7 @@ suite("test_es_query", "p0") { ); """ - sql """create resource if not exists es8_resource properties( + sql """create catalog if not exists es8 properties( "type"="es", "hosts"="http://127.0.0.1:$es_8_port", "nodes_discovery"="false", @@ -141,9 +138,6 @@ suite("test_es_query", "p0") { order_qt_sql54 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')""" order_qt_sql55 """select test4,test5,test6,test7,test8 from test_v2 order by test8""" - sql """create catalog if not exists es6 with resource es6_resource;""" - sql """create catalog if not exists es7 with resource es7_resource;""" - sql """create catalog if not exists es8 with resource es8_resource;""" sql """switch es6""" // order_qt_sql61 """show tables""" order_qt_sql62 """select * from test1 where test2='text#1'""" @@ -171,8 +165,5 @@ suite("test_es_query", "p0") { sql """drop catalog if exists es6;""" sql """drop catalog if exists es7;""" sql """drop catalog if exists es8;""" - sql """drop resource if exists es6_resource;""" - sql """drop resource if exists es7_resource;""" - sql """drop resource if exists es8_resource;""" } } diff --git a/regression-test/suites/es_p0/test_es_query_nereids.groovy b/regression-test/suites/es_p0/test_es_query_nereids.groovy index 40faa957fc..407d6c2d17 100644 --- a/regression-test/suites/es_p0/test_es_query_nereids.groovy +++ b/regression-test/suites/es_p0/test_es_query_nereids.groovy @@ -26,9 +26,6 @@ suite("test_es_query_nereids", "p0") { sql """drop catalog if exists es6_nereids;""" sql """drop catalog if exists es7_nereids;""" sql """drop catalog if exists es8_nereids;""" - sql """drop resource if exists es6_resource_nereids;""" - sql """drop resource if exists es7_resource_nereids;""" - sql """drop resource if exists es8_resource_nereids;""" sql """drop table if exists test_v1_nereids;""" sql """drop table if exists test_v2_nereids;""" sql """set enable_nereids_planner=true;""" @@ -46,7 +43,7 @@ suite("test_es_query_nereids", "p0") { """ // test new create catalog syntax - sql """create resource if not exists es7_resource_nereids properties( + sql """create catalog if not exists es7_nereids properties( "type"="es", "hosts"="http://127.0.0.1:$es_7_port", "nodes_discovery"="false", @@ -54,7 +51,7 @@ suite("test_es_query_nereids", "p0") { ); """ - sql """create resource if not exists es8_resource_nereids properties( + sql """create catalog if not exists es8_nereids properties( "type"="es", "hosts"="http://127.0.0.1:$es_8_port", "nodes_discovery"="false", @@ -138,9 +135,6 @@ suite("test_es_query_nereids", "p0") { order_qt_sql51 """select * from test_v2_nereids where test2='text#1'""" - sql """create catalog if not exists es6_nereids with resource es6_resource_nereids;""" - sql """create catalog if not exists es7_nereids with resource es7_resource_nereids;""" - sql """create catalog if not exists es8_nereids with resource es8_resource_nereids;""" sql """switch es6_nereids""" // order_qt_sql61 """show tables""" order_qt_sql62 """select * from test1 where test2='text#1'""" @@ -165,8 +159,5 @@ suite("test_es_query_nereids", "p0") { sql """drop catalog if exists es6_nereids;""" sql """drop catalog if exists es7_nereids;""" sql """drop catalog if exists es8_nereids;""" - sql """drop resource if exists es6_resource_nereids;""" - sql """drop resource if exists es7_resource_nereids;""" - sql """drop resource if exists es8_resource_nereids;""" } } diff --git a/regression-test/suites/es_p0/test_es_query_no_http_url.groovy b/regression-test/suites/es_p0/test_es_query_no_http_url.groovy index 2651626672..e7e6af5b95 100644 --- a/regression-test/suites/es_p0/test_es_query_no_http_url.groovy +++ b/regression-test/suites/es_p0/test_es_query_no_http_url.groovy @@ -26,9 +26,6 @@ suite("test_es_query_no_http_url", "p0") { sql """drop catalog if exists es6;""" sql """drop catalog if exists es7;""" sql """drop catalog if exists es8;""" - sql """drop resource if exists es6_resource;""" - sql """drop resource if exists es7_resource;""" - sql """drop resource if exists es8_resource;""" sql """drop table if exists test_v1;""" sql """drop table if exists test_v2;""" @@ -44,7 +41,7 @@ suite("test_es_query_no_http_url", "p0") { """ // test new create catalog syntax - sql """create resource if not exists es7_resource properties( + sql """create catalog if not exists es7 properties( "type"="es", "hosts"="127.0.0.1:$es_7_port", "nodes_discovery"="false", @@ -52,7 +49,7 @@ suite("test_es_query_no_http_url", "p0") { ); """ - sql """create resource if not exists es8_resource properties( + sql """create catalog if not exists es8 properties( "type"="es", "hosts"="127.0.0.1:$es_8_port", "nodes_discovery"="false", @@ -137,9 +134,6 @@ suite("test_es_query_no_http_url", "p0") { """ order_qt_sql52 """select * from test_v2 where test2='text#1'""" - sql """create catalog if not exists es6 with resource es6_resource;""" - sql """create catalog if not exists es7 with resource es7_resource;""" - sql """create catalog if not exists es8 with resource es8_resource;""" // es6 sql """switch es6""" order_qt_sql61 """select * from test1 where test2='text#1'""" @@ -153,8 +147,5 @@ suite("test_es_query_no_http_url", "p0") { sql """drop catalog if exists es6;""" sql """drop catalog if exists es7;""" sql """drop catalog if exists es8;""" - sql """drop resource if exists es6_resource;""" - sql """drop resource if exists es7_resource;""" - sql """drop resource if exists es8_resource;""" } } diff --git a/regression-test/suites/external_catalog_p0/hive/test_catalog_with_resource.groovy b/regression-test/suites/external_catalog_p0/hive/test_catalog_with_resource.groovy deleted file mode 100644 index 9b9320a2fd..0000000000 --- a/regression-test/suites/external_catalog_p0/hive/test_catalog_with_resource.groovy +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_catalog_with_resource", "p0") { - String enabledJdbc = context.config.otherConfigs.get("enableJdbcTest") - String enabledHive = context.config.otherConfigs.get("enableHiveTest") - String enabledEs = context.config.otherConfigs.get("enableEsTest") - if (enabledJdbc != null && enabledJdbc.equalsIgnoreCase("true") - && enabledHive != null && enabledHive.equalsIgnoreCase("true") - && enabledEs != null && enabledEs.equalsIgnoreCase("true")) { - - String jdbc_resource_name = "jdbc1_resource" - String hive_resource_name = "hive1_resource" - String es_resource_name = "es1_resource" - - String jdbc_catalog_name = "jdbc1_catalog"; - String hive_catalog_name = "hive1_catalog"; - String es_catalog_name = "es1_catalog"; - - String mysql_port = context.config.otherConfigs.get("mysql_57_port"); - String hms_port = context.config.otherConfigs.get("hms_port") - String es_7_port = context.config.otherConfigs.get("es_7_port") - - sql """drop catalog if exists ${jdbc_catalog_name} """ - sql """drop resource if exists ${jdbc_resource_name} """ - sql """drop catalog if exists ${hive_catalog_name} """ - sql """drop resource if exists ${hive_resource_name} """ - sql """drop catalog if exists ${es_catalog_name} """ - sql """drop resource if exists ${es_resource_name} """ - - // jdbc - sql """create resource if not exists ${jdbc_resource_name} properties( - "type"="jdbc", - "user"="root", - "password"="123456", - "jdbc_url" = "jdbc:mysql://127.0.0.1:${mysql_port}/doris_test?useSSL=false", - "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", - "driver_class" = "com.mysql.cj.jdbc.Driver" - );""" - sql """CREATE CATALOG ${jdbc_catalog_name} WITH RESOURCE ${jdbc_resource_name}""" - - qt_sql11 """show create catalog ${jdbc_catalog_name}""" - sql """alter catalog ${jdbc_catalog_name} set properties("user"="root2")""" - qt_sql12 """show create catalog ${jdbc_catalog_name}""" - sql """alter resource ${jdbc_resource_name} properties("user"="root3")""" - qt_sql13 """show create catalog ${jdbc_catalog_name}""" - sql """alter resource ${jdbc_resource_name} properties("driver_class"="com.mysql.jdbc.Driver")""" - qt_sql14 """show create catalog ${jdbc_catalog_name}""" - - // hive - sql """drop catalog if exists ${hive_catalog_name} """ - sql """drop resource if exists ${hive_resource_name} """ - sql """create resource if not exists ${hive_resource_name} properties ( - "type"="hms", - 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' - );""" - sql """create catalog if not exists ${hive_catalog_name} with resource ${hive_resource_name};""" - qt_sql21 """show create catalog ${hive_catalog_name}""" - sql """alter catalog ${hive_catalog_name} set properties("new_config"="value1")""" - qt_sql22 """show create catalog ${hive_catalog_name}""" - sql """alter resource ${hive_resource_name} properties('hive.metastore.uris' = 'thrift://127.0.0.2:${hms_port}')""" - qt_sql23 """show create catalog ${hive_catalog_name}""" - - // es - sql """drop catalog if exists ${es_catalog_name} """ - sql """drop resource if exists ${es_resource_name} """ - - sql """create resource if not exists ${es_resource_name} properties( - "type"="es", - "hosts"="http://127.0.0.1:$es_7_port", - "nodes_discovery"="false", - "enable_keyword_sniff"="true" - ); - """ - sql """create catalog ${es_catalog_name} with resource ${es_resource_name} properties - ("nodes_discovery"="true"); - """ - qt_sql31 """show create catalog ${es_catalog_name}""" - sql """alter catalog ${es_catalog_name} set properties("enable_keyword_sniff"="false")""" - qt_sql22 """show create catalog ${es_catalog_name}""" - sql """alter resource ${es_resource_name} properties('hosts' = 'http://127.0.0.2:${es_7_port}')""" - qt_sql23 """show create catalog ${es_catalog_name}""" - } -} diff --git a/regression-test/suites/external_catalog_p0/hive/test_different_parquet_types.groovy b/regression-test/suites/external_catalog_p0/hive/test_different_parquet_types.groovy index 9c4f969511..e26864e66c 100644 --- a/regression-test/suites/external_catalog_p0/hive/test_different_parquet_types.groovy +++ b/regression-test/suites/external_catalog_p0/hive/test_different_parquet_types.groovy @@ -180,16 +180,13 @@ suite("test_different_parquet_types", "p0") { String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { try { - String resource_name = "hive_different_parquet_types_resource" String catalog_name = "hive_different_parquet_types" sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists ${resource_name}""" - sql """create resource if not exists ${resource_name} properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource ${resource_name};""" sql """use `${catalog_name}`.`default`""" q01() @@ -202,7 +199,6 @@ suite("test_different_parquet_types", "p0") { q08() q09() sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists ${resource_name}""" } finally { } } diff --git a/regression-test/suites/external_catalog_p0/hive/test_hive_orc.groovy b/regression-test/suites/external_catalog_p0/hive/test_hive_orc.groovy index 5e58e42ef5..35877387f6 100644 --- a/regression-test/suites/external_catalog_p0/hive/test_hive_orc.groovy +++ b/regression-test/suites/external_catalog_p0/hive/test_hive_orc.groovy @@ -72,11 +72,10 @@ suite("test_hive_orc", "all_types") { String hms_port = context.config.otherConfigs.get("hms_port") String catalog_name = "hive_test_orc" sql """drop catalog if exists ${catalog_name}""" - sql """create resource if not exists hms_resource_hive_orc properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource hms_resource_hive_orc;""" sql """use `${catalog_name}`.`default`""" select_top50() @@ -87,7 +86,6 @@ suite("test_hive_orc", "all_types") { only_partition_col() sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists hms_resource_hive_orc""" // test old create-catalog syntax for compatibility sql """ diff --git a/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy b/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy index 5d91ea7e5c..3954afcb45 100644 --- a/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy +++ b/regression-test/suites/external_catalog_p0/hive/test_hive_other.groovy @@ -56,11 +56,10 @@ suite("test_hive_other", "p0") { String catalog_name = "hive_test_other" sql """drop catalog if exists ${catalog_name}""" - sql """create resource if not exists hms_resource_hive_other properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource hms_resource_hive_other;""" // test user's grants on external catalog sql """drop user if exists ext_catalog_user""" @@ -87,6 +86,5 @@ suite("test_hive_other", "p0") { order_qt_after_refresh """ select dt, dt, k2, k5, dt from table_with_vertical_line where dt in ('2022-11-25') or dt in ('2022-11-24') order by k2 desc limit 10;""" sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists hms_resource_hive_other""" } } diff --git a/regression-test/suites/external_catalog_p0/hive/test_hive_parquet.groovy b/regression-test/suites/external_catalog_p0/hive/test_hive_parquet.groovy index 9e2eb5bfd3..bbe324ff2e 100644 --- a/regression-test/suites/external_catalog_p0/hive/test_hive_parquet.groovy +++ b/regression-test/suites/external_catalog_p0/hive/test_hive_parquet.groovy @@ -145,11 +145,10 @@ suite("test_hive_parquet", "p0") { String hms_port = context.config.otherConfigs.get("hms_port") String catalog_name = "hive_test_parquet" sql """drop catalog if exists ${catalog_name}""" - sql """create resource if not exists hms_resource_hive_parquet properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource hms_resource_hive_parquet;""" sql """use `${catalog_name}`.`default`""" q01() @@ -174,7 +173,6 @@ suite("test_hive_parquet", "p0") { q20() sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists hms_resource_hive_parquet""" } finally { } } diff --git a/regression-test/suites/external_catalog_p0/hive/test_hive_partitions.groovy b/regression-test/suites/external_catalog_p0/hive/test_hive_partitions.groovy index 39c575f7c1..7e70f5729a 100644 --- a/regression-test/suites/external_catalog_p0/hive/test_hive_partitions.groovy +++ b/regression-test/suites/external_catalog_p0/hive/test_hive_partitions.groovy @@ -43,17 +43,15 @@ suite("test_hive_partitions", "p0") { String hms_port = context.config.otherConfigs.get("hms_port") String catalog_name = "hive_test_partitions" sql """drop catalog if exists ${catalog_name}""" - sql """create resource if not exists hms_resource_hive_partitions properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource hms_resource_hive_partitions;""" sql """use `${catalog_name}`.`default`""" q01() sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists hms_resource_hive_partitions""" } finally { } } diff --git a/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy b/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy index b938f8c8ed..c85641c141 100644 --- a/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy +++ b/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy @@ -24,7 +24,6 @@ suite("test_external_catalog_mysql", "p2") { String extMysqlUser = context.config.otherConfigs.get("extMysqlUser") String extMysqlPassword = context.config.otherConfigs.get("extMysqlPassword") String mysqlDatabaseName01 = "external_mysql_catalog_database01" - String mysqlResource01 = "mysql_catalog_resource_01" String mysqlCatalogName = "mysql_jdbc" String mysqlTableName01 = "lineorder" String mysqlTableName02 = "customer" @@ -36,9 +35,8 @@ suite("test_external_catalog_mysql", "p2") { sql """drop catalog if exists ${mysqlCatalogName};""" - sql """drop resource if exists ${mysqlResource01};""" sql """ - CREATE RESOURCE ${mysqlResource01} + CREATE CATALOG ${mysqlCatalogName} properties ( "type"="jdbc", "user"="${extMysqlUser}", @@ -49,22 +47,6 @@ suite("test_external_catalog_mysql", "p2") { ); """ - sql """CREATE CATALOG ${mysqlCatalogName} WITH RESOURCE ${mysqlResource01};""" -// sql """drop catalog if exists ${mysqlCatalogName};""" - - -// sql """ -// CREATE CATALOG ${mysqlCatalogName} -// PROPERTIES( -// "type"="jdbc", -// "jdbc.user"="${extMysqlUser}", -// "jdbc.password"="${extMysqlPassword}", -// "jdbc.jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false", -// "jdbc.driver_url"="https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", -// "jdbc.driver_class"="com.mysql.cj.jdbc.Driver" -// ); -// """ - sql """switch ${mysqlCatalogName}""" sql """use ssb;""" diff --git a/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy index fc13620288..d2f67340e3 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy @@ -18,7 +18,6 @@ suite("test_clickhouse_jdbc_catalog", "p0") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { - String resource_name = "clickhouse_catalog_resource"; String catalog_name = "clickhouse_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_db_name = "doris_test"; @@ -27,9 +26,8 @@ suite("test_clickhouse_jdbc_catalog", "p0") { String inDorisTable = "doris_in_tb"; sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ - sql """ create resource if not exists ${resource_name} properties( + sql """ create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="default", "password"="123456", @@ -38,8 +36,6 @@ suite("test_clickhouse_jdbc_catalog", "p0") { "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver" );""" - sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ - sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( @@ -60,6 +56,5 @@ suite("test_clickhouse_jdbc_catalog", "p0") { order_qt_in_tb """ select id, name, age from internal.${internal_db_name}.${inDorisTable} order by id; """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ } -} \ No newline at end of file +} diff --git a/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy index 2f503ea68d..e5ef743a9c 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy @@ -20,7 +20,6 @@ suite("test_mysql_jdbc_catalog", "p0") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { - String resource_name = "jdbc_resource_catalog_mysql" String catalog_name = "mysql_jdbc_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_db_name = "doris_test"; @@ -51,9 +50,8 @@ suite("test_mysql_jdbc_catalog", "p0") { String test_insert2 = "test_insert2"; sql """drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root", "password"="123456", @@ -62,9 +60,6 @@ suite("test_mysql_jdbc_catalog", "p0") { "driver_class" = "com.mysql.cj.jdbc.Driver" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" - - sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( @@ -122,10 +117,9 @@ suite("test_mysql_jdbc_catalog", "p0") { order_qt_test_insert4 """ select k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15 from ${test_insert2} where id = '${uuid3}' """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ // test only_specified_database argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root", "password"="123456", @@ -135,16 +129,14 @@ suite("test_mysql_jdbc_catalog", "p0") { "only_specified_database" = "true" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" sql """switch ${catalog_name}""" qt_specified_database_1 """ show databases; """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ // test only_specified_database and include_database_list argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root", "password"="123456", @@ -155,16 +147,14 @@ suite("test_mysql_jdbc_catalog", "p0") { "include_database_list" = "doris_test" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" sql """switch ${catalog_name}""" qt_specified_database_2 """ show databases; """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ // test only_specified_database and exclude_database_list argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root", "password"="123456", @@ -175,16 +165,14 @@ suite("test_mysql_jdbc_catalog", "p0") { "exclude_database_list" = "doris_test" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" sql """switch ${catalog_name}""" qt_specified_database_3 """ show databases; """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ // test include_database_list and exclude_database_list have overlapping items case - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root", "password"="123456", @@ -196,13 +184,11 @@ suite("test_mysql_jdbc_catalog", "p0") { "exclude_database_list" = "doris_test" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" sql """switch ${catalog_name}""" qt_specified_database_4 """ show databases; """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ // test old create-catalog syntax for compatibility sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( diff --git a/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.groovy b/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.groovy index f974d5171c..5b23fc51da 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.groovy @@ -18,7 +18,6 @@ suite("test_mysql_jdbc_catalog_nereids", "p0") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { - String resource_name = "jdbc_resource_catalog_mysql_nereids" String catalog_name = "mysql_jdbc_catalog_nereids"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_db_name = "doris_test"; @@ -50,12 +49,11 @@ suite("test_mysql_jdbc_catalog_nereids", "p0") { sql """ADMIN SET FRONTEND CONFIG ("enable_decimal_conversion" = "true");""" sql """drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ - sql """set enable_nereids_planner=true;""" - sql """set enable_fallback_to_original_planner=false;""" + sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root", "password"="123456", @@ -64,9 +62,6 @@ suite("test_mysql_jdbc_catalog_nereids", "p0") { "driver_class" = "com.mysql.cj.jdbc.Driver" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" - - sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( @@ -122,7 +117,6 @@ suite("test_mysql_jdbc_catalog_nereids", "p0") { order_qt_test_insert4 """ select k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15 from ${test_insert2} where id = '${uuid3}' """ sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ // test old create-catalog syntax for compatibility sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( diff --git a/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy index 12223568e1..1d31af4a00 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy @@ -18,7 +18,6 @@ suite("test_oracle_jdbc_catalog", "p0") { String enabled = context.config.otherConfigs.get("enableJdbcTest"); if (enabled != null && enabled.equalsIgnoreCase("true")) { - String resource_name = "oracle_catalog_resource"; String catalog_name = "oracle_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_db_name = "DORIS_TEST"; @@ -30,9 +29,8 @@ suite("test_oracle_jdbc_catalog", "p0") { String inDorisTable = "doris_in_tb"; sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="doris_test", "password"="123456", @@ -41,8 +39,6 @@ suite("test_oracle_jdbc_catalog", "p0") { "driver_class" = "oracle.jdbc.driver.OracleDriver" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" - sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( @@ -87,10 +83,9 @@ suite("test_oracle_jdbc_catalog", "p0") { order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test only_specified_database argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="doris_test", "password"="123456", @@ -99,15 +94,13 @@ suite("test_oracle_jdbc_catalog", "p0") { "driver_class" = "oracle.jdbc.driver.OracleDriver", "only_specified_database" = "true" );""" - sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """ switch ${catalog_name} """ qt_specified_database """ show databases; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test only_specified_database and specified_database_list argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="doris_test", "password"="123456", @@ -117,15 +110,13 @@ suite("test_oracle_jdbc_catalog", "p0") { "only_specified_database" = "true", "include_database_list" = "${ex_db_name}" );""" - sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """ switch ${catalog_name} """ qt_specified_database """ show databases; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test lower_case_table_names argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="doris_test", "password"="123456", @@ -134,7 +125,6 @@ suite("test_oracle_jdbc_catalog", "p0") { "driver_class" = "oracle.jdbc.driver.OracleDriver", "lower_case_table_names" = "true" );""" - sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """ switch ${catalog_name} """ sql """ use ${ex_db_name_lower_case}""" @@ -143,7 +133,6 @@ suite("test_oracle_jdbc_catalog", "p0") { qt_lower_case_table_names3 """ select * from test_int order by ID; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" } } diff --git a/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy index 1676d3d807..62e30509cc 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy @@ -18,7 +18,6 @@ suite("test_pg_jdbc_catalog", "p0") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { - String resource_name = "jdbc_resource_catalog_pg"; String catalog_name = "pg_jdbc_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_schema_name = "doris_test"; @@ -28,9 +27,8 @@ suite("test_pg_jdbc_catalog", "p0") { String test_insert = "test_insert"; sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="postgres", "password"="123456", @@ -39,8 +37,6 @@ suite("test_pg_jdbc_catalog", "p0") { "driver_class" = "org.postgresql.Driver" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" - sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( @@ -88,10 +84,9 @@ suite("test_pg_jdbc_catalog", "p0") { order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test only_specified_database argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="postgres", "password"="123456", @@ -100,15 +95,13 @@ suite("test_pg_jdbc_catalog", "p0") { "driver_class" = "org.postgresql.Driver", "only_specified_database" = "true" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """switch ${catalog_name} """ qt_specified_database_1 """ show databases; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test only_specified_database and include_database_list argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="postgres", "password"="123456", @@ -118,15 +111,13 @@ suite("test_pg_jdbc_catalog", "p0") { "only_specified_database" = "true", "include_database_list" = "doris_test" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """switch ${catalog_name} """ qt_specified_database_2 """ show databases; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test only_specified_database and exclude_database_list argument - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="postgres", "password"="123456", @@ -136,15 +127,13 @@ suite("test_pg_jdbc_catalog", "p0") { "only_specified_database" = "true", "exclude_database_list" = "doris_test" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """switch ${catalog_name} """ qt_specified_database_3 """ show databases; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" // test include_database_list and exclude_database_list have overlapping items case - sql """create resource if not exists ${resource_name} properties( + sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="postgres", "password"="123456", @@ -155,14 +144,11 @@ suite("test_pg_jdbc_catalog", "p0") { "include_database_list" = "doris_test", "exclude_database_list" = "doris_test" );""" - sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ sql """switch ${catalog_name} """ qt_specified_database_4 """ show databases; """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists ${resource_name}""" - // test old create-catalog syntax for compatibility sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( "type"="jdbc", "jdbc.user"="postgres", diff --git a/regression-test/suites/jdbc_catalog_p0/test_sqlserver_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_sqlserver_jdbc_catalog.groovy index 4301a80b29..5babedbd7c 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_sqlserver_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_sqlserver_jdbc_catalog.groovy @@ -18,7 +18,6 @@ suite("test_sqlserver_jdbc_catalog", "p0") { String enabled = context.config.otherConfigs.get("enableJdbcTest"); if (enabled != null && enabled.equalsIgnoreCase("true")) { - String resource_name = "sqlserver_catalog_resource"; String catalog_name = "sqlserver_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_db_name = "dbo"; @@ -27,9 +26,8 @@ suite("test_sqlserver_jdbc_catalog", "p0") { String inDorisTable = "doris_in_tb"; sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ - sql """ create resource if not exists ${resource_name} properties( + sql """ create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="SA", "password"="Doris123456", @@ -38,8 +36,6 @@ suite("test_sqlserver_jdbc_catalog", "p0") { "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" );""" - sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ - sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( @@ -67,6 +63,5 @@ suite("test_sqlserver_jdbc_catalog", "p0") { sql """ drop catalog if exists ${catalog_name} """ - sql """ drop resource if exists ${resource_name} """ } -} \ No newline at end of file +} diff --git a/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_orc.groovy b/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_orc.groovy index 38941bf7c4..8902190a88 100644 --- a/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_orc.groovy +++ b/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_orc.groovy @@ -803,11 +803,10 @@ order by String catalog_name = "test_catalog_hive_orc" sql """drop catalog if exists ${catalog_name}""" - sql """create resource if not exists hms_resource_catalog_orc properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource hms_resource_catalog_orc;""" sql """switch ${catalog_name}""" sql """use `tpch1_orc`""" @@ -835,7 +834,6 @@ order by q22() sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists hms_resource_catalog_orc""" } } diff --git a/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_parquet.groovy b/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_parquet.groovy index f5a1dc3f01..31acb64310 100644 --- a/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_parquet.groovy +++ b/regression-test/suites/tpch_sf1_p0/multi_catalog_query/hive_catalog_parquet.groovy @@ -803,11 +803,10 @@ order by String catalog_name = "test_catalog_hive_parquet" sql """drop catalog if exists ${catalog_name}""" - sql """create resource if not exists hms_resource_catalog_parquet properties ( + sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://127.0.0.1:${hms_port}' );""" - sql """create catalog if not exists ${catalog_name} with resource hms_resource_catalog_parquet;""" sql """switch ${catalog_name}""" sql """use `tpch1_parquet`""" @@ -835,7 +834,6 @@ order by q22() sql """drop catalog if exists ${catalog_name}""" - sql """drop resource if exists hms_resource_catalog_parquet""" } }