From cb6875b5a4bee4fffc9311c982af1bd3b8b5dab0 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Thu, 2 Feb 2023 17:35:48 +0800 Subject: [PATCH] [improvement](multi-catalog) use date/datetimev2 as default col type for catalog table (#16304) 1. When mapping column from external datasource, use date/datetimev2 as default type 2. check `is_cancelled` when read data, to avoid endless loop after query is cancelled --- be/src/runtime/buffer_control_block.cpp | 2 +- be/src/vec/exec/scan/vscanner.cpp | 7 ++++++- .../developer-guide/regression-testing.md | 4 ++-- .../catalog/HiveMetaStoreClientHelper.java | 4 ++-- .../doris/external/elasticsearch/EsUtil.java | 2 +- .../apache/doris/external/jdbc/JdbcClient.java | 18 +++++++++--------- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/be/src/runtime/buffer_control_block.cpp b/be/src/runtime/buffer_control_block.cpp index c745bc8750..ef09d7465e 100644 --- a/be/src/runtime/buffer_control_block.cpp +++ b/be/src/runtime/buffer_control_block.cpp @@ -110,7 +110,7 @@ Status BufferControlBlock::add_batch(std::unique_ptr& result) int num_rows = result->result_batch.rows.size(); while ((!_batch_queue.empty() && _buffer_rows > _buffer_limit) && !_is_cancelled) { - _data_removal.wait(l); + _data_removal.wait_for(l, std::chrono::seconds(1), [&]() { return _is_cancelled.load(); }); } if (_is_cancelled) { diff --git a/be/src/vec/exec/scan/vscanner.cpp b/be/src/vec/exec/scan/vscanner.cpp index 7a956ecdb5..dfb260de50 100644 --- a/be/src/vec/exec/scan/vscanner.cpp +++ b/be/src/vec/exec/scan/vscanner.cpp @@ -70,7 +70,12 @@ Status VScanner::get_block(RuntimeState* state, Block* block, bool* eof) { // record rows return (after filter) for _limit check _num_rows_return += block->rows(); } - } while (block->rows() == 0 && !(*eof) && raw_rows_read() < raw_rows_threshold); + } while (!state->is_cancelled() && block->rows() == 0 && !(*eof) && + raw_rows_read() < raw_rows_threshold); + } + + if (state->is_cancelled()) { + return Status::Cancelled("cancelled"); } // set eof to true if per scanner limit is reached diff --git a/docs/zh-CN/community/developer-guide/regression-testing.md b/docs/zh-CN/community/developer-guide/regression-testing.md index 60469ddbb3..5d8a64caab 100644 --- a/docs/zh-CN/community/developer-guide/regression-testing.md +++ b/docs/zh-CN/community/developer-guide/regression-testing.md @@ -705,7 +705,7 @@ Doris 支持一些外部署数据源的查询。所以回归框架也提供了 ``` create database db1; show databases; - craete table db1.test1(k1 bigint, k2 bigint, k3 string) partition by (k1); + create table db1.test1(k1 bigint, k2 bigint, k3 string) partitioned by (k1); insert into db1.test1 values(1,2,'abc'); select * from db1.test1; quit; @@ -718,7 +718,7 @@ Doris 支持一些外部署数据源的查询。所以回归框架也提供了 spark.sql(s"create database db1") spark.sql(s"show databases").show() - spark.sql(s"craete table db1.test1(k1 bigint, k2 bigint, k3 string) partition by (k1)").show() + spark.sql(s"create table db1.test1(k1 bigint, k2 bigint, k3 string) partitioned by (k1)").show() spark.sql(s"show tables from db1").show() spark.sql(s"insert into db1.test1 values(1,2,'abc')").show() spark.sql(s"select * from db1.test1").show() diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java index adf5dccc0a..0ff5ee4363 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java @@ -686,9 +686,9 @@ public class HiveMetaStoreClientHelper { case "bigint": return Type.BIGINT; case "date": - return ScalarType.getDefaultDateType(Type.DATE); + return ScalarType.createDateV2Type(); case "timestamp": - return ScalarType.getDefaultDateType(Type.DATETIME); + return ScalarType.createDatetimeV2Type(0); case "float": return Type.FLOAT; case "double": diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index 10c9a675c2..2276356fa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -446,7 +446,7 @@ public class EsUtil { case "scaled_float": return Type.DOUBLE; case "date": - return ScalarType.getDefaultDateType(Type.DATE); + return ScalarType.createDateV2Type(); case "keyword": case "text": case "ip": diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java index 0f75b4755a..3605fefdd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java @@ -390,10 +390,10 @@ public class JdbcClient { case "BIGINT": return Type.BIGINT; case "DATE": - return ScalarType.getDefaultDateType(Type.DATE); + return ScalarType.createDateV2Type(); case "TIMESTAMP": case "DATETIME": - return ScalarType.getDefaultDateType(Type.DATETIME); + return ScalarType.createDatetimeV2Type(0); case "FLOAT": return Type.FLOAT; case "DOUBLE": @@ -473,9 +473,9 @@ public class JdbcClient { return charType; case "timestamp": case "timestamptz": - return ScalarType.getDefaultDateType(Type.DATETIME); + return ScalarType.createDatetimeV2Type(0); case "date": - return ScalarType.getDefaultDateType(Type.DATE); + return ScalarType.createDateV2Type(); case "bool": return Type.BOOLEAN; case "bit": @@ -530,7 +530,7 @@ public class JdbcClient { || ckType.startsWith("FixedString")) { return ScalarType.createStringType(); } else if (ckType.startsWith("DateTime")) { - return ScalarType.getDefaultDateType(Type.DATETIME); + return ScalarType.createDatetimeV2Type(0); } switch (ckType) { case "Bool": @@ -559,7 +559,7 @@ public class JdbcClient { return Type.DOUBLE; case "Date": case "Date32": - return ScalarType.getDefaultDateType(Type.DATE); + return ScalarType.createDateV2Type(); default: return Type.UNSUPPORTED; } @@ -574,7 +574,7 @@ public class JdbcClient { if (oracleType.equals("TIMESTAMPTZ") || oracleType.equals("TIMESTAMPLTZ")) { return Type.UNSUPPORTED; } - return ScalarType.getDefaultDateType(Type.DATETIME); + return ScalarType.createDatetimeV2Type(0); } switch (oracleType) { case "NUMBER": @@ -605,7 +605,7 @@ public class JdbcClient { case "FLOAT": return Type.DOUBLE; case "DATE": - return ScalarType.getDefaultDateType(Type.DATETIME); + return ScalarType.createDatetimeV2Type(0); case "VARCHAR2": case "NVARCHAR2": case "CHAR": @@ -650,7 +650,7 @@ public class JdbcClient { int scale = fieldSchema.getDecimalDigits(); return ScalarType.createDecimalV3Type(precision, scale); case "date": - return ScalarType.getDefaultDateType(Type.DATE); + return ScalarType.createDateV2Type(); case "datetime": case "datetime2": case "smalldatetime":