From 7df60a49806ccb901375329d9e47d4a6f12b22fc Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:06:09 +0800 Subject: [PATCH] [Refactor](Tvf) delete some unused code of tvf and add doc for `queries` tvf (#26460) 1. delete some unused code of tvf 2. add doc for `queries` tvf: #25051 --- .../sql-functions/table-functions/queries.md | 79 +++++++++++++++++++ docs/sidebars.json | 5 +- .../sql-functions/table-functions/queries.md | 79 +++++++++++++++++++ .../org/apache/doris/load/ExportFailMsg.java | 13 --- .../java/org/apache/doris/load/ExportJob.java | 1 - .../ExternalFileTableValuedFunction.java | 15 ++-- .../HdfsTableValuedFunction.java | 1 - .../tablefunction/MetadataGenerator.java | 25 ------ 8 files changed, 168 insertions(+), 50 deletions(-) create mode 100644 docs/en/docs/sql-manual/sql-functions/table-functions/queries.md create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/table-functions/queries.md diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/queries.md b/docs/en/docs/sql-manual/sql-functions/table-functions/queries.md new file mode 100644 index 0000000000..26527c3968 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/queries.md @@ -0,0 +1,79 @@ +--- +{ + "title": "QUERIES", + "language": "en" +} +--- + + + +## `queries` + +### Name + + + +queries + + + +### description + +Table-Value-Function, generate a temporary table named queries. This tvf is used to view the information of running queries and history queries in doris cluster. + +This function is used in FROM clauses. + +#### syntax +`queries()` + +queries() table schema： +``` +mysql> desc function queries(); ++------------------+--------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++------------------+--------+------+-------+---------+-------+ +| QueryId | TEXT | No | false | NULL | NONE | +| StartTime | BIGINT | No | false | NULL | NONE | +| EndTime | BIGINT | Yes | false | NULL | NONE | +| EventTime | BIGINT | Yes | false | NULL | NONE | +| Latency | BIGINT | No | false | NULL | NONE | +| State | TEXT | No | false | NULL | NONE | +| Database | TEXT | Yes | false | NULL | NONE | +| Sql | TEXT | No | false | NULL | NONE | +| FrontendInstance | TEXT | No | false | NULL | NONE | ++------------------+--------+------+-------+---------+-------+ +9 rows in set (0.00 sec) +``` + +### example +``` +mysql> select* from queries(); ++-----------------------------------+---------------+---------------+---------------+---------+----------+----------+------------------------+------------------+ +| QueryId | StartTime | EndTime | EventTime | Latency | State | Database | Sql | FrontendInstance | ++-----------------------------------+---------------+---------------+---------------+---------+----------+----------+------------------------+------------------+ +| e1293f2ed2a5427a-982301c462586043 | 1699255138730 | 1699255139823 | 1699255139823 | 1093 | FINISHED | demo | select* from queries() | localhost | +| 46fa3ad0e7814ebd-b1cd34940a29b1e9 | 1699255143588 | -1 | 1699255143588 | 20 | RUNNING | demo | select* from queries() | localhost | ++-----------------------------------+---------------+---------------+---------------+---------+----------+----------+------------------------+------------------+ +2 rows in set (0.04 sec) +``` + +### keywords + + queries diff --git a/docs/sidebars.json b/docs/sidebars.json index 9fd9cf9755..e436221e69 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -749,7 +749,8 @@ "sql-manual/sql-functions/table-functions/frontends", "sql-manual/sql-functions/table-functions/workload-group", "sql-manual/sql-functions/table-functions/catalogs", - "sql-manual/sql-functions/table-functions/frontends_disks" + "sql-manual/sql-functions/table-functions/frontends_disks", + "sql-manual/sql-functions/table-functions/queries" ] }, { @@ -1342,4 +1343,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/queries.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/queries.md new file mode 100644 index 0000000000..cdfd3e75fd --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/queries.md @@ -0,0 +1,79 @@ +--- +{ + "title": "QUERIES", + "language": "zh-CN" +} +--- + + + +## `queries` + +### Name + + + +queries + + + +### description + +表函数，生成queries临时表，可以查看当前doris集群中正在运行的以及历史的 query 信息。 + +该函数用于from子句中。 + +#### syntax +`queries()` + +queries()表结构： +``` +mysql> desc function queries(); ++------------------+--------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++------------------+--------+------+-------+---------+-------+ +| QueryId | TEXT | No | false | NULL | NONE | +| StartTime | BIGINT | No | false | NULL | NONE | +| EndTime | BIGINT | Yes | false | NULL | NONE | +| EventTime | BIGINT | Yes | false | NULL | NONE | +| Latency | BIGINT | No | false | NULL | NONE | +| State | TEXT | No | false | NULL | NONE | +| Database | TEXT | Yes | false | NULL | NONE | +| Sql | TEXT | No | false | NULL | NONE | +| FrontendInstance | TEXT | No | false | NULL | NONE | ++------------------+--------+------+-------+---------+-------+ +9 rows in set (0.00 sec) +``` + +### example +``` +mysql> select* from queries(); ++-----------------------------------+---------------+---------------+---------------+---------+----------+----------+------------------------+------------------+ +| QueryId | StartTime | EndTime | EventTime | Latency | State | Database | Sql | FrontendInstance | ++-----------------------------------+---------------+---------------+---------------+---------+----------+----------+------------------------+------------------+ +| e1293f2ed2a5427a-982301c462586043 | 1699255138730 | 1699255139823 | 1699255139823 | 1093 | FINISHED | demo | select* from queries() | localhost | +| 46fa3ad0e7814ebd-b1cd34940a29b1e9 | 1699255143588 | -1 | 1699255143588 | 20 | RUNNING | demo | select* from queries() | localhost | ++-----------------------------------+---------------+---------------+---------------+---------+----------+----------+------------------------+------------------+ +2 rows in set (0.04 sec) +``` + +### keywords + + queries diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportFailMsg.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportFailMsg.java index 591abe462d..48df894000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportFailMsg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportFailMsg.java @@ -40,11 +40,6 @@ public class ExportFailMsg implements Writable { @SerializedName("msg") private String msg; - public ExportFailMsg() { - this.cancelType = CancelType.UNKNOWN; - this.msg = ""; - } - public ExportFailMsg(CancelType cancelType, String msg) { this.cancelType = cancelType; this.msg = msg; @@ -54,18 +49,10 @@ public class ExportFailMsg implements Writable { return cancelType; } - public void setCancelType(CancelType cancelType) { - this.cancelType = cancelType; - } - public String getMsg() { return msg; } - public void setMsg(String msg) { - this.msg = msg; - } - @Override public String toString() { return "ExportFailMsg [cancelType=" + cancelType + ", msg=" + msg + "]"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index b0df62afb0..5d31053017 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -227,7 +227,6 @@ public class ExportJob implements Writable { public void generateOutfileStatement() throws UserException { exportTable.readLock(); try { - // generateQueryStmtOld generateQueryStmt(); } finally { exportTable.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index ffe6f7109a..3569c4f447 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -33,7 +33,6 @@ import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.BrokerUtil; @@ -311,13 +310,13 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio if (!csvSchema.isEmpty()) { return csvSchema; } - if (FeConstants.runningUnitTest) { - Object mockedUtObj = FeConstants.unitTestConstant; - if (mockedUtObj instanceof List) { - return ((List) mockedUtObj); - } - return new ArrayList<>(); - } + // if (FeConstants.runningUnitTest) { + // Object mockedUtObj = FeConstants.unitTestConstant; + // if (mockedUtObj instanceof List) { + // return ((List) mockedUtObj); + // } + // return new ArrayList<>(); + // } if (this.columns != null) { return columns; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index 051706ae47..dd85ec55a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -60,7 +60,6 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction { // 3. analyze other properties for (String key : otherProps.keySet()) { if (HdfsResource.HADOOP_FS_NAME.equalsIgnoreCase(key)) { - // because HADOOP_FS_NAME contains upper and lower case locationProperties.put(HdfsResource.HADOOP_FS_NAME, otherProps.get(key)); } else { locationProperties.put(key, otherProps.get(key)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 46443ca64a..c38744ba35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -20,15 +20,12 @@ package org.apache.doris.tablefunction; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ClientPool; -import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.FrontendsProcNode; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.HMSExternalCatalog; -import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.planner.external.iceberg.IcebergMetadataCache; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.QueryDetail; @@ -55,10 +52,7 @@ import com.google.common.base.Stopwatch; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.gson.Gson; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.iceberg.Snapshot; -import org.apache.iceberg.catalog.TableIdentifier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; @@ -67,13 +61,10 @@ import org.jetbrains.annotations.NotNull; import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; - - public class MetadataGenerator { private static final Logger LOG = LogManager.getLogger(MetadataGenerator.class); @@ -480,22 +471,6 @@ public class MetadataGenerator { result.setDataBatch(filterColumnsRows); } - private static org.apache.iceberg.Table getIcebergTable(HMSExternalCatalog catalog, String db, String tbl) - throws MetaNotFoundException { - org.apache.iceberg.hive.HiveCatalog hiveCatalog = new org.apache.iceberg.hive.HiveCatalog(); - Configuration conf = new HdfsConfiguration(); - Map properties = catalog.getCatalogProperty().getHadoopProperties(); - for (Map.Entry entry : properties.entrySet()) { - conf.set(entry.getKey(), entry.getValue()); - } - hiveCatalog.setConf(conf); - Map catalogProperties = new HashMap<>(); - catalogProperties.put(HMSProperties.HIVE_METASTORE_URIS, catalog.getHiveMetastoreUris()); - catalogProperties.put("uri", catalog.getHiveMetastoreUris()); - hiveCatalog.initialize("hive", catalogProperties); - return hiveCatalog.loadTable(TableIdentifier.of(db, tbl)); - } - private static long convertToDateTimeV2( int year, int month, int day, int hour, int minute, int second, int microsecond) { return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32