diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index f0adefef90..aea604e8f5 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -213,6 +213,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) { case TMetadataType::FRONTENDS: RETURN_IF_ERROR(_build_frontends_metadata_request(meta_scan_range, &request)); break; + case TMetadataType::FRONTENDS_DISKS: + RETURN_IF_ERROR(_build_frontends_disks_metadata_request(meta_scan_range, &request)); + break; case TMetadataType::WORKLOAD_GROUPS: RETURN_IF_ERROR(_build_workload_groups_metadata_request(meta_scan_range, &request)); break; @@ -309,6 +312,25 @@ Status VMetaScanner::_build_frontends_metadata_request(const TMetaScanRange& met return Status::OK(); } +Status VMetaScanner::_build_frontends_disks_metadata_request( + const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) { + VLOG_CRITICAL << "VMetaScanner::_build_frontends_metadata_request"; + if (!meta_scan_range.__isset.frontends_params) { + return Status::InternalError("Can not find TFrontendsMetadataParams from meta_scan_range."); + } + // create request + request->__set_cluster_name(""); + request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE); + + // create TMetadataTableRequestParams + TMetadataTableRequestParams metadata_table_params; + metadata_table_params.__set_metadata_type(TMetadataType::FRONTENDS_DISKS); + metadata_table_params.__set_frontends_metadata_params(meta_scan_range.frontends_params); + + request->__set_metada_table_params(metadata_table_params); + return Status::OK(); +} + Status VMetaScanner::_build_workload_groups_metadata_request( const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) { VLOG_CRITICAL << "VMetaScanner::_build_workload_groups_metadata_request"; diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index 4c2dd00584..5eb2296d27 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -71,6 +71,8 @@ private: TFetchSchemaTableDataRequest* request); Status _build_frontends_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); + Status _build_frontends_disks_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request); Status _build_workload_groups_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); Status _build_catalogs_metadata_request(const TMetaScanRange& meta_scan_range, diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md b/docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md new file mode 100644 index 0000000000..c92a076f5f --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md @@ -0,0 +1,93 @@ +--- +{ + "title": "frontends_disks", + "language": "en" +} +--- + + + +## `frontends` + +### Name + + + +frontends + + + +### description + +Table-Value-Function, generate a temporary table named `frontends_disks`. This tvf is used to view the information of FE nodes 's disks in the doris cluster. + +This function is used in `FROM` clauses. + +#### syntax + +`frontends_disks()` + +The table schema of `frontends_disks()` tvf： +``` +mysql> desc function frontends_disks(); ++-------------+------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+------+------+-------+---------+-------+ +| Name | TEXT | No | false | NULL | NONE | +| Host | TEXT | No | false | NULL | NONE | +| EditLogPort | TEXT | No | false | NULL | NONE | +| DirType | TEXT | No | false | NULL | NONE | +| Dir | TEXT | No | false | NULL | NONE | +| Filesystem | TEXT | No | false | NULL | NONE | +| Capacity | TEXT | No | false | NULL | NONE | +| Used | TEXT | No | false | NULL | NONE | +| Available | TEXT | No | false | NULL | NONE | +| UseRate | TEXT | No | false | NULL | NONE | +| MountOn | TEXT | No | false | NULL | NONE | ++-------------+------+------+-------+---------+-------+ +11 rows in set (0.14 sec) +``` + +The information displayed by the `frontends_disks` tvf is basically consistent with the information displayed by the `show frontends disks` statement. However, the types of each field in the `frontends_disks` tvf are more specific, and you can use the `frontends_disks` tvf to perform operations such as filtering and joining. + +The information displayed by the `frontends_disks` tvf is authenticated, which is consistent with the behavior of `show frontends disks`, user must have ADMIN/OPERATOR privelege. + +### example +``` +mysql> select * from frontends_disk()\G +*************************** 1. row *************************** + Name: fe_fe1d5bd9_d1e5_4ccc_9b03_ca79b95c9941 + Host: 172.XX.XX.1 +EditLogPort: 9010 + DirType: log + Dir: /data/doris/fe-github/log + Filesystem: /dev/sdc5 + Capacity: 366G + Used: 119G + Available: 228G + UseRate: 35% + MountOn: /data +...... +12 row in set (0.03 sec) +``` + +### keywords + + frontends_disks \ No newline at end of file diff --git a/docs/sidebars.json b/docs/sidebars.json index c43e6d1e86..160d665490 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -726,7 +726,8 @@ "sql-manual/sql-functions/table-functions/backends", "sql-manual/sql-functions/table-functions/frontends", "sql-manual/sql-functions/table-functions/workload-group", - "sql-manual/sql-functions/table-functions/catalogs" + "sql-manual/sql-functions/table-functions/catalogs", + "sql-manual/sql-functions/table-functions/frontends_disks" ] }, { diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md new file mode 100644 index 0000000000..fcba673a0a --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md @@ -0,0 +1,92 @@ +--- +{ + "title": "frontends_disks", + "language": "zh-CN" +} +--- + + + +## `frontends_disks` + +### Name + + + +frontends_disks + + + +### description + +表函数，生成frontends_disks临时表，可以查看当前doris集群中的 FE 节点的磁盘信息。 + +该函数用于from子句中。 + +#### syntax +`frontends_disks()` + +frontends_disks()表结构： +``` +mysql> desc function frontends_disks(); ++-------------+------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+------+------+-------+---------+-------+ +| Name | TEXT | No | false | NULL | NONE | +| Host | TEXT | No | false | NULL | NONE | +| EditLogPort | TEXT | No | false | NULL | NONE | +| DirType | TEXT | No | false | NULL | NONE | +| Dir | TEXT | No | false | NULL | NONE | +| Filesystem | TEXT | No | false | NULL | NONE | +| Capacity | TEXT | No | false | NULL | NONE | +| Used | TEXT | No | false | NULL | NONE | +| Available | TEXT | No | false | NULL | NONE | +| UseRate | TEXT | No | false | NULL | NONE | +| MountOn | TEXT | No | false | NULL | NONE | ++-------------+------+------+-------+---------+-------+ +11 rows in set (0.14 sec) +``` + +`frontends_disks()` tvf展示出来的信息基本与 `show frontends disks` 语句展示出的信息一致,但是 `frontends_disks()` tvf的各个字段类型更加明确，且可以利用tvf生成的表去做过滤、join等操作。 + +对 `frontends_disks()` tvf信息展示进行了鉴权，与 `show frontends disks` 行为保持一致，要求用户具有 ADMIN/OPERATOR 权限。 + +### example +``` +mysql> select * from frontends_disk()\G +*************************** 1. row *************************** + Name: fe_fe1d5bd9_d1e5_4ccc_9b03_ca79b95c9941 + Host: 172.XX.XX.1 +EditLogPort: 9010 + DirType: log + Dir: /data/doris/fe-github/log + Filesystem: /dev/sdc5 + Capacity: 366G + Used: 119G + Available: 228G + UseRate: 35% + MountOn: /data +...... +12 row in set (0.03 sec) +``` + +### keywords + + frontends_disks \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java new file mode 100644 index 0000000000..f7f9f9e7a7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.thrift.TFrontendsMetadataParams; +import org.apache.doris.thrift.TMetaScanRange; +import org.apache.doris.thrift.TMetadataType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; + +/** + * The Implement of table valued function + * frontends_disks(). + */ +public class FrontendsDisksTableValuedFunction extends MetadataTableValuedFunction { + public static final String NAME = "frontends_disks"; + + private static final ImmutableList SCHEMA = ImmutableList.of( + new Column("Name", ScalarType.createStringType()), + new Column("Host", ScalarType.createStringType()), + new Column("EditLogPort", ScalarType.createStringType()), + new Column("DirType", ScalarType.createStringType()), + new Column("Dir", ScalarType.createStringType()), + new Column("Filesystem", ScalarType.createStringType()), + new Column("Capacity", ScalarType.createStringType()), + new Column("Used", ScalarType.createStringType()), + new Column("Available", ScalarType.createStringType()), + new Column("UseRate", ScalarType.createStringType()), + new Column("MountOn", ScalarType.createStringType())); + + private static final ImmutableMap COLUMN_TO_INDEX; + + static { + ImmutableMap.Builder builder = new ImmutableMap.Builder(); + for (int i = 0; i < SCHEMA.size(); i++) { + builder.put(SCHEMA.get(i).getName().toLowerCase(), i); + } + COLUMN_TO_INDEX = builder.build(); + } + + public static Integer getColumnIndexFromColumnName(String columnName) { + return COLUMN_TO_INDEX.get(columnName.toLowerCase()); + } + + public FrontendsDisksTableValuedFunction(Map params) throws AnalysisException { + if (params.size() != 0) { + throw new AnalysisException("frontends_disks table-valued-function does not support any params"); + } + } + + @Override + public TMetadataType getMetadataType() { + return TMetadataType.FRONTENDS_DISKS; + } + + @Override + public TMetaScanRange getMetaScanRange() { + TMetaScanRange metaScanRange = new TMetaScanRange(); + metaScanRange.setMetadataType(TMetadataType.FRONTENDS_DISKS); + TFrontendsMetadataParams frontendsMetadataParams = new TFrontendsMetadataParams(); + frontendsMetadataParams.setClusterName(""); + metaScanRange.setFrontendsParams(frontendsMetadataParams); + return metaScanRange; + } + + @Override + public String getTableName() { + return "FrontendsDisksTableValuedFunction"; + } + + @Override + public List getTableColumns() throws AnalysisException { + return SCHEMA; + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 00bcf168cd..dfab4bb8fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -83,6 +83,9 @@ public class MetadataGenerator { case FRONTENDS: result = frontendsMetadataResult(params); break; + case FRONTENDS_DISKS: + result = frontendsDisksMetadataResult(params); + break; case WORKLOAD_GROUPS: result = workloadGroupsMetadataResult(params); break; @@ -266,6 +269,29 @@ public class MetadataGenerator { return result; } + private static TFetchSchemaTableDataResult frontendsDisksMetadataResult(TMetadataTableRequestParams params) { + if (!params.isSetFrontendsMetadataParams()) { + return errorResult("frontends metadata param is not set."); + } + + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + + List dataBatch = Lists.newArrayList(); + List> infos = Lists.newArrayList(); + FrontendsProcNode.getFrontendsDiskInfo(Env.getCurrentEnv(), infos); + for (List info : infos) { + TRow trow = new TRow(); + for (String item : info) { + trow.addToColumnValue(new TCell().setStringVal(item)); + } + dataBatch.add(trow); + } + + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + private static TFetchSchemaTableDataResult catalogsMetadataResult(TMetadataTableRequestParams params) { TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); List info = Env.getCurrentEnv().getCatalogMgr().listCatalogs(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java index c5740c739b..c09687c9b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java @@ -33,6 +33,8 @@ public abstract class MetadataTableValuedFunction extends TableValuedFunctionIf return BackendsTableValuedFunction.getColumnIndexFromColumnName(columnName); case FRONTENDS: return FrontendsTableValuedFunction.getColumnIndexFromColumnName(columnName); + case FRONTENDS_DISKS: + return FrontendsDisksTableValuedFunction.getColumnIndexFromColumnName(columnName); case ICEBERG: return IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName); case WORKLOAD_GROUPS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 6deb85cf51..5d0b3b00bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -57,6 +57,8 @@ public abstract class TableValuedFunctionIf { return new BackendsTableValuedFunction(params); case FrontendsTableValuedFunction.NAME: return new FrontendsTableValuedFunction(params); + case FrontendsDisksTableValuedFunction.NAME: + return new FrontendsDisksTableValuedFunction(params); case WorkloadGroupsTableValuedFunction.NAME: return new WorkloadGroupsTableValuedFunction(params); case CatalogsTableValuedFunction.NAME: diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 522b80c0d8..baca98b228 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -690,7 +690,8 @@ enum TMetadataType { BACKENDS, WORKLOAD_GROUPS, FRONTENDS, - CATALOGS + CATALOGS, + FRONTENDS_DISKS, } enum TIcebergQueryType { diff --git a/regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy b/regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy new file mode 100644 index 0000000000..93bd2490e8 --- /dev/null +++ b/regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This suit test the `frontends_disks` tvf +suite("test_frontends_disks_tvf") { + List> table = sql """ select * from `frontends_disks`(); """ + assertTrue(table.size() > 0) + assertTrue(table[0].size == 11) + + // filter columns + table = sql """ select Name from `frontends_disks`();""" + assertTrue(table.size() > 0) + assertTrue(table[0].size == 1) + + // case insensitive + table = sql """ select name, host, editlogport, dirtype, dir from frontends_disks() order by dirtype;""" + assertTrue(table.size() > 0) + assertTrue(table[0].size == 5) + assertEquals("audit-log", table[0][3]) + + // test aliase columns + table = sql """ select name as n, host as h, dirtype as a, editlogport as e from frontends_disks() order by dirtype; """ + assertTrue(table.size() > 0) + assertTrue(table[0].size == 4) + assertEquals("audit-log", table[0][2]) + + // test changing position of columns + def res = sql """ select count(*) from frontends_disks() where dirtype = 'audit-log'; """ + assertTrue(res[0][0] > 0) + + sql """ select Name, Host, EditLogPort, + DirType, Dir, Filesystem, Capacity, Used + Available, UseRate, MountOn from frontends_disks(); + """ +}