database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhancement](tvf) Add frontends_disks table-valued-function (#22568)

Browse Source 
---------

Co-authored-by: yuxianbing <yuxianbing@yy.com>
Co-authored-by: yuxianbing <iloveqaz123>
This commit is contained in:
herry2038
2023-08-10 10:40:24 +08:00
committed by GitHub
parent b90a7748a6
commit eafdab0cfd
11 changed files with 390 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
be/src/vec/exec/scan/vmeta_scanner.cpp
View File

@ -213,6 +213,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) {
case TMetadataType::FRONTENDS:
RETURN_IF_ERROR(_build_frontends_metadata_request(meta_scan_range, &request));
break;
case TMetadataType::FRONTENDS_DISKS:
RETURN_IF_ERROR(_build_frontends_disks_metadata_request(meta_scan_range, &request));
break;
case TMetadataType::WORKLOAD_GROUPS:
RETURN_IF_ERROR(_build_workload_groups_metadata_request(meta_scan_range, &request));
break;
@ -309,6 +312,25 @@ Status VMetaScanner::_build_frontends_metadata_request(const TMetaScanRange& met
return Status::OK();
}
Status VMetaScanner::_build_frontends_disks_metadata_request(
const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) {
VLOG_CRITICAL << "VMetaScanner::_build_frontends_metadata_request";
if (!meta_scan_range.__isset.frontends_params) {
return Status::InternalError("Can not find TFrontendsMetadataParams from meta_scan_range.");
}
// create request
request->__set_cluster_name("");
request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE);
// create TMetadataTableRequestParams
TMetadataTableRequestParams metadata_table_params;
metadata_table_params.__set_metadata_type(TMetadataType::FRONTENDS_DISKS);
metadata_table_params.__set_frontends_metadata_params(meta_scan_range.frontends_params);
request->__set_metada_table_params(metadata_table_params);
return Status::OK();
}
Status VMetaScanner::_build_workload_groups_metadata_request(
const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) {
VLOG_CRITICAL << "VMetaScanner::_build_workload_groups_metadata_request";

2
be/src/vec/exec/scan/vmeta_scanner.h
View File

@ -71,6 +71,8 @@ private:
TFetchSchemaTableDataRequest* request);
Status _build_frontends_metadata_request(const TMetaScanRange& meta_scan_range,
TFetchSchemaTableDataRequest* request);
Status _build_frontends_disks_metadata_request(const TMetaScanRange& meta_scan_range,
TFetchSchemaTableDataRequest* request);
Status _build_workload_groups_metadata_request(const TMetaScanRange& meta_scan_range,
TFetchSchemaTableDataRequest* request);
Status _build_catalogs_metadata_request(const TMetaScanRange& meta_scan_range,

93
docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md Normal file
View File

@ -0,0 +1,93 @@
---
{
"title": "frontends_disks",
"language": "en"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## `frontends`
### Name
<version since="dev">
frontends
</version>
### description
Table-Value-Function, generate a temporary table named `frontends_disks`. This tvf is used to view the information of FE nodes 's disks in the doris cluster.
This function is used in `FROM` clauses.
#### syntax
`frontends_disks()`
The table schema of `frontends_disks()` tvf：
```
mysql> desc function frontends_disks();
+-------------+------+------+-------+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+-------------+------+------+-------+---------+-------+
| Name | TEXT | No | false | NULL | NONE |
| Host | TEXT | No | false | NULL | NONE |
| EditLogPort | TEXT | No | false | NULL | NONE |
| DirType | TEXT | No | false | NULL | NONE |
| Dir | TEXT | No | false | NULL | NONE |
| Filesystem | TEXT | No | false | NULL | NONE |
| Capacity | TEXT | No | false | NULL | NONE |
| Used | TEXT | No | false | NULL | NONE |
| Available | TEXT | No | false | NULL | NONE |
| UseRate | TEXT | No | false | NULL | NONE |
| MountOn | TEXT | No | false | NULL | NONE |
+-------------+------+------+-------+---------+-------+
11 rows in set (0.14 sec)
```
The information displayed by the `frontends_disks` tvf is basically consistent with the information displayed by the `show frontends disks` statement. However, the types of each field in the `frontends_disks` tvf are more specific, and you can use the `frontends_disks` tvf to perform operations such as filtering and joining.
The information displayed by the `frontends_disks` tvf is authenticated, which is consistent with the behavior of `show frontends disks`, user must have ADMIN/OPERATOR privelege.
### example
```
mysql> select * from frontends_disk()\G
*************************** 1. row ***************************
Name: fe_fe1d5bd9_d1e5_4ccc_9b03_ca79b95c9941
Host: 172.XX.XX.1
EditLogPort: 9010
DirType: log
Dir: /data/doris/fe-github/log
Filesystem: /dev/sdc5
Capacity: 366G
Used: 119G
Available: 228G
UseRate: 35%
MountOn: /data
......
12 row in set (0.03 sec)
```
### keywords
frontends_disks

3
docs/sidebars.json
View File

@ -726,7 +726,8 @@
"sql-manual/sql-functions/table-functions/backends",
"sql-manual/sql-functions/table-functions/frontends",
"sql-manual/sql-functions/table-functions/workload-group",
"sql-manual/sql-functions/table-functions/catalogs"
"sql-manual/sql-functions/table-functions/catalogs",
"sql-manual/sql-functions/table-functions/frontends_disks"
]
},
{

92
docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md Normal file
View File

@ -0,0 +1,92 @@
---
{
"title": "frontends_disks",
"language": "zh-CN"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## `frontends_disks`
### Name
<version since="dev">
frontends_disks
</version>
### description
表函数，生成frontends_disks临时表，可以查看当前doris集群中的 FE 节点的磁盘信息。
该函数用于from子句中。
#### syntax
`frontends_disks()`
frontends_disks()表结构：
```
mysql> desc function frontends_disks();
+-------------+------+------+-------+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+-------------+------+------+-------+---------+-------+
| Name | TEXT | No | false | NULL | NONE |
| Host | TEXT | No | false | NULL | NONE |
| EditLogPort | TEXT | No | false | NULL | NONE |
| DirType | TEXT | No | false | NULL | NONE |
| Dir | TEXT | No | false | NULL | NONE |
| Filesystem | TEXT | No | false | NULL | NONE |
| Capacity | TEXT | No | false | NULL | NONE |
| Used | TEXT | No | false | NULL | NONE |
| Available | TEXT | No | false | NULL | NONE |
| UseRate | TEXT | No | false | NULL | NONE |
| MountOn | TEXT | No | false | NULL | NONE |
+-------------+------+------+-------+---------+-------+
11 rows in set (0.14 sec)
```
`frontends_disks()` tvf展示出来的信息基本与 `show frontends disks` 语句展示出的信息一致,但是 `frontends_disks()` tvf的各个字段类型更加明确，且可以利用tvf生成的表去做过滤、join等操作。
对 `frontends_disks()` tvf信息展示进行了鉴权，与 `show frontends disks` 行为保持一致，要求用户具有 ADMIN/OPERATOR 权限。
### example
```
mysql> select * from frontends_disk()\G
*************************** 1. row ***************************
Name: fe_fe1d5bd9_d1e5_4ccc_9b03_ca79b95c9941
Host: 172.XX.XX.1
EditLogPort: 9010
DirType: log
Dir: /data/doris/fe-github/log
Filesystem: /dev/sdc5
Capacity: 366G
Used: 119G
Available: 228G
UseRate: 35%
MountOn: /data
......
12 row in set (0.03 sec)
```
### keywords
frontends_disks

98
fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java Normal file
View File

@ -0,0 +1,98 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.tablefunction;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.thrift.TFrontendsMetadataParams;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
/**
* The Implement of table valued function
* frontends_disks().
*/
public class FrontendsDisksTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "frontends_disks";
private static final ImmutableList<Column> SCHEMA = ImmutableList.of(
new Column("Name", ScalarType.createStringType()),
new Column("Host", ScalarType.createStringType()),
new Column("EditLogPort", ScalarType.createStringType()),
new Column("DirType", ScalarType.createStringType()),
new Column("Dir", ScalarType.createStringType()),
new Column("Filesystem", ScalarType.createStringType()),
new Column("Capacity", ScalarType.createStringType()),
new Column("Used", ScalarType.createStringType()),
new Column("Available", ScalarType.createStringType()),
new Column("UseRate", ScalarType.createStringType()),
new Column("MountOn", ScalarType.createStringType()));
private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
static {
ImmutableMap.Builder<String, Integer> builder = new ImmutableMap.Builder();
for (int i = 0; i < SCHEMA.size(); i++) {
builder.put(SCHEMA.get(i).getName().toLowerCase(), i);
}
COLUMN_TO_INDEX = builder.build();
}
public static Integer getColumnIndexFromColumnName(String columnName) {
return COLUMN_TO_INDEX.get(columnName.toLowerCase());
}
public FrontendsDisksTableValuedFunction(Map<String, String> params) throws AnalysisException {
if (params.size() != 0) {
throw new AnalysisException("frontends_disks table-valued-function does not support any params");
}
}
@Override
public TMetadataType getMetadataType() {
return TMetadataType.FRONTENDS_DISKS;
}
@Override
public TMetaScanRange getMetaScanRange() {
TMetaScanRange metaScanRange = new TMetaScanRange();
metaScanRange.setMetadataType(TMetadataType.FRONTENDS_DISKS);
TFrontendsMetadataParams frontendsMetadataParams = new TFrontendsMetadataParams();
frontendsMetadataParams.setClusterName("");
metaScanRange.setFrontendsParams(frontendsMetadataParams);
return metaScanRange;
}
@Override
public String getTableName() {
return "FrontendsDisksTableValuedFunction";
}
@Override
public List<Column> getTableColumns() throws AnalysisException {
return SCHEMA;
}
}

26
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
View File

@ -83,6 +83,9 @@ public class MetadataGenerator {
case FRONTENDS:
result = frontendsMetadataResult(params);
break;
case FRONTENDS_DISKS:
result = frontendsDisksMetadataResult(params);
break;
case WORKLOAD_GROUPS:
result = workloadGroupsMetadataResult(params);
break;
@ -266,6 +269,29 @@ public class MetadataGenerator {
return result;
}
private static TFetchSchemaTableDataResult frontendsDisksMetadataResult(TMetadataTableRequestParams params) {
if (!params.isSetFrontendsMetadataParams()) {
return errorResult("frontends metadata param is not set.");
}
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
List<TRow> dataBatch = Lists.newArrayList();
List<List<String>> infos = Lists.newArrayList();
FrontendsProcNode.getFrontendsDiskInfo(Env.getCurrentEnv(), infos);
for (List<String> info : infos) {
TRow trow = new TRow();
for (String item : info) {
trow.addToColumnValue(new TCell().setStringVal(item));
}
dataBatch.add(trow);
}
result.setDataBatch(dataBatch);
result.setStatus(new TStatus(TStatusCode.OK));
return result;
}
private static TFetchSchemaTableDataResult catalogsMetadataResult(TMetadataTableRequestParams params) {
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
List<CatalogIf> info = Env.getCurrentEnv().getCatalogMgr().listCatalogs();

2
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
View File

@ -33,6 +33,8 @@ public abstract class MetadataTableValuedFunction extends TableValuedFunctionIf
return BackendsTableValuedFunction.getColumnIndexFromColumnName(columnName);
case FRONTENDS:
return FrontendsTableValuedFunction.getColumnIndexFromColumnName(columnName);
case FRONTENDS_DISKS:
return FrontendsDisksTableValuedFunction.getColumnIndexFromColumnName(columnName);
case ICEBERG:
return IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName);
case WORKLOAD_GROUPS:

2
fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
View File

@ -57,6 +57,8 @@ public abstract class TableValuedFunctionIf {
return new BackendsTableValuedFunction(params);
case FrontendsTableValuedFunction.NAME:
return new FrontendsTableValuedFunction(params);
case FrontendsDisksTableValuedFunction.NAME:
return new FrontendsDisksTableValuedFunction(params);
case WorkloadGroupsTableValuedFunction.NAME:
return new WorkloadGroupsTableValuedFunction(params);
case CatalogsTableValuedFunction.NAME:

3
gensrc/thrift/Types.thrift
View File

@ -690,7 +690,8 @@ enum TMetadataType {
BACKENDS,
WORKLOAD_GROUPS,
FRONTENDS,
CATALOGS
CATALOGS,
FRONTENDS_DISKS,
}
enum TIcebergQueryType {

49
regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy Normal file
View File

@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This suit test the `frontends_disks` tvf
suite("test_frontends_disks_tvf") {
List<List<Object>> table = sql """ select * from `frontends_disks`(); """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 11)
// filter columns
table = sql """ select Name from `frontends_disks`();"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 1)
// case insensitive
table = sql """ select name, host, editlogport, dirtype, dir from frontends_disks() order by dirtype;"""
assertTrue(table.size() > 0)
assertTrue(table[0].size == 5)
assertEquals("audit-log", table[0][3])
// test aliase columns
table = sql """ select name as n, host as h, dirtype as a, editlogport as e from frontends_disks() order by dirtype; """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 4)
assertEquals("audit-log", table[0][2])
// test changing position of columns
def res = sql """ select count(*) from frontends_disks() where dirtype = 'audit-log'; """
assertTrue(res[0][0] > 0)
sql """ select Name, Host, EditLogPort,
DirType, Dir, Filesystem, Capacity, Used
Available, UseRate, MountOn from frontends_disks();
"""
}