database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](resource-group) Support resource group tvf (#18519)

Browse Source 
related: #18098
This commit is contained in:
yongjinhou
2023-04-13 20:11:20 +08:00
committed by GitHub
parent 33eec9096f
commit 281ceee3cc
10 changed files with 299 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
be/src/vec/exec/scan/vmeta_scanner.cpp
View File

@ -176,6 +176,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) {
case TMetadataType::BACKENDS:
RETURN_IF_ERROR(_build_backends_metadata_request(meta_scan_range, &request));
break;
case TMetadataType::RESOURCE_GROUPS:
RETURN_IF_ERROR(_build_resource_groups_metadata_request(meta_scan_range, &request));
break;
default:
_meta_eos = true;
return Status::OK();
@ -240,6 +243,22 @@ Status VMetaScanner::_build_backends_metadata_request(const TMetaScanRange& meta
return Status::OK();
}
Status VMetaScanner::_build_resource_groups_metadata_request(
const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) {
VLOG_CRITICAL << "VMetaScanner::_build_resource_groups_metadata_request";
// create request
request->__set_cluster_name("");
request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE);
// create TMetadataTableRequestParams
TMetadataTableRequestParams metadata_table_params;
metadata_table_params.__set_metadata_type(TMetadataType::RESOURCE_GROUPS);
request->__set_metada_table_params(metadata_table_params);
return Status::OK();
}
Status VMetaScanner::close(RuntimeState* state) {
VLOG_CRITICAL << "VMetaScanner::close";
RETURN_IF_ERROR(VScanner::close(state));

2
be/src/vec/exec/scan/vmeta_scanner.h
View File

@ -42,6 +42,8 @@ private:
TFetchSchemaTableDataRequest* request);
Status _build_backends_metadata_request(const TMetaScanRange& meta_scan_range,
TFetchSchemaTableDataRequest* request);
Status _build_resource_groups_metadata_request(const TMetaScanRange& meta_scan_range,
TFetchSchemaTableDataRequest* request);
bool _meta_eos;
TupleId _tuple_id;

75
docs/en/docs/sql-manual/sql-functions/table-functions/resource-group.md Normal file
View File

@ -0,0 +1,75 @@
---
{
"title": "resource_groups",
"language": "en"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## `resource_groups`
### Name
<version since="dev">
resource_groups
</version>
### description
Table-Value-Function, generate a temporary table named `resource_groups`. This tvf is used to view informations about current resource groups.
This function is used in `FROM` clauses.
grammar:
```
resource_groups();
```
The table schema of `resource_groups()` tvf:
```
mysql> desc function resource_groups();
+-------+-------------+------+-------+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+-------+-------------+------+-------+---------+-------+
| Id | BIGINT | No | false | NULL | NONE |
| Name | VARCHAR(64) | No | false | NULL | NONE |
| Item | VARCHAR(64) | No | false | NULL | NONE |
| Value | INT | No | false | NULL | NONE |
+-------+-------------+------+-------+---------+-------+
```
### example
```
mysql> select * from resource_groups()\G
+-------+------------+-----------+-------+
| Id | Name | Item | Value |
+-------+------------+-----------+-------+
| 10076 | group_name | cpu_share | 1 |
| 10077 | group_test | cpu_share | 10 |
+-------+------------+-----------+-------+
```
### keywords
resource_groups

3
docs/sidebars.json
View File

@ -684,7 +684,8 @@
"sql-manual/sql-functions/table-functions/s3",
"sql-manual/sql-functions/table-functions/hdfs",
"sql-manual/sql-functions/table-functions/iceberg_meta",
"sql-manual/sql-functions/table-functions/backends"
"sql-manual/sql-functions/table-functions/backends",
"sql-manual/sql-functions/table-functions/resource-group"
]
},
{

75
docs/zh-CN/docs/sql-manual/sql-functions/table-functions/resource-group.md Normal file
View File

@ -0,0 +1,75 @@
---
{
"title": "resource_groups",
"language": "zh-CN"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## `resource_groups`
### Name
<version since="dev">
resource_groups
</version>
### description
表函数，生成 resource_groups 临时表，可以查看当前资源组信息。
该函数用于from子句中。
语法：
```
resource_groups();
```
resource_groups()表结构：
```
mysql> desc function resource_groups();
+-------+-------------+------+-------+---------+-------+
| Field | Type | Null | Key | Default | Extra |
+-------+-------------+------+-------+---------+-------+
| Id | BIGINT | No | false | NULL | NONE |
| Name | VARCHAR(64) | No | false | NULL | NONE |
| Item | VARCHAR(64) | No | false | NULL | NONE |
| Value | INT | No | false | NULL | NONE |
+-------+-------------+------+-------+---------+-------+
```
### example
```
mysql> select * from resource_groups()\G
+-------+------------+-----------+-------+
| Id | Name | Item | Value |
+-------+------------+-----------+-------+
| 10076 | group_name | cpu_share | 1 |
| 10077 | group_test | cpu_share | 10 |
+-------+------------+-----------+-------+
```
### keywords
resource_groups

27
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
View File

@ -68,6 +68,8 @@ public class MetadataGenerator {
return icebergMetadataResult(request.getMetadaTableParams());
case BACKENDS:
return backendsMetadataResult(request.getMetadaTableParams());
case RESOURCE_GROUPS:
return resourceGroupsMetadataResult(request.getMetadaTableParams());
default:
break;
}
@ -84,7 +86,7 @@ public class MetadataGenerator {
private static TFetchSchemaTableDataResult icebergMetadataResult(TMetadataTableRequestParams params) {
if (!params.isSetIcebergMetadataParams()) {
return errorResult("Iceberg metadata params is not set. ");
return errorResult("Iceberg metadata params is not set.");
}
TIcebergMetadataParams icebergMetadataParams = params.getIcebergMetadataParams();
HMSExternalCatalog catalog = (HMSExternalCatalog) Env.getCurrentEnv().getCatalogMgr()
@ -130,7 +132,7 @@ public class MetadataGenerator {
private static TFetchSchemaTableDataResult backendsMetadataResult(TMetadataTableRequestParams params) {
if (!params.isSetBackendsMetadataParams()) {
return errorResult("backends metadata param is not set. ");
return errorResult("backends metadata param is not set.");
}
TBackendsMetadataParams backendsParam = params.getBackendsMetadataParams();
final SystemInfoService clusterInfoService = Env.getCurrentSystemInfo();
@ -242,6 +244,27 @@ public class MetadataGenerator {
return result;
}
private static TFetchSchemaTableDataResult resourceGroupsMetadataResult(TMetadataTableRequestParams params) {
List<List<String>> resourceGroupsInfo = Env.getCurrentEnv().getResourceGroupMgr()
.getResourcesInfo();
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
List<TRow> dataBatch = Lists.newArrayList();
for (List<String> rGroupsInfo : resourceGroupsInfo) {
TRow trow = new TRow();
Long id = Long.valueOf(rGroupsInfo.get(0));
int value = Integer.valueOf(rGroupsInfo.get(3));
trow.addToColumnValue(new TCell().setLongVal(id));
trow.addToColumnValue(new TCell().setStringVal(rGroupsInfo.get(1)));
trow.addToColumnValue(new TCell().setStringVal(rGroupsInfo.get(2)));
trow.addToColumnValue(new TCell().setIntVal(value));
dataBatch.add(trow);
}
result.setDataBatch(dataBatch);
result.setStatus(new TStatus(TStatusCode.OK));
return result;
}
private static org.apache.iceberg.Table getIcebergTable(HMSExternalCatalog catalog, String db, String tbl)
throws MetaNotFoundException {
org.apache.iceberg.hive.HiveCatalog hiveCatalog = new org.apache.iceberg.hive.HiveCatalog();

71
fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java Normal file
View File

@ -0,0 +1,71 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.tablefunction;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
import com.google.common.collect.Lists;
import java.util.List;
import java.util.Map;
/**
* The Implement of table valued function
* resource_groups().
*/
public class ResourceGroupsTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "resource_groups";
public ResourceGroupsTableValuedFunction(Map<String, String> params) throws AnalysisException {
if (params.size() != 0) {
throw new AnalysisException("resource groups table-valued-function does not support any params");
}
}
@Override
public TMetadataType getMetadataType() {
return TMetadataType.RESOURCE_GROUPS;
}
@Override
public TMetaScanRange getMetaScanRange() {
TMetaScanRange metaScanRange = new TMetaScanRange();
metaScanRange.setMetadataType(TMetadataType.RESOURCE_GROUPS);
return metaScanRange;
}
@Override
public String getTableName() {
return "ResourceGroupsTableValuedFunction";
}
@Override
public List<Column> getTableColumns() throws AnalysisException {
List<Column> resColumns = Lists.newArrayList();
resColumns.add(new Column("Id", ScalarType.createType(PrimitiveType.BIGINT)));
resColumns.add(new Column("Name", ScalarType.createVarchar(64)));
resColumns.add(new Column("Item", ScalarType.createVarchar(64)));
resColumns.add(new Column("Value", ScalarType.createType(PrimitiveType.INT)));
return resColumns;
}
}

2
fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
View File

@ -53,6 +53,8 @@ public abstract class TableValuedFunctionIf {
return new IcebergTableValuedFunction(params);
case BackendsTableValuedFunction.NAME:
return new BackendsTableValuedFunction(params);
case ResourceGroupsTableValuedFunction.NAME:
return new ResourceGroupsTableValuedFunction(params);
default:
throw new AnalysisException("Could not find table function " + funcName);
}

3
gensrc/thrift/Types.thrift
View File

@ -662,7 +662,8 @@ enum TSortType {
enum TMetadataType {
ICEBERG,
BACKENDS
BACKENDS,
RESOURCE_GROUPS
}
enum TIcebergQueryType {

26
regression-test/suites/correctness_p0/table_valued_function/test_resource_group_tvf.groovy Normal file
View File

@ -0,0 +1,26 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This suit test the `resource_groups` tvf
// TO DO
suite("test_resource_groups_tvf") {
def name1 = "test";
sql "create resource group if not exists ${name1} properties('cpu_share'='10');"
List<List<Object>> table = sql """ select * from resource_groups(); """
assertTrue(table.size() > 0)
assertTrue(table[0].size == 4) // column should be 4
}