From f9dfcb923d392fc7f1c74f15c698422322596655 Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Wed, 31 May 2023 15:23:24 +0800 Subject: [PATCH] [Enhancement] Change Create Resource Group Grammar (#20249) --- be/src/runtime/fragment_mgr.cpp | 4 +- be/src/runtime/task_group/task_group.cpp | 6 +- be/src/runtime/task_group/task_group.h | 6 +- be/src/vec/exec/scan/vmeta_scanner.cpp | 10 +- be/src/vec/exec/scan/vmeta_scanner.h | 2 +- docs/en/docs/admin-manual/resource-group.md | 71 ---- docs/en/docs/admin-manual/workload-group.md | 71 ++++ docs/en/docs/lakehouse/multi-catalog/jdbc.md | 1 - .../{resource-group.md => workload-group.md} | 18 +- ...OURCE-GROUP.md => ALTER-WORKLOAD-GROUP.md} | 16 +- ...URCE-GROUP.md => CREATE-WORKLOAD-GROUP.md} | 22 +- ...SOURCE-GROUP.md => DROP-WORKLOAD-GROUP.md} | 16 +- ...URCE-GROUPS.md => SHOW-WORKLOAD-GROUPS.md} | 18 +- docs/sidebars.json | 12 +- .../{resource-group.md => workload-group.md} | 10 +- .../{resource-group.md => workload-group.md} | 18 +- ...OURCE-GROUP.md => ALTER-WORKLOAD-GROUP.md} | 12 +- ...URCE-GROUP.md => CREATE-WORKLOAD-GROUP.md} | 12 +- ...SOURCE-GROUP.md => DROP-WORKLOAD-GROUP.md} | 12 +- ...URCE-GROUPS.md => SHOW-WORKLOAD-GROUPS.md} | 14 +- .../java/org/apache/doris/common/Config.java | 4 +- fe/fe-core/src/main/cup/sql_parser.cup | 19 +- ...pStmt.java => AlterWorkloadGroupStmt.java} | 14 +- .../org/apache/doris/analysis/Analyzer.java | 12 +- ...Stmt.java => CreateWorkloadGroupStmt.java} | 16 +- ...upStmt.java => DropWorkloadGroupStmt.java} | 16 +- ...sStmt.java => ShowWorkloadGroupsStmt.java} | 8 +- .../java/org/apache/doris/catalog/Env.java | 22 +- .../ColocateTableCheckerAndBalancer.java | 8 +- .../apache/doris/clone/TabletSchedCtx.java | 2 +- .../org/apache/doris/common/FeNameFormat.java | 4 +- .../apache/doris/journal/JournalEntity.java | 14 +- .../org/apache/doris/mysql/MysqlProto.java | 4 +- .../mysql/privilege/CommonUserProperties.java | 4 +- ...ava => DropWorkloadGroupOperationLog.java} | 10 +- .../org/apache/doris/persist/EditLog.java | 34 +- .../apache/doris/persist/OperationType.java | 8 +- .../doris/persist/meta/MetaPersistMethod.java | 6 +- .../persist/meta/PersistMetaModules.java | 2 +- .../java/org/apache/doris/qe/Coordinator.java | 10 +- .../java/org/apache/doris/qe/DdlExecutor.java | 18 +- .../org/apache/doris/qe/SessionVariable.java | 12 + .../org/apache/doris/qe/ShowExecutor.java | 14 +- .../org/apache/doris/qe/StmtExecutor.java | 16 +- .../resourcegroup/ResourceGroupMgr.java | 325 ------------------ .../QueryQueue.java | 2 +- .../QueueOfferToken.java | 2 +- .../WorkloadGroup.java} | 40 +-- .../workloadgroup/WorkloadGroupMgr.java | 325 ++++++++++++++++++ .../tablefunction/MetadataGenerator.java | 10 +- .../MetadataTableValuedFunction.java | 4 +- .../tablefunction/TableValuedFunctionIf.java | 4 +- ...=> WorkloadGroupsTableValuedFunction.java} | 16 +- .../resourcegroup/ResourceGroupMgrTest.java | 219 ------------ .../workloadgroup/WorkloadGroupMgrTest.java | 219 ++++++++++++ .../WorkloadGroupTest.java} | 36 +- gensrc/thrift/PaloInternalService.thrift | 4 +- gensrc/thrift/Types.thrift | 2 +- 58 files changed, 924 insertions(+), 912 deletions(-) delete mode 100644 docs/en/docs/admin-manual/resource-group.md create mode 100644 docs/en/docs/admin-manual/workload-group.md rename docs/en/docs/sql-manual/sql-functions/table-functions/{resource-group.md => workload-group.md} (81%) rename docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/{ALTER-RESOURCE-GROUP.md => ALTER-WORKLOAD-GROUP.md} (83%) rename docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/{CREATE-RESOURCE-GROUP.md => CREATE-WORKLOAD-GROUP.md} (55%) rename docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/{DROP-RESOURCE-GROUP.md => DROP-WORKLOAD-GROUP.md} (77%) rename docs/en/docs/sql-manual/sql-reference/Show-Statements/{SHOW-RESOURCE-GROUPS.md => SHOW-WORKLOAD-GROUPS.md} (80%) rename docs/zh-CN/docs/admin-manual/{resource-group.md => workload-group.md} (83%) rename docs/zh-CN/docs/sql-manual/sql-functions/table-functions/{resource-group.md => workload-group.md} (85%) rename docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/{ALTER-RESOURCE-GROUP.md => ALTER-WORKLOAD-GROUP.md} (89%) rename docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/{CREATE-RESOURCE-GROUP.md => CREATE-WORKLOAD-GROUP.md} (93%) rename docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/{DROP-RESOURCE-GROUP.md => DROP-WORKLOAD-GROUP.md} (84%) rename docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/{SHOW-RESOURCE-GROUPS.md => SHOW-WORKLOAD-GROUPS.md} (89%) rename fe/fe-core/src/main/java/org/apache/doris/analysis/{AlterResourceGroupStmt.java => AlterWorkloadGroupStmt.java} (86%) rename fe/fe-core/src/main/java/org/apache/doris/analysis/{CreateResourceGroupStmt.java => CreateWorkloadGroupStmt.java} (83%) rename fe/fe-core/src/main/java/org/apache/doris/analysis/{DropResourceGroupStmt.java => DropWorkloadGroupStmt.java} (81%) rename fe/fe-core/src/main/java/org/apache/doris/analysis/{ShowResourceGroupsStmt.java => ShowWorkloadGroupsStmt.java} (89%) rename fe/fe-core/src/main/java/org/apache/doris/persist/{DropResourceGroupOperationLog.java => DropWorkloadGroupOperationLog.java} (86%) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgr.java rename fe/fe-core/src/main/java/org/apache/doris/resource/{resourcegroup => workloadgroup}/QueryQueue.java (98%) rename fe/fe-core/src/main/java/org/apache/doris/resource/{resourcegroup => workloadgroup}/QueueOfferToken.java (96%) rename fe/fe-core/src/main/java/org/apache/doris/resource/{resourcegroup/ResourceGroup.java => workloadgroup/WorkloadGroup.java} (88%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java rename fe/fe-core/src/main/java/org/apache/doris/tablefunction/{ResourceGroupsTableValuedFunction.java => WorkloadGroupsTableValuedFunction.java} (86%) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgrTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java rename fe/fe-core/src/test/java/org/apache/doris/resource/{resourcegroup/ResourceGroupTest.java => workloadgroup/WorkloadGroupTest.java} (72%) diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index c1413b0145..5ac0554104 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -685,9 +685,9 @@ Status FragmentMgr::_get_query_ctx(const Params& params, TUniqueId query_id, boo } if constexpr (std::is_same_v) { - if (params.__isset.resource_groups && !params.resource_groups.empty()) { + if (params.__isset.workload_groups && !params.workload_groups.empty()) { taskgroup::TaskGroupInfo task_group_info; - auto status = taskgroup::TaskGroupInfo::parse_group_info(params.resource_groups[0], + auto status = taskgroup::TaskGroupInfo::parse_group_info(params.workload_groups[0], &task_group_info); if (status.ok()) { auto tg = taskgroup::TaskGroupManager::instance()->get_or_create_task_group( diff --git a/be/src/runtime/task_group/task_group.cpp b/be/src/runtime/task_group/task_group.cpp index dad31cae1e..37149ddfd8 100644 --- a/be/src/runtime/task_group/task_group.cpp +++ b/be/src/runtime/task_group/task_group.cpp @@ -156,7 +156,7 @@ void TaskGroup::task_group_info(TaskGroupInfo* tg_info) const { tg_info->version = _version; } -Status TaskGroupInfo::parse_group_info(const TPipelineResourceGroup& resource_group, +Status TaskGroupInfo::parse_group_info(const TPipelineWorkloadGroup& resource_group, TaskGroupInfo* task_group_info) { if (UNLIKELY(!check_group_info(resource_group))) { std::stringstream ss; @@ -181,7 +181,7 @@ Status TaskGroupInfo::parse_group_info(const TPipelineResourceGroup& resource_gr ParseUtil::parse_mem_spec(mem_limit_str, -1, MemInfo::mem_limit(), &is_percent); if (UNLIKELY(mem_limit <= 0)) { std::stringstream ss; - ss << "parse memory limit from TPipelineResourceGroup error, " << MEMORY_LIMIT << ": " + ss << "parse memory limit from TPipelineWorkloadGroup error, " << MEMORY_LIMIT << ": " << mem_limit_str; LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); @@ -196,7 +196,7 @@ Status TaskGroupInfo::parse_group_info(const TPipelineResourceGroup& resource_gr return Status::OK(); } -bool TaskGroupInfo::check_group_info(const TPipelineResourceGroup& resource_group) { +bool TaskGroupInfo::check_group_info(const TPipelineWorkloadGroup& resource_group) { return resource_group.__isset.id && resource_group.__isset.version && resource_group.__isset.name && resource_group.__isset.properties && resource_group.properties.count(CPU_SHARE) > 0 && diff --git a/be/src/runtime/task_group/task_group.h b/be/src/runtime/task_group/task_group.h index 4fe33799b9..876f981cba 100644 --- a/be/src/runtime/task_group/task_group.h +++ b/be/src/runtime/task_group/task_group.h @@ -35,7 +35,7 @@ namespace pipeline { class PipelineTask; } -class TPipelineResourceGroup; +class TPipelineWorkloadGroup; class MemTrackerLimiter; namespace taskgroup { @@ -138,11 +138,11 @@ struct TaskGroupInfo { bool enable_memory_overcommit; int64_t version; - static Status parse_group_info(const TPipelineResourceGroup& resource_group, + static Status parse_group_info(const TPipelineWorkloadGroup& resource_group, TaskGroupInfo* task_group_info); private: - static bool check_group_info(const TPipelineResourceGroup& resource_group); + static bool check_group_info(const TPipelineWorkloadGroup& resource_group); }; } // namespace taskgroup diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index 41114eba76..67c421b00b 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -210,8 +210,8 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) { case TMetadataType::BACKENDS: RETURN_IF_ERROR(_build_backends_metadata_request(meta_scan_range, &request)); break; - case TMetadataType::RESOURCE_GROUPS: - RETURN_IF_ERROR(_build_resource_groups_metadata_request(meta_scan_range, &request)); + case TMetadataType::WORKLOAD_GROUPS: + RETURN_IF_ERROR(_build_workload_groups_metadata_request(meta_scan_range, &request)); break; default: _meta_eos = true; @@ -284,9 +284,9 @@ Status VMetaScanner::_build_backends_metadata_request(const TMetaScanRange& meta return Status::OK(); } -Status VMetaScanner::_build_resource_groups_metadata_request( +Status VMetaScanner::_build_workload_groups_metadata_request( const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request) { - VLOG_CRITICAL << "VMetaScanner::_build_resource_groups_metadata_request"; + VLOG_CRITICAL << "VMetaScanner::_build_workload_groups_metadata_request"; // create request request->__set_cluster_name(""); @@ -294,7 +294,7 @@ Status VMetaScanner::_build_resource_groups_metadata_request( // create TMetadataTableRequestParams TMetadataTableRequestParams metadata_table_params; - metadata_table_params.__set_metadata_type(TMetadataType::RESOURCE_GROUPS); + metadata_table_params.__set_metadata_type(TMetadataType::WORKLOAD_GROUPS); request->__set_metada_table_params(metadata_table_params); return Status::OK(); diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index de6e5cbd41..d566225b60 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -67,7 +67,7 @@ private: TFetchSchemaTableDataRequest* request); Status _build_backends_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); - Status _build_resource_groups_metadata_request(const TMetaScanRange& meta_scan_range, + Status _build_workload_groups_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); bool _meta_eos; diff --git a/docs/en/docs/admin-manual/resource-group.md b/docs/en/docs/admin-manual/resource-group.md deleted file mode 100644 index 377b64ffdc..0000000000 --- a/docs/en/docs/admin-manual/resource-group.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -{ - "title": "RESOURCE GROUP", - "language": "en" -} ---- - - - -# RESOURCE GROUP - - - -Resource groups can limit the use of compute and memory resources on a single be node for tasks within the group, thus achieving resource isolation. - -## Resource group properties - -* cpu_share: Required, used to set how much cpu time the resource group can acquire, which can achieve soft isolation of cpu resources. cpu_share is a relative value indicating the weight of cpu resources available to the running resource group. For example, if a user creates 3 resource groups rg-a, rg-b and rg-c with cpu_share of 10, 30 and 40 respectively, and at a certain moment rg-a and rg-b are running tasks while rg-c has no tasks, then rg-a can get 25% (10 / (10 + 30)) of the cpu resources while resource group rg-b can get 75% of the cpu resources. If the system has only one resource group running, it gets all the cpu resources regardless of the value of its cpu_share. - -* memory_limit: Required, set the percentage of be memory that can be used by the resource group. The absolute value of the resource group memory limit is: physical_memory * mem_limit * memory_limit, where mem_limit is a be configuration item. The total memory_limit of all resource groups in the system must not exceed 100%. Resource groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the resource group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. - -* enable_memory_overcommit: Optional, enable soft memory isolation for the resource group, default is false. if set to false, the resource group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the resource group memory usage exceeds the limit to release the excess memory. if set to true, the resource group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the resource group memory usage exceeds the limit to release the excess memory. if set to true, the resource group is softly isolated, if the system has free memory resources, the resource group can continue to use system memory after exceeding the memory_limit limit, and when the total system memory is tight, it will cancel several tasks in the group with the largest memory occupation, releasing part of the excess memory to relieve the system memory pressure. It is recommended that when this configuration is enabled for a resource group, the total memory_limit of all resource groups should be less than 100%, and the remaining portion should be used for resource group memory overcommit. - -## Resource group usage - -1. Enable the experimental_enable_resource_group configuration, set in fe.conf to -``` -experimental_enable_resource_group=true -``` -The system will automatically create a default resource group named ``normal`` after this configuration is enabled. 2. - -2. To create a resource group: -``` -create resource group if not exists g1 -properties ( - "cpu_share"="10". - "memory_limit"="30%". - "enable_memory_overcommit"="true" -). -``` -For details on creating a resource group, see [CREATE-RESOURCE-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md), and to delete a resource group, refer to [DROP-RESOURCE-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md); to modify a resource group, refer to [ALTER-RESOURCE-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md); to view the resource group, refer to: [RESOURCE_GROUPS()](../sql-manual/sql-functions/table-functions/resource-group.md) and [SHOW-RESOURCE-GROUPS](../sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md). - - -3. turn on the pipeline execution engine, the resource group cpu isolation is based on the implementation of the pipeline execution engine, so you need to turn on the session variable: -``` -set experimental_enable_pipeline_engine = true. -``` - -4. Queries bind to resource groups. Currently, queries are mainly bound to resource groups by specifying session variables. If the user does not specify a resource group, the query will be submitted to the `normal` resource group by default. -``` -set resource_group = g1. -``` - -5. Execute the query, which will be associated with the g1 resource group. \ No newline at end of file diff --git a/docs/en/docs/admin-manual/workload-group.md b/docs/en/docs/admin-manual/workload-group.md new file mode 100644 index 0000000000..2b6ae1338b --- /dev/null +++ b/docs/en/docs/admin-manual/workload-group.md @@ -0,0 +1,71 @@ +--- +{ + "title": "WORKLOAD GROUP", + "language": "en" +} +--- + + + +# WORKLOAD GROUP + + + +Workload groups can limit the use of compute and memory resources on a single be node for tasks within the group, thus achieving resource isolation. + +## Workload group properties + +* cpu_share: Required, used to set how much cpu time the workload group can acquire, which can achieve soft isolation of cpu resources. cpu_share is a relative value indicating the weight of cpu resources available to the running workload group. For example, if a user creates 3 workload groups rg-a, rg-b and rg-c with cpu_share of 10, 30 and 40 respectively, and at a certain moment rg-a and rg-b are running tasks while rg-c has no tasks, then rg-a can get 25% (10 / (10 + 30)) of the cpu resources while workload group rg-b can get 75% of the cpu resources. If the system has only one workload group running, it gets all the cpu resources regardless of the value of its cpu_share. + +* memory_limit: Required, set the percentage of be memory that can be used by the workload group. The absolute value of the workload group memory limit is: physical_memory * mem_limit * memory_limit, where mem_limit is a be configuration item. The total memory_limit of all workload groups in the system must not exceed 100%. Workload groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the workload group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. + +* enable_memory_overcommit: Optional, enable soft memory isolation for the workload group, default is false. if set to false, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is softly isolated, if the system has free memory resources, the workload group can continue to use system memory after exceeding the memory_limit limit, and when the total system memory is tight, it will cancel several tasks in the group with the largest memory occupation, releasing part of the excess memory to relieve the system memory pressure. It is recommended that when this configuration is enabled for a workload group, the total memory_limit of all workload groups should be less than 100%, and the remaining portion should be used for workload group memory overcommit. + +## Workload group usage + +1. Enable the experimental_enable_workload_group configuration, set in fe.conf to +``` +experimental_enable_workload_group=true +``` +The system will automatically create a default workload group named ``normal`` after this configuration is enabled. 2. + +2. To create a workload group: +``` +create workload group if not exists g1 +properties ( + "cpu_share"="10". + "memory_limit"="30%". + "enable_memory_overcommit"="true" +). +``` +For details on creating a workload group, see [CREATE-WORKLOAD-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md), and to delete a workload group, refer to [DROP-WORKLOAD-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md); to modify a workload group, refer to [ALTER-WORKLOAD-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md); to view the workload group, refer to: [WORKLOAD_GROUPS()](../sql-manual/sql-functions/table-functions/workload-group.md) and [SHOW-WORKLOAD-GROUPS](../sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md). + + +3. turn on the pipeline execution engine, the workload group cpu isolation is based on the implementation of the pipeline execution engine, so you need to turn on the session variable: +``` +set experimental_enable_pipeline_engine = true. +``` + +4. Queries bind to workload groups. Currently, queries are mainly bound to workload groups by specifying session variables. If the user does not specify a workload group, the query will be submitted to the `normal` workload group by default. +``` +set workload_group = g1. +``` + +5. Execute the query, which will be associated with the g1 workload group. \ No newline at end of file diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index df8be16bdc..8e3ab3eb60 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -205,7 +205,6 @@ When Trino is mapped, Doris's Database corresponds to a Schema in Trino that spe 9. OceanBase -<<<<<<< HEAD ```sql diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/resource-group.md b/docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md similarity index 81% rename from docs/en/docs/sql-manual/sql-functions/table-functions/resource-group.md rename to docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md index e0b1f75147..0e222e7a0a 100644 --- a/docs/en/docs/sql-manual/sql-functions/table-functions/resource-group.md +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/workload-group.md @@ -1,6 +1,6 @@ --- { - "title": "resource_groups", + "title": "workload_groups", "language": "en" } --- @@ -24,29 +24,29 @@ specific language governing permissions and limitations under the License. --> -## `resource_groups` +## `workload_groups` ### Name -resource_groups +workload_groups ### description -Table-Value-Function, generate a temporary table named `resource_groups`. This tvf is used to view informations about current resource groups. +Table-Value-Function, generate a temporary table named `workload_groups`. This tvf is used to view informations about current workload groups. This function is used in `FROM` clauses. #### syntax -`resource_groups()` +`workload_groups()` -The table schema of `resource_groups()` tvf: +The table schema of `workload_groups()` tvf: ``` -mysql> desc function resource_groups(); +mysql> desc function workload_groups(); +-------+-------------+------+-------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-------------+------+-------+---------+-------+ @@ -59,7 +59,7 @@ mysql> desc function resource_groups(); ### example ``` -mysql> select * from resource_groups()\G +mysql> select * from workload_groups()\G +-------+--------+--------------+-------+ | Id | Name | Item | Value | +-------+--------+--------------+-------+ @@ -70,4 +70,4 @@ mysql> select * from resource_groups()\G ### keywords - resource_groups \ No newline at end of file + workload_groups \ No newline at end of file diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md similarity index 83% rename from docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md rename to docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md index f9b6fb7df9..c46044b9be 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md @@ -1,6 +1,6 @@ --- { -"title": "ALTER-RESOURCE-GROUP", +"title": "ALTER-WORKLOAD-GROUP", "language": "en" } --- @@ -24,22 +24,22 @@ specific language governing permissions and limitations under the License. --> -## ALTER-RESOURCE-GROUP +## ALTER-WORKLOAD-GROUP ### Name -ALTER RESOURCE GROUP +ALTER WORKLOAD GROUP ### Description -This statement is used to modify the resource group. +This statement is used to modify the workload group. Syntax: ```sql -ALTER RESOURCE GROUP "rg_name" +ALTER WORKLOAD GROUP "rg_name" PROPERTIES ( property_list ); @@ -52,10 +52,10 @@ NOTE: ### Example -1. Modify the resource group named g1: +1. Modify the workload group named g1: ```sql - alter resource group g1 + alter workload group g1 properties ( "cpu_share"="30", "memory_limit"="30%" @@ -65,7 +65,7 @@ NOTE: ### Keywords ```sql -ALTER, RESOURCE, GROUP +ALTER, WORKLOAD, GROUP ``` ### Best Practice diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md similarity index 55% rename from docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md rename to docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md index be9e50ca02..eda88cad5d 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md @@ -1,6 +1,6 @@ --- { - "title": "CREATE-RESOURCE-GORUP", + "title": "CREATE-WORKLOAD-GORUP", "language": "en" } --- @@ -24,22 +24,22 @@ specific language governing permissions and limitations under the License. --> -## CREATE-RESOURCE-GROUP +## CREATE-WORKLOAD-GROUP ### Name -CREATE RESOURCE GROUP +CREATE WORKLOAD GROUP ### Description -This statement is used to create a resource group. Resource groups enable the isolation of cpu resources and memory resources on a single be. +This statement is used to create a workload group. Workload groups enable the isolation of cpu resources and memory resources on a single be. grammar: ```sql -CREATE RESOURCE GROUP [IF NOT EXISTS] "rg_name" +CREATE WORKLOAD GROUP [IF NOT EXISTS] "rg_name" PROPERTIES ( property_list ); @@ -49,18 +49,18 @@ illustrate: Properties supported by property_list: -* cpu_share: Required, used to set how much cpu time the resource group can acquire, which can achieve soft isolation of cpu resources. cpu_share is a relative value indicating the weight of cpu resources available to the running resource group. For example, if a user creates 3 resource groups rg-a, rg-b and rg-c with cpu_share of 10, 30 and 40 respectively, and at a certain moment rg-a and rg-b are running tasks while rg-c has no tasks, then rg-a can get 25% (10 / (10 + 30)) of the cpu resources while resource group rg-b can get 75% of the cpu resources. If the system has only one resource group running, it gets all the cpu resources regardless of the value of its cpu_share. +* cpu_share: Required, used to set how much cpu time the workload group can acquire, which can achieve soft isolation of cpu resources. cpu_share is a relative value indicating the weight of cpu resources available to the running workload group. For example, if a user creates 3 workload groups rg-a, rg-b and rg-c with cpu_share of 10, 30 and 40 respectively, and at a certain moment rg-a and rg-b are running tasks while rg-c has no tasks, then rg-a can get 25% (10 / (10 + 30)) of the cpu resources while workload group rg-b can get 75% of the cpu resources. If the system has only one workload group running, it gets all the cpu resources regardless of the value of its cpu_share. -* memory_limit: Required, set the percentage of be memory that can be used by the resource group. The absolute value of the resource group memory limit is: physical_memory * mem_limit * memory_limit, where mem_limit is a be configuration item. The total memory_limit of all resource groups in the system must not exceed 100%. Resource groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the resource group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. +* memory_limit: Required, set the percentage of be memory that can be used by the workload group. The absolute value of the workload group memory limit is: physical_memory * mem_limit * memory_limit, where mem_limit is a be configuration item. The total memory_limit of all workload groups in the system must not exceed 100%. Workload groups are guaranteed to use the memory_limit for the tasks in the group in most cases. When the workload group memory usage exceeds this limit, tasks in the group with larger memory usage may be canceled to release the excess memory, refer to enable_memory_overcommit. -* enable_memory_overcommit: Optional, enable soft memory isolation for the resource group, default is false. if set to false, the resource group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the resource group memory usage exceeds the limit to release the excess memory. if set to true, the resource group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the resource group memory usage exceeds the limit to release the excess memory. if set to true, the resource group is softly isolated, if the system has free memory resources, the resource group can continue to use system memory after exceeding the memory_limit limit, and when the total system memory is tight, it will cancel several tasks in the group with the largest memory occupation, releasing part of the excess memory to relieve the system memory pressure. It is recommended that when this configuration is enabled for a resource group, the total memory_limit of all resource groups should be less than 100%, and the remaining portion should be used for resource group memory overcommit. +* enable_memory_overcommit: Optional, enable soft memory isolation for the workload group, default is false. if set to false, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is hard memory isolated and the tasks with the largest memory usage will be canceled immediately after the workload group memory usage exceeds the limit to release the excess memory. if set to true, the workload group is softly isolated, if the system has free memory resources, the workload group can continue to use system memory after exceeding the memory_limit limit, and when the total system memory is tight, it will cancel several tasks in the group with the largest memory occupation, releasing part of the excess memory to relieve the system memory pressure. It is recommended that when this configuration is enabled for a workload group, the total memory_limit of all workload groups should be less than 100%, and the remaining portion should be used for workload group memory overcommit. ### Example -1. Create a resource group named g1: +1. Create a workload group named g1: ```sql - create resource group if not exists g1 + create workload group if not exists g1 properties ( "cpu_share"="10", "memory_limit"="30%", @@ -70,7 +70,7 @@ Properties supported by property_list: ### Keywords - CREATE, RESOURCE, GROUP + CREATE, WORKLOAD, GROUP ### Best Practice diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md similarity index 77% rename from docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md rename to docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md index 33f2718f10..e87ff941de 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md @@ -1,6 +1,6 @@ --- { - "title": "DROP-RESOURCE-GROUP", + "title": "DROP-WORKLOAD-GROUP", "language": "en" } --- @@ -24,33 +24,33 @@ specific language governing permissions and limitations under the License. --> -## DROP-RESOURCE-GROUP +## DROP-WORKLOAD-GROUP ### Name -DROP RESOURCE GROUP +DROP WORKLOAD GROUP ### Description -This statement is used to delete a resource group. +This statement is used to delete a workload group. ```sql -DROP RESOURCE GROUP [IF EXISTS] 'rg_name' +DROP WORKLOAD GROUP [IF EXISTS] 'rg_name' ``` ### Example -1. Delete the resource group named g1: +1. Delete the workload group named g1: ```sql - drop resource group if exists g1; + drop workload group if exists g1; ``` ### Keywords - DROP, RESOURCE, GROUP + DROP, WORKLOAD, GROUP ### Best Practice diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md similarity index 80% rename from docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md rename to docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md index 5908a05265..e892b31a57 100644 --- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md +++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md @@ -1,6 +1,6 @@ --- { - "title": "SHOW-RESOURCE-GROUPS", + "title": "SHOW-workload-GROUPS", "language": "en" } --- @@ -24,34 +24,34 @@ specific language governing permissions and limitations under the License. --> -## SHOW-RESOURCE-GROUPS +## SHOW-workload-GROUPS ### Name -SHOW RESOURCE GROUPS +SHOW workload GROUPS ### Description -This statement is used to display all resource groups. +This statement is used to display all workload groups. grammar: ```sql -SHOW RESOURCE GROUPS; +SHOW workload GROUPS; ``` Description: -This statement only does a simple display of resource groups, for a more complex display refer to tvf resource_groups(). +This statement only does a simple display of workload groups, for a more complex display refer to tvf workload_groups(). ### Example -1. Show all resource groups: +1. Show all workload groups: ```sql - mysql> show resource groups; + mysql> show workload groups; +----------+--------+--------------------------+---------+ | Id | Name | Item | Value | +----------+--------+--------------------------+---------+ @@ -65,6 +65,6 @@ This statement only does a simple display of resource groups, for a more complex ### Keywords - SHOW, RESOURCE, GROUPS, GROUP + SHOW, workload, GROUPS, GROUP ### Best Practice diff --git a/docs/sidebars.json b/docs/sidebars.json index 0a88724ac4..de9748cc61 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -705,7 +705,7 @@ "sql-manual/sql-functions/table-functions/hdfs", "sql-manual/sql-functions/table-functions/iceberg_meta", "sql-manual/sql-functions/table-functions/backends", - "sql-manual/sql-functions/table-functions/resource-group" + "sql-manual/sql-functions/table-functions/workload-group" ] }, { @@ -786,7 +786,7 @@ "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-PARTITION", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-COLUMN", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE", - "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP", + "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP", "sql-manual/sql-reference/Data-Definition-Statements/Alter/CANCEL-ALTER-TABLE", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-COMMENT", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-VIEW", @@ -822,7 +822,7 @@ "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-FILE", "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX", "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE", - "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP", + "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP", "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE-LIKE", "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-MATERIALIZED-VIEW", "sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-EXTERNAL-TABLE", @@ -838,7 +838,7 @@ "items": [ "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX", "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE", - "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP", + "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP", "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-FILE", "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ENCRYPT-KEY", "sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-DATABASE", @@ -981,7 +981,7 @@ "sql-manual/sql-reference/Show-Statements/SHOW-LOAD", "sql-manual/sql-reference/Show-Statements/SHOW-TABLES", "sql-manual/sql-reference/Show-Statements/SHOW-RESOURCES", - "sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS", + "sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS", "sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS", "sql-manual/sql-reference/Show-Statements/SHOW-FRONTENDS", "sql-manual/sql-reference/Show-Statements/SHOW-RESTORE", @@ -1138,7 +1138,7 @@ "admin-manual/system-table/rowsets" ] }, - "admin-manual/resource-group", + "admin-manual/workload-group", "admin-manual/multi-tenant", { "type": "category", diff --git a/docs/zh-CN/docs/admin-manual/resource-group.md b/docs/zh-CN/docs/admin-manual/workload-group.md similarity index 83% rename from docs/zh-CN/docs/admin-manual/resource-group.md rename to docs/zh-CN/docs/admin-manual/workload-group.md index 641ec0a715..9ebe72e87e 100644 --- a/docs/zh-CN/docs/admin-manual/resource-group.md +++ b/docs/zh-CN/docs/admin-manual/workload-group.md @@ -40,22 +40,22 @@ under the License. ## 资源组使用 -1. 开启 experimental_enable_resource_group 配置项，在fe.conf中设置： +1. 开启 experimental_enable_workload_group 配置项，在fe.conf中设置： ``` -experimental_enable_resource_group=true +experimental_enable_workload_group=true ``` 在开启该配置后系统会自动创建名为`normal`的默认资源组。 2. 创建资源组： ``` -create resource group if not exists g1 +create workload group if not exists g1 properties ( "cpu_share"="10", "memory_limit"="30%", "enable_memory_overcommit"="true" ); ``` -创建资源组详细可参考：[CREATE-RESOURCE-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md)，另删除资源组可参考[DROP-RESOURCE-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md)；修改资源组可参考：[ALTER-RESOURCE-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md)；查看资源组可参考：[RESOURCE_GROUPS()](../sql-manual/sql-functions/table-functions/resource-group.md)和[SHOW-RESOURCE-GROUPS](../sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md)。 +创建资源组详细可参考：[CREATE-WORKLOAD-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md)，另删除资源组可参考[DROP-WORKLOAD-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md)；修改资源组可参考：[ALTER-WORKLOAD-GROUP](../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md)；查看资源组可参考：[WORKLOAD_GROUPS()](../sql-manual/sql-functions/table-functions/workload-group.md)和[SHOW-WORKLOAD-GROUPS](../sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md)。 3. 开启pipeline执行引擎，资源组cpu隔离基于pipeline执行引擎实现，因此需开启session变量： ``` @@ -64,7 +64,7 @@ set experimental_enable_pipeline_engine = true; 4. 查询绑定资源组。目前主要通过指定session变量的方式绑定查询到资源组。如果用户不指定资源组，那么查询默认会提交到`normal`资源组: ``` -set resource_group = g1; +set workload_group = g1; ``` 5. 执行查询，查询将关联到 g1 资源组。 diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/resource-group.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md similarity index 85% rename from docs/zh-CN/docs/sql-manual/sql-functions/table-functions/resource-group.md rename to docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md index 5bc895f7a7..9a4802559b 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/resource-group.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/workload-group.md @@ -1,6 +1,6 @@ --- { - "title": "resource_groups", + "title": "workload_groups", "language": "zh-CN" } --- @@ -24,28 +24,28 @@ specific language governing permissions and limitations under the License. --> -## `resource_groups` +## `workload_groups` ### Name -resource_groups +workload_groups ### description -表函数，生成 resource_groups 临时表，可以查看当前资源组信息。 +表函数，生成 workload_groups 临时表，可以查看当前资源组信息。 该函数用于from子句中。 #### syntax -`resource_groups()` +`workload_groups()` -resource_groups()表结构： +workload_groups()表结构： ``` -mysql> desc function resource_groups(); +mysql> desc function workload_groups(); +-------+-------------+------+-------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-------------+------+-------+---------+-------+ @@ -58,7 +58,7 @@ mysql> desc function resource_groups(); ### example ``` -mysql> select * from resource_groups()\G +mysql> select * from workload_groups()\G +-------+--------+--------------+-------+ | Id | Name | Item | Value | +-------+--------+--------------+-------+ @@ -69,4 +69,4 @@ mysql> select * from resource_groups()\G ### keywords - resource_groups \ No newline at end of file + workload_groups \ No newline at end of file diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md similarity index 89% rename from docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md rename to docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md index 80446c5fd9..1bc19780f6 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-RESOURCE-GROUP.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-WORKLOAD-GROUP.md @@ -1,6 +1,6 @@ --- { -"title": "ALTER-RESOURCE-GROUP", +"title": "ALTER-WORKLOAD -GROUP", "language": "zh-CN" } --- @@ -24,11 +24,11 @@ specific language governing permissions and limitations under the License. --> -## ALTER-RESOURCE-GROUP +## ALTER-WORKLOAD -GROUP ### Name -ALTER RESOURCE GROUP +ALTER WORKLOAD GROUP @@ -39,7 +39,7 @@ ALTER RESOURCE GROUP 语法： ```sql -ALTER RESOURCE GROUP "rg_name" +ALTER WORKLOAD GROUP "rg_name" PROPERTIES ( property_list ); @@ -55,7 +55,7 @@ PROPERTIES ( 1. 修改名为 g1 的资源组： ```sql - alter resource group g1 + alter workload group g1 properties ( "cpu_share"="30", "memory_limit"="30%" @@ -65,7 +65,7 @@ PROPERTIES ( ### Keywords ```sql -ALTER, RESOURCE, GROUP +ALTER, WORKLOAD , GROUP ``` ### Best Practice diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md similarity index 93% rename from docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md rename to docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md index 4c02b130d5..b3cfb004ae 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-RESOURCE-GROUP.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-WORKLOAD-GROUP.md @@ -1,6 +1,6 @@ --- { - "title": "CREATE-RESOURCE-GROUP", + "title": "CREATE-WORKLOAD-GROUP", "language": "zh-CN" } --- @@ -24,11 +24,11 @@ specific language governing permissions and limitations under the License. --> -## CREATE-RESOURCE-GROUP +## CREATE-WORKLOAD-GROUP ### Name -CREATE RESOURCE GROUP +CREATE WORKLOAD GROUP @@ -39,7 +39,7 @@ CREATE RESOURCE GROUP 语法： ```sql -CREATE RESOURCE GROUP [IF NOT EXISTS] "rg_name" +CREATE WORKLOAD GROUP [IF NOT EXISTS] "rg_name" PROPERTIES ( property_list ); @@ -60,7 +60,7 @@ property_list 支持的属性： 1. 创建名为g1的资源组： ```sql - create resource group if not exists g1 + create workload group if not exists g1 properties ( "cpu_share"="10", "memory_limit"="30%", @@ -70,7 +70,7 @@ property_list 支持的属性： ### Keywords - CREATE, RESOURCE, GROUP + CREATE, WORKLOAD, GROUP ### Best Practice diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md similarity index 84% rename from docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md rename to docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md index 010645169e..d6c0ceab6f 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-RESOURCE-GROUP.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-WORKLOAD-GROUP.md @@ -1,6 +1,6 @@ --- { - "title": "DROP-RESOURCE-GROUP", + "title": "DROP-WORKLOAD-GROUP", "language": "zh-CN" } --- @@ -24,11 +24,11 @@ specific language governing permissions and limitations under the License. --> -## DROP-RESOURCE-GROUP +## DROP-WORKLOAD-GROUP ### Name -DROP RESOURCE GROUP +DROP WORKLOAD GROUP ### Description @@ -37,7 +37,7 @@ DROP RESOURCE GROUP 该语句用于删除资源组。 ```sql -DROP RESOURCE GROUP [IF EXISTS] 'rg_name' +DROP WORKLOAD GROUP [IF EXISTS] 'rg_name' ``` ### Example @@ -45,12 +45,12 @@ DROP RESOURCE GROUP [IF EXISTS] 'rg_name' 1. 删除名为 g1 的资源组： ```sql - drop resource group if exists g1; + drop workload group if exists g1; ``` ### Keywords - DROP, RESOURCE, GROUP + DROP, WORKLOAD, GROUP ### Best Practice diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md similarity index 89% rename from docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md rename to docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md index eb3f574740..36d0640553 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-RESOURCE-GROUPS.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-WORKLOAD-GROUPS.md @@ -1,6 +1,6 @@ --- { - "title": "SHOW-RESOURCE-GROUPS", + "title": "SHOW-WORKLOAD-GROUPS", "language": "zh-CN" } --- @@ -24,11 +24,11 @@ specific language governing permissions and limitations under the License. --> -## SHOW-RESOURCE-GROUPS +## SHOW-WORKLOAD-GROUPS ### Name -SHOW RESOURCE GROUPS +SHOW WORKLOAD GROUPS @@ -39,19 +39,19 @@ SHOW RESOURCE GROUPS 语法： ```sql -SHOW RESOURCE GROUPS; +SHOW WORKLOAD GROUPS; ``` 说明： -该语句仅做资源组简单展示，更复杂的展示可参考 tvf resource_groups(). +该语句仅做资源组简单展示，更复杂的展示可参考 tvf workload_groups(). ### Example 1. 展示所有资源组： ```sql - mysql> show resource groups; + mysql> show workload groups; +----------+--------+--------------------------+---------+ | Id | Name | Item | Value | +----------+--------+--------------------------+---------+ @@ -65,6 +65,6 @@ SHOW RESOURCE GROUPS; ### Keywords - SHOW, RESOURCE, GROUPS, GROUP + SHOW, WORKLOAD, GROUPS, GROUP ### Best Practice diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 23ecd187ab..602341afaa 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1488,9 +1488,9 @@ public class Config extends ConfigBase { @ConfField public static boolean enable_pipeline_load = false; - // enable_resource_group should be immutable and temporarily set to mutable during the development test phase + // enable_workload_group should be immutable and temporarily set to mutable during the development test phase @ConfField(mutable = true, masterOnly = true, expType = ExperimentalType.EXPERIMENTAL) - public static boolean enable_resource_group = false; + public static boolean enable_workload_group = false; @ConfField(mutable = true) public static boolean enable_query_queue = true; diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 305e033427..cc07ea2847 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -625,6 +625,7 @@ terminal String KW_WHITELIST, KW_WITH, KW_WORK, + KW_WORKLOAD, KW_WRITE, KW_YEAR, KW_MTMV, @@ -1298,9 +1299,9 @@ alter_stmt ::= {: RESULT = new AlterResourceStmt(resourceName, properties); :} - | KW_ALTER KW_RESOURCE KW_GROUP ident_or_text:resourceGroupName opt_properties:properties + | KW_ALTER KW_WORKLOAD KW_GROUP ident_or_text:workloadGroupName opt_properties:properties {: - RESULT = new AlterResourceGroupStmt(resourceGroupName, properties); + RESULT = new AlterWorkloadGroupStmt(workloadGroupName, properties); :} | KW_ALTER KW_ROUTINE KW_LOAD KW_FOR job_label:jobLabel opt_properties:jobProperties opt_datasource_properties:datasourceProperties @@ -1929,10 +1930,10 @@ create_stmt ::= {: RESULT = new CreateResourceStmt(false, ifNotExists, resourceName, properties); :} - /* resource group*/ - | KW_CREATE KW_RESOURCE KW_GROUP opt_if_not_exists:ifNotExists ident_or_text:resourceGroupName opt_properties:properties + /* workload group*/ + | KW_CREATE KW_WORKLOAD KW_GROUP opt_if_not_exists:ifNotExists ident_or_text:workloadGroupName opt_properties:properties {: - RESULT = new CreateResourceGroupStmt(ifNotExists, resourceGroupName, properties); + RESULT = new CreateWorkloadGroupStmt(ifNotExists, workloadGroupName, properties); :} /* encryptkey */ | KW_CREATE KW_ENCRYPTKEY opt_if_not_exists:ifNotExists encryptkey_name:keyName KW_AS STRING_LITERAL:keyString @@ -2968,9 +2969,9 @@ drop_stmt ::= {: RESULT = new DropResourceStmt(ifExists, resourceName); :} - | KW_DROP KW_RESOURCE KW_GROUP opt_if_exists:ifExists ident_or_text:resourceGroupName + | KW_DROP KW_WORKLOAD KW_GROUP opt_if_exists:ifExists ident_or_text:workloadGroupName {: - RESULT = new DropResourceGroupStmt(ifExists, resourceGroupName); + RESULT = new DropWorkloadGroupStmt(ifExists, workloadGroupName); :} | KW_DROP KW_ENCRYPTKEY opt_if_exists:ifExists encryptkey_name:keyName {: @@ -3869,9 +3870,9 @@ show_param ::= {: RESULT = new ShowResourcesStmt(parser.where, orderByClause, limitClause); :} - | KW_RESOURCE KW_GROUPS + | KW_WORKLOAD KW_GROUPS {: - RESULT = new ShowResourceGroupsStmt(); + RESULT = new ShowWorkloadGroupsStmt(); :} | KW_BACKENDS {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java similarity index 86% rename from fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceGroupStmt.java rename to fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java index bbec596b66..f612033833 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceGroupStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java @@ -28,17 +28,17 @@ import org.apache.doris.qe.ConnectContext; import java.util.Map; -public class AlterResourceGroupStmt extends DdlStmt { - private final String resourceGroupName; +public class AlterWorkloadGroupStmt extends DdlStmt { + private final String workloadGroupName; private final Map properties; - public AlterResourceGroupStmt(String resourceGroupName, Map properties) { - this.resourceGroupName = resourceGroupName; + public AlterWorkloadGroupStmt(String workloadGroupName, Map properties) { + this.workloadGroupName = workloadGroupName; this.properties = properties; } - public String getResourceGroupName() { - return resourceGroupName; + public String getWorkloadGroupName() { + return workloadGroupName; } public Map getProperties() { @@ -62,7 +62,7 @@ public class AlterResourceGroupStmt extends DdlStmt { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("ALTER RESOURCE GROUP '").append(resourceGroupName).append("' "); + sb.append("ALTER RESOURCE GROUP '").append(workloadGroupName).append("' "); sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index 6549282dab..117124f6ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -69,7 +69,7 @@ import org.apache.doris.rewrite.mvrewrite.ExprToSlotRefRule; import org.apache.doris.rewrite.mvrewrite.HLLHashToSlotRefRule; import org.apache.doris.rewrite.mvrewrite.NDVToHll; import org.apache.doris.rewrite.mvrewrite.ToBitmapToSlotRefRule; -import org.apache.doris.thrift.TPipelineResourceGroup; +import org.apache.doris.thrift.TPipelineWorkloadGroup; import org.apache.doris.thrift.TQueryGlobals; import com.google.common.base.Joiner; @@ -408,7 +408,7 @@ public class Analyzer { private final Map> migrateFailedConjuncts = Maps.newHashMap(); - public List tResourceGroups; + public List tWorkloadGroups; public GlobalState(Env env, ConnectContext context) { this.env = env; @@ -597,12 +597,12 @@ public class Analyzer { return explicitViewAlias; } - public void setResourceGroups(List tResourceGroups) { - globalState.tResourceGroups = tResourceGroups; + public void setWorkloadGroups(List tWorkloadGroups) { + globalState.tWorkloadGroups = tWorkloadGroups; } - public List getResourceGroups() { - return globalState.tResourceGroups; + public List getWorkloadGroups() { + return globalState.tWorkloadGroups; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java similarity index 83% rename from fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceGroupStmt.java rename to fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java index b09de46412..3d48cad5cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceGroupStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java @@ -29,16 +29,16 @@ import org.apache.doris.qe.ConnectContext; import java.util.Map; -public class CreateResourceGroupStmt extends DdlStmt { +public class CreateWorkloadGroupStmt extends DdlStmt { private final boolean ifNotExists; - private final String resourceGroupName; + private final String workloadGroupName; private final Map properties; - public CreateResourceGroupStmt(boolean ifNotExists, String resourceGroupName, Map properties) { + public CreateWorkloadGroupStmt(boolean ifNotExists, String workloadGroupName, Map properties) { this.ifNotExists = ifNotExists; - this.resourceGroupName = resourceGroupName; + this.workloadGroupName = workloadGroupName; this.properties = properties; } @@ -46,8 +46,8 @@ public class CreateResourceGroupStmt extends DdlStmt { return ifNotExists; } - public String getResourceGroupName() { - return resourceGroupName; + public String getWorkloadGroupName() { + return workloadGroupName; } public Map getProperties() { @@ -64,7 +64,7 @@ public class CreateResourceGroupStmt extends DdlStmt { } // check name - FeNameFormat.checkResourceGroupName(resourceGroupName); + FeNameFormat.checkWorkloadGroupName(workloadGroupName); if (properties == null || properties.isEmpty()) { throw new AnalysisException("Resource group properties can't be null"); @@ -75,7 +75,7 @@ public class CreateResourceGroupStmt extends DdlStmt { public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("CREATE "); - sb.append("RESOURCE GROUP '").append(resourceGroupName).append("' "); + sb.append("RESOURCE GROUP '").append(workloadGroupName).append("' "); sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")"); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java similarity index 81% rename from fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceGroupStmt.java rename to fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java index 3da3f5a683..d7a1703771 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceGroupStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java @@ -25,21 +25,21 @@ import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; -public class DropResourceGroupStmt extends DdlStmt { +public class DropWorkloadGroupStmt extends DdlStmt { private boolean ifExists; - private String resourceGroupName; + private String workloadGroupName; - public DropResourceGroupStmt(boolean ifExists, String resourceGroupName) { + public DropWorkloadGroupStmt(boolean ifExists, String workloadGroupName) { this.ifExists = ifExists; - this.resourceGroupName = resourceGroupName; + this.workloadGroupName = workloadGroupName; } public boolean isIfExists() { return ifExists; } - public String getResourceGroupName() { - return resourceGroupName; + public String getWorkloadGroupName() { + return workloadGroupName; } @Override @@ -51,14 +51,14 @@ public class DropResourceGroupStmt extends DdlStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - FeNameFormat.checkResourceGroupName(resourceGroupName); + FeNameFormat.checkWorkloadGroupName(workloadGroupName); } @Override public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("DROP "); - sb.append("RESOURCE GROUP '").append(resourceGroupName).append("' "); + sb.append("RESOURCE GROUP '").append(workloadGroupName).append("' "); return sb.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourceGroupsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourceGroupsStmt.java rename to fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java index 41a2d34d47..bb0d0bb566 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourceGroupsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadGroupsStmt.java @@ -22,11 +22,11 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -import org.apache.doris.resource.resourcegroup.ResourceGroupMgr; +import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; -public class ShowResourceGroupsStmt extends ShowStmt { +public class ShowWorkloadGroupsStmt extends ShowStmt { - public ShowResourceGroupsStmt() {} + public ShowWorkloadGroupsStmt() {} @Override public void analyze(Analyzer analyzer) throws UserException { @@ -46,7 +46,7 @@ public class ShowResourceGroupsStmt extends ShowStmt { @Override public ShowResultSetMetaData getMetaData() { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - for (String title : ResourceGroupMgr.RESOURCE_GROUP_PROC_NODE_TITLE_NAMES) { + for (String title : WorkloadGroupMgr.WORKLOAD_GROUP_PROC_NODE_TITLE_NAMES) { builder.addColumn(new Column(title, ScalarType.createVarchar(30))); } return builder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index b87f21de5e..11cd3f7612 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -205,7 +205,7 @@ import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.JournalObservable; import org.apache.doris.qe.VariableMgr; import org.apache.doris.resource.Tag; -import org.apache.doris.resource.resourcegroup.ResourceGroupMgr; +import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; import org.apache.doris.service.FrontendOptions; import org.apache.doris.statistics.AnalysisManager; import org.apache.doris.statistics.AnalysisTaskScheduler; @@ -440,7 +440,7 @@ public class Env { private AtomicLong stmtIdCounter; - private ResourceGroupMgr resourceGroupMgr; + private WorkloadGroupMgr workloadGroupMgr; private QueryStats queryStats; @@ -663,7 +663,7 @@ public class Env { this.statisticsAutoAnalyzer = new StatisticsAutoAnalyzer(); } this.globalFunctionMgr = new GlobalFunctionMgr(); - this.resourceGroupMgr = new ResourceGroupMgr(); + this.workloadGroupMgr = new WorkloadGroupMgr(); this.queryStats = new QueryStats(); this.loadManagerAdapter = new LoadManagerAdapter(); this.hiveTransactionMgr = new HiveTransactionMgr(); @@ -735,8 +735,8 @@ public class Env { return auditEventProcessor; } - public ResourceGroupMgr getResourceGroupMgr() { - return resourceGroupMgr; + public WorkloadGroupMgr getWorkloadGroupMgr() { + return workloadGroupMgr; } // use this to get correct ClusterInfoService instance @@ -1360,7 +1360,7 @@ public class Env { editLog.logMasterInfo(masterInfo); LOG.info("logMasterInfo:{}", masterInfo); - this.resourceGroupMgr.init(); + this.workloadGroupMgr.init(); // for master, the 'isReady' is set behind. // but we are sure that all metadata is replayed if we get here. @@ -1945,9 +1945,9 @@ public class Env { return checksum; } - public long loadResourceGroups(DataInputStream in, long checksum) throws IOException { - resourceGroupMgr = ResourceGroupMgr.read(in); - LOG.info("finished replay resource groups from image"); + public long loadWorkloadGroups(DataInputStream in, long checksum) throws IOException { + workloadGroupMgr = WorkloadGroupMgr.read(in); + LOG.info("finished replay workload groups from image"); return checksum; } @@ -2216,8 +2216,8 @@ public class Env { return checksum; } - public long saveResourceGroups(CountingDataOutputStream dos, long checksum) throws IOException { - Env.getCurrentEnv().getResourceGroupMgr().write(dos); + public long saveWorkloadGroups(CountingDataOutputStream dos, long checksum) throws IOException { + Env.getCurrentEnv().getWorkloadGroupMgr().write(dos); return checksum; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java index fec7da93a3..b3fdc3ca71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java @@ -295,7 +295,7 @@ public class ColocateTableCheckerAndBalancer extends MasterDaemon { } /* - * Each balance is performed for a single resource group in a colocate group. + * Each balance is performed for a single workload group in a colocate group. * For example, if the replica allocation of a colocate group is {TagA: 2, TagB: 1}, * So the backend bucket seq may be like: * @@ -304,10 +304,10 @@ public class ColocateTableCheckerAndBalancer extends MasterDaemon { * TagA B C A B * TagB D D D D * - * First, we will handle resource group of TagA, then TagB. + * First, we will handle workload group of TagA, then TagB. * - * For a single resource group, the balance logic is as follow - * (Suppose there is only one resource group with 3 replicas): + * For a single workload group, the balance logic is as follow + * (Suppose there is only one workload group with 3 replicas): * * All backends: A,B,C,D,E,F,G,H,I,J * diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 6e2dc1e895..58c411c40f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -220,7 +220,7 @@ public class TabletSchedCtx implements Comparable { // replicaAlloc is only set for REPAIR task private ReplicaAllocation replicaAlloc; - // tag is only set for BALANCE task, used to identify which resource group this Balance job is in + // tag is only set for BALANCE task, used to identify which workload group this Balance job is in private Tag tag; public TabletSchedCtx(Type type, long dbId, long tblId, long partId, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index 8addebfcb9..483855d8d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -129,8 +129,8 @@ public class FeNameFormat { checkCommonName("resource", resourceName); } - public static void checkResourceGroupName(String resourceGroupName) throws AnalysisException { - checkCommonName("resource group", resourceGroupName); + public static void checkWorkloadGroupName(String workloadGroupName) throws AnalysisException { + checkCommonName("workload group", workloadGroupName); } public static void checkCommonName(String type, String name) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 006930dbea..00906c2c8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -81,9 +81,9 @@ import org.apache.doris.persist.DatabaseInfo; import org.apache.doris.persist.DropDbInfo; import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.DropPartitionInfo; -import org.apache.doris.persist.DropResourceGroupOperationLog; import org.apache.doris.persist.DropResourceOperationLog; import org.apache.doris.persist.DropSqlBlockRuleOperationLog; +import org.apache.doris.persist.DropWorkloadGroupOperationLog; import org.apache.doris.persist.GlobalVarPersistInfo; import org.apache.doris.persist.HbPackage; import org.apache.doris.persist.LdapInfo; @@ -113,7 +113,7 @@ import org.apache.doris.plugin.PluginInfo; import org.apache.doris.policy.DropPolicyLog; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; -import org.apache.doris.resource.resourcegroup.ResourceGroup; +import org.apache.doris.resource.workloadgroup.WorkloadGroup; import org.apache.doris.system.Backend; import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; @@ -774,14 +774,14 @@ public class JournalEntity implements Writable { isRead = true; break; } - case OperationType.OP_CREATE_RESOURCE_GROUP: - case OperationType.OP_ALTER_RESOURCE_GROUP: { - data = ResourceGroup.read(in); + case OperationType.OP_CREATE_WORKLOAD_GROUP: + case OperationType.OP_ALTER_WORKLOAD_GROUP: { + data = WorkloadGroup.read(in); isRead = true; break; } - case OperationType.OP_DROP_RESOURCE_GROUP: { - data = DropResourceGroupOperationLog.read(in); + case OperationType.OP_DROP_WORKLOAD_GROUP: { + data = DropWorkloadGroupOperationLog.read(in); isRead = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java index 8514f14f24..a9b82166c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlProto.java @@ -78,10 +78,10 @@ public class MysqlProto { context.setCluster(SystemInfoService.DEFAULT_CLUSTER); - // check resource group level. user name may contains resource group level. + // check workload group level. user name may contains workload group level. // eg: // ...@user_name#HIGH - // set resource group if it is valid, or just ignore it + // set workload group if it is valid, or just ignore it String[] strList = tmpUser.split("#", 2); if (strList.length > 1) { tmpUser = strList[0]; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index b1e0dca03b..cf36639c0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -21,7 +21,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.resource.Tag; -import org.apache.doris.resource.resourcegroup.ResourceGroupMgr; +import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; @@ -59,7 +59,7 @@ public class CommonUserProperties implements Writable { private int insertTimeout = -1; @SerializedName("workloadGroup") - private String workloadGroup = ResourceGroupMgr.DEFAULT_GROUP_NAME; + private String workloadGroup = WorkloadGroupMgr.DEFAULT_GROUP_NAME; private String[] sqlBlockRulesSplit = {}; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropResourceGroupOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropWorkloadGroupOperationLog.java similarity index 86% rename from fe/fe-core/src/main/java/org/apache/doris/persist/DropResourceGroupOperationLog.java rename to fe/fe-core/src/main/java/org/apache/doris/persist/DropWorkloadGroupOperationLog.java index 5a912ab0f1..5dc6cc527a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropResourceGroupOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropWorkloadGroupOperationLog.java @@ -28,13 +28,13 @@ import java.io.DataOutput; import java.io.IOException; /** - * For resource group drop + * For workload group drop */ -public class DropResourceGroupOperationLog implements Writable { +public class DropWorkloadGroupOperationLog implements Writable { @SerializedName(value = "id") private long id; - public DropResourceGroupOperationLog(long id) { + public DropWorkloadGroupOperationLog(long id) { this.id = id; } @@ -47,7 +47,7 @@ public class DropResourceGroupOperationLog implements Writable { Text.writeString(out, GsonUtils.GSON.toJson(this)); } - public static DropResourceGroupOperationLog read(DataInput in) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(in), DropResourceGroupOperationLog.class); + public static DropWorkloadGroupOperationLog read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), DropWorkloadGroupOperationLog.class); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 55eb7759b5..5cbb47b93e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -79,7 +79,7 @@ import org.apache.doris.plugin.PluginInfo; import org.apache.doris.policy.DropPolicyLog; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; -import org.apache.doris.resource.resourcegroup.ResourceGroup; +import org.apache.doris.resource.workloadgroup.WorkloadGroup; import org.apache.doris.system.Backend; import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; @@ -990,20 +990,20 @@ public class EditLog { env.getCatalogMgr().replayRefreshExternalPartitions(log); break; } - case OperationType.OP_CREATE_RESOURCE_GROUP: { - final ResourceGroup resourceGroup = (ResourceGroup) journal.getData(); - env.getResourceGroupMgr().replayCreateResourceGroup(resourceGroup); + case OperationType.OP_CREATE_WORKLOAD_GROUP: { + final WorkloadGroup workloadGroup = (WorkloadGroup) journal.getData(); + env.getWorkloadGroupMgr().replayCreateWorkloadGroup(workloadGroup); break; } - case OperationType.OP_DROP_RESOURCE_GROUP: { - final DropResourceGroupOperationLog operationLog = - (DropResourceGroupOperationLog) journal.getData(); - env.getResourceGroupMgr().replayDropResourceGroup(operationLog); + case OperationType.OP_DROP_WORKLOAD_GROUP: { + final DropWorkloadGroupOperationLog operationLog = + (DropWorkloadGroupOperationLog) journal.getData(); + env.getWorkloadGroupMgr().replayDropWorkloadGroup(operationLog); break; } - case OperationType.OP_ALTER_RESOURCE_GROUP: { - final ResourceGroup resource = (ResourceGroup) journal.getData(); - env.getResourceGroupMgr().replayAlterResourceGroup(resource); + case OperationType.OP_ALTER_WORKLOAD_GROUP: { + final WorkloadGroup resource = (WorkloadGroup) journal.getData(); + env.getWorkloadGroupMgr().replayAlterWorkloadGroup(resource); break; } case OperationType.OP_INIT_EXTERNAL_TABLE: { @@ -1525,16 +1525,16 @@ public class EditLog { logEdit(OperationType.OP_ALTER_RESOURCE, resource); } - public void logAlterResourceGroup(ResourceGroup resourceGroup) { - logEdit(OperationType.OP_ALTER_RESOURCE_GROUP, resourceGroup); + public void logAlterWorkloadGroup(WorkloadGroup workloadGroup) { + logEdit(OperationType.OP_ALTER_WORKLOAD_GROUP, workloadGroup); } - public void logCreateResourceGroup(ResourceGroup resourceGroup) { - logEdit(OperationType.OP_CREATE_RESOURCE_GROUP, resourceGroup); + public void logCreateWorkloadGroup(WorkloadGroup workloadGroup) { + logEdit(OperationType.OP_CREATE_WORKLOAD_GROUP, workloadGroup); } - public void logDropResourceGroup(DropResourceGroupOperationLog operationLog) { - logEdit(OperationType.OP_DROP_RESOURCE_GROUP, operationLog); + public void logDropWorkloadGroup(DropWorkloadGroupOperationLog operationLog) { + logEdit(OperationType.OP_DROP_WORKLOAD_GROUP, operationLog); } public void logAlterStoragePolicy(StoragePolicy storagePolicy) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 8ffa0b609a..8261cb94d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -284,10 +284,10 @@ public class OperationType { public static final short OP_COOLDOWN_DELETE = 402; public static final short OP_ALTER_LIGHT_SCHEMA_CHANGE = 403; - // resource group 410 ~ 419 - public static final short OP_CREATE_RESOURCE_GROUP = 410; - public static final short OP_DROP_RESOURCE_GROUP = 411; - public static final short OP_ALTER_RESOURCE_GROUP = 412; + // workload group 410 ~ 419 + public static final short OP_CREATE_WORKLOAD_GROUP = 410; + public static final short OP_DROP_WORKLOAD_GROUP = 411; + public static final short OP_ALTER_WORKLOAD_GROUP = 412; // query stats 440 ~ 424 public static final short OP_CLEAN_QUERY_STATS = 420; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java index a142253ab8..65242d8822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java @@ -209,11 +209,11 @@ public class MetaPersistMethod { metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveGlobalFunction", CountingDataOutputStream.class, long.class); break; - case "resourceGroups": + case "workloadGroups": metaPersistMethod.readMethod = - Env.class.getDeclaredMethod("loadResourceGroups", DataInputStream.class, long.class); + Env.class.getDeclaredMethod("loadWorkloadGroups", DataInputStream.class, long.class); metaPersistMethod.writeMethod = - Env.class.getDeclaredMethod("saveResourceGroups", CountingDataOutputStream.class, long.class); + Env.class.getDeclaredMethod("saveWorkloadGroups", CountingDataOutputStream.class, long.class); break; case "binlogs": metaPersistMethod.readMethod = diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java index 31154c9743..366eda7655 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java @@ -38,7 +38,7 @@ public class PersistMetaModules { "masterInfo", "frontends", "backends", "datasource", "db", "alterJob", "recycleBin", "globalVariable", "cluster", "broker", "resources", "exportJob", "syncJob", "backupHandler", "paloAuth", "transactionState", "colocateTableIndex", "routineLoadJobs", "loadJobV2", "smallFiles", - "plugins", "deleteHandler", "sqlBlockRule", "policy", "mtmvJobManager", "globalFunction", "resourceGroups", + "plugins", "deleteHandler", "sqlBlockRule", "policy", "mtmvJobManager", "globalFunction", "workloadGroups", "binlogs"); // Modules in this list is deprecated and will not be saved in meta file. (also should not be in MODULE_NAMES) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index f37de81217..4d3c30b7f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -91,7 +91,7 @@ import org.apache.doris.thrift.TPaloScanRange; import org.apache.doris.thrift.TPipelineFragmentParams; import org.apache.doris.thrift.TPipelineFragmentParamsList; import org.apache.doris.thrift.TPipelineInstanceParams; -import org.apache.doris.thrift.TPipelineResourceGroup; +import org.apache.doris.thrift.TPipelineWorkloadGroup; import org.apache.doris.thrift.TPlanFragmentDestination; import org.apache.doris.thrift.TPlanFragmentExecParams; import org.apache.doris.thrift.TQueryGlobals; @@ -260,7 +260,7 @@ public class Coordinator { private StatsErrorEstimator statsErrorEstimator; - private List tResourceGroups = Lists.newArrayList(); + private List tWorkloadGroups = Lists.newArrayList(); private final ExecutionProfile executionProfile; @@ -348,7 +348,7 @@ public class Coordinator { nextInstanceId.setHi(queryId.hi); nextInstanceId.setLo(queryId.lo + 1); this.assignedRuntimeFilters = planner.getRuntimeFilters(); - this.tResourceGroups = analyzer == null ? null : analyzer.getResourceGroups(); + this.tWorkloadGroups = analyzer == null ? null : analyzer.getWorkloadGroups(); this.executionProfile = new ExecutionProfile(queryId, fragments.size()); } @@ -3219,8 +3219,8 @@ public class Coordinator { fragment.isTransferQueryStatisticsWithEveryBatch()); params.setFragment(fragment.toThrift()); params.setLocalParams(Lists.newArrayList()); - if (tResourceGroups != null) { - params.setResourceGroups(tResourceGroups); + if (tWorkloadGroups != null) { + params.setWorkloadGroups(tWorkloadGroups); } res.put(instanceExecParam.host, params); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 4ac228cf10..6aa6a9c616 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -34,7 +34,6 @@ import org.apache.doris.analysis.AlterDatabaseQuotaStmt; import org.apache.doris.analysis.AlterDatabaseRename; import org.apache.doris.analysis.AlterMaterializedViewStmt; import org.apache.doris.analysis.AlterPolicyStmt; -import org.apache.doris.analysis.AlterResourceGroupStmt; import org.apache.doris.analysis.AlterResourceStmt; import org.apache.doris.analysis.AlterRoutineLoadStmt; import org.apache.doris.analysis.AlterSqlBlockRuleStmt; @@ -43,6 +42,7 @@ import org.apache.doris.analysis.AlterTableStatsStmt; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.AlterUserStmt; import org.apache.doris.analysis.AlterViewStmt; +import org.apache.doris.analysis.AlterWorkloadGroupStmt; import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.BackupStmt; import org.apache.doris.analysis.CancelAlterSystemStmt; @@ -63,7 +63,6 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; import org.apache.doris.analysis.CreatePolicyStmt; import org.apache.doris.analysis.CreateRepositoryStmt; -import org.apache.doris.analysis.CreateResourceGroupStmt; import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.analysis.CreateRoleStmt; import org.apache.doris.analysis.CreateRoutineLoadStmt; @@ -73,6 +72,7 @@ import org.apache.doris.analysis.CreateTableLikeStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.CreateUserStmt; import org.apache.doris.analysis.CreateViewStmt; +import org.apache.doris.analysis.CreateWorkloadGroupStmt; import org.apache.doris.analysis.DdlStmt; import org.apache.doris.analysis.DeleteStmt; import org.apache.doris.analysis.DropCatalogStmt; @@ -83,13 +83,13 @@ import org.apache.doris.analysis.DropFunctionStmt; import org.apache.doris.analysis.DropMaterializedViewStmt; import org.apache.doris.analysis.DropPolicyStmt; import org.apache.doris.analysis.DropRepositoryStmt; -import org.apache.doris.analysis.DropResourceGroupStmt; import org.apache.doris.analysis.DropResourceStmt; import org.apache.doris.analysis.DropRoleStmt; import org.apache.doris.analysis.DropSqlBlockRuleStmt; import org.apache.doris.analysis.DropStatsStmt; import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.DropUserStmt; +import org.apache.doris.analysis.DropWorkloadGroupStmt; import org.apache.doris.analysis.GrantStmt; import org.apache.doris.analysis.InstallPluginStmt; import org.apache.doris.analysis.KillAnalysisJobStmt; @@ -257,10 +257,10 @@ public class DdlExecutor { env.getResourceMgr().createResource((CreateResourceStmt) ddlStmt); } else if (ddlStmt instanceof DropResourceStmt) { env.getResourceMgr().dropResource((DropResourceStmt) ddlStmt); - } else if (ddlStmt instanceof CreateResourceGroupStmt) { - env.getResourceGroupMgr().createResourceGroup((CreateResourceGroupStmt) ddlStmt); - } else if (ddlStmt instanceof DropResourceGroupStmt) { - env.getResourceGroupMgr().dropResourceGroup((DropResourceGroupStmt) ddlStmt); + } else if (ddlStmt instanceof CreateWorkloadGroupStmt) { + env.getWorkloadGroupMgr().createWorkloadGroup((CreateWorkloadGroupStmt) ddlStmt); + } else if (ddlStmt instanceof DropWorkloadGroupStmt) { + env.getWorkloadGroupMgr().dropWorkloadGroup((DropWorkloadGroupStmt) ddlStmt); } else if (ddlStmt instanceof CreateDataSyncJobStmt) { CreateDataSyncJobStmt createSyncJobStmt = (CreateDataSyncJobStmt) ddlStmt; SyncJobManager syncJobMgr = env.getSyncJobManager(); @@ -298,8 +298,8 @@ public class DdlExecutor { env.createAnalysisJob((AnalyzeStmt) ddlStmt); } else if (ddlStmt instanceof AlterResourceStmt) { env.getResourceMgr().alterResource((AlterResourceStmt) ddlStmt); - } else if (ddlStmt instanceof AlterResourceGroupStmt) { - env.getResourceGroupMgr().alterResourceGroup((AlterResourceGroupStmt) ddlStmt); + } else if (ddlStmt instanceof AlterWorkloadGroupStmt) { + env.getWorkloadGroupMgr().alterWorkloadGroup((AlterWorkloadGroupStmt) ddlStmt); } else if (ddlStmt instanceof CreatePolicyStmt) { env.getPolicyMgr().createPolicy((CreatePolicyStmt) ddlStmt); } else if (ddlStmt instanceof DropPolicyStmt) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index aadb76b158..00ed7a501d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -69,6 +69,7 @@ public class SessionVariable implements Serializable, Writable { public static final String INSERT_TIMEOUT = "insert_timeout"; public static final String ENABLE_PROFILE = "enable_profile"; public static final String SQL_MODE = "sql_mode"; + public static final String WORKLOAD_VARIABLE = "workload_group"; public static final String RESOURCE_VARIABLE = "resource_group"; public static final String AUTO_COMMIT = "autocommit"; public static final String TX_ISOLATION = "tx_isolation"; @@ -404,6 +405,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = SQL_MODE, needForward = true) public long sqlMode = SqlModeHelper.MODE_DEFAULT; + @VariableMgr.VarAttr(name = WORKLOAD_VARIABLE) + public String workloadGroup = ""; + @VariableMgr.VarAttr(name = RESOURCE_VARIABLE) public String resourceGroup = ""; @@ -1270,6 +1274,14 @@ public class SessionVariable implements Serializable, Writable { this.queryTimeoutS = this.maxExecutionTimeMS / 1000; } + public String getWorkloadGroup() { + return workloadGroup; + } + + public void setWorkloadGroup(String workloadGroup) { + this.workloadGroup = workloadGroup; + } + public String getResourceGroup() { return resourceGroup; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 42b7d52b2a..093722ff66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -76,7 +76,6 @@ import org.apache.doris.analysis.ShowProcesslistStmt; import org.apache.doris.analysis.ShowQueryProfileStmt; import org.apache.doris.analysis.ShowQueryStatsStmt; import org.apache.doris.analysis.ShowRepositoriesStmt; -import org.apache.doris.analysis.ShowResourceGroupsStmt; import org.apache.doris.analysis.ShowResourcesStmt; import org.apache.doris.analysis.ShowRestoreStmt; import org.apache.doris.analysis.ShowRolesStmt; @@ -102,6 +101,7 @@ import org.apache.doris.analysis.ShowTypeCastStmt; import org.apache.doris.analysis.ShowUserPropertyStmt; import org.apache.doris.analysis.ShowVariablesStmt; import org.apache.doris.analysis.ShowViewStmt; +import org.apache.doris.analysis.ShowWorkloadGroupsStmt; import org.apache.doris.analysis.TableName; import org.apache.doris.backup.AbstractJob; import org.apache.doris.backup.BackupJob; @@ -332,8 +332,8 @@ public class ShowExecutor { handleShowBroker(); } else if (stmt instanceof ShowResourcesStmt) { handleShowResources(); - } else if (stmt instanceof ShowResourceGroupsStmt) { - handleShowResourceGroups(); + } else if (stmt instanceof ShowWorkloadGroupsStmt) { + handleShowWorkloadGroups(); } else if (stmt instanceof ShowExportStmt) { handleShowExport(); } else if (stmt instanceof ShowBackendsStmt) { @@ -1807,11 +1807,11 @@ public class ShowExecutor { resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } - private void handleShowResourceGroups() { - ShowResourceGroupsStmt showStmt = (ShowResourceGroupsStmt) stmt; - List> resourceGroupsInfos = Env.getCurrentEnv().getResourceGroupMgr().getResourcesInfo(); + private void handleShowWorkloadGroups() { + ShowWorkloadGroupsStmt showStmt = (ShowWorkloadGroupsStmt) stmt; + List> workloadGroupsInfos = Env.getCurrentEnv().getWorkloadGroupMgr().getResourcesInfo(); - resultSet = new ShowResultSet(showStmt.getMetaData(), resourceGroupsInfos); + resultSet = new ShowResultSet(showStmt.getMetaData(), workloadGroupsInfos); } private void handleShowExport() throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index a890b43d4e..a759d4de0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -135,8 +135,8 @@ import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.cache.Cache; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.qe.cache.CacheAnalyzer.CacheMode; -import org.apache.doris.resource.resourcegroup.QueryQueue; -import org.apache.doris.resource.resourcegroup.QueueOfferToken; +import org.apache.doris.resource.workloadgroup.QueryQueue; +import org.apache.doris.resource.workloadgroup.QueueOfferToken; import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import org.apache.doris.rpc.RpcException; @@ -558,9 +558,9 @@ public class StmtExecutor { private void handleQueryWithRetry(TUniqueId queryId) throws Exception { // queue query here - if (!parsedStmt.isExplain() && Config.enable_resource_group && Config.enable_query_queue) { - this.queryQueue = analyzer.getEnv().getResourceGroupMgr() - .getResourceGroupQueryQueue(context.sessionVariable.resourceGroup); + if (!parsedStmt.isExplain() && Config.enable_workload_group && Config.enable_query_queue) { + this.queryQueue = analyzer.getEnv().getWorkloadGroupMgr() + .getWorkloadGroupQueryQueue(context.sessionVariable.workloadGroup); try { this.offerRet = queryQueue.offer(); } catch (InterruptedException e) { @@ -1094,10 +1094,10 @@ public class StmtExecutor { parsedStmt.setIsExplain(explainOptions); } } - if (parsedStmt instanceof QueryStmt && Config.enable_resource_group + if (parsedStmt instanceof QueryStmt && Config.enable_workload_group && context.sessionVariable.enablePipelineEngine()) { - analyzer.setResourceGroups(analyzer.getEnv().getResourceGroupMgr() - .getResourceGroup(context)); + analyzer.setWorkloadGroups(analyzer.getEnv().getWorkloadGroupMgr() + .getWorkloadGroup(context)); } } profile.getSummaryProfile().setQueryAnalysisFinishTime(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgr.java deleted file mode 100644 index d764c64ce8..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgr.java +++ /dev/null @@ -1,325 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.resource.resourcegroup; - -import org.apache.doris.analysis.AlterResourceGroupStmt; -import org.apache.doris.analysis.CreateResourceGroupStmt; -import org.apache.doris.analysis.DropResourceGroupStmt; -import org.apache.doris.catalog.Env; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.UserException; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.common.proc.BaseProcResult; -import org.apache.doris.common.proc.ProcNodeInterface; -import org.apache.doris.common.proc.ProcResult; -import org.apache.doris.persist.DropResourceGroupOperationLog; -import org.apache.doris.persist.gson.GsonPostProcessable; -import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TPipelineResourceGroup; - -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.gson.annotations.SerializedName; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -public class ResourceGroupMgr implements Writable, GsonPostProcessable { - - private static final Logger LOG = LogManager.getLogger(ResourceGroupMgr.class); - - public static final String DEFAULT_GROUP_NAME = "normal"; - - public static final ImmutableList RESOURCE_GROUP_PROC_NODE_TITLE_NAMES = new ImmutableList.Builder() - .add("Id").add("Name").add("Item").add("Value") - .build(); - - @SerializedName(value = "idToResourceGroup") - private final Map idToResourceGroup = Maps.newHashMap(); - - private final Map nameToResourceGroup = Maps.newHashMap(); - - private final ResourceProcNode procNode = new ResourceProcNode(); - - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - public ResourceGroupMgr() { - } - - private void readLock() { - lock.readLock().lock(); - } - - private void readUnlock() { - lock.readLock().unlock(); - } - - private void writeLock() { - lock.writeLock().lock(); - } - - private void writeUnlock() { - lock.writeLock().unlock(); - } - - private void checkResourceGroupEnabled() throws DdlException { - if (!Config.enable_resource_group) { - throw new DdlException("unsupported feature now, coming soon."); - } - } - - public void init() { - if (Config.enable_resource_group || Config.use_fuzzy_session_variable /* for github workflow */) { - checkAndCreateDefaultGroup(); - } - } - - public List getResourceGroup(ConnectContext context) throws UserException { - String groupName = context.getSessionVariable().getResourceGroup(); - if (Strings.isNullOrEmpty(groupName)) { - groupName = Env.getCurrentEnv().getAuth().getWorkloadGroup(context.getQualifiedUser()); - } - List resourceGroups = Lists.newArrayList(); - readLock(); - try { - ResourceGroup resourceGroup = nameToResourceGroup.get(groupName); - if (resourceGroup == null) { - throw new UserException("Resource group " + groupName + " does not exist"); - } - resourceGroups.add(resourceGroup.toThrift()); - } finally { - readUnlock(); - } - return resourceGroups; - } - - public QueryQueue getResourceGroupQueryQueue(String groupName) throws UserException { - readLock(); - try { - ResourceGroup resourceGroup = nameToResourceGroup.get(groupName); - if (resourceGroup == null) { - throw new UserException("Resource group " + groupName + " does not exist"); - } - return resourceGroup.getQueryQueue(); - } finally { - readUnlock(); - } - } - - private void checkAndCreateDefaultGroup() { - ResourceGroup defaultResourceGroup = null; - writeLock(); - try { - if (nameToResourceGroup.containsKey(DEFAULT_GROUP_NAME)) { - return; - } - Map properties = Maps.newHashMap(); - properties.put(ResourceGroup.CPU_SHARE, "10"); - properties.put(ResourceGroup.MEMORY_LIMIT, "30%"); - properties.put(ResourceGroup.ENABLE_MEMORY_OVERCOMMIT, "true"); - defaultResourceGroup = ResourceGroup.create(DEFAULT_GROUP_NAME, properties); - nameToResourceGroup.put(DEFAULT_GROUP_NAME, defaultResourceGroup); - idToResourceGroup.put(defaultResourceGroup.getId(), defaultResourceGroup); - Env.getCurrentEnv().getEditLog().logCreateResourceGroup(defaultResourceGroup); - } catch (DdlException e) { - LOG.warn("Create resource group " + DEFAULT_GROUP_NAME + " fail"); - } finally { - writeUnlock(); - } - LOG.info("Create resource group success: {}", defaultResourceGroup); - } - - public void createResourceGroup(CreateResourceGroupStmt stmt) throws DdlException { - checkResourceGroupEnabled(); - - ResourceGroup resourceGroup = ResourceGroup.create(stmt.getResourceGroupName(), stmt.getProperties()); - String resourceGroupName = resourceGroup.getName(); - writeLock(); - try { - if (nameToResourceGroup.containsKey(resourceGroupName)) { - if (stmt.isIfNotExists()) { - return; - } - throw new DdlException("Resource group " + resourceGroupName + " already exist"); - } - checkGlobalUnlock(resourceGroup, null); - nameToResourceGroup.put(resourceGroupName, resourceGroup); - idToResourceGroup.put(resourceGroup.getId(), resourceGroup); - Env.getCurrentEnv().getEditLog().logCreateResourceGroup(resourceGroup); - } finally { - writeUnlock(); - } - LOG.info("Create resource group success: {}", resourceGroup); - } - - private void checkGlobalUnlock(ResourceGroup resourceGroup, ResourceGroup old) throws DdlException { - double totalMemoryLimit = idToResourceGroup.values().stream().mapToDouble(ResourceGroup::getMemoryLimitPercent) - .sum() + resourceGroup.getMemoryLimitPercent(); - if (!Objects.isNull(old)) { - totalMemoryLimit -= old.getMemoryLimitPercent(); - } - if (totalMemoryLimit > 100.0 + 1e-6) { - throw new DdlException( - "The sum of all resource group " + ResourceGroup.MEMORY_LIMIT + " cannot be greater than 100.0%."); - } - } - - public void alterResourceGroup(AlterResourceGroupStmt stmt) throws DdlException { - checkResourceGroupEnabled(); - - String resourceGroupName = stmt.getResourceGroupName(); - Map properties = stmt.getProperties(); - ResourceGroup newResourceGroup; - writeLock(); - try { - if (!nameToResourceGroup.containsKey(resourceGroupName)) { - throw new DdlException("Resource Group(" + resourceGroupName + ") does not exist."); - } - ResourceGroup resourceGroup = nameToResourceGroup.get(resourceGroupName); - newResourceGroup = ResourceGroup.copyAndUpdate(resourceGroup, properties); - checkGlobalUnlock(newResourceGroup, resourceGroup); - nameToResourceGroup.put(resourceGroupName, newResourceGroup); - idToResourceGroup.put(newResourceGroup.getId(), newResourceGroup); - Env.getCurrentEnv().getEditLog().logAlterResourceGroup(newResourceGroup); - } finally { - writeUnlock(); - } - LOG.info("Alter resource success: {}", newResourceGroup); - } - - public void dropResourceGroup(DropResourceGroupStmt stmt) throws DdlException { - checkResourceGroupEnabled(); - - String resourceGroupName = stmt.getResourceGroupName(); - if (DEFAULT_GROUP_NAME.equals(resourceGroupName)) { - throw new DdlException("Dropping default resource group " + resourceGroupName + " is not allowed"); - } - - writeLock(); - try { - if (!nameToResourceGroup.containsKey(resourceGroupName)) { - if (stmt.isIfExists()) { - return; - } - throw new DdlException("Resource group " + resourceGroupName + " does not exist"); - } - ResourceGroup resourceGroup = nameToResourceGroup.get(resourceGroupName); - long groupId = resourceGroup.getId(); - idToResourceGroup.remove(groupId); - nameToResourceGroup.remove(resourceGroupName); - Env.getCurrentEnv().getEditLog().logDropResourceGroup(new DropResourceGroupOperationLog(groupId)); - } finally { - writeUnlock(); - } - LOG.info("Drop resource group success: {}", resourceGroupName); - } - - private void insertResourceGroup(ResourceGroup resourceGroup) { - writeLock(); - try { - nameToResourceGroup.put(resourceGroup.getName(), resourceGroup); - idToResourceGroup.put(resourceGroup.getId(), resourceGroup); - } finally { - writeUnlock(); - } - } - - public void replayCreateResourceGroup(ResourceGroup resourceGroup) { - insertResourceGroup(resourceGroup); - } - - public void replayAlterResourceGroup(ResourceGroup resourceGroup) { - insertResourceGroup(resourceGroup); - } - - public void replayDropResourceGroup(DropResourceGroupOperationLog operationLog) { - long id = operationLog.getId(); - writeLock(); - try { - if (!idToResourceGroup.containsKey(id)) { - return; - } - ResourceGroup resourceGroup = idToResourceGroup.get(id); - nameToResourceGroup.remove(resourceGroup.getName()); - idToResourceGroup.remove(id); - } finally { - writeUnlock(); - } - } - - public List> getResourcesInfo() { - return procNode.fetchResult().getRows(); - } - - // for ut - public Map getNameToResourceGroup() { - return nameToResourceGroup; - } - - // for ut - public Map getIdToResourceGroup() { - return idToResourceGroup; - } - - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } - - public static ResourceGroupMgr read(DataInput in) throws IOException { - String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, ResourceGroupMgr.class); - } - - @Override - public void gsonPostProcess() throws IOException { - idToResourceGroup.forEach( - (id, resourceGroup) -> nameToResourceGroup.put(resourceGroup.getName(), resourceGroup)); - } - - public class ResourceProcNode implements ProcNodeInterface { - @Override - public ProcResult fetchResult() { - BaseProcResult result = new BaseProcResult(); - result.setNames(RESOURCE_GROUP_PROC_NODE_TITLE_NAMES); - readLock(); - try { - for (ResourceGroup resourceGroup : idToResourceGroup.values()) { - // need to check resource group privs - resourceGroup.getProcNodeData(result); - } - } finally { - readUnlock(); - } - return result; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/QueryQueue.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/QueryQueue.java rename to fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java index 4b464bd94d..8f364fc4a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/QueryQueue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueryQueue.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.resource.resourcegroup; +package org.apache.doris.resource.workloadgroup; import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/QueueOfferToken.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueueOfferToken.java similarity index 96% rename from fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/QueueOfferToken.java rename to fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueueOfferToken.java index 4096a1095d..adf16e21b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/QueueOfferToken.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/QueueOfferToken.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.resource.resourcegroup; +package org.apache.doris.resource.workloadgroup; // used to mark QueryQueue offer result // if offer failed, then need to cancel query diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroup.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java similarity index 88% rename from fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroup.java rename to fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java index 039209df44..bf2e99c992 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.resource.resourcegroup; +package org.apache.doris.resource.workloadgroup; import org.apache.doris.catalog.Env; import org.apache.doris.common.DdlException; @@ -24,7 +24,7 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.thrift.TPipelineResourceGroup; +import org.apache.doris.thrift.TPipelineWorkloadGroup; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; @@ -40,8 +40,8 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -public class ResourceGroup implements Writable, GsonPostProcessable { - private static final Logger LOG = LogManager.getLogger(ResourceGroup.class); +public class WorkloadGroup implements Writable, GsonPostProcessable { + private static final Logger LOG = LogManager.getLogger(WorkloadGroup.class); public static final String CPU_SHARE = "cpu_share"; @@ -82,11 +82,11 @@ public class ResourceGroup implements Writable, GsonPostProcessable { private int maxQueueSize = 0; private int queueTimeout = 0; - private ResourceGroup(long id, String name, Map properties) { + private WorkloadGroup(long id, String name, Map properties) { this(id, name, properties, 0); } - private ResourceGroup(long id, String name, Map properties, long version) { + private WorkloadGroup(long id, String name, Map properties, long version) { this.id = id; this.name = name; this.properties = properties; @@ -138,17 +138,17 @@ public class ResourceGroup implements Writable, GsonPostProcessable { } // new resource group - public static ResourceGroup create(String name, Map properties) throws DdlException { + public static WorkloadGroup create(String name, Map properties) throws DdlException { checkProperties(properties); - ResourceGroup newResourceGroup = new ResourceGroup(Env.getCurrentEnv().getNextId(), name, properties); - newResourceGroup.initQueryQueue(); - return newResourceGroup; + WorkloadGroup newWorkloadGroup = new WorkloadGroup(Env.getCurrentEnv().getNextId(), name, properties); + newWorkloadGroup.initQueryQueue(); + return newWorkloadGroup; } // alter resource group - public static ResourceGroup copyAndUpdate(ResourceGroup resourceGroup, Map updateProperties) + public static WorkloadGroup copyAndUpdate(WorkloadGroup workloadGroup, Map updateProperties) throws DdlException { - Map newProperties = new HashMap<>(resourceGroup.getProperties()); + Map newProperties = new HashMap<>(workloadGroup.getProperties()); for (Map.Entry kv : updateProperties.entrySet()) { if (!Strings.isNullOrEmpty(kv.getValue())) { newProperties.put(kv.getKey(), kv.getValue()); @@ -156,13 +156,13 @@ public class ResourceGroup implements Writable, GsonPostProcessable { } checkProperties(newProperties); - ResourceGroup newResourceGroup = new ResourceGroup( - resourceGroup.getId(), resourceGroup.getName(), newProperties, resourceGroup.getVersion() + 1); + WorkloadGroup newWorkloadGroup = new WorkloadGroup( + workloadGroup.getId(), workloadGroup.getName(), newProperties, workloadGroup.getVersion() + 1); // note(wb) query queue should be unique and can not be copy - newResourceGroup.resetQueryQueue(resourceGroup.getQueryQueue()); + newWorkloadGroup.resetQueryQueue(workloadGroup.getQueryQueue()); - return newResourceGroup; + return newWorkloadGroup; } private static void checkProperties(Map properties) throws DdlException { @@ -264,8 +264,8 @@ public class ResourceGroup implements Writable, GsonPostProcessable { return GsonUtils.GSON.toJson(this); } - public TPipelineResourceGroup toThrift() { - return new TPipelineResourceGroup().setId(id).setName(name).setProperties(properties).setVersion(version); + public TPipelineWorkloadGroup toThrift() { + return new TPipelineWorkloadGroup().setId(id).setName(name).setProperties(properties).setVersion(version); } @Override @@ -274,9 +274,9 @@ public class ResourceGroup implements Writable, GsonPostProcessable { Text.writeString(out, json); } - public static ResourceGroup read(DataInput in) throws IOException { + public static WorkloadGroup read(DataInput in) throws IOException { String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, ResourceGroup.class); + return GsonUtils.GSON.fromJson(json, WorkloadGroup.class); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java new file mode 100644 index 0000000000..1813271a4d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java @@ -0,0 +1,325 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.resource.workloadgroup; + +import org.apache.doris.analysis.AlterWorkloadGroupStmt; +import org.apache.doris.analysis.CreateWorkloadGroupStmt; +import org.apache.doris.analysis.DropWorkloadGroupStmt; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.proc.BaseProcResult; +import org.apache.doris.common.proc.ProcNodeInterface; +import org.apache.doris.common.proc.ProcResult; +import org.apache.doris.persist.DropWorkloadGroupOperationLog; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TPipelineWorkloadGroup; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class WorkloadGroupMgr implements Writable, GsonPostProcessable { + + private static final Logger LOG = LogManager.getLogger(WorkloadGroupMgr.class); + + public static final String DEFAULT_GROUP_NAME = "normal"; + + public static final ImmutableList WORKLOAD_GROUP_PROC_NODE_TITLE_NAMES = new ImmutableList.Builder() + .add("Id").add("Name").add("Item").add("Value") + .build(); + + @SerializedName(value = "idToWorkloadGroup") + private final Map idToWorkloadGroup = Maps.newHashMap(); + + private final Map nameToWorkloadGroup = Maps.newHashMap(); + + private final ResourceProcNode procNode = new ResourceProcNode(); + + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + public WorkloadGroupMgr() { + } + + private void readLock() { + lock.readLock().lock(); + } + + private void readUnlock() { + lock.readLock().unlock(); + } + + private void writeLock() { + lock.writeLock().lock(); + } + + private void writeUnlock() { + lock.writeLock().unlock(); + } + + private void checkWorkloadGroupEnabled() throws DdlException { + if (!Config.enable_workload_group) { + throw new DdlException("unsupported feature now, coming soon."); + } + } + + public void init() { + if (Config.enable_workload_group || Config.use_fuzzy_session_variable /* for github workflow */) { + checkAndCreateDefaultGroup(); + } + } + + public List getWorkloadGroup(ConnectContext context) throws UserException { + String groupName = context.getSessionVariable().getWorkloadGroup(); + if (Strings.isNullOrEmpty(groupName)) { + groupName = Env.getCurrentEnv().getAuth().getWorkloadGroup(context.getQualifiedUser()); + } + List workloadGroups = Lists.newArrayList(); + readLock(); + try { + WorkloadGroup workloadGroup = nameToWorkloadGroup.get(groupName); + if (workloadGroup == null) { + throw new UserException("Workload group " + groupName + " does not exist"); + } + workloadGroups.add(workloadGroup.toThrift()); + } finally { + readUnlock(); + } + return workloadGroups; + } + + public QueryQueue getWorkloadGroupQueryQueue(String groupName) throws UserException { + readLock(); + try { + WorkloadGroup workloadGroup = nameToWorkloadGroup.get(groupName); + if (workloadGroup == null) { + throw new UserException("Workload group " + groupName + " does not exist"); + } + return workloadGroup.getQueryQueue(); + } finally { + readUnlock(); + } + } + + private void checkAndCreateDefaultGroup() { + WorkloadGroup defaultWorkloadGroup = null; + writeLock(); + try { + if (nameToWorkloadGroup.containsKey(DEFAULT_GROUP_NAME)) { + return; + } + Map properties = Maps.newHashMap(); + properties.put(WorkloadGroup.CPU_SHARE, "10"); + properties.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + properties.put(WorkloadGroup.ENABLE_MEMORY_OVERCOMMIT, "true"); + defaultWorkloadGroup = WorkloadGroup.create(DEFAULT_GROUP_NAME, properties); + nameToWorkloadGroup.put(DEFAULT_GROUP_NAME, defaultWorkloadGroup); + idToWorkloadGroup.put(defaultWorkloadGroup.getId(), defaultWorkloadGroup); + Env.getCurrentEnv().getEditLog().logCreateWorkloadGroup(defaultWorkloadGroup); + } catch (DdlException e) { + LOG.warn("Create workload group " + DEFAULT_GROUP_NAME + " fail"); + } finally { + writeUnlock(); + } + LOG.info("Create workload group success: {}", defaultWorkloadGroup); + } + + public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws DdlException { + checkWorkloadGroupEnabled(); + + WorkloadGroup workloadGroup = WorkloadGroup.create(stmt.getWorkloadGroupName(), stmt.getProperties()); + String workloadGroupName = workloadGroup.getName(); + writeLock(); + try { + if (nameToWorkloadGroup.containsKey(workloadGroupName)) { + if (stmt.isIfNotExists()) { + return; + } + throw new DdlException("workload group " + workloadGroupName + " already exist"); + } + checkGlobalUnlock(workloadGroup, null); + nameToWorkloadGroup.put(workloadGroupName, workloadGroup); + idToWorkloadGroup.put(workloadGroup.getId(), workloadGroup); + Env.getCurrentEnv().getEditLog().logCreateWorkloadGroup(workloadGroup); + } finally { + writeUnlock(); + } + LOG.info("Create workload group success: {}", workloadGroup); + } + + private void checkGlobalUnlock(WorkloadGroup workloadGroup, WorkloadGroup old) throws DdlException { + double totalMemoryLimit = idToWorkloadGroup.values().stream().mapToDouble(WorkloadGroup::getMemoryLimitPercent) + .sum() + workloadGroup.getMemoryLimitPercent(); + if (!Objects.isNull(old)) { + totalMemoryLimit -= old.getMemoryLimitPercent(); + } + if (totalMemoryLimit > 100.0 + 1e-6) { + throw new DdlException( + "The sum of all workload group " + WorkloadGroup.MEMORY_LIMIT + " cannot be greater than 100.0%."); + } + } + + public void alterWorkloadGroup(AlterWorkloadGroupStmt stmt) throws DdlException { + checkWorkloadGroupEnabled(); + + String workloadGroupName = stmt.getWorkloadGroupName(); + Map properties = stmt.getProperties(); + WorkloadGroup newWorkloadGroup; + writeLock(); + try { + if (!nameToWorkloadGroup.containsKey(workloadGroupName)) { + throw new DdlException("workload group(" + workloadGroupName + ") does not exist."); + } + WorkloadGroup workloadGroup = nameToWorkloadGroup.get(workloadGroupName); + newWorkloadGroup = WorkloadGroup.copyAndUpdate(workloadGroup, properties); + checkGlobalUnlock(newWorkloadGroup, workloadGroup); + nameToWorkloadGroup.put(workloadGroupName, newWorkloadGroup); + idToWorkloadGroup.put(newWorkloadGroup.getId(), newWorkloadGroup); + Env.getCurrentEnv().getEditLog().logAlterWorkloadGroup(newWorkloadGroup); + } finally { + writeUnlock(); + } + LOG.info("Alter resource success: {}", newWorkloadGroup); + } + + public void dropWorkloadGroup(DropWorkloadGroupStmt stmt) throws DdlException { + checkWorkloadGroupEnabled(); + + String workloadGroupName = stmt.getWorkloadGroupName(); + if (DEFAULT_GROUP_NAME.equals(workloadGroupName)) { + throw new DdlException("Dropping default workload group " + workloadGroupName + " is not allowed"); + } + + writeLock(); + try { + if (!nameToWorkloadGroup.containsKey(workloadGroupName)) { + if (stmt.isIfExists()) { + return; + } + throw new DdlException("workload group " + workloadGroupName + " does not exist"); + } + WorkloadGroup workloadGroup = nameToWorkloadGroup.get(workloadGroupName); + long groupId = workloadGroup.getId(); + idToWorkloadGroup.remove(groupId); + nameToWorkloadGroup.remove(workloadGroupName); + Env.getCurrentEnv().getEditLog().logDropWorkloadGroup(new DropWorkloadGroupOperationLog(groupId)); + } finally { + writeUnlock(); + } + LOG.info("Drop workload group success: {}", workloadGroupName); + } + + private void insertWorkloadGroup(WorkloadGroup workloadGroup) { + writeLock(); + try { + nameToWorkloadGroup.put(workloadGroup.getName(), workloadGroup); + idToWorkloadGroup.put(workloadGroup.getId(), workloadGroup); + } finally { + writeUnlock(); + } + } + + public void replayCreateWorkloadGroup(WorkloadGroup workloadGroup) { + insertWorkloadGroup(workloadGroup); + } + + public void replayAlterWorkloadGroup(WorkloadGroup workloadGroup) { + insertWorkloadGroup(workloadGroup); + } + + public void replayDropWorkloadGroup(DropWorkloadGroupOperationLog operationLog) { + long id = operationLog.getId(); + writeLock(); + try { + if (!idToWorkloadGroup.containsKey(id)) { + return; + } + WorkloadGroup workloadGroup = idToWorkloadGroup.get(id); + nameToWorkloadGroup.remove(workloadGroup.getName()); + idToWorkloadGroup.remove(id); + } finally { + writeUnlock(); + } + } + + public List> getResourcesInfo() { + return procNode.fetchResult().getRows(); + } + + // for ut + public Map getNameToWorkloadGroup() { + return nameToWorkloadGroup; + } + + // for ut + public Map getIdToWorkloadGroup() { + return idToWorkloadGroup; + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } + + public static WorkloadGroupMgr read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, WorkloadGroupMgr.class); + } + + @Override + public void gsonPostProcess() throws IOException { + idToWorkloadGroup.forEach( + (id, workloadGroup) -> nameToWorkloadGroup.put(workloadGroup.getName(), workloadGroup)); + } + + public class ResourceProcNode implements ProcNodeInterface { + @Override + public ProcResult fetchResult() { + BaseProcResult result = new BaseProcResult(); + result.setNames(WORKLOAD_GROUP_PROC_NODE_TITLE_NAMES); + readLock(); + try { + for (WorkloadGroup workloadGroup : idToWorkloadGroup.values()) { + // need to check workload group privs + workloadGroup.getProcNodeData(result); + } + } finally { + readUnlock(); + } + return result; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index f288172692..e43e619542 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -73,8 +73,8 @@ public class MetadataGenerator { case BACKENDS: result = backendsMetadataResult(params); break; - case RESOURCE_GROUPS: - result = resourceGroupsMetadataResult(params); + case WORKLOAD_GROUPS: + result = workloadGroupsMetadataResult(params); break; default: return errorResult("Metadata table params is not set."); @@ -229,12 +229,12 @@ public class MetadataGenerator { return result; } - private static TFetchSchemaTableDataResult resourceGroupsMetadataResult(TMetadataTableRequestParams params) { - List> resourceGroupsInfo = Env.getCurrentEnv().getResourceGroupMgr() + private static TFetchSchemaTableDataResult workloadGroupsMetadataResult(TMetadataTableRequestParams params) { + List> workloadGroupsInfo = Env.getCurrentEnv().getWorkloadGroupMgr() .getResourcesInfo(); TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); List dataBatch = Lists.newArrayList(); - for (List rGroupsInfo : resourceGroupsInfo) { + for (List rGroupsInfo : workloadGroupsInfo) { TRow trow = new TRow(); Long id = Long.valueOf(rGroupsInfo.get(0)); trow.addToColumnValue(new TCell().setLongVal(id)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java index 45e521a9e5..dc5f68a6db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java @@ -33,8 +33,8 @@ public abstract class MetadataTableValuedFunction extends TableValuedFunctionIf return BackendsTableValuedFunction.getColumnIndexFromColumnName(columnName); case ICEBERG: return IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName); - case RESOURCE_GROUPS: - return ResourceGroupsTableValuedFunction.getColumnIndexFromColumnName(columnName); + case WORKLOAD_GROUPS: + return WorkloadGroupsTableValuedFunction.getColumnIndexFromColumnName(columnName); default: throw new AnalysisException("Unknown Metadata TableValuedFunction type"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index d6ade9b5e9..5a44e7cd5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -53,8 +53,8 @@ public abstract class TableValuedFunctionIf { return new IcebergTableValuedFunction(params); case BackendsTableValuedFunction.NAME: return new BackendsTableValuedFunction(params); - case ResourceGroupsTableValuedFunction.NAME: - return new ResourceGroupsTableValuedFunction(params); + case WorkloadGroupsTableValuedFunction.NAME: + return new WorkloadGroupsTableValuedFunction(params); default: throw new AnalysisException("Could not find table function " + funcName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/WorkloadGroupsTableValuedFunction.java similarity index 86% rename from fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java rename to fe/fe-core/src/main/java/org/apache/doris/tablefunction/WorkloadGroupsTableValuedFunction.java index 0ded6c90df..90f7ea1109 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/WorkloadGroupsTableValuedFunction.java @@ -32,10 +32,10 @@ import java.util.Map; /** * The Implement of table valued function - * resource_groups(). + * workload_groups(). */ -public class ResourceGroupsTableValuedFunction extends MetadataTableValuedFunction { - public static final String NAME = "resource_groups"; +public class WorkloadGroupsTableValuedFunction extends MetadataTableValuedFunction { + public static final String NAME = "workload_groups"; private static final ImmutableList SCHEMA = ImmutableList.of( new Column("Id", ScalarType.createType(PrimitiveType.BIGINT)), @@ -57,27 +57,27 @@ public class ResourceGroupsTableValuedFunction extends MetadataTableValuedFuncti return COLUMN_TO_INDEX.get(columnName.toLowerCase()); } - public ResourceGroupsTableValuedFunction(Map params) throws AnalysisException { + public WorkloadGroupsTableValuedFunction(Map params) throws AnalysisException { if (params.size() != 0) { - throw new AnalysisException("resource groups table-valued-function does not support any params"); + throw new AnalysisException("workload groups table-valued-function does not support any params"); } } @Override public TMetadataType getMetadataType() { - return TMetadataType.RESOURCE_GROUPS; + return TMetadataType.WORKLOAD_GROUPS; } @Override public TMetaScanRange getMetaScanRange() { TMetaScanRange metaScanRange = new TMetaScanRange(); - metaScanRange.setMetadataType(TMetadataType.RESOURCE_GROUPS); + metaScanRange.setMetadataType(TMetadataType.WORKLOAD_GROUPS); return metaScanRange; } @Override public String getTableName() { - return "ResourceGroupsTableValuedFunction"; + return "WorkloadGroupsTableValuedFunction"; } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgrTest.java deleted file mode 100644 index 562e71017d..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgrTest.java +++ /dev/null @@ -1,219 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.resource.resourcegroup; - -import org.apache.doris.analysis.AlterResourceGroupStmt; -import org.apache.doris.analysis.CreateResourceGroupStmt; -import org.apache.doris.analysis.DropResourceGroupStmt; -import org.apache.doris.catalog.Env; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.UserException; -import org.apache.doris.persist.EditLog; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TPipelineResourceGroup; - -import com.google.common.collect.Maps; -import mockit.Delegate; -import mockit.Expectations; -import mockit.Injectable; -import mockit.Mocked; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -public class ResourceGroupMgrTest { - - @Injectable - private EditLog editLog; - - @Mocked - private Env env; - - private AtomicLong id = new AtomicLong(10); - - @Before - public void setUp() throws DdlException { - new Expectations() { - { - env.getEditLog(); - minTimes = 0; - result = editLog; - - env.getNextId(); - minTimes = 0; - result = new Delegate() { - long delegate() { - return id.addAndGet(1); - } - }; - - editLog.logCreateResourceGroup((ResourceGroup) any); - minTimes = 0; - - Env.getCurrentEnv(); - minTimes = 0; - result = env; - } - }; - } - - @Test - public void testCreateResourceGroup() throws DdlException { - Config.enable_resource_group = true; - ResourceGroupMgr resourceGroupMgr = new ResourceGroupMgr(); - Map properties1 = Maps.newHashMap(); - properties1.put(ResourceGroup.CPU_SHARE, "10"); - properties1.put(ResourceGroup.MEMORY_LIMIT, "30%"); - String name1 = "g1"; - CreateResourceGroupStmt stmt1 = new CreateResourceGroupStmt(false, name1, properties1); - resourceGroupMgr.createResourceGroup(stmt1); - - Map nameToRG = resourceGroupMgr.getNameToResourceGroup(); - Assert.assertEquals(1, nameToRG.size()); - Assert.assertTrue(nameToRG.containsKey(name1)); - ResourceGroup group1 = nameToRG.get(name1); - Assert.assertEquals(name1, group1.getName()); - - Map idToRG = resourceGroupMgr.getIdToResourceGroup(); - Assert.assertEquals(1, idToRG.size()); - Assert.assertTrue(idToRG.containsKey(group1.getId())); - - Map properties2 = Maps.newHashMap(); - properties2.put(ResourceGroup.CPU_SHARE, "20"); - properties2.put(ResourceGroup.MEMORY_LIMIT, "30%"); - String name2 = "g2"; - CreateResourceGroupStmt stmt2 = new CreateResourceGroupStmt(false, name2, properties2); - resourceGroupMgr.createResourceGroup(stmt2); - - nameToRG = resourceGroupMgr.getNameToResourceGroup(); - Assert.assertEquals(2, nameToRG.size()); - Assert.assertTrue(nameToRG.containsKey(name2)); - ResourceGroup group2 = nameToRG.get(name2); - idToRG = resourceGroupMgr.getIdToResourceGroup(); - Assert.assertEquals(2, idToRG.size()); - Assert.assertTrue(idToRG.containsKey(group2.getId())); - - try { - resourceGroupMgr.createResourceGroup(stmt2); - Assert.fail(); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("already exist")); - } - - CreateResourceGroupStmt stmt3 = new CreateResourceGroupStmt(true, name2, properties2); - resourceGroupMgr.createResourceGroup(stmt3); - Assert.assertEquals(2, resourceGroupMgr.getIdToResourceGroup().size()); - Assert.assertEquals(2, resourceGroupMgr.getNameToResourceGroup().size()); - } - - @Test - public void testGetResourceGroup() throws UserException { - Config.enable_resource_group = true; - ConnectContext context = new ConnectContext(); - ResourceGroupMgr resourceGroupMgr = new ResourceGroupMgr(); - Map properties1 = Maps.newHashMap(); - properties1.put(ResourceGroup.CPU_SHARE, "10"); - properties1.put(ResourceGroup.MEMORY_LIMIT, "30%"); - String name1 = "g1"; - CreateResourceGroupStmt stmt1 = new CreateResourceGroupStmt(false, name1, properties1); - resourceGroupMgr.createResourceGroup(stmt1); - context.getSessionVariable().setResourceGroup(name1); - List tResourceGroups1 = resourceGroupMgr.getResourceGroup(context); - Assert.assertEquals(1, tResourceGroups1.size()); - TPipelineResourceGroup tResourceGroup1 = tResourceGroups1.get(0); - Assert.assertEquals(name1, tResourceGroup1.getName()); - Assert.assertTrue(tResourceGroup1.getProperties().containsKey(ResourceGroup.CPU_SHARE)); - - try { - context.getSessionVariable().setResourceGroup("g2"); - resourceGroupMgr.getResourceGroup(context); - Assert.fail(); - } catch (UserException e) { - Assert.assertTrue(e.getMessage().contains("does not exist")); - } - } - - @Test - public void testDropResourceGroup() throws UserException { - Config.enable_resource_group = true; - ConnectContext context = new ConnectContext(); - ResourceGroupMgr resourceGroupMgr = new ResourceGroupMgr(); - Map properties = Maps.newHashMap(); - properties.put(ResourceGroup.CPU_SHARE, "10"); - properties.put(ResourceGroup.MEMORY_LIMIT, "30%"); - String name = "g1"; - CreateResourceGroupStmt createStmt = new CreateResourceGroupStmt(false, name, properties); - resourceGroupMgr.createResourceGroup(createStmt); - context.getSessionVariable().setResourceGroup(name); - Assert.assertEquals(1, resourceGroupMgr.getResourceGroup(context).size()); - - DropResourceGroupStmt dropStmt = new DropResourceGroupStmt(false, name); - resourceGroupMgr.dropResourceGroup(dropStmt); - try { - context.getSessionVariable().setResourceGroup(name); - resourceGroupMgr.getResourceGroup(context); - Assert.fail(); - } catch (UserException e) { - Assert.assertTrue(e.getMessage().contains("does not exist")); - } - - DropResourceGroupStmt dropDefaultStmt = new DropResourceGroupStmt(false, ResourceGroupMgr.DEFAULT_GROUP_NAME); - try { - resourceGroupMgr.dropResourceGroup(dropDefaultStmt); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("is not allowed")); - } - } - - @Test - public void testAlterResourceGroup() throws UserException { - Config.enable_resource_group = true; - ConnectContext context = new ConnectContext(); - ResourceGroupMgr resourceGroupMgr = new ResourceGroupMgr(); - Map properties = Maps.newHashMap(); - String name = "g1"; - try { - AlterResourceGroupStmt stmt1 = new AlterResourceGroupStmt(name, properties); - resourceGroupMgr.alterResourceGroup(stmt1); - } catch (DdlException e) { - Assert.assertTrue(e.getMessage().contains("does not exist")); - } - - properties.put(ResourceGroup.CPU_SHARE, "10"); - properties.put(ResourceGroup.MEMORY_LIMIT, "30%"); - CreateResourceGroupStmt createStmt = new CreateResourceGroupStmt(false, name, properties); - resourceGroupMgr.createResourceGroup(createStmt); - - Map newProperties = Maps.newHashMap(); - newProperties.put(ResourceGroup.CPU_SHARE, "5"); - newProperties.put(ResourceGroup.MEMORY_LIMIT, "30%"); - AlterResourceGroupStmt stmt2 = new AlterResourceGroupStmt(name, newProperties); - resourceGroupMgr.alterResourceGroup(stmt2); - - context.getSessionVariable().setResourceGroup(name); - List tResourceGroups = resourceGroupMgr.getResourceGroup(context); - Assert.assertEquals(1, tResourceGroups.size()); - TPipelineResourceGroup tResourceGroup1 = tResourceGroups.get(0); - Assert.assertEquals(tResourceGroup1.getProperties().get(ResourceGroup.CPU_SHARE), "5"); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java new file mode 100644 index 0000000000..31e6030c84 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgrTest.java @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.resource.workloadgroup; + +import org.apache.doris.analysis.AlterWorkloadGroupStmt; +import org.apache.doris.analysis.CreateWorkloadGroupStmt; +import org.apache.doris.analysis.DropWorkloadGroupStmt; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.UserException; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TPipelineWorkloadGroup; + +import com.google.common.collect.Maps; +import mockit.Delegate; +import mockit.Expectations; +import mockit.Injectable; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +public class WorkloadGroupMgrTest { + + @Injectable + private EditLog editLog; + + @Mocked + private Env env; + + private AtomicLong id = new AtomicLong(10); + + @Before + public void setUp() throws DdlException { + new Expectations() { + { + env.getEditLog(); + minTimes = 0; + result = editLog; + + env.getNextId(); + minTimes = 0; + result = new Delegate() { + long delegate() { + return id.addAndGet(1); + } + }; + + editLog.logCreateWorkloadGroup((WorkloadGroup) any); + minTimes = 0; + + Env.getCurrentEnv(); + minTimes = 0; + result = env; + } + }; + } + + @Test + public void testCreateWorkloadGroup() throws DdlException { + Config.enable_workload_group = true; + WorkloadGroupMgr workloadGroupMgr = new WorkloadGroupMgr(); + Map properties1 = Maps.newHashMap(); + properties1.put(WorkloadGroup.CPU_SHARE, "10"); + properties1.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + String name1 = "g1"; + CreateWorkloadGroupStmt stmt1 = new CreateWorkloadGroupStmt(false, name1, properties1); + workloadGroupMgr.createWorkloadGroup(stmt1); + + Map nameToRG = workloadGroupMgr.getNameToWorkloadGroup(); + Assert.assertEquals(1, nameToRG.size()); + Assert.assertTrue(nameToRG.containsKey(name1)); + WorkloadGroup group1 = nameToRG.get(name1); + Assert.assertEquals(name1, group1.getName()); + + Map idToRG = workloadGroupMgr.getIdToWorkloadGroup(); + Assert.assertEquals(1, idToRG.size()); + Assert.assertTrue(idToRG.containsKey(group1.getId())); + + Map properties2 = Maps.newHashMap(); + properties2.put(WorkloadGroup.CPU_SHARE, "20"); + properties2.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + String name2 = "g2"; + CreateWorkloadGroupStmt stmt2 = new CreateWorkloadGroupStmt(false, name2, properties2); + workloadGroupMgr.createWorkloadGroup(stmt2); + + nameToRG = workloadGroupMgr.getNameToWorkloadGroup(); + Assert.assertEquals(2, nameToRG.size()); + Assert.assertTrue(nameToRG.containsKey(name2)); + WorkloadGroup group2 = nameToRG.get(name2); + idToRG = workloadGroupMgr.getIdToWorkloadGroup(); + Assert.assertEquals(2, idToRG.size()); + Assert.assertTrue(idToRG.containsKey(group2.getId())); + + try { + workloadGroupMgr.createWorkloadGroup(stmt2); + Assert.fail(); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("already exist")); + } + + CreateWorkloadGroupStmt stmt3 = new CreateWorkloadGroupStmt(true, name2, properties2); + workloadGroupMgr.createWorkloadGroup(stmt3); + Assert.assertEquals(2, workloadGroupMgr.getIdToWorkloadGroup().size()); + Assert.assertEquals(2, workloadGroupMgr.getNameToWorkloadGroup().size()); + } + + @Test + public void testGetWorkloadGroup() throws UserException { + Config.enable_workload_group = true; + ConnectContext context = new ConnectContext(); + WorkloadGroupMgr workloadGroupMgr = new WorkloadGroupMgr(); + Map properties1 = Maps.newHashMap(); + properties1.put(WorkloadGroup.CPU_SHARE, "10"); + properties1.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + String name1 = "g1"; + CreateWorkloadGroupStmt stmt1 = new CreateWorkloadGroupStmt(false, name1, properties1); + workloadGroupMgr.createWorkloadGroup(stmt1); + context.getSessionVariable().setWorkloadGroup(name1); + List tWorkloadGroups1 = workloadGroupMgr.getWorkloadGroup(context); + Assert.assertEquals(1, tWorkloadGroups1.size()); + TPipelineWorkloadGroup tWorkloadGroup1 = tWorkloadGroups1.get(0); + Assert.assertEquals(name1, tWorkloadGroup1.getName()); + Assert.assertTrue(tWorkloadGroup1.getProperties().containsKey(WorkloadGroup.CPU_SHARE)); + + try { + context.getSessionVariable().setWorkloadGroup("g2"); + workloadGroupMgr.getWorkloadGroup(context); + Assert.fail(); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("does not exist")); + } + } + + @Test + public void testDropWorkloadGroup() throws UserException { + Config.enable_workload_group = true; + ConnectContext context = new ConnectContext(); + WorkloadGroupMgr workloadGroupMgr = new WorkloadGroupMgr(); + Map properties = Maps.newHashMap(); + properties.put(WorkloadGroup.CPU_SHARE, "10"); + properties.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + String name = "g1"; + CreateWorkloadGroupStmt createStmt = new CreateWorkloadGroupStmt(false, name, properties); + workloadGroupMgr.createWorkloadGroup(createStmt); + context.getSessionVariable().setWorkloadGroup(name); + Assert.assertEquals(1, workloadGroupMgr.getWorkloadGroup(context).size()); + + DropWorkloadGroupStmt dropStmt = new DropWorkloadGroupStmt(false, name); + workloadGroupMgr.dropWorkloadGroup(dropStmt); + try { + context.getSessionVariable().setWorkloadGroup(name); + workloadGroupMgr.getWorkloadGroup(context); + Assert.fail(); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("does not exist")); + } + + DropWorkloadGroupStmt dropDefaultStmt = new DropWorkloadGroupStmt(false, WorkloadGroupMgr.DEFAULT_GROUP_NAME); + try { + workloadGroupMgr.dropWorkloadGroup(dropDefaultStmt); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("is not allowed")); + } + } + + @Test + public void testAlterWorkloadGroup() throws UserException { + Config.enable_workload_group = true; + ConnectContext context = new ConnectContext(); + WorkloadGroupMgr workloadGroupMgr = new WorkloadGroupMgr(); + Map properties = Maps.newHashMap(); + String name = "g1"; + try { + AlterWorkloadGroupStmt stmt1 = new AlterWorkloadGroupStmt(name, properties); + workloadGroupMgr.alterWorkloadGroup(stmt1); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("does not exist")); + } + + properties.put(WorkloadGroup.CPU_SHARE, "10"); + properties.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + CreateWorkloadGroupStmt createStmt = new CreateWorkloadGroupStmt(false, name, properties); + workloadGroupMgr.createWorkloadGroup(createStmt); + + Map newProperties = Maps.newHashMap(); + newProperties.put(WorkloadGroup.CPU_SHARE, "5"); + newProperties.put(WorkloadGroup.MEMORY_LIMIT, "30%"); + AlterWorkloadGroupStmt stmt2 = new AlterWorkloadGroupStmt(name, newProperties); + workloadGroupMgr.alterWorkloadGroup(stmt2); + + context.getSessionVariable().setWorkloadGroup(name); + List tWorkloadGroups = workloadGroupMgr.getWorkloadGroup(context); + Assert.assertEquals(1, tWorkloadGroups.size()); + TPipelineWorkloadGroup tWorkloadGroup1 = tWorkloadGroups.get(0); + Assert.assertEquals(tWorkloadGroup1.getProperties().get(WorkloadGroup.CPU_SHARE), "5"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupTest.java b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupTest.java similarity index 72% rename from fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupTest.java rename to fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupTest.java index ff7199aa67..7009e55e38 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/resource/resourcegroup/ResourceGroupTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/resource/workloadgroup/WorkloadGroupTest.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.resource.resourcegroup; +package org.apache.doris.resource.workloadgroup; import org.apache.doris.common.DdlException; import org.apache.doris.common.proc.BaseProcResult; @@ -27,54 +27,54 @@ import org.junit.Test; import java.util.List; import java.util.Map; -public class ResourceGroupTest { +public class WorkloadGroupTest { @Test public void testCreateNormal() throws DdlException { Map properties1 = Maps.newHashMap(); - properties1.put(ResourceGroup.CPU_SHARE, "10"); - properties1.put(ResourceGroup.MEMORY_LIMIT, "30%"); + properties1.put(WorkloadGroup.CPU_SHARE, "10"); + properties1.put(WorkloadGroup.MEMORY_LIMIT, "30%"); String name1 = "g1"; - ResourceGroup group1 = ResourceGroup.create(name1, properties1); + WorkloadGroup group1 = WorkloadGroup.create(name1, properties1); Assert.assertEquals(name1, group1.getName()); Assert.assertEquals(5, group1.getProperties().size()); - Assert.assertTrue(group1.getProperties().containsKey(ResourceGroup.CPU_SHARE)); + Assert.assertTrue(group1.getProperties().containsKey(WorkloadGroup.CPU_SHARE)); Assert.assertTrue(Math.abs(group1.getMemoryLimitPercent() - 30) < 1e-6); } @Test(expected = DdlException.class) public void testNotSupportProperty() throws DdlException { Map properties1 = Maps.newHashMap(); - properties1.put(ResourceGroup.CPU_SHARE, "10"); - properties1.put(ResourceGroup.MEMORY_LIMIT, "30%"); + properties1.put(WorkloadGroup.CPU_SHARE, "10"); + properties1.put(WorkloadGroup.MEMORY_LIMIT, "30%"); properties1.put("share", "10"); String name1 = "g1"; - ResourceGroup.create(name1, properties1); + WorkloadGroup.create(name1, properties1); } @Test(expected = DdlException.class) public void testRequiredProperty() throws DdlException { Map properties1 = Maps.newHashMap(); String name1 = "g1"; - ResourceGroup.create(name1, properties1); + WorkloadGroup.create(name1, properties1); } @Test public void testCpuShareValue() { Map properties1 = Maps.newHashMap(); - properties1.put(ResourceGroup.CPU_SHARE, "0"); - properties1.put(ResourceGroup.MEMORY_LIMIT, "30%"); + properties1.put(WorkloadGroup.CPU_SHARE, "0"); + properties1.put(WorkloadGroup.MEMORY_LIMIT, "30%"); String name1 = "g1"; try { - ResourceGroup.create(name1, properties1); + WorkloadGroup.create(name1, properties1); Assert.fail(); } catch (DdlException e) { Assert.assertTrue(e.getMessage().contains("requires a positive integer.")); } - properties1.put(ResourceGroup.CPU_SHARE, "cpu"); + properties1.put(WorkloadGroup.CPU_SHARE, "cpu"); try { - ResourceGroup.create(name1, properties1); + WorkloadGroup.create(name1, properties1); Assert.fail(); } catch (DdlException e) { Assert.assertTrue(e.getMessage().contains("requires a positive integer.")); @@ -84,10 +84,10 @@ public class ResourceGroupTest { @Test public void testGetProcNodeData() throws DdlException { Map properties1 = Maps.newHashMap(); - properties1.put(ResourceGroup.CPU_SHARE, "10"); - properties1.put(ResourceGroup.MEMORY_LIMIT, "30%"); + properties1.put(WorkloadGroup.CPU_SHARE, "10"); + properties1.put(WorkloadGroup.MEMORY_LIMIT, "30%"); String name1 = "g1"; - ResourceGroup group1 = ResourceGroup.create(name1, properties1); + WorkloadGroup group1 = WorkloadGroup.create(name1, properties1); BaseProcResult result = new BaseProcResult(); group1.getProcNodeData(result); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 3b77b325b9..0c1a4feee8 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -589,7 +589,7 @@ struct TPipelineInstanceParams { 7: optional map per_node_shared_scans } -struct TPipelineResourceGroup { +struct TPipelineWorkloadGroup { 1: optional i64 id 2: optional string name 3: optional map properties @@ -624,7 +624,7 @@ struct TPipelineFragmentParams { 22: optional TGlobalDict global_dict // scan node could use the global dict to encode the string value to an integer 23: optional Planner.TPlanFragment fragment 24: list local_params - 26: optional list resource_groups + 26: optional list workload_groups } struct TPipelineFragmentParamsList { diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index ffaabf51f3..b0b5da4550 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -683,7 +683,7 @@ enum TSortType { enum TMetadataType { ICEBERG, BACKENDS, - RESOURCE_GROUPS + WORKLOAD_GROUPS } enum TIcebergQueryType {