From d24a1645d94fcdf64688c78b8f4978d059236f50 Mon Sep 17 00:00:00 2001 From: xueweizhang Date: Tue, 19 Dec 2023 11:44:33 +0800 Subject: [PATCH] [improvement](http) add show_table_data http api (#28380) In some cases, users need to get the data size of single replica of a table, and evaluate certain actions based on this, such as estimating the precise backup size. Signed-off-by: nextdreamblue --- .../http-actions/fe/show-table-data-action.md | 163 ++++++++++++++++++ docs/sidebars.json | 1 + .../http-actions/fe/show-table-data-action.md | 163 ++++++++++++++++++ .../doris/httpv2/rest/RestBaseController.java | 1 + .../apache/doris/httpv2/rest/ShowAction.java | 74 ++++++++ 5 files changed, 402 insertions(+) create mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md create mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md diff --git a/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md b/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md new file mode 100644 index 0000000000..9cdbe2c49a --- /dev/null +++ b/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md @@ -0,0 +1,163 @@ +--- +{ + "title": "Show Table Data Action", + "language": "en" +} +--- + + + +# Show Table Data Action + +## Request + +`GET /api/show_table_data` + +## Description + +Used to get the data size of all tables in all databases under all internal catalog, or the data size of the specified database or table. Unit byte. + +## Path parameters + +NULL + +## Query parameters + +* `db` + + Optional. If specified, get the data size of the tables under the specified database. + +* `table` + + Optional. If specified, get the data size of the specified table. + +* `single_replica` + + Optional. If specified, get the data size of the single replica of the table. + +## Request body + +NULL + +## Response + +1. The data size of all tables in the specified database. + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. The data size of the specified table of the specified db. + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` + +3. The data size of the single replica of the specified table of the specified db. + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` + +## Examples + +1. The data size of all tables in the specified database. + + ``` + GET /api/show_table_data?db=tpch + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. The data size of the specified table of the specified db. + + ``` + GET /api/show_table_data?db=tpch&table=partsupp + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` +3. The data size of the single replica of the specified table of the specified db. + + ``` + GET /api/show_table_data?db=tpch&table=partsupp&single_replica=true + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` diff --git a/docs/sidebars.json b/docs/sidebars.json index ea56ee81e1..0ba8f71b49 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -1264,6 +1264,7 @@ "admin-manual/http-actions/fe/show-meta-info-action", "admin-manual/http-actions/fe/show-proc-action", "admin-manual/http-actions/fe/show-runtime-info-action", + "admin-manual/http-actions/fe/show-table-data-action", "admin-manual/http-actions/fe/statement-execution-action", "admin-manual/http-actions/fe/table-query-plan-action", "admin-manual/http-actions/fe/table-row-count-action", diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md new file mode 100644 index 0000000000..97bb3dcf3b --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md @@ -0,0 +1,163 @@ +--- +{ + "title": "Show Table Data Action", + "language": "zh-CN" +} +--- + + + +# Show Table Data Action + +## Request + +`GET /api/show_table_data` + +## Description + +用于获取所有internal源下所有数据库所有表的数据量，或者指定数据库或指定表的数据量。单位字节。 + +## Path parameters + +无 + +## Query parameters + +* `db` + + 可选。如果指定，则获取指定数据库下表的数据量。 + +* `table` + + 可选。如果指定，则获取指定表的数据量。 + +* `single_replica` + + 可选。如果指定，则获取表单副本所占用的数据量。 + +## Request body + +无 + +## Response + +1. 指定数据库所有表的数据量。 + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. 指定数据库指定表的数据量。 + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` + +3. 指定数据库指定表单副本的数据量。 + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` + +## Examples + +1. 获取指定数据库的数据量 + + ``` + GET /api/show_table_data?db=tpch + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. 指定数据库指定表的数据量。 + + ``` + GET /api/show_table_data?db=tpch&table=partsupp + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` +3. 指定数据库指定表单副本的数据量。 + + ``` + GET /api/show_table_data?db=tpch&table=partsupp&single_replica=true + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java index faa7c7b72e..5621ffaccd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java @@ -54,6 +54,7 @@ public class RestBaseController extends BaseController { protected static final String LABEL_KEY = "label"; protected static final String TXN_ID_KEY = "txn_id"; protected static final String TXN_OPERATION_KEY = "txn_operation"; + protected static final String SINGLE_REPLICA_KEY = "single_replica"; private static final Logger LOG = LogManager.getLogger(RestBaseController.class); public ActionAuthorizationInfo executeCheckPassword(HttpServletRequest request, diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java index c1dc8f4380..6db2a7e7b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.proc.ProcNodeInterface; @@ -208,6 +209,39 @@ public class ShowAction extends RestBaseController { return ResponseEntityBuilder.ok(oneEntry); } + @RequestMapping(path = "/api/show_table_data", method = RequestMethod.GET) + public Object show_table_data(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + } + + String dbName = request.getParameter(DB_KEY); + String tableName = request.getParameter(TABLE_KEY); + String singleReplica = request.getParameter(SINGLE_REPLICA_KEY); + boolean singleReplicaBool = Boolean.parseBoolean(singleReplica); + Map> oneEntry = Maps.newHashMap(); + if (dbName != null) { + String fullDbName = getFullDbName(dbName); + DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("database " + fullDbName + " not found."); + } + Map tablesEntry = getDataSizeOfTables(db, tableName, singleReplicaBool); + oneEntry.put(ClusterNamespace.getNameFromFullName(fullDbName), tablesEntry); + } else { + for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) { + DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null || !(db instanceof Database) || ((Database) db).isMysqlCompatibleDatabase()) { + continue; + } + Map tablesEntry = getDataSizeOfTables(db, tableName, singleReplicaBool); + oneEntry.put(ClusterNamespace.getNameFromFullName(db.getFullName()), tablesEntry); + } + } + return ResponseEntityBuilder.ok(oneEntry); + } + private Map getHaInfo() throws IOException { HashMap feInfo = new HashMap(); feInfo.put("role", Env.getCurrentEnv().getFeType().toString()); @@ -289,6 +323,46 @@ public class ShowAction extends RestBaseController { return totalSize; } + public Map getDataSizeOfTables(DatabaseIf db, String tableName, boolean singleReplica) { + Map oneEntry = Maps.newHashMap(); + db.readLock(); + try { + if (Strings.isNullOrEmpty(tableName)) { + List tables = db.getTables(); + for (Table table : tables) { + Map tableEntry = getDataSizeOfTable(table, singleReplica); + oneEntry.putAll(tableEntry); + } + } else { + Table table = ((Database) db).getTableNullable(tableName); + if (table == null) { + return oneEntry; + } + Map tableEntry = getDataSizeOfTable(table, singleReplica); + oneEntry.putAll(tableEntry); + } + } finally { + db.readUnlock(); + } + return oneEntry; + } + + public Map getDataSizeOfTable(Table table, boolean singleReplica) { + Map oneEntry = Maps.newHashMap(); + if (table.getType() == TableType.VIEW || table.getType() == TableType.ODBC) { + oneEntry.put(table.getName(), 0L); + } else if (table.getType() == TableType.OLAP) { + table.readLock(); + try { + long tableSize = ((OlapTable) table).getDataSize(singleReplica); + oneEntry.put(table.getName(), tableSize); + } finally { + table.readUnlock(); + } + } + return oneEntry; + } + private Map getDataSize() { Map result = new HashMap(); List dbNames = Env.getCurrentInternalCatalog().getDbNames();