diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 9a6d13552d..26b9bbd2e1 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -251,7 +251,16 @@ import org.apache.doris.task.PriorityMasterTaskExecutor; import org.apache.doris.thrift.BackendService; import org.apache.doris.thrift.TCompressionType; import org.apache.doris.thrift.TFrontendInfo; +import org.apache.doris.thrift.TGetMetaDBMeta; +import org.apache.doris.thrift.TGetMetaIndexMeta; +import org.apache.doris.thrift.TGetMetaPartitionMeta; +import org.apache.doris.thrift.TGetMetaReplicaMeta; +import org.apache.doris.thrift.TGetMetaResult; +import org.apache.doris.thrift.TGetMetaTableMeta; +import org.apache.doris.thrift.TGetMetaTabletMeta; import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TStatus; +import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.transaction.DbUsedDataQuotaInfoCollector; import org.apache.doris.transaction.GlobalTransactionMgr; @@ -5625,6 +5634,89 @@ public class Env { return GlobalVariable.lowerCaseTableNames == 2; } + private static void getTableMeta(OlapTable olapTable, TGetMetaDBMeta dbMeta) { + LOG.debug("get table meta. table: {}", olapTable.getName()); + + TGetMetaTableMeta tableMeta = new TGetMetaTableMeta(); + olapTable.readLock(); + try { + tableMeta.setId(olapTable.getId()); + tableMeta.setName(olapTable.getName()); + + PartitionInfo tblPartitionInfo = olapTable.getPartitionInfo(); + + Collection partitions = olapTable.getAllPartitions(); + for (Partition partition : partitions) { + TGetMetaPartitionMeta partitionMeta = new TGetMetaPartitionMeta(); + long partitionId = partition.getId(); + partitionMeta.setId(partitionId); + partitionMeta.setName(partition.getName()); + String partitionRange = ""; + if (tblPartitionInfo.getType() == PartitionType.RANGE + || tblPartitionInfo.getType() == PartitionType.LIST) { + partitionRange = tblPartitionInfo.getItem(partitionId).getItems().toString(); + } + partitionMeta.setRange(partitionRange); + partitionMeta.setVisibleVersion(partition.getVisibleVersion()); + // partitionMeta.setTemp（partition.isTemp()); + + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { + TGetMetaIndexMeta indexMeta = new TGetMetaIndexMeta(); + indexMeta.setId(index.getId()); + indexMeta.setName(olapTable.getIndexNameById(index.getId())); + + for (Tablet tablet : index.getTablets()) { + TGetMetaTabletMeta tabletMeta = new TGetMetaTabletMeta(); + tabletMeta.setId(tablet.getId()); + + for (Replica replica : tablet.getReplicas()) { + TGetMetaReplicaMeta replicaMeta = new TGetMetaReplicaMeta(); + replicaMeta.setId(replica.getId()); + replicaMeta.setBackendId(replica.getBackendId()); + replicaMeta.setVersion(replica.getVersion()); + tabletMeta.addToReplicas(replicaMeta); + } + + indexMeta.addToTablets(tabletMeta); + } + + partitionMeta.addToIndexes(indexMeta); + } + tableMeta.addToPartitions(partitionMeta); + } + dbMeta.addToTables(tableMeta); + } finally { + olapTable.readUnlock(); + } + } + + public static TGetMetaResult getMeta(Database db, List tables) throws MetaNotFoundException { + TGetMetaResult result = new TGetMetaResult(); + result.setStatus(new TStatus(TStatusCode.OK)); + + TGetMetaDBMeta dbMeta = new TGetMetaDBMeta(); + dbMeta.setId(db.getId()); + dbMeta.setName(db.getFullName()); + + if (tables == null) { + db.readLock(); + tables = db.getTables(); + db.readUnlock(); + } + + for (Table table : tables) { + if (table.getType() != TableType.OLAP) { + continue; + } + + OlapTable olapTable = (OlapTable) table; + getTableMeta(olapTable, dbMeta); + } + + result.setDbMeta(dbMeta); + return result; + } + public void compactTable(AdminCompactTableStmt stmt) throws DdlException { String dbName = stmt.getDbName(); String tableName = stmt.getTblName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index febd638e73..e6a883e07c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -107,6 +107,7 @@ import org.apache.doris.thrift.TAddColumnsRequest; import org.apache.doris.thrift.TAddColumnsResult; import org.apache.doris.thrift.TAutoIncrementRangeRequest; import org.apache.doris.thrift.TAutoIncrementRangeResult; +import org.apache.doris.thrift.TBackend; import org.apache.doris.thrift.TBeginTxnRequest; import org.apache.doris.thrift.TBeginTxnResult; import org.apache.doris.thrift.TBinlog; @@ -134,6 +135,8 @@ import org.apache.doris.thrift.TFinishTaskRequest; import org.apache.doris.thrift.TFrontendPingFrontendRequest; import org.apache.doris.thrift.TFrontendPingFrontendResult; import org.apache.doris.thrift.TFrontendPingFrontendStatusCode; +import org.apache.doris.thrift.TGetBackendMetaRequest; +import org.apache.doris.thrift.TGetBackendMetaResult; import org.apache.doris.thrift.TGetBinlogLagResult; import org.apache.doris.thrift.TGetBinlogRequest; import org.apache.doris.thrift.TGetBinlogResult; @@ -141,6 +144,10 @@ import org.apache.doris.thrift.TGetDbsParams; import org.apache.doris.thrift.TGetDbsResult; import org.apache.doris.thrift.TGetMasterTokenRequest; import org.apache.doris.thrift.TGetMasterTokenResult; +import org.apache.doris.thrift.TGetMetaDB; +import org.apache.doris.thrift.TGetMetaRequest; +import org.apache.doris.thrift.TGetMetaResult; +import org.apache.doris.thrift.TGetMetaTable; import org.apache.doris.thrift.TGetQueryStatsRequest; import org.apache.doris.thrift.TGetSnapshotRequest; import org.apache.doris.thrift.TGetSnapshotResult; @@ -3331,4 +3338,171 @@ public class FrontendServiceImpl implements FrontendService.Iface { LOG.debug("send create partition result: {}", result); return result; } + + public TGetMetaResult getMeta(TGetMetaRequest request) throws TException { + String clientAddr = getClientAddrAsString(); + LOG.debug("receive get meta request: {}", request); + + TGetMetaResult result = new TGetMetaResult(); + TStatus status = new TStatus(TStatusCode.OK); + result.setStatus(status); + try { + result = getMetaImpl(request, clientAddr); + } catch (UserException e) { + LOG.warn("failed to get meta: {}", e.getMessage()); + status.setStatusCode(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs(e.getMessage()); + } catch (Throwable e) { + LOG.warn("catch unknown result.", e); + status.setStatusCode(TStatusCode.INTERNAL_ERROR); + status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage())); + } + return result; + } + + private TGetMetaResult getMetaImpl(TGetMetaRequest request, String clientIp) + throws Exception { + // Step 1: check fields + if (!request.isSetUser()) { + throw new UserException("user is not set"); + } + if (!request.isSetPasswd()) { + throw new UserException("passwd is not set"); + } + if (!request.isSetDb()) { + throw new UserException("db is not set"); + } + + // Step 2: check auth + TGetMetaResult result = new TGetMetaResult(); + result.setStatus(new TStatus(TStatusCode.OK)); + Database db = null; + List
tables = null; + + String cluster = request.getCluster(); + if (Strings.isNullOrEmpty(cluster)) { + cluster = SystemInfoService.DEFAULT_CLUSTER; + } + if (Strings.isNullOrEmpty(request.getToken())) { + TGetMetaDB getMetaDb = request.getDb(); + + if (getMetaDb.isSetId()) { + db = Env.getCurrentInternalCatalog().getDbNullable(getMetaDb.getId()); + } else if (getMetaDb.isSetName()) { + db = Env.getCurrentInternalCatalog().getDbNullable(getMetaDb.getName()); + } + + if (db == null) { + LOG.warn("db not found {}", getMetaDb); + return result; + } + + if (getMetaDb.isSetTables()) { + tables = Lists.newArrayList(); + List getMetaTables = getMetaDb.getTables(); + for (TGetMetaTable getMetaTable : getMetaTables) { + Table table = null; + if (getMetaTable.isSetId()) { + table = db.getTableNullable(getMetaTable.getId()); + } else { + table = db.getTableNullable(getMetaTable.getName()); + } + + if (table == null) { + LOG.warn("table not found {}", getMetaTable); + continue; + } + + tables.add(table); + } + } + + if (tables == null) { + checkDbPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), db.getFullName(), clientIp, + PrivPredicate.SELECT); + } else { + List tableList = Lists.newArrayList(); + for (Table table : tables) { + tableList.add(table.getName()); + } + checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), db.getFullName(), tableList, + clientIp, + PrivPredicate.SELECT); + } + } + + // Step 3: get meta + try { + return Env.getMeta(db, tables); + } catch (Throwable e) { + throw e; + } + } + + public TGetBackendMetaResult getBackendMeta(TGetBackendMetaRequest request) throws TException { + String clientAddr = getClientAddrAsString(); + LOG.debug("receive get backend meta request: {}", request); + + TGetBackendMetaResult result = new TGetBackendMetaResult(); + TStatus status = new TStatus(TStatusCode.OK); + result.setStatus(status); + try { + result = getBackendMetaImpl(request, clientAddr); + } catch (UserException e) { + LOG.warn("failed to get backend meta: {}", e.getMessage()); + status.setStatusCode(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs(e.getMessage()); + } catch (Throwable e) { + LOG.warn("catch unknown result.", e); + status.setStatusCode(TStatusCode.INTERNAL_ERROR); + status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage())); + } + + return result; + } + + private TGetBackendMetaResult getBackendMetaImpl(TGetBackendMetaRequest request, String clientAddr) + throws UserException { + // Step 1: Check fields + if (!request.isSetUser()) { + throw new UserException("user is not set"); + } + if (!request.isSetPasswd()) { + throw new UserException("passwd is not set"); + } + + // Step 2: check auth + String cluster = request.getCluster(); + if (Strings.isNullOrEmpty(cluster)) { + cluster = SystemInfoService.DEFAULT_CLUSTER; + } + checkPassword(request.getCluster(), request.getUser(), request.getPasswd(), clientAddr); + + + // TODO: check getBackendMeta privilege, which privilege should be checked? + + // Step 3: get meta + try { + TGetBackendMetaResult result = new TGetBackendMetaResult(); + result.setStatus(new TStatus(TStatusCode.OK)); + + final SystemInfoService systemInfoService = Env.getCurrentSystemInfo(); + List backends = systemInfoService.getAllBackends(); + + for (Backend backend : backends) { + TBackend tBackend = new TBackend(); + tBackend.setId(backend.getId()); + tBackend.setHost(backend.getHost()); + tBackend.setHttpPort(backend.getHttpPort()); + tBackend.setBrpcPort(backend.getBrpcPort()); + tBackend.setBePort(backend.getBePort()); + tBackend.setIsAlive(backend.isAlive()); + result.addToBackends(tBackend); + } + + return result; + } catch (Throwable e) { + throw e; + } + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index ee593083fe..1e0ad2d462 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1183,6 +1183,113 @@ struct TCreatePartitionResult { 4: optional list nodes } +struct TGetMetaReplica { + 1: optional i64 id +} + +struct TGetMetaTablet { + 1: optional i64 id + 2: optional list replicas +} + +struct TGetMetaIndex { + 1: optional i64 id + 2: optional string name + 3: optional list tablets +} + +struct TGetMetaPartition { + 1: optional i64 id + 2: optional string name + 3: optional string key + 4: optional string range + 5: optional bool is_temp + 6: optional list indexes +} + +struct TGetMetaTable { + 1: optional i64 id + 2: optional string name + 3: optional bool in_trash + 4: optional list partitions +} + +struct TGetMetaDB { + 1: optional i64 id + 2: optional string name + 3: optional bool only_table_names + 4: optional list tables +} + +struct TGetMetaRequest { + 1: optional string cluster + 2: optional string user + 3: optional string passwd + 4: optional string user_ip + 5: optional string token + 6: optional TGetMetaDB db + // trash +} + +struct TGetMetaReplicaMeta { + 1: optional i64 id + 2: optional i64 backend_id + 3: optional i64 version +} + +struct TGetMetaTabletMeta { + 1: optional i64 id + 2: optional list replicas +} + +struct TGetMetaIndexMeta { + 1: optional i64 id + 2: optional string name + 3: optional list tablets +} + +struct TGetMetaPartitionMeta { + 1: optional i64 id + 2: optional string name + 3: optional string key + 4: optional string range + 5: optional i64 visible_version + 6: optional bool is_temp + 7: optional list indexes +} + +struct TGetMetaTableMeta { + 1: optional i64 id + 2: optional string name + 3: optional bool in_trash + 4: optional list partitions +} + +struct TGetMetaDBMeta { + 1: optional i64 id + 2: optional string name + 3: optional list tables +} + +struct TGetMetaResult { + 1: required Status.TStatus status + 2: optional TGetMetaDBMeta db_meta +} + +struct TGetBackendMetaRequest { + 1: optional string cluster + 2: optional string user + 3: optional string passwd + 4: optional string user_ip + 5: optional string token + 6: optional i64 backend_id +} + +struct TGetBackendMetaResult { + 1: required Status.TStatus status + 2: optional list backends +} + service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) @@ -1254,4 +1361,8 @@ service FrontendService { TAutoIncrementRangeResult getAutoIncrementRange(1: TAutoIncrementRangeRequest request) TCreatePartitionResult createPartition(1: TCreatePartitionRequest request) + + TGetMetaResult getMeta(1: TGetMetaRequest request) + + TGetBackendMetaResult getBackendMeta(1: TGetBackendMetaRequest request) } diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 7da21f30bc..69ad600952 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -621,6 +621,9 @@ struct TBackend { 1: required string host 2: required TPort be_port 3: required TPort http_port + 4: optional TPort brpc_port + 5: optional bool is_alive + 6: optional i64 id } struct TReplicaInfo {