database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](meta) Add thrift rpc to get db/table/backend meta (#25943)

Browse Source 
Signed-off-by: Jack Drogon <jack.xsuperman@gmail.com>
This commit is contained in:
Jack Drogon
2023-10-27 10:12:58 +08:00
committed by GitHub
parent b32557e195
commit fa06c7517a
4 changed files with 380 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -251,7 +251,16 @@ import org.apache.doris.task.PriorityMasterTaskExecutor;
import org.apache.doris.thrift.BackendService;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TFrontendInfo;
import org.apache.doris.thrift.TGetMetaDBMeta;
import org.apache.doris.thrift.TGetMetaIndexMeta;
import org.apache.doris.thrift.TGetMetaPartitionMeta;
import org.apache.doris.thrift.TGetMetaReplicaMeta;
import org.apache.doris.thrift.TGetMetaResult;
import org.apache.doris.thrift.TGetMetaTableMeta;
import org.apache.doris.thrift.TGetMetaTabletMeta;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TStatus;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.transaction.DbUsedDataQuotaInfoCollector;
import org.apache.doris.transaction.GlobalTransactionMgr;
@ -5625,6 +5634,89 @@ public class Env {
return GlobalVariable.lowerCaseTableNames == 2;
}
private static void getTableMeta(OlapTable olapTable, TGetMetaDBMeta dbMeta) {
LOG.debug("get table meta. table: {}", olapTable.getName());
TGetMetaTableMeta tableMeta = new TGetMetaTableMeta();
olapTable.readLock();
try {
tableMeta.setId(olapTable.getId());
tableMeta.setName(olapTable.getName());
PartitionInfo tblPartitionInfo = olapTable.getPartitionInfo();
Collection<Partition> partitions = olapTable.getAllPartitions();
for (Partition partition : partitions) {
TGetMetaPartitionMeta partitionMeta = new TGetMetaPartitionMeta();
long partitionId = partition.getId();
partitionMeta.setId(partitionId);
partitionMeta.setName(partition.getName());
String partitionRange = "";
if (tblPartitionInfo.getType() == PartitionType.RANGE
|| tblPartitionInfo.getType() == PartitionType.LIST) {
partitionRange = tblPartitionInfo.getItem(partitionId).getItems().toString();
}
partitionMeta.setRange(partitionRange);
partitionMeta.setVisibleVersion(partition.getVisibleVersion());
// partitionMeta.setTemp（partition.isTemp());
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) {
TGetMetaIndexMeta indexMeta = new TGetMetaIndexMeta();
indexMeta.setId(index.getId());
indexMeta.setName(olapTable.getIndexNameById(index.getId()));
for (Tablet tablet : index.getTablets()) {
TGetMetaTabletMeta tabletMeta = new TGetMetaTabletMeta();
tabletMeta.setId(tablet.getId());
for (Replica replica : tablet.getReplicas()) {
TGetMetaReplicaMeta replicaMeta = new TGetMetaReplicaMeta();
replicaMeta.setId(replica.getId());
replicaMeta.setBackendId(replica.getBackendId());
replicaMeta.setVersion(replica.getVersion());
tabletMeta.addToReplicas(replicaMeta);
}
indexMeta.addToTablets(tabletMeta);
}
partitionMeta.addToIndexes(indexMeta);
}
tableMeta.addToPartitions(partitionMeta);
}
dbMeta.addToTables(tableMeta);
} finally {
olapTable.readUnlock();
}
}
public static TGetMetaResult getMeta(Database db, List<Table> tables) throws MetaNotFoundException {
TGetMetaResult result = new TGetMetaResult();
result.setStatus(new TStatus(TStatusCode.OK));
TGetMetaDBMeta dbMeta = new TGetMetaDBMeta();
dbMeta.setId(db.getId());
dbMeta.setName(db.getFullName());
if (tables == null) {
db.readLock();
tables = db.getTables();
db.readUnlock();
}
for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
continue;
}
OlapTable olapTable = (OlapTable) table;
getTableMeta(olapTable, dbMeta);
}
result.setDbMeta(dbMeta);
return result;
}
public void compactTable(AdminCompactTableStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
String tableName = stmt.getTblName();

174
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -107,6 +107,7 @@ import org.apache.doris.thrift.TAddColumnsRequest;
import org.apache.doris.thrift.TAddColumnsResult;
import org.apache.doris.thrift.TAutoIncrementRangeRequest;
import org.apache.doris.thrift.TAutoIncrementRangeResult;
import org.apache.doris.thrift.TBackend;
import org.apache.doris.thrift.TBeginTxnRequest;
import org.apache.doris.thrift.TBeginTxnResult;
import org.apache.doris.thrift.TBinlog;
@ -134,6 +135,8 @@ import org.apache.doris.thrift.TFinishTaskRequest;
import org.apache.doris.thrift.TFrontendPingFrontendRequest;
import org.apache.doris.thrift.TFrontendPingFrontendResult;
import org.apache.doris.thrift.TFrontendPingFrontendStatusCode;
import org.apache.doris.thrift.TGetBackendMetaRequest;
import org.apache.doris.thrift.TGetBackendMetaResult;
import org.apache.doris.thrift.TGetBinlogLagResult;
import org.apache.doris.thrift.TGetBinlogRequest;
import org.apache.doris.thrift.TGetBinlogResult;
@ -141,6 +144,10 @@ import org.apache.doris.thrift.TGetDbsParams;
import org.apache.doris.thrift.TGetDbsResult;
import org.apache.doris.thrift.TGetMasterTokenRequest;
import org.apache.doris.thrift.TGetMasterTokenResult;
import org.apache.doris.thrift.TGetMetaDB;
import org.apache.doris.thrift.TGetMetaRequest;
import org.apache.doris.thrift.TGetMetaResult;
import org.apache.doris.thrift.TGetMetaTable;
import org.apache.doris.thrift.TGetQueryStatsRequest;
import org.apache.doris.thrift.TGetSnapshotRequest;
import org.apache.doris.thrift.TGetSnapshotResult;
@ -3331,4 +3338,171 @@ public class FrontendServiceImpl implements FrontendService.Iface {
LOG.debug("send create partition result: {}", result);
return result;
}
public TGetMetaResult getMeta(TGetMetaRequest request) throws TException {
String clientAddr = getClientAddrAsString();
LOG.debug("receive get meta request: {}", request);
TGetMetaResult result = new TGetMetaResult();
TStatus status = new TStatus(TStatusCode.OK);
result.setStatus(status);
try {
result = getMetaImpl(request, clientAddr);
} catch (UserException e) {
LOG.warn("failed to get meta: {}", e.getMessage());
status.setStatusCode(TStatusCode.ANALYSIS_ERROR);
status.addToErrorMsgs(e.getMessage());
} catch (Throwable e) {
LOG.warn("catch unknown result.", e);
status.setStatusCode(TStatusCode.INTERNAL_ERROR);
status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage()));
}
return result;
}
private TGetMetaResult getMetaImpl(TGetMetaRequest request, String clientIp)
throws Exception {
// Step 1: check fields
if (!request.isSetUser()) {
throw new UserException("user is not set");
}
if (!request.isSetPasswd()) {
throw new UserException("passwd is not set");
}
if (!request.isSetDb()) {
throw new UserException("db is not set");
}
// Step 2: check auth
TGetMetaResult result = new TGetMetaResult();
result.setStatus(new TStatus(TStatusCode.OK));
Database db = null;
List<Table> tables = null;
String cluster = request.getCluster();
if (Strings.isNullOrEmpty(cluster)) {
cluster = SystemInfoService.DEFAULT_CLUSTER;
}
if (Strings.isNullOrEmpty(request.getToken())) {
TGetMetaDB getMetaDb = request.getDb();
if (getMetaDb.isSetId()) {
db = Env.getCurrentInternalCatalog().getDbNullable(getMetaDb.getId());
} else if (getMetaDb.isSetName()) {
db = Env.getCurrentInternalCatalog().getDbNullable(getMetaDb.getName());
}
if (db == null) {
LOG.warn("db not found {}", getMetaDb);
return result;
}
if (getMetaDb.isSetTables()) {
tables = Lists.newArrayList();
List<TGetMetaTable> getMetaTables = getMetaDb.getTables();
for (TGetMetaTable getMetaTable : getMetaTables) {
Table table = null;
if (getMetaTable.isSetId()) {
table = db.getTableNullable(getMetaTable.getId());
} else {
table = db.getTableNullable(getMetaTable.getName());
}
if (table == null) {
LOG.warn("table not found {}", getMetaTable);
continue;
}
tables.add(table);
}
}
if (tables == null) {
checkDbPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), db.getFullName(), clientIp,
PrivPredicate.SELECT);
} else {
List<String> tableList = Lists.newArrayList();
for (Table table : tables) {
tableList.add(table.getName());
}
checkPasswordAndPrivs(cluster, request.getUser(), request.getPasswd(), db.getFullName(), tableList,
clientIp,
PrivPredicate.SELECT);
}
}
// Step 3: get meta
try {
return Env.getMeta(db, tables);
} catch (Throwable e) {
throw e;
}
}
public TGetBackendMetaResult getBackendMeta(TGetBackendMetaRequest request) throws TException {
String clientAddr = getClientAddrAsString();
LOG.debug("receive get backend meta request: {}", request);
TGetBackendMetaResult result = new TGetBackendMetaResult();
TStatus status = new TStatus(TStatusCode.OK);
result.setStatus(status);
try {
result = getBackendMetaImpl(request, clientAddr);
} catch (UserException e) {
LOG.warn("failed to get backend meta: {}", e.getMessage());
status.setStatusCode(TStatusCode.ANALYSIS_ERROR);
status.addToErrorMsgs(e.getMessage());
} catch (Throwable e) {
LOG.warn("catch unknown result.", e);
status.setStatusCode(TStatusCode.INTERNAL_ERROR);
status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage()));
}
return result;
}
private TGetBackendMetaResult getBackendMetaImpl(TGetBackendMetaRequest request, String clientAddr)
throws UserException {
// Step 1: Check fields
if (!request.isSetUser()) {
throw new UserException("user is not set");
}
if (!request.isSetPasswd()) {
throw new UserException("passwd is not set");
}
// Step 2: check auth
String cluster = request.getCluster();
if (Strings.isNullOrEmpty(cluster)) {
cluster = SystemInfoService.DEFAULT_CLUSTER;
}
checkPassword(request.getCluster(), request.getUser(), request.getPasswd(), clientAddr);
// TODO: check getBackendMeta privilege, which privilege should be checked?
// Step 3: get meta
try {
TGetBackendMetaResult result = new TGetBackendMetaResult();
result.setStatus(new TStatus(TStatusCode.OK));
final SystemInfoService systemInfoService = Env.getCurrentSystemInfo();
List<Backend> backends = systemInfoService.getAllBackends();
for (Backend backend : backends) {
TBackend tBackend = new TBackend();
tBackend.setId(backend.getId());
tBackend.setHost(backend.getHost());
tBackend.setHttpPort(backend.getHttpPort());
tBackend.setBrpcPort(backend.getBrpcPort());
tBackend.setBePort(backend.getBePort());
tBackend.setIsAlive(backend.isAlive());
result.addToBackends(tBackend);
}
return result;
} catch (Throwable e) {
throw e;
}
}
}