database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhencement](Backends) add HostName filed in backends table and delete backends table in information_schema (#18156)

Browse Source 
1.  Add `HostName` field for `show backends` statement and `backends()` tvf.
2. delete the `backends` table in `information_schema` database
This commit is contained in:
Tiewei Fang
2023-04-07 08:30:42 +08:00
committed by GitHub
parent 22deeecbe1
commit 759f1da32e
70 changed files with 324 additions and 682 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
View File

@ -67,8 +67,7 @@ public enum SchemaTableType {
SCH_VIEWS("VIEWS", "VIEWS", TSchemaTableType.SCH_VIEWS),
SCH_CREATE_TABLE("CREATE_TABLE", "CREATE_TABLE", TSchemaTableType.SCH_CREATE_TABLE),
SCH_INVALID("NULL", "NULL", TSchemaTableType.SCH_INVALID),
SCH_ROWSETS("ROWSETS", "ROWSETS", TSchemaTableType.SCH_ROWSETS),
SCH_BACKENDS("BACKENDS", "BACKENDS", TSchemaTableType.SCH_BACKENDS);
SCH_ROWSETS("ROWSETS", "ROWSETS", TSchemaTableType.SCH_ROWSETS);
private static final String dbName = "INFORMATION_SCHEMA";
private static SelectList fullSelectLists;

4
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java
View File

@ -46,10 +46,6 @@ public class ShowBackendsStmt extends ShowStmt {
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
for (String title : BackendsProcDir.TITLE_NAMES) {
// hide hostname for SHOW BACKENDS stmt
if (title.equals("HostName")) {
continue;
}
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
}
return builder.build();

4
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java
View File

@ -46,10 +46,6 @@ public class ShowFrontendsStmt extends ShowStmt {
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
for (String title : FrontendsProcNode.TITLE_NAMES) {
// hide hostname for SHOW FRONTENDS stmt
if (title.equals("HostName")) {
continue;
}
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
}
return builder.build();

47
fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.SchemaTableType;
import org.apache.doris.common.SystemIdGenerator;
import org.apache.doris.thrift.TSchemaTable;
import org.apache.doris.thrift.TSchemaTableStructure;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
@ -393,54 +392,8 @@ public class SchemaTable extends Table {
.column("CREATION_TIME", ScalarType.createType(PrimitiveType.BIGINT))
.column("NEWEST_WRITE_TIMESTAMP", ScalarType.createType(PrimitiveType.BIGINT))
.build()))
.put("backends", new SchemaTable(SystemIdGenerator.getNextId(), "backends", TableType.SCHEMA,
builder().column("BackendId", ScalarType.createType(PrimitiveType.BIGINT))
.column("Cluster", ScalarType.createVarchar(64))
.column("IP", ScalarType.createVarchar(16))
.column("HeartbeatPort", ScalarType.createType(PrimitiveType.INT))
.column("BePort", ScalarType.createType(PrimitiveType.INT))
.column("HttpPort", ScalarType.createType(PrimitiveType.INT))
.column("BrpcPort", ScalarType.createType(PrimitiveType.INT))
.column("LastStartTime", ScalarType.createVarchar(32))
.column("LastHeartbeat", ScalarType.createVarchar(32))
.column("Alive", ScalarType.createVarchar(8))
.column("SystemDecommissioned", ScalarType.createVarchar(8))
.column("ClusterDecommissioned", ScalarType.createVarchar(8))
.column("TabletNum", ScalarType.createType(PrimitiveType.BIGINT))
.column("DataUsedCapacity", ScalarType.createType(PrimitiveType.BIGINT))
.column("AvailCapacity", ScalarType.createType(PrimitiveType.BIGINT))
.column("TotalCapacity", ScalarType.createType(PrimitiveType.BIGINT))
.column("UsedPct", ScalarType.createType(PrimitiveType.DOUBLE))
.column("MaxDiskUsedPct", ScalarType.createType(PrimitiveType.DOUBLE))
.column("RemoteUsedCapacity", ScalarType.createType(PrimitiveType.BIGINT))
.column("Tag", ScalarType.createVarchar(128))
.column("ErrMsg", ScalarType.createVarchar(2048))
.column("Version", ScalarType.createVarchar(64))
.column("Status", ScalarType.createVarchar(1024))
.build()))
.build();
public static List<TSchemaTableStructure> getTableStructure(String tableName) {
List<TSchemaTableStructure> tSchemaTableStructureList = Lists.newArrayList();
switch (tableName) {
case "backends": {
Table table = TABLE_MAP.get(tableName);
for (Column column : table.getFullSchema()) {
TSchemaTableStructure tSchemaTableStructure = new TSchemaTableStructure();
tSchemaTableStructure.setColumnName(column.getName());
tSchemaTableStructure.setType(column.getDataType().toThrift());
tSchemaTableStructure.setLen(column.getDataType().getSlotSize());
tSchemaTableStructure.setIsNull(column.isAllowNull());
tSchemaTableStructureList.add(tSchemaTableStructure);
}
break;
}
default:
break;
}
return tSchemaTableStructureList;
}
protected SchemaTable(long id, String name, TableType type, List<Column> baseSchema) {
super(id, name, type, baseSchema);
}

2
fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
View File

@ -50,8 +50,6 @@ public class FrontendsProcNode implements ProcNodeInterface {
.add("CurrentConnected")
.build();
public static final int HOSTNAME_INDEX = 2;
private Env env;
public FrontendsProcNode(Env env) {

2
fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java
View File

@ -122,8 +122,6 @@ public class SchemaScanNode extends ScanNode {
TUserIdentity tCurrentUser = ConnectContext.get().getCurrentUserIdentity().toThrift();
msg.schema_scan_node.setCurrentUserIdent(tCurrentUser);
msg.schema_scan_node.setTableStructure(SchemaTable.getTableStructure(tableName));
}
/**

7
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -1820,9 +1820,6 @@ public class ShowExecutor {
final ShowBackendsStmt showStmt = (ShowBackendsStmt) stmt;
List<List<String>> backendInfos = BackendsProcDir.getClusterBackendInfos(showStmt.getClusterName());
for (List<String> row : backendInfos) {
row.remove(BackendsProcDir.HOSTNAME_INDEX);
}
backendInfos.sort(new Comparator<List<String>>() {
@Override
public int compare(List<String> o1, List<String> o2) {
@ -1838,10 +1835,6 @@ public class ShowExecutor {
List<List<String>> infos = Lists.newArrayList();
FrontendsProcNode.getFrontendsInfo(Env.getCurrentEnv(), infos);
for (List<String> row : infos) {
row.remove(FrontendsProcNode.HOSTNAME_INDEX);
}
resultSet = new ShowResultSet(showStmt.getMetaData(), infos);
}

2
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -1340,8 +1340,6 @@ public class FrontendServiceImpl implements FrontendService.Iface {
@Override
public TFetchSchemaTableDataResult fetchSchemaTableData(TFetchSchemaTableDataRequest request) throws TException {
switch (request.getSchemaTableName()) {
case BACKENDS:
return MetadataGenerator.getBackendsSchemaTable(request);
case METADATA_TABLE:
return MetadataGenerator.getMetadataTable(request);
default:

25
fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
View File

@ -67,17 +67,18 @@ public class BackendsTableValuedFunction extends MetadataTableValuedFunction {
public List<Column> getTableColumns() throws AnalysisException {
List<Column> resColumns = Lists.newArrayList();
resColumns.add(new Column("BackendId", ScalarType.createType(PrimitiveType.BIGINT)));
resColumns.add(new Column("Cluster", ScalarType.createVarchar(64)));
resColumns.add(new Column("IP", ScalarType.createVarchar(16)));
resColumns.add(new Column("Cluster", ScalarType.createStringType()));
resColumns.add(new Column("IP", ScalarType.createStringType()));
resColumns.add(new Column("HostName", ScalarType.createStringType()));
resColumns.add(new Column("HeartbeatPort", ScalarType.createType(PrimitiveType.INT)));
resColumns.add(new Column("BePort", ScalarType.createType(PrimitiveType.INT)));
resColumns.add(new Column("HttpPort", ScalarType.createType(PrimitiveType.INT)));
resColumns.add(new Column("BrpcPort", ScalarType.createType(PrimitiveType.INT)));
resColumns.add(new Column("LastStartTime", ScalarType.createVarchar(32)));
resColumns.add(new Column("LastHeartbeat", ScalarType.createVarchar(32)));
resColumns.add(new Column("Alive", ScalarType.createVarchar(8)));
resColumns.add(new Column("SystemDecommissioned", ScalarType.createVarchar(8)));
resColumns.add(new Column("ClusterDecommissioned", ScalarType.createVarchar(8)));
resColumns.add(new Column("LastStartTime", ScalarType.createStringType()));
resColumns.add(new Column("LastHeartbeat", ScalarType.createStringType()));
resColumns.add(new Column("Alive", ScalarType.createStringType()));
resColumns.add(new Column("SystemDecommissioned", ScalarType.createStringType()));
resColumns.add(new Column("ClusterDecommissioned", ScalarType.createStringType()));
resColumns.add(new Column("TabletNum", ScalarType.createType(PrimitiveType.BIGINT)));
resColumns.add(new Column("DataUsedCapacity", ScalarType.createType(PrimitiveType.BIGINT)));
resColumns.add(new Column("AvailCapacity", ScalarType.createType(PrimitiveType.BIGINT)));
@ -85,12 +86,12 @@ public class BackendsTableValuedFunction extends MetadataTableValuedFunction {
resColumns.add(new Column("UsedPct", ScalarType.createType(PrimitiveType.DOUBLE)));
resColumns.add(new Column("MaxDiskUsedPct", ScalarType.createType(PrimitiveType.DOUBLE)));
resColumns.add(new Column("RemoteUsedCapacity", ScalarType.createType(PrimitiveType.BIGINT)));
resColumns.add(new Column("Tag", ScalarType.createVarchar(128)));
resColumns.add(new Column("ErrMsg", ScalarType.createVarchar(2048)));
resColumns.add(new Column("Version", ScalarType.createVarchar(64)));
resColumns.add(new Column("Status", ScalarType.createVarchar(1024)));
resColumns.add(new Column("Tag", ScalarType.createStringType()));
resColumns.add(new Column("ErrMsg", ScalarType.createStringType()));
resColumns.add(new Column("Version", ScalarType.createStringType()));
resColumns.add(new Column("Status", ScalarType.createStringType()));
resColumns.add(new Column("HeartbeatFailureCounter", ScalarType.createType(PrimitiveType.INT)));
resColumns.add(new Column("NodeRole", ScalarType.createVarchar(64)));
resColumns.add(new Column("NodeRole", ScalarType.createStringType()));
return resColumns;
}
}

106
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
View File

@ -74,107 +74,6 @@ public class MetadataGenerator {
return errorResult("Metadata table params is not set. ");
}
// deprecated
public static TFetchSchemaTableDataResult getBackendsSchemaTable(TFetchSchemaTableDataRequest request) {
final SystemInfoService clusterInfoService = Env.getCurrentSystemInfo();
List<Long> backendIds = null;
if (!Strings.isNullOrEmpty(request.cluster_name)) {
final Cluster cluster = Env.getCurrentEnv().getCluster(request.cluster_name);
// root not in any cluster
if (null == cluster) {
return errorResult("Cluster is not existed.");
}
backendIds = cluster.getBackendIdList();
} else {
backendIds = clusterInfoService.getBackendIds(false);
}
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
long start = System.currentTimeMillis();
Stopwatch watch = Stopwatch.createUnstarted();
List<TRow> dataBatch = Lists.newArrayList();
for (long backendId : backendIds) {
Backend backend = clusterInfoService.getBackend(backendId);
if (backend == null) {
continue;
}
watch.start();
Integer tabletNum = Env.getCurrentInvertedIndex().getTabletNumByBackendId(backendId);
watch.stop();
TRow trow = new TRow();
trow.addToColumnValue(new TCell().setLongVal(backendId));
trow.addToColumnValue(new TCell().setStringVal(backend.getOwnerClusterName()));
trow.addToColumnValue(new TCell().setStringVal(backend.getIp()));
if (Strings.isNullOrEmpty(request.cluster_name)) {
trow.addToColumnValue(new TCell().setIntVal(backend.getHeartbeatPort()));
trow.addToColumnValue(new TCell().setIntVal(backend.getBePort()));
trow.addToColumnValue(new TCell().setIntVal(backend.getHttpPort()));
trow.addToColumnValue(new TCell().setIntVal(backend.getBrpcPort()));
}
trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(backend.getLastStartTime())));
trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(backend.getLastUpdateMs())));
trow.addToColumnValue(new TCell().setStringVal(String.valueOf(backend.isAlive())));
if (backend.isDecommissioned() && backend.getDecommissionType() == DecommissionType.ClusterDecommission) {
trow.addToColumnValue(new TCell().setStringVal("false"));
trow.addToColumnValue(new TCell().setStringVal("true"));
} else if (backend.isDecommissioned()
&& backend.getDecommissionType() == DecommissionType.SystemDecommission) {
trow.addToColumnValue(new TCell().setStringVal("true"));
trow.addToColumnValue(new TCell().setStringVal("false"));
} else {
trow.addToColumnValue(new TCell().setStringVal("false"));
trow.addToColumnValue(new TCell().setStringVal("false"));
}
trow.addToColumnValue(new TCell().setLongVal(tabletNum));
// capacity
// data used
trow.addToColumnValue(new TCell().setLongVal(backend.getDataUsedCapacityB()));
// available
long availB = backend.getAvailableCapacityB();
trow.addToColumnValue(new TCell().setLongVal(availB));
// total
long totalB = backend.getTotalCapacityB();
trow.addToColumnValue(new TCell().setLongVal(totalB));
// used percent
double used = 0.0;
if (totalB <= 0) {
used = 0.0;
} else {
used = (double) (totalB - availB) * 100 / totalB;
}
trow.addToColumnValue(new TCell().setDoubleVal(used));
trow.addToColumnValue(new TCell().setDoubleVal(backend.getMaxDiskUsedPct() * 100));
// remote used capacity
trow.addToColumnValue(new TCell().setLongVal(backend.getRemoteUsedCapacityB()));
// tags
trow.addToColumnValue(new TCell().setStringVal(backend.getTagMapString()));
// err msg
trow.addToColumnValue(new TCell().setStringVal(backend.getHeartbeatErrMsg()));
// version
trow.addToColumnValue(new TCell().setStringVal(backend.getVersion()));
// status
trow.addToColumnValue(new TCell().setStringVal(new Gson().toJson(backend.getBackendStatus())));
dataBatch.add(trow);
}
// backends proc node get result too slow, add log to observer.
LOG.debug("backends proc get tablet num cost: {}, total cost: {}",
watch.elapsed(TimeUnit.MILLISECONDS), (System.currentTimeMillis() - start));
result.setDataBatch(dataBatch);
result.setStatus(new TStatus(TStatusCode.OK));
return result;
}
@NotNull
public static TFetchSchemaTableDataResult errorResult(String msg) {
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
@ -266,6 +165,11 @@ public class MetadataGenerator {
trow.addToColumnValue(new TCell().setLongVal(backendId));
trow.addToColumnValue(new TCell().setStringVal(backend.getOwnerClusterName()));
trow.addToColumnValue(new TCell().setStringVal(backend.getIp()));
if (backend.getHostName() != null) {
trow.addToColumnValue(new TCell().setStringVal(backend.getHostName()));
} else {
trow.addToColumnValue(new TCell().setStringVal(backend.getIp()));
}
if (Strings.isNullOrEmpty(backendsParam.cluster_name)) {
trow.addToColumnValue(new TCell().setIntVal(backend.getHeartbeatPort()));
trow.addToColumnValue(new TCell().setIntVal(backend.getBePort()));