[fix](mtmv) use isManagedTable instead of check table type (#34287) (#37822)

pick: https://github.com/apache/doris/pull/34287
This commit is contained in:
zhangdong
2024-07-16 15:01:28 +08:00
committed by GitHub
parent 2c4b5519e9
commit 2c80259f66
19 changed files with 152 additions and 55 deletions

View File

@ -841,7 +841,7 @@ public class Analyzer {
.getDbOrAnalysisException(tableName.getDb());
TableIf table = database.getTableOrAnalysisException(tableName.getTbl());
if (table.getType() == TableType.OLAP && (((OlapTable) table).getState() == OlapTableState.RESTORE
if (table.isManagedTable() && (((OlapTable) table).getState() == OlapTableState.RESTORE
|| ((OlapTable) table).getState() == OlapTableState.RESTORE_WITH_LOAD)) {
Boolean isNotRestoring = ((OlapTable) table).getPartitions().stream()
.filter(partition -> partition.getState() == PartitionState.RESTORE).collect(Collectors.toList())

View File

@ -30,7 +30,6 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
@ -1087,7 +1086,7 @@ public class SelectStmt extends QueryStmt {
break;
}
long rowCount = 0;
if (tblRef.getTable().getType() == TableType.OLAP) {
if (tblRef.getTable().isManagedTable()) {
rowCount = ((OlapTable) (tblRef.getTable())).getRowCount();
if (LOG.isDebugEnabled()) {
LOG.debug("tableName={} rowCount={}", tblRef.getAlias(), rowCount);

View File

@ -112,7 +112,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
}
private void addRecycledTabletsForTable(Set<Long> recycledTabletSet, Table table) {
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
OlapTable olapTable = (OlapTable) table;
Collection<Partition> allPartitions = olapTable.getAllPartitions();
for (Partition partition : allPartitions) {
@ -321,7 +321,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
}
Table table = tableInfo.getTable();
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, false);
}
iterator.remove();
@ -352,7 +352,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
long tableId = table.getId();
if (isExpire(tableId, currentTimeMs)) {
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, false);
}
@ -434,7 +434,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
continue;
}
Table table = tableInfo.getTable();
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, false);
}
@ -455,7 +455,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
return;
}
Table table = tableInfo.getTable();
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, true);
}
LOG.info("replay erase table[{}]", tableId);
@ -646,8 +646,12 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
}
Table table = tableInfo.getTable();
db.registerTable(table);
LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName());
if (table.getType() == TableType.OLAP) {
db.registerTable(table);
LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName());
} else {
LOG.info("ignore recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName());
}
iterator.remove();
idToRecycleTime.remove(table.getId());
tableNames.remove(table.getName());
@ -694,6 +698,11 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
+ db.getFullName());
}
if (table.getType() == TableType.MATERIALIZED_VIEW) {
throw new DdlException("Can not recover materialized view '" + tableName + "' or table id '"
+ tableId + "' in " + db.getFullName());
}
innerRecoverTable(db, table, tableName, newTableName, null, false);
LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName());
return true;
@ -732,7 +741,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
throw new DdlException("Table name[" + newTableName + "] is already used");
}
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
// olap table should also check if any rollup has same name as "newTableName"
((OlapTable) table).checkAndSetName(newTableName, false);
} else {
@ -756,7 +765,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
Env.getCurrentEnv().getEditLog().logRecoverTable(recoverInfo);
}
// Only olap table need recover dynamic partition, other table like jdbc odbc view.. do not need it
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), (OlapTable) table, isReplay);
}
} finally {
@ -767,6 +776,10 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
public synchronized void recoverPartition(long dbId, OlapTable table, String partitionName,
long partitionIdToRecover, String newPartitionName) throws DdlException {
if (table.getType() == TableType.MATERIALIZED_VIEW) {
throw new DdlException("Can not recover partition in materialized view: " + table.getName());
}
long recycleTime = -1;
// make sure to get db write lock
RecyclePartitionInfo recoverPartitionInfo = null;
@ -903,7 +916,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
// idToTable
for (RecycleTableInfo tableInfo : idToTable.values()) {
Table table = tableInfo.getTable();
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}

View File

@ -286,7 +286,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
}
for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}
@ -307,7 +307,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
try {
long usedReplicaCount = 0;
for (Table table : this.idToTable.values()) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}
@ -545,7 +545,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
readLock();
try {
for (Table table : idToTable.values()) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}
OlapTable olapTable = (OlapTable) table;
@ -866,7 +866,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
if (newBinlogConfig.isEnable() && !oldBinlogConfig.isEnable()) {
// check all tables binlog enable is true
for (Table table : idToTable.values()) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}

View File

@ -3532,7 +3532,7 @@ public class Env {
}
// There MUST BE 2 space in front of each column description line
// sqlalchemy requires this to parse SHOW CREATE TABLE stmt.
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
sb.append(" ").append(
column.toSql(((OlapTable) table).getKeysType() == KeysType.UNIQUE_KEYS, true));
} else {
@ -3994,7 +3994,7 @@ public class Env {
}
List<Table> tableList = db.getTables();
for (Table table : tableList) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}
@ -4492,7 +4492,7 @@ public class Env {
throw new DdlException("Table name[" + newTableName + "] is already used");
}
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
// olap table should also check if any rollup has same name as "newTableName"
((OlapTable) table).checkAndSetName(newTableName, false);
} else {
@ -5568,7 +5568,7 @@ public class Env {
List<ReplicaPersistInfo> replicaPersistInfos = backendTabletsInfo.getReplicaPersistInfos();
for (ReplicaPersistInfo info : replicaPersistInfos) {
OlapTable olapTable = (OlapTable) getInternalCatalog().getDb(info.getDbId())
.flatMap(db -> db.getTable(info.getTableId())).filter(t -> t.getType() == TableType.OLAP)
.flatMap(db -> db.getTable(info.getTableId())).filter(t -> t.isManagedTable())
.orElse(null);
if (olapTable == null) {
continue;
@ -6111,7 +6111,7 @@ public class Env {
}
for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}

View File

@ -22,7 +22,6 @@ import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.qe.SessionVariable;
@ -60,7 +59,7 @@ public class IndexInfoProcDir implements ProcDirInterface {
result.setNames(TITLE_NAMES);
table.readLock();
try {
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
OlapTable olapTable = (OlapTable) table;
// indices order
@ -122,7 +121,7 @@ public class IndexInfoProcDir implements ProcDirInterface {
try {
List<Column> schema = null;
Set<String> bfColumns = null;
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
OlapTable olapTable = (OlapTable) table;
schema = olapTable.getSchemaByIndexId(idxId);
if (schema == null) {

View File

@ -21,7 +21,6 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.FetchRemoteTabletSchemaUtil;
@ -90,7 +89,7 @@ public class RemoteIndexSchemaProcDir implements ProcDirInterface {
List<Partition> partitions = Lists.newArrayList();
table.readLock();
try {
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
OlapTable olapTable = (OlapTable) table;
for (String partitionName : partitionNameList) {
Partition partition = olapTable.getPartition(partitionName);

View File

@ -24,7 +24,6 @@ import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.AnalysisException;
@ -95,7 +94,7 @@ public class StatisticProcNode implements ProcNodeInterface {
this.db.getTables().stream().filter(Objects::nonNull).forEach(t -> {
++tableNum;
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
OlapTable olapTable = (OlapTable) t;
olapTable.readLock();
try {

View File

@ -25,7 +25,6 @@ import org.apache.doris.catalog.MetaObject;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.Config;
import org.apache.doris.common.MetaNotFoundException;
@ -268,7 +267,7 @@ public class ConsistencyChecker extends MasterDaemon {
// sort tables
Queue<MetaObject> tableQueue = new PriorityQueue<>(Math.max(tables.size(), 1), COMPARATOR);
for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}
tableQueue.add(table);

View File

@ -956,11 +956,12 @@ public class InternalCatalog implements CatalogIf<Database> {
long recycleTime) {
if (table.getType() == TableType.ELASTICSEARCH) {
esRepository.deRegisterTable(table.getId());
} else if (table.getType() == TableType.OLAP) {
} else if (table.isManagedTable()) {
// drop all temp partitions of this table, so that there is no temp partitions in recycle bin,
// which make things easier.
((OlapTable) table).dropAllTempPartitions();
} else if (table.getType() == TableType.MATERIALIZED_VIEW) {
}
if (table.getType() == TableType.MATERIALIZED_VIEW) {
Env.getCurrentEnv().getMtmvService().deregisterMTMV((MTMV) table);
}
@ -1176,7 +1177,7 @@ public class InternalCatalog implements CatalogIf<Database> {
List<String> createTableStmt = Lists.newArrayList();
table.readLock();
try {
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
if (!CollectionUtils.isEmpty(stmt.getRollupNames())) {
OlapTable olapTable = (OlapTable) table;
for (String rollupIndexName : stmt.getRollupNames()) {
@ -1380,7 +1381,7 @@ public class InternalCatalog implements CatalogIf<Database> {
try {
Table table = db.getTableOrDdlException(tableName);
if (table.getType() != TableType.OLAP && table.getType() != TableType.MATERIALIZED_VIEW) {
if (!table.isManagedTable()) {
throw new DdlException("Only support create partition from a OLAP table");
}

View File

@ -307,7 +307,7 @@ public class ShowAction extends RestBaseController {
// sort by table name
List<Table> tables = db.getTables();
for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
if (!table.isManagedTable()) {
continue;
}
table.readLock();
@ -352,7 +352,7 @@ public class ShowAction extends RestBaseController {
Map<String, Long> oneEntry = Maps.newHashMap();
if (table.getType() == TableType.VIEW || table.getType() == TableType.ODBC) {
oneEntry.put(table.getName(), 0L);
} else if (table.getType() == TableType.OLAP) {
} else if (table.isManagedTable()) {
table.readLock();
try {
long tableSize = ((OlapTable) table).getDataSize(singleReplica);

View File

@ -22,7 +22,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.mysql.privilege.PrivPredicate;
@ -64,7 +63,7 @@ public class StorageTypeCheckAction extends RestBaseController {
Map<String, Map<String, String>> result = Maps.newHashMap();
List<Table> tbls = db.getTables();
for (Table tbl : tbls) {
if (tbl.getType() != TableType.OLAP) {
if (!tbl.isManagedTable()) {
continue;
}

View File

@ -273,7 +273,7 @@ public class ExportJob implements Writable {
if (exportTable.getType() == TableType.VIEW) {
// view table
generateViewOrExternalTableOutfile(qualifiedTableName);
} else if (exportTable.getType() == TableType.OLAP) {
} else if (exportTable.isManagedTable()) {
// olap table
generateOlapTableOutfile(qualifiedTableName);
} else {

View File

@ -24,7 +24,6 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.load.ExportFailMsg.CancelType;
@ -88,7 +87,7 @@ public class ExportTaskExecutor implements TransientTaskExecutor {
throw new JobException("Export executor has been canceled, task id: {}", taskId);
}
// check the version of tablets, skip if the consistency is in partition level.
if (exportJob.getExportTable().getType() == TableType.OLAP && !exportJob.isPartitionConsistency()) {
if (exportJob.getExportTable().isManagedTable() && !exportJob.isPartitionConsistency()) {
try {
Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException(
exportJob.getTableName().getDb());

View File

@ -69,7 +69,7 @@ public class DropMTMVInfo {
*/
public DropTableStmt translateToLegacyStmt() {
TableName tableName = mvName.transferToTableName();
DropTableStmt dropTableStmt = new DropTableStmt(ifExists, tableName, false);
DropTableStmt dropTableStmt = new DropTableStmt(ifExists, tableName, true);
dropTableStmt.setMaterializedView(true);
return dropTableStmt;
}

View File

@ -20,7 +20,6 @@ package org.apache.doris.statistics.query;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.util.Util;
import com.google.common.collect.ImmutableMap;
@ -138,7 +137,7 @@ public class IndexStats {
*/
public Map<String, Map> getStats(boolean summary) {
List<Column> indexColumns;
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
OlapTable olapTable = (OlapTable) table;
indexColumns = olapTable.getSchemaByIndexId(indexId);
} else {

View File

@ -21,7 +21,6 @@ import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
@ -187,7 +186,7 @@ public class QueryStats {
DatabaseIf d = c.getDbOrAnalysisException(database);
TableIf t = d.getTableOrAnalysisException(table);
long indexId = TableStats.DEFAULT_INDEX_ID;
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
indexId = ((OlapTable) t).getIndexIdByName(index);
}
return getStats(c.getId(), d.getId(), t.getId(), indexId, summary);
@ -266,7 +265,7 @@ public class QueryStats {
ConcurrentHashMap<Long, IndexStats> indexStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId())
.getTableStats().get(t.getId()).getIndexStats();
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
for (Map.Entry<Long, IndexStats> entry : indexStats.entrySet()) {
for (Map.Entry<String, AtomicLong> indexEntry : entry.getValue().getColumnQueryStats().entrySet()) {
if (result.containsKey(indexEntry.getKey())) {
@ -300,7 +299,7 @@ public class QueryStats {
CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog);
DatabaseIf d = c.getDbOrAnalysisException(db);
TableIf t = d.getTableOrAnalysisException(tbl);
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
((OlapTable) t).getIndexNameToId().keySet().forEach(indexName -> result.put(indexName, 0L));
} else {
result.put(tbl, 0L);
@ -316,7 +315,7 @@ public class QueryStats {
}
ConcurrentHashMap<Long, IndexStats> indexStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId())
.getTableStats().get(t.getId()).getIndexStats();
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
for (Map.Entry<Long, IndexStats> entry : indexStats.entrySet()) {
result.put(((OlapTable) t).getIndexNameById(entry.getKey()), entry.getValue().getQueryStats());
}
@ -334,7 +333,7 @@ public class QueryStats {
CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog);
DatabaseIf d = c.getDbOrAnalysisException(db);
TableIf t = d.getTableOrAnalysisException(tbl);
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
((OlapTable) t).getIndexNameToId().forEach((indexName, indexId) -> {
Map<String, Pair<Long, Long>> indexResult = new LinkedHashMap<>();
((OlapTable) t).getSchemaByIndexId(indexId)
@ -358,7 +357,7 @@ public class QueryStats {
ConcurrentHashMap<Long, IndexStats> indexStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId())
.getTableStats().get(t.getId()).getIndexStats();
for (Map.Entry<Long, IndexStats> entry : indexStats.entrySet()) {
String indexName = t.getType() == TableType.OLAP ? ((OlapTable) t).getIndexNameById(entry.getKey()) : tbl;
String indexName = t.isManagedTable() ? ((OlapTable) t).getIndexNameById(entry.getKey()) : tbl;
if (!result.containsKey(indexName)) {
continue;
}
@ -373,7 +372,7 @@ public class QueryStats {
indexResult.get(indexEntry.getKey()).second = indexEntry.getValue().get();
}
}
if (t.getType() == TableType.OLAP) {
if (t.isManagedTable()) {
result.get(((OlapTable) t).getIndexNameById(entry.getKey())).putAll(indexResult);
} else {
result.put(tbl, indexResult);

View File

@ -20,7 +20,6 @@ package org.apache.doris.statistics.query;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import com.google.common.collect.ImmutableMap;
@ -126,7 +125,7 @@ public class TableStats {
Map<String, Map> stat = new HashMap<>();
stat.put("summary", ImmutableMap.of("query", getQueryStats()));
Map<String, Map> dstat = new HashMap<>();
if (table.getType() == TableType.OLAP) {
if (table.isManagedTable()) {
OlapTable olapTable = (OlapTable) table;
for (Map.Entry<String, Long> entry : olapTable.getIndexNameToId().entrySet()) {
if (indexStats.containsKey(entry.getValue())) {