database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](statistics)Fix drop stats log editlog bug. Catch drop stats exception while truncate table. (#40738) (#40853)

Browse Source 
backport: https://github.com/apache/doris/pull/40738
This commit is contained in:
Jibing-Li
2024-09-14 23:08:24 +08:00
committed by GitHub
parent cecd214345
commit b7103bf00c
6 changed files with 220 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -2756,11 +2756,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropInvertedIndices(info);
// Drop table column stats after light schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
} catch (Exception e) {
LOG.info("Failed to drop stats after light schema change. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
if (isDropIndex) {
// send drop rpc to be
@ -2788,11 +2784,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropColumns(info);
// Drop table column stats after light schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
} catch (Exception e) {
LOG.info("Failed to drop stats after light schema change. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
}
LOG.info("finished modify table's add or drop or modify columns. table: {}, job: {}, is replay: {}",
olapTable.getName(), jobId, isReplay);

6
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -600,11 +600,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
changeTableState(dbId, tableId, OlapTableState.NORMAL);
LOG.info("set table's state to NORMAL, table id: {}, job id: {}", tableId, jobId);
// Drop table column stats after schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(tbl);
} catch (Exception e) {
LOG.info("Failed to drop stats after schema change finished. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(tbl);
}
private void onFinished(OlapTable tbl) {

6
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -4942,11 +4942,7 @@ public class Env {
indexIdToSchemaVersion);
editLog.logColumnRename(info);
LOG.info("rename coloumn[{}] to {}", colName, newColName);
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(table);
} catch (Exception e) {
LOG.info("Failed to drop stats after rename column. Reason: {}", e.getMessage());
}
Env.getCurrentEnv().getAnalysisManager().dropStats(table);
}
}

3
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -3421,9 +3421,6 @@ public class FrontendServiceImpl implements FrontendService.Iface {
InvalidateStatsTarget target = GsonUtils.GSON.fromJson(request.key, InvalidateStatsTarget.class);
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableStats = analysisManager.findTableStatsStatus(target.tableId);
if (tableStats == null) {
return new TStatus(TStatusCode.OK);
}
analysisManager.invalidateLocalStats(target.catalogId, target.dbId, target.tableId, target.columns, tableStats);
return new TStatus(TStatusCode.OK);
}

71
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -50,6 +50,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.AnalyzeDeletionLog;
import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
@ -632,13 +633,18 @@ public class AnalysisManager implements Writable {
return;
}
TableStatsMeta tableStats = findTableStatsStatus(dropStatsStmt.getTblId());
if (tableStats == null) {
return;
}
Set<String> cols = dropStatsStmt.getColumnNames();
long catalogId = dropStatsStmt.getCatalogIdId();
long dbId = dropStatsStmt.getDbId();
long tblId = dropStatsStmt.getTblId();
TableStatsMeta tableStats = findTableStatsStatus(dropStatsStmt.getTblId());
if (tableStats == null) {
return;
// Remove tableMetaStats if drop whole table stats.
if (dropStatsStmt.isAllColumns()) {
removeTableStats(tblId);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(tblId));
}
invalidateLocalStats(catalogId, dbId, tblId, dropStatsStmt.isAllColumns() ? null : cols, tableStats);
// Drop stats ddl is master only operation.
@ -646,19 +652,26 @@ public class AnalysisManager implements Writable {
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, tblId, cols);
}
public void dropStats(TableIf table) throws DdlException {
TableStatsMeta tableStats = findTableStatsStatus(table.getId());
if (tableStats == null) {
return;
public void dropStats(TableIf table) {
try {
TableStatsMeta tableStats = findTableStatsStatus(table.getId());
if (tableStats == null) {
return;
}
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
removeTableStats(tableId);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(tableId));
Set<String> cols = table.getSchemaAllIndexes(false).stream().map(Column::getName)
.collect(Collectors.toSet());
invalidateLocalStats(catalogId, dbId, tableId, null, tableStats);
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tableId, cols, true);
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols);
} catch (Throwable e) {
LOG.warn("Failed to drop stats for table {}", table.getName(), e);
}
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
Set<String> cols = table.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet());
invalidateLocalStats(catalogId, dbId, tableId, null, tableStats);
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tableId, cols, true);
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols);
}
public void dropCachedStats(long catalogId, long dbId, long tableId) {
@ -681,14 +694,9 @@ public class AnalysisManager implements Writable {
public void invalidateLocalStats(long catalogId, long dbId, long tableId,
Set<String> columns, TableStatsMeta tableStats) {
if (tableStats == null) {
return;
}
TableIf table = StatisticsUtil.findTable(catalogId, dbId, tableId);
StatisticsCache statsCache = Env.getCurrentEnv().getStatisticsCache();
boolean allColumn = false;
if (columns == null) {
allColumn = true;
columns = table.getSchemaAllIndexes(false)
.stream().map(Column::getName).collect(Collectors.toSet());
}
@ -710,18 +718,16 @@ public class AnalysisManager implements Writable {
indexName = olapTable.getIndexNameById(indexId);
}
}
tableStats.removeColumn(indexName, column);
if (tableStats != null) {
tableStats.removeColumn(indexName, column);
}
statsCache.invalidate(catalogId, dbId, tableId, indexId, column);
}
}
// To remove stale column name that is changed before.
if (allColumn) {
tableStats.removeAllColumn();
tableStats.clearIndexesRowCount();
removeTableStats(tableId);
if (tableStats != null) {
tableStats.updatedTime = 0;
tableStats.userInjected = false;
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;
}
public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
@ -731,18 +737,15 @@ public class AnalysisManager implements Writable {
request.key = GsonUtils.GSON.toJson(target);
StatisticsCache statisticsCache = Env.getCurrentEnv().getStatisticsCache();
SystemInfoService.HostInfo selfNode = Env.getCurrentEnv().getSelfNode();
boolean success = true;
for (Frontend frontend : Env.getCurrentEnv().getFrontends(null)) {
// Skip master
if (selfNode.getHost().equals(frontend.getHost())) {
continue;
}
success = success && statisticsCache.invalidateStats(frontend, request);
statisticsCache.invalidateStats(frontend, request);
}
if (!success) {
// If any rpc failed, use edit log to sync table stats to non-master FEs.
LOG.warn("Failed to invalidate all remote stats by rpc for table {}, use edit log.", tableId);
TableStatsMeta tableStats = findTableStatsStatus(tableId);
TableStatsMeta tableStats = findTableStatsStatus(tableId);
if (tableStats != null) {
logCreateTableStats(tableStats);
}
}