database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](statistics)Drop column stats after schema change. (#39101) (#39401)

Browse Source 
backport: https://github.com/apache/doris/pull/39101
This commit is contained in:
Jibing-Li
2024-08-15 17:02:42 +08:00
committed by GitHub
parent 01090cf61f
commit b3597ea898
6 changed files with 208 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -2766,6 +2766,12 @@ public class SchemaChangeHandler extends AlterHandler {
LOG.debug("logModifyTableAddOrDropInvertedIndices info:{}", info);
}
Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropInvertedIndices(info);
// Drop table column stats after light schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
} catch (Exception e) {
LOG.info("Failed to drop stats after light schema change. Reason: {}", e.getMessage());
}
if (isDropIndex) {
// send drop rpc to be
@ -2792,6 +2798,12 @@ public class SchemaChangeHandler extends AlterHandler {
LOG.debug("logModifyTableAddOrDropColumns info:{}", info);
}
Env.getCurrentEnv().getEditLog().logModifyTableAddOrDropColumns(info);
// Drop table column stats after light schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable);
} catch (Exception e) {
LOG.info("Failed to drop stats after light schema change. Reason: {}", e.getMessage());
}
}
LOG.info("finished modify table's add or drop or modify columns. table: {}, job: {}, is replay: {}",
olapTable.getName(), jobId, isReplay);

6
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -599,6 +599,12 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
changeTableState(dbId, tableId, OlapTableState.NORMAL);
LOG.info("set table's state to NORMAL, table id: {}, job id: {}", tableId, jobId);
// Drop table column stats after schema change finished.
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(tbl);
} catch (Exception e) {
LOG.info("Failed to drop stats after schema change finished. Reason: {}", e.getMessage());
}
}
private void onFinished(OlapTable tbl) {

5
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -4900,6 +4900,11 @@ public class Env {
indexIdToSchemaVersion);
editLog.logColumnRename(info);
LOG.info("rename coloumn[{}] to {}", colName, newColName);
try {
Env.getCurrentEnv().getAnalysisManager().dropStats(table);
} catch (Exception e) {
LOG.info("Failed to drop stats after rename column. Reason: {}", e.getMessage());
}
}
}

10
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -658,8 +658,10 @@ public class AnalysisManager implements Writable {
return;
}
TableIf table = StatisticsUtil.findTable(catalogId, dbId, tableId);
StatisticsCache statisticsCache = Env.getCurrentEnv().getStatisticsCache();
StatisticsCache statsCache = Env.getCurrentEnv().getStatisticsCache();
boolean allColumn = false;
if (columns == null) {
allColumn = true;
columns = table.getSchemaAllIndexes(false)
.stream().map(Column::getName).collect(Collectors.toSet());
}
@ -682,9 +684,13 @@ public class AnalysisManager implements Writable {
}
}
tableStats.removeColumn(indexName, column);
statisticsCache.invalidate(catalogId, dbId, tableId, indexId, column);
statsCache.invalidate(catalogId, dbId, tableId, indexId, column);
}
}
// To remove stale column name that is changed before.
if (allColumn) {
tableStats.removeAllColumn();
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;
}

7
fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
View File

@ -124,6 +124,10 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
colToColStatsMeta.remove(Pair.of(indexName, colName));
}
public void removeAllColumn() {
colToColStatsMeta.clear();
}
public Set<Pair<String, String>> analyzeColumns() {
return colToColStatsMeta.keySet();
}
@ -174,6 +178,9 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
if (newPartitionLoaded == null) {
newPartitionLoaded = new AtomicBoolean(false);
}
if (colToColStatsMeta == null) {
colToColStatsMeta = new ConcurrentHashMap<>();
}
}
public long getRowCount(long indexId) {