database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] Fix bug that truncate table may change the storage medium property (#6905)

Browse Source
This commit is contained in:
Mingyu Chen
2021-10-25 10:07:27 +08:00
committed by GitHub
parent ed7a873a44
commit adb6bfdf74
13 changed files with 56 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
View File

@ -49,9 +49,6 @@ import org.apache.doris.thrift.TFinishTaskRequest;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TTaskType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
@ -61,6 +58,9 @@ import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
@ -505,7 +505,7 @@ public class BackupJob extends AbstractJob {
// only copy visible indexes
List<String> reservedPartitions = tableRef.getPartitionNames() == null ? null
: tableRef.getPartitionNames().getPartitionNames();
OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions, true, IndexExtState.VISIBLE);
OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions, IndexExtState.VISIBLE, true);
if (copiedTbl == null) {
status = new Status(ErrCode.COMMON_ERROR, "failed to copy table: " + tblName);
return;

3
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -777,6 +777,7 @@ public class Catalog {
// 1. check and create dirs and files
File meta = new File(metaDir);
if (!meta.exists()) {
LOG.warn("Doris' meta dir {} does not exist. You need to create it before starting FE", meta.getAbsolutePath());
throw new Exception(meta.getAbsolutePath() + " does not exist, will exit");
}
@ -6680,7 +6681,7 @@ public class Catalog {
partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo());
}
}
copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), true, IndexExtState.VISIBLE);
copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), IndexExtState.VISIBLE, false);
} finally {
olapTable.readUnlock();
}

47
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -1278,34 +1278,37 @@ public class OlapTable extends Table {
return table instanceof OlapTable;
}
public OlapTable selectiveCopy(Collection<String> reservedPartitions, boolean resetState, IndexExtState extState) {
public OlapTable selectiveCopy(Collection<String> reservedPartitions, IndexExtState extState, boolean isForBackup) {
OlapTable copied = new OlapTable();
if (!DeepCopy.copy(this, copied, OlapTable.class, FeConstants.meta_version)) {
LOG.warn("failed to copy olap table: " + getName());
return null;
}
if (resetState) {
// remove shadow index from copied table
List<MaterializedIndex> shadowIndex = copied.getPartitions().stream().findFirst().get().getMaterializedIndices(IndexExtState.SHADOW);
// remove shadow index from copied table
List<MaterializedIndex> shadowIndex = copied.getPartitions().stream().findFirst().get().getMaterializedIndices(IndexExtState.SHADOW);
for (MaterializedIndex deleteIndex : shadowIndex) {
LOG.debug("copied table delete shadow index : {}", deleteIndex.getId());
copied.deleteIndexInfo(copied.getIndexNameById(deleteIndex.getId()));
}
copied.setState(OlapTableState.NORMAL);
for (Partition partition : copied.getPartitions()) {
// remove shadow index from partition
for (MaterializedIndex deleteIndex : shadowIndex) {
LOG.debug("copied table delete shadow index : {}", deleteIndex.getId());
copied.deleteIndexInfo(copied.getIndexNameById(deleteIndex.getId()));
partition.deleteRollupIndex(deleteIndex.getId());
}
copied.setState(OlapTableState.NORMAL);
for (Partition partition : copied.getPartitions()) {
// remove shadow index from partition
for (MaterializedIndex deleteIndex : shadowIndex) {
partition.deleteRollupIndex(deleteIndex.getId());
}
partition.setState(PartitionState.NORMAL);
partition.setState(PartitionState.NORMAL);
if (isForBackup) {
// set storage medium to HDD for backup job, because we want that the backuped table
// can be able to restored to another Doris cluster without SSD disk.
// But for other operation such as truncate table, keep the origin storage medium.
copied.getPartitionInfo().setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD));
for (MaterializedIndex idx : partition.getMaterializedIndices(extState)) {
idx.setState(IndexState.NORMAL);
for (Tablet tablet : idx.getTablets()) {
for (Replica replica : tablet.getReplicas()) {
replica.setState(ReplicaState.NORMAL);
}
}
for (MaterializedIndex idx : partition.getMaterializedIndices(extState)) {
idx.setState(IndexState.NORMAL);
for (Tablet tablet : idx.getTablets()) {
for (Replica replica : tablet.getReplicas()) {
replica.setState(ReplicaState.NORMAL);
}
}
}
@ -1315,10 +1318,10 @@ public class OlapTable extends Table {
// reserve all
return copied;
}
Set<String> partNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
partNames.addAll(copied.getPartitionNames());
for (String partName : partNames) {
if (!reservedPartitions.contains(partName)) {
copied.dropPartitionAndReserveTablet(partName);