database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] Fix bug that can not repair replica in DECOMMISSION state (#6560)

Browse Source 
Fix bug that if a tablet belongs to a colocation table, and one of its
replica is in DECOMMISSION state. This tablet can not be repaired.

Also fix a bug that quota does not escape in show create table result.

```
COMMENT "a"bc" to COMMENT "a\"bc"
```
This commit is contained in:
Mingyu Chen
2021-09-07 11:55:23 +08:00
committed by GitHub
parent 80cb22c880
commit bf6d043a7e
8 changed files with 70 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View File

@ -25,7 +25,7 @@ body:
value: |
Thank you very much for submitting feedback to Doris to help Doris develop better.
If it is a problem with Doris, please go to the [Discussion](https://github.com/apache/incubator-doris/discussions)
If it is an idea or help wanted, please go to the [Discussion](https://github.com/apache/incubator-doris/discussions)
or [Dev Mail List](mailto:dev@doris.apache.org)
- type: checkboxes

27
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.analysis.AddPartitionClause;
import org.apache.doris.analysis.AddRollupClause;
import org.apache.doris.analysis.AdminCheckTabletsStmt;
import org.apache.doris.analysis.AdminCheckTabletsStmt.CheckType;
import org.apache.doris.analysis.AdminCleanTrashStmt;
import org.apache.doris.analysis.AdminSetConfigStmt;
import org.apache.doris.analysis.AdminSetReplicaStatusStmt;
import org.apache.doris.analysis.AlterClause;
@ -43,7 +44,6 @@ import org.apache.doris.analysis.BackupStmt;
import org.apache.doris.analysis.CancelAlterSystemStmt;
import org.apache.doris.analysis.CancelAlterTableStmt;
import org.apache.doris.analysis.CancelBackupStmt;
import org.apache.doris.analysis.AdminCleanTrashStmt;
import org.apache.doris.analysis.ColumnRenameClause;
import org.apache.doris.analysis.CreateClusterStmt;
import org.apache.doris.analysis.CreateDbStmt;
@ -105,9 +105,9 @@ import org.apache.doris.cluster.BaseParam;
import org.apache.doris.cluster.Cluster;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Config;
import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@ -225,12 +225,12 @@ import org.apache.doris.task.CreateReplicaTask;
import org.apache.doris.task.DropReplicaTask;
import org.apache.doris.task.MasterTaskExecutor;
import org.apache.doris.thrift.BackendService;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TTabletType;
import org.apache.doris.thrift.TTaskType;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.transaction.DbUsedDataQuotaInfoCollector;
import org.apache.doris.transaction.GlobalTransactionMgr;
import org.apache.doris.transaction.PublishVersionDaemon;
@ -244,10 +244,14 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Queues;
import com.google.common.collect.Sets;
import com.sleepycat.je.rep.InsufficientLogException;
import com.sleepycat.je.rep.NetworkRestore;
import com.sleepycat.je.rep.NetworkRestoreConfig;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import javax.annotation.Nullable;
import java.io.BufferedReader;
@ -275,11 +279,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import com.sleepycat.je.rep.InsufficientLogException;
import com.sleepycat.je.rep.NetworkRestore;
import com.sleepycat.je.rep.NetworkRestoreConfig;
import org.codehaus.jackson.map.ObjectMapper;
public class Catalog {
private static final Logger LOG = LogManager.getLogger(Catalog.class);
// 0 ~ 9999 used for qe
@ -4023,7 +4022,7 @@ public class Catalog {
sb.append(Joiner.on(", ").join(keysColumnNames)).append(")");
if (!Strings.isNullOrEmpty(table.getComment())) {
sb.append("\nCOMMENT \"").append(table.getComment()).append("\"");
sb.append("\nCOMMENT \"").append(table.getComment(true)).append("\"");
}
// partition
@ -4150,7 +4149,7 @@ public class Catalog {
} else if (table.getType() == TableType.MYSQL) {
MysqlTable mysqlTable = (MysqlTable) table;
if (!Strings.isNullOrEmpty(table.getComment())) {
sb.append("\nCOMMENT \"").append(table.getComment()).append("\"");
sb.append("\nCOMMENT \"").append(table.getComment(true)).append("\"");
}
// properties
sb.append("\nPROPERTIES (\n");
@ -4168,7 +4167,7 @@ public class Catalog {
} else if (table.getType() == TableType.ODBC) {
OdbcTable odbcTable = (OdbcTable) table;
if (!Strings.isNullOrEmpty(table.getComment())) {
sb.append("\nCOMMENT \"").append(table.getComment()).append("\"");
sb.append("\nCOMMENT \"").append(table.getComment(true)).append("\"");
}
// properties
sb.append("\nPROPERTIES (\n");
@ -4188,7 +4187,7 @@ public class Catalog {
} else if (table.getType() == TableType.BROKER) {
BrokerTable brokerTable = (BrokerTable) table;
if (!Strings.isNullOrEmpty(table.getComment())) {
sb.append("\nCOMMENT \"").append(table.getComment()).append("\"");
sb.append("\nCOMMENT \"").append(table.getComment(true)).append("\"");
}
// properties
sb.append("\nPROPERTIES (\n");
@ -4206,7 +4205,7 @@ public class Catalog {
} else if (table.getType() == TableType.ELASTICSEARCH) {
EsTable esTable = (EsTable) table;
if (!Strings.isNullOrEmpty(table.getComment())) {
sb.append("\nCOMMENT \"").append(table.getComment()).append("\"");
sb.append("\nCOMMENT \"").append(table.getComment(true)).append("\"");
}
// partition
@ -4242,7 +4241,7 @@ public class Catalog {
} else if (table.getType() == TableType.HIVE) {
HiveTable hiveTable = (HiveTable) table;
if (!Strings.isNullOrEmpty(table.getComment())) {
sb.append("\nCOMMENT \"").append(table.getComment()).append("\"");
sb.append("\nCOMMENT \"").append(table.getComment(true)).append("\"");
}
// properties
sb.append("\nPROPERTIES (\n");

12
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
@ -298,7 +299,14 @@ public class Column implements Writable {
}
public String getComment() {
return comment;
return getComment(false);
}
public String getComment(boolean escapeQuota) {
if (!escapeQuota) {
return comment;
}
return SqlUtils.escapeQuota(comment);
}
public int getOlapColumnIndexSize() {
@ -488,7 +496,7 @@ public class Column implements Writable {
if (defaultValue != null && getDataType() != PrimitiveType.HLL && getDataType() != PrimitiveType.BITMAP) {
sb.append("DEFAULT \"").append(defaultValue).append("\" ");
}
sb.append("COMMENT \"").append(comment).append("\"");
sb.append("COMMENT \"").append(getComment(true)).append("\"");
return sb.toString();
}

18
fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
View File

@ -21,18 +21,19 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@ -396,8 +397,15 @@ public class Table extends MetaObject implements Writable {
}
public String getComment() {
return getComment(false);
}
public String getComment(boolean escapeQuota) {
if (!Strings.isNullOrEmpty(comment)) {
return comment;
if (!escapeQuota) {
return comment;
}
return SqlUtils.escapeQuota(comment);
}
return type.name();
}

3
fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
View File

@ -205,8 +205,7 @@ public class Tablet extends MetaObject implements Writable {
}
ReplicaState state = replica.getState();
if (infoService.checkBackendAlive(replica.getBackendId())
&& (state == ReplicaState.NORMAL || state == ReplicaState.ALTER)) {
if (infoService.checkBackendAlive(replica.getBackendId()) && state.canLoad()) {
map.put(replica.getBackendId(), replica.getPathHash());
}
}

3
fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
View File

@ -422,7 +422,8 @@ public class ColocateTableCheckerAndBalancer extends MasterDaemon {
if (!backendsSet.contains(destBeId) && !hostsSet.contains(destBe.getHost())) {
Preconditions.checkState(backendsSet.contains(srcBeId), srcBeId);
flatBackendsPerBucketSeq.set(seqIndex, destBeId);
LOG.info("replace backend {} with backend {} in colocate group {}", srcBeId, destBeId, groupId);
LOG.info("replace backend {} with backend {} in colocate group {}, idx: {}",
srcBeId, destBeId, groupId, seqIndex);
// just replace one backend at a time, src and dest BE id should be recalculated because
// flatBackendsPerBucketSeq is changed.
isChanged = true;

23
fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
View File

@ -612,12 +612,31 @@ public class TabletSchedCtx implements Comparable<TabletSchedCtx> {
if (slot == null) {
throw new SchedException(Status.SCHEDULE_FAILED, "backend of dest replica is missing");
}
long destPathHash = slot.takeSlot(chosenReplica.getPathHash());
if (destPathHash == -1) {
throw new SchedException(Status.SCHEDULE_FAILED, "unable to take slot of dest path");
}
if (chosenReplica.getState() == ReplicaState.DECOMMISSION) {
// Since this replica is selected as the repair object of VERSION_INCOMPLETE,
// it means that this replica needs to be able to accept loading data.
// So if this replica was previously set to DECOMMISSION, this state needs to be reset to NORMAL.
// It may happen as follows:
// 1. A tablet of colocation table is in COLOCATION_REDUNDANT state
// 2. The tablet is being scheduled and set one of replica as DECOMMISSION in TabletScheduler.deleteReplicaInternal()
// 3. The tablet will then be scheduled again
// 4. But at that time, the BE node of the replica that was
// set to the DECOMMISSION state in step 2 is returned to the colocation group.
// So the tablet's health status becomes VERSION_INCOMPLETE.
//
// If we do not reset this replica state to NORMAL, the tablet's health status will be in VERSION_INCOMPLETE
// forever, because the replica in the DECOMMISSION state will not receive the load task.
chosenReplica.setWatermarkTxnId(-1);
chosenReplica.setState(ReplicaState.NORMAL);
LOG.info("choose replica {} on backend {} of tablet {} as dest replica for version incomplete," +
" and change state from DECOMMISSION to NORMAL", chosenReplica.getId(), chosenReplica.getBackendId(), tabletId);
}
setDest(chosenReplica.getBackendId(), chosenReplica.getPathHash());
}

9
fe/fe-core/src/main/java/org/apache/doris/common/util/SqlUtils.java
View File

@ -17,6 +17,8 @@
package org.apache.doris.common.util;
import org.apache.parquet.Strings;
public class SqlUtils {
public static String escapeUnquote(String ident) {
return ident.replaceAll("``", "`");
@ -35,4 +37,11 @@ public class SqlUtils {
sb.append('`');
return sb.toString();
}
public static String escapeQuota(String str) {
if (Strings.isNullOrEmpty(str)) {
return str;
}
return str.replaceAll("\"", "\\\\\"");
}
}