database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Support check committed txns before catalog drop meta, like db, table, partition etc (#4029)

Browse Source 
This PR is to ensure that dropped db , table or partition can be with normal state after recovered by user. Commited txns can not be aborted, because the partitions's commited versions have been changed, and some tablets may already have new visible versions. If user just don't want the meta(db, table or partition) anymore, just use drop force instead of drop to skip committed txn check.
This commit is contained in:
caiconghui
2020-07-28 15:18:52 +08:00
committed by GitHub
parent 90eaa514ba
commit 150f8e0e2b
25 changed files with 419 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -235,7 +235,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A
KW_DELETE, KW_DISTINCT, KW_DISTINCTPC, KW_DISTINCTPCSA, KW_DISTRIBUTED, KW_DISTRIBUTION, KW_DYNAMIC, KW_BUCKETS, KW_DIV, KW_DOUBLE, KW_DROP, KW_DROPP, KW_DUPLICATE,
KW_ELSE, KW_END, KW_ENGINE, KW_ENGINES, KW_ENTER, KW_ERRORS, KW_EVENTS, KW_EXCEPT, KW_EXISTS, KW_EXPORT,
KW_EXTERNAL, KW_EXTRACT,
KW_FALSE, KW_FOLLOWER, KW_FOLLOWING, KW_FREE, KW_FROM, KW_FILE, KW_FIRST, KW_FLOAT, KW_FOR, KW_FORMAT, KW_FRONTEND, KW_FRONTENDS, KW_FULL, KW_FUNCTION, KW_FUNCTIONS,
KW_FALSE, KW_FOLLOWER, KW_FOLLOWING, KW_FREE, KW_FROM, KW_FILE, KW_FIRST, KW_FLOAT, KW_FOR, KW_FORCE, KW_FORMAT, KW_FRONTEND, KW_FRONTENDS, KW_FULL, KW_FUNCTION, KW_FUNCTIONS,
KW_GLOBAL, KW_GRANT, KW_GRANTS, KW_GROUP, KW_GROUPING,
KW_HASH, KW_HAVING, KW_HELP,KW_HLL, KW_HLL_UNION, KW_HOUR, KW_HUB,
KW_IDENTIFIED, KW_IF, KW_IN, KW_INDEX, KW_INDEXES, KW_INFILE, KW_INSTALL,
@ -463,6 +463,7 @@ nonterminal String opt_db, procedure_or_function, opt_comment, opt_engine;
nonterminal ColumnDef.DefaultValue opt_default_value;
nonterminal Boolean opt_if_exists, opt_if_not_exists;
nonterminal Boolean opt_external;
nonterminal Boolean opt_force;
nonterminal IndexDef.IndexType opt_index_type;
nonterminal ShowAlterStmt.AlterType opt_alter_type;
@ -882,9 +883,9 @@ alter_table_clause ::=
{:
RESULT = new AddPartitionClause(desc, distribution, properties, isTempPartition);
:}
| KW_DROP opt_tmp:isTempPartition KW_PARTITION opt_if_exists:ifExists ident:partitionName
| KW_DROP opt_tmp:isTempPartition KW_PARTITION opt_force:force opt_if_exists:ifExists ident:partitionName
{:
RESULT = new DropPartitionClause(ifExists, partitionName, isTempPartition);
RESULT = new DropPartitionClause(ifExists, partitionName, isTempPartition, force ? !force : !isTempPartition);
:}
| KW_MODIFY KW_PARTITION ident:partitionName KW_SET LPAREN key_value_map:properties RPAREN
{:
@ -1575,13 +1576,13 @@ revoke_stmt ::=
// Drop statement
drop_stmt ::=
/* Database */
KW_DROP KW_DATABASE opt_if_exists:ifExists ident:db
KW_DROP KW_DATABASE opt_force:force opt_if_exists:ifExists ident:db
{:
RESULT = new DropDbStmt(ifExists, db);
RESULT = new DropDbStmt(ifExists, db, !force);
:}
| KW_DROP KW_SCHEMA opt_if_exists:ifExists ident:db
| KW_DROP KW_SCHEMA opt_force:force opt_if_exists:ifExists ident:db
{:
RESULT = new DropDbStmt(ifExists, db);
RESULT = new DropDbStmt(ifExists, db, !force);
:}
/* cluster */
| KW_DROP KW_CLUSTER opt_if_exists:ifExists ident:cluster
@ -1594,9 +1595,9 @@ drop_stmt ::=
RESULT = new DropFunctionStmt(functionName, args);
:}
/* Table */
| KW_DROP KW_TABLE opt_if_exists:ifExists table_name:name
| KW_DROP KW_TABLE opt_force:force opt_if_exists:ifExists table_name:name
{:
RESULT = new DropTableStmt(ifExists, name);
RESULT = new DropTableStmt(ifExists, name, !force);
:}
/* User */
| KW_DROP KW_USER user_identity:userId
@ -1606,7 +1607,7 @@ drop_stmt ::=
/* View */
| KW_DROP KW_VIEW opt_if_exists:ifExists table_name:name
{:
RESULT = new DropTableStmt(ifExists, name, true);
RESULT = new DropTableStmt(ifExists, name, true, false);
:}
| KW_DROP KW_REPOSITORY ident:repoName
{:
@ -2025,6 +2026,17 @@ opt_external ::=
:}
;
opt_force ::=
/* empty */
{:
RESULT = false;
:}
| KW_FORCE
{:
RESULT = true;
:}
;
// Show statement
show_stmt ::=
KW_SHOW show_param:stmt

1
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View File

@ -143,7 +143,6 @@ public class Alter {
throw new DdlException("Table[" + table.getName() + "]'s state is not NORMAL. "
+ "Do not allow doing DROP ops");
}
// drop materialized view
((MaterializedViewHandler)materializedViewHandler).processDropMaterializedView(stmt, db, olapTable);

6
fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -701,7 +701,7 @@ public class MaterializedViewHandler extends AlterHandler {
public void processDropMaterializedView(DropMaterializedViewStmt dropMaterializedViewStmt, Database db,
OlapTable olapTable) throws DdlException, MetaNotFoundException {
db.writeLock();
Preconditions.checkState(db.isWriteLockHeldByCurrentThread());
try {
String mvName = dropMaterializedViewStmt.getMvName();
// Step1: check drop mv index operation
@ -710,7 +710,7 @@ public class MaterializedViewHandler extends AlterHandler {
long mvIndexId = dropMaterializedView(mvName, olapTable);
// Step3: log drop mv operation
EditLog editLog = Catalog.getCurrentCatalog().getEditLog();
editLog.logDropRollup(new DropInfo(db.getId(), olapTable.getId(), mvIndexId));
editLog.logDropRollup(new DropInfo(db.getId(), olapTable.getId(), mvIndexId, false));
LOG.info("finished drop materialized view [{}] in table [{}]", mvName, olapTable.getName());
} catch (MetaNotFoundException e) {
if (dropMaterializedViewStmt.isIfExists()) {
@ -718,8 +718,6 @@ public class MaterializedViewHandler extends AlterHandler {
} else {
throw e;
}
} finally {
db.writeUnlock();
}
}

1
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
View File

@ -48,7 +48,6 @@ public class AlterTableStmt extends DdlStmt {
return ops;
}
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);

8
fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
View File

@ -33,10 +33,12 @@ import com.google.common.base.Strings;
public class DropDbStmt extends DdlStmt {
private boolean ifExists;
private String dbName;
private boolean needCheckCommittedTxns;
public DropDbStmt(boolean ifExists, String dbName) {
public DropDbStmt(boolean ifExists, String dbName, boolean needCheckCommittedTxns) {
this.ifExists = ifExists;
this.dbName = dbName;
this.needCheckCommittedTxns = needCheckCommittedTxns;
}
public boolean isSetIfExists() {
@ -47,6 +49,10 @@ public class DropDbStmt extends DdlStmt {
return this.dbName;
}
public boolean isNeedCheckCommittedTxns() {
return this.needCheckCommittedTxns;
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);

8
fe/fe-core/src/main/java/org/apache/doris/analysis/DropPartitionClause.java
View File

@ -32,13 +32,15 @@ public class DropPartitionClause extends AlterTableClause {
private String partitionName;
// true if this is to drop a temp partition
private boolean isTempPartition;
private boolean needCheckCommittedTxns;
public DropPartitionClause(boolean ifExists, String partitionName, boolean isTempPartition) {
public DropPartitionClause(boolean ifExists, String partitionName, boolean isTempPartition, boolean needCheckCommittedTxns) {
super(AlterOpType.DROP_PARTITION);
this.ifExists = ifExists;
this.partitionName = partitionName;
this.isTempPartition = isTempPartition;
this.needTableStable = false;
this.needCheckCommittedTxns = needCheckCommittedTxns;
}
public boolean isSetIfExists() {
@ -53,6 +55,10 @@ public class DropPartitionClause extends AlterTableClause {
return isTempPartition;
}
public boolean isNeedCheckCommittedTxns() {
return needCheckCommittedTxns;
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
if (Strings.isNullOrEmpty(partitionName)) {

11
fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStmt.java
View File

@ -32,17 +32,20 @@ public class DropTableStmt extends DdlStmt {
private boolean ifExists;
private final TableName tableName;
private final boolean isView;
private boolean needCheckCommittedTxns;
public DropTableStmt(boolean ifExists, TableName tableName) {
public DropTableStmt(boolean ifExists, TableName tableName, boolean needCheckCommittedTxns) {
this.ifExists = ifExists;
this.tableName = tableName;
this.isView = false;
this.needCheckCommittedTxns = needCheckCommittedTxns;
}
public DropTableStmt(boolean ifExists, TableName tableName, boolean isView) {
public DropTableStmt(boolean ifExists, TableName tableName, boolean isView, boolean needCheckCommittedTxns) {
this.ifExists = ifExists;
this.tableName = tableName;
this.isView = isView;
this.needCheckCommittedTxns = needCheckCommittedTxns;
}
public boolean isSetIfExists() {
@ -61,6 +64,10 @@ public class DropTableStmt extends DdlStmt {
return isView;
}
public boolean isNeedCheckCommittedTxns() {
return this.needCheckCommittedTxns;
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
if (Strings.isNullOrEmpty(tableName.getDb())) {

72
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -2678,6 +2678,13 @@ public class Catalog {
Database db = this.fullNameToDb.get(dbName);
db.writeLock();
try {
if (stmt.isNeedCheckCommittedTxns()) {
if (Catalog.getCurrentCatalog().getGlobalTransactionMgr().existCommittedTxns(db.getId(), null, null)) {
throw new DdlException("There are still some transactions in the COMMITTED state waiting to be completed. " +
"The database [" + dbName +"] cannot be dropped. If you want to forcibly drop(cannot be recovered)," +
" please use \"DROP database FORCE\".");
}
}
if (db.getDbState() == DbState.LINK && dbName.equals(db.getAttachDb())) {
// We try to drop a hard link.
final DropLinkDbAndUpdateDbInfo info = new DropLinkDbAndUpdateDbInfo();
@ -2713,8 +2720,10 @@ public class Catalog {
// save table names for recycling
Set<String> tableNames = db.getTableNamesWithLock();
unprotectDropDb(db);
Catalog.getCurrentRecycleBin().recycleDatabase(db, tableNames);
unprotectDropDb(db, !stmt.isNeedCheckCommittedTxns());
if (stmt.isNeedCheckCommittedTxns()) {
Catalog.getCurrentRecycleBin().recycleDatabase(db, tableNames);
}
} finally {
db.writeUnlock();
}
@ -2724,17 +2733,17 @@ public class Catalog {
fullNameToDb.remove(db.getFullName());
final Cluster cluster = nameToCluster.get(db.getClusterName());
cluster.removeDb(dbName, db.getId());
editLog.logDropDb(dbName);
editLog.logDropDb(dbName, !stmt.isNeedCheckCommittedTxns());
} finally {
unlock();
}
LOG.info("finish drop database[{}]", dbName);
LOG.info("finish drop database[{}], is force : {}", dbName, !stmt.isNeedCheckCommittedTxns());
}
public void unprotectDropDb(Database db) {
public void unprotectDropDb(Database db, boolean isForeDrop) {
for (Table table : db.getTables()) {
unprotectDropTable(db, table.getId());
unprotectDropTable(db, table.getId(), isForeDrop);
}
}
@ -2754,15 +2763,17 @@ public class Catalog {
}
}
public void replayDropDb(String dbName) throws DdlException {
public void replayDropDb(String dbName, boolean isForceDrop) throws DdlException {
tryLock(true);
try {
Database db = fullNameToDb.get(dbName);
db.writeLock();
try {
Set<String> tableNames = db.getTableNamesWithLock();
unprotectDropDb(db);
Catalog.getCurrentRecycleBin().recycleDatabase(db, tableNames);
unprotectDropDb(db, isForceDrop);
if (!isForceDrop) {
Catalog.getCurrentRecycleBin().recycleDatabase(db, tableNames);
}
} finally {
db.writeUnlock();
}
@ -3322,14 +3333,24 @@ public class Catalog {
if (isTempPartition) {
olapTable.dropTempPartition(partitionName, true);
} else {
olapTable.dropPartition(db.getId(), partitionName);
if (clause.isNeedCheckCommittedTxns()) {
Partition partition = olapTable.getPartition(partitionName);
if (partition != null) {
if (Catalog.getCurrentCatalog().getGlobalTransactionMgr().existCommittedTxns(db.getId(), olapTable.getId(), partition.getId())) {
throw new DdlException("There are still some transactions in the COMMITTED state waiting to be completed." +
" The partition [" + partitionName + "] cannot be dropped. If you want to forcibly drop(cannot be recovered)," +
" please use \"DROP partition FORCE\".");
}
}
}
olapTable.dropPartition(db.getId(), partitionName, !clause.isNeedCheckCommittedTxns());
}
// log
DropPartitionInfo info = new DropPartitionInfo(db.getId(), olapTable.getId(), partitionName, isTempPartition);
DropPartitionInfo info = new DropPartitionInfo(db.getId(), olapTable.getId(), partitionName, isTempPartition, !clause.isNeedCheckCommittedTxns());
editLog.logDropPartition(info);
LOG.info("succeed in droping partition[{}]", partitionName);
LOG.info("succeed in droping partition[{}], is temp : {}, is force : {}", partitionName, isTempPartition, !clause.isNeedCheckCommittedTxns());
}
public void replayDropPartition(DropPartitionInfo info) {
@ -3340,7 +3361,7 @@ public class Catalog {
if (info.isTempPartition()) {
olapTable.dropTempPartition(info.getPartitionName(), true);
} else {
olapTable.dropPartition(info.getDbId(), info.getPartitionName());
olapTable.dropPartition(info.getDbId(), info.getPartitionName(), info.isForceDrop());
}
} finally {
db.writeUnlock();
@ -4284,18 +4305,24 @@ public class Catalog {
}
}
unprotectDropTable(db, table.getId());
DropInfo info = new DropInfo(db.getId(), table.getId(), -1L);
if (stmt.isNeedCheckCommittedTxns()) {
if (Catalog.getCurrentCatalog().getGlobalTransactionMgr().existCommittedTxns(db.getId(), table.getId(), null)) {
throw new DdlException("There are still some transactions in the COMMITTED state waiting to be completed. " +
"The table [" + tableName +"] cannot be dropped. If you want to forcibly drop(cannot be recovered)," +
" please use \"DROP table FORCE\".");
}
}
unprotectDropTable(db, table.getId(), !stmt.isNeedCheckCommittedTxns());
DropInfo info = new DropInfo(db.getId(), table.getId(), -1L, !stmt.isNeedCheckCommittedTxns());
editLog.logDropTable(info);
} finally {
db.writeUnlock();
}
LOG.info("finished dropping table: {} from db: {}", tableName, dbName);
LOG.info("finished dropping table: {} from db: {}, is force: {}", tableName, dbName, !stmt.isNeedCheckCommittedTxns());
}
public boolean unprotectDropTable(Database db, long tableId) {
public boolean unprotectDropTable(Database db, long tableId, boolean isForceDrop) {
Table table = db.getTable(tableId);
// delete from db meta
if (table == null) {
@ -4311,17 +4338,18 @@ public class Catalog {
}
db.dropTable(table.getName());
Catalog.getCurrentRecycleBin().recycleTable(db.getId(), table);
if (!isForceDrop) {
Catalog.getCurrentRecycleBin().recycleTable(db.getId(), table);
}
LOG.info("finished dropping table[{}] in db[{}]", table.getName(), db.getFullName());
return true;
}
public void replayDropTable(Database db, long tableId) {
public void replayDropTable(Database db, long tableId, boolean isForceDrop) {
db.writeLock();
try {
unprotectDropTable(db, tableId);
unprotectDropTable(db, tableId, isForceDrop);
} finally {
db.writeUnlock();
}

8
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -589,11 +589,7 @@ public class OlapTable extends Table {
nameToPartition.put(partition.getName(), partition);
}
public Partition dropPartition(long dbId, String partitionName) {
return dropPartition(dbId, partitionName, false);
}
public Partition dropPartition(long dbId, String partitionName, boolean isRestore) {
public Partition dropPartition(long dbId, String partitionName, boolean isForceDrop) {
Partition partition = nameToPartition.get(partitionName);
if (partition != null) {
idToPartition.remove(partition.getId());
@ -602,7 +598,7 @@ public class OlapTable extends Table {
Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE);
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo;
if (!isRestore) {
if (!isForceDrop) {
// recycle partition
Catalog.getCurrentRecycleBin().recyclePartition(dbId, id, partition,
rangePartitionInfo.getRange(partition.getId()),

2
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View File

@ -237,7 +237,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
RangeUtils.checkRangeIntersect(reservePartitionKeyRange, checkDropPartitionKey);
if (checkDropPartitionKey.upperEndpoint().compareTo(reservePartitionKeyRange.lowerEndpoint()) <= 0) {
String dropPartitionName = olapTable.getPartition(checkDropPartitionId).getName();
dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, false));
dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, false, true));
}
} catch (DdlException e) {
break;

5
fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java
View File

@ -187,6 +187,9 @@ public final class FeMetaVersion {
public static final int VERSION_87 = 87;
// add partition visibleVersionTime
public static final int VERSION_88 = 88;
// force drop db, force drop table, force drop partition
// make force drop operation do not recycle meta
public static final int VERSION_89 = 89;
// note: when increment meta version, should assign the latest version to VERSION_CURRENT
public static final int VERSION_CURRENT = VERSION_88;
public static final int VERSION_CURRENT = VERSION_89;
}

4
fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
View File

@ -56,6 +56,7 @@ import org.apache.doris.persist.ColocatePersistInfo;
import org.apache.doris.persist.ConsistencyCheckInfo;
import org.apache.doris.persist.CreateTableInfo;
import org.apache.doris.persist.DatabaseInfo;
import org.apache.doris.persist.DropDbInfo;
import org.apache.doris.persist.DropInfo;
import org.apache.doris.persist.DropLinkDbAndUpdateDbInfo;
import org.apache.doris.persist.DropPartitionInfo;
@ -149,8 +150,7 @@ public class JournalEntity implements Writable {
break;
}
case OperationType.OP_DROP_DB: {
data = new Text();
((Text) data).readFields(in);
data = DropDbInfo.read(in);
isRead = true;
break;
}

90
fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java Normal file
View File

@ -0,0 +1,90 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.persist;
import com.google.gson.annotations.SerializedName;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class DropDbInfo implements Writable {
@SerializedName(value = "dbName")
private String dbName;
@SerializedName(value = "forceDrop")
private boolean forceDrop = false;
public DropDbInfo() {
this("", false);
}
public DropDbInfo(String dbName, boolean forceDrop) {
this.dbName = dbName;
this.forceDrop = forceDrop;
}
public String getDbName() {
return dbName;
}
public boolean isForceDrop() {
return forceDrop;
}
private void readFields(DataInput in) throws IOException {
dbName = Text.readString(in);
}
public static DropDbInfo read(DataInput in) throws IOException {
if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_88) {
DropDbInfo info = new DropDbInfo();
info.readFields(in);
return info;
} else {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, DropDbInfo.class);
}
}
@Override
public void write(DataOutput out) throws IOException {
String json = GsonUtils.GSON.toJson(this);
Text.writeString(out, json);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!(obj instanceof DropDbInfo)) {
return false;
}
DropDbInfo info = (DropDbInfo) obj;
return (dbName.equals(info.getDbName()))
&& (forceDrop == info.isForceDrop());
}
}

20
fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java
View File

@ -17,6 +17,8 @@
package org.apache.doris.persist;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Writable;
import java.io.DataInput;
@ -28,14 +30,16 @@ public class DropInfo implements Writable {
private long tableId;
private long indexId;
private boolean forceDrop = false;
public DropInfo() {
}
public DropInfo(long dbId, long tableId, long indexId) {
public DropInfo(long dbId, long tableId, long indexId, boolean forceDrop) {
this.dbId = dbId;
this.tableId = tableId;
this.indexId = indexId;
this.forceDrop = forceDrop;
}
public long getDbId() {
@ -49,11 +53,16 @@ public class DropInfo implements Writable {
public long getIndexId() {
return this.indexId;
}
public boolean isForceDrop() {
return forceDrop;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(dbId);
out.writeLong(tableId);
out.writeBoolean(forceDrop);
if (indexId == -1L) {
out.writeBoolean(false);
} else {
@ -65,7 +74,9 @@ public class DropInfo implements Writable {
public void readFields(DataInput in) throws IOException {
dbId = in.readLong();
tableId = in.readLong();
if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_89) {
forceDrop = in.readBoolean();
}
boolean hasIndexId = in.readBoolean();
if (hasIndexId) {
indexId = in.readLong();
@ -73,7 +84,7 @@ public class DropInfo implements Writable {
indexId = -1L;
}
}
public static DropInfo read(DataInput in) throws IOException {
DropInfo dropInfo = new DropInfo();
dropInfo.readFields(in);
@ -91,6 +102,7 @@ public class DropInfo implements Writable {
DropInfo info = (DropInfo) obj;
return dbId == info.dbId && tableId == info.tableId;
return (dbId == info.dbId) && (tableId == info.tableId) && (indexId == info.indexId)
&& (forceDrop == info.forceDrop);
}
}

13
fe/fe-core/src/main/java/org/apache/doris/persist/DropPartitionInfo.java
View File

@ -38,15 +38,18 @@ public class DropPartitionInfo implements Writable {
private String partitionName;
@SerializedName(value = "isTempPartition")
private boolean isTempPartition = false;
@SerializedName(value = "forceDrop")
private boolean forceDrop = false;
private DropPartitionInfo() {
}
public DropPartitionInfo(Long dbId, Long tableId, String partitionName, boolean isTempPartition) {
public DropPartitionInfo(Long dbId, Long tableId, String partitionName, boolean isTempPartition, boolean forceDrop) {
this.dbId = dbId;
this.tableId = tableId;
this.partitionName = partitionName;
this.isTempPartition = isTempPartition;
this.forceDrop = forceDrop;
}
public Long getDbId() {
@ -65,6 +68,10 @@ public class DropPartitionInfo implements Writable {
return isTempPartition;
}
public boolean isForceDrop() {
return forceDrop;
}
private void readFields(DataInput in) throws IOException {
dbId = in.readLong();
tableId = in.readLong();
@ -101,6 +108,8 @@ public class DropPartitionInfo implements Writable {
return (dbId.equals(info.dbId))
&& (tableId.equals(info.tableId))
&& (partitionName.equals(info.partitionName));
&& (partitionName.equals(info.partitionName))
&& (isTempPartition == info.isTempPartition)
&& (forceDrop == info.forceDrop);
}
}

10
fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
View File

@ -145,8 +145,8 @@ public class EditLog {
break;
}
case OperationType.OP_DROP_DB: {
String dbName = ((Text) journal.getData()).toString();
catalog.replayDropDb(dbName);
DropDbInfo dropDbInfo = (DropDbInfo) journal.getData();
catalog.replayDropDb(dropDbInfo.getDbName(), dropDbInfo.isForceDrop());
break;
}
case OperationType.OP_ALTER_DB: {
@ -189,7 +189,7 @@ public class EditLog {
}
LOG.info("Begin to unprotect drop table. db = "
+ db.getFullName() + " table = " + info.getTableId());
catalog.replayDropTable(db, info.getTableId());
catalog.replayDropTable(db, info.getTableId(), info.isForceDrop());
break;
}
case OperationType.OP_ADD_PARTITION: {
@ -297,7 +297,7 @@ public class EditLog {
BatchDropInfo batchDropInfo = (BatchDropInfo) journal.getData();
for (long indexId : batchDropInfo.getIndexIdSet()) {
catalog.getRollupHandler().replayDropRollup(
new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(), indexId), catalog);
new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(), indexId, false), catalog);
}
break;
}
@ -889,7 +889,7 @@ public class EditLog {
logEdit(OperationType.OP_CREATE_DB, db);
}
public void logDropDb(String dbName) {
public void logDropDb(String dbName, boolean isForceDrop) {
logEdit(OperationType.OP_DROP_DB, new Text(dbName));
}

22
fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
View File

@ -217,14 +217,32 @@ public class GlobalTransactionMgr implements Writable {
* get all txns which is ready to publish
* a ready-to-publish txn's partition's visible version should be ONE less than txn's commit version.
*/
public List<TransactionState> getReadyToPublishTransactions() throws UserException {
public List<TransactionState> getReadyToPublishTransactions() {
List<TransactionState> transactionStateList = Lists.newArrayList();
for (DatabaseTransactionMgr dbTransactionMgr : dbIdToDatabaseTransactionMgrs.values()) {
transactionStateList.addAll(dbTransactionMgr.getCommittedTxnList());
}
return transactionStateList;
}
public boolean existCommittedTxns(Long dbId, Long tableId, Long partitionId) {
DatabaseTransactionMgr dbTransactionMgr = dbIdToDatabaseTransactionMgrs.get(dbId);
if (tableId == null && partitionId == null) {
return !dbTransactionMgr.getCommittedTxnList().isEmpty();
}
for (TransactionState transactionState : dbTransactionMgr.getCommittedTxnList()) {
if (transactionState.getTableIdList().contains(tableId)) {
if (partitionId == null) {
return true;
} else if (transactionState.getTableCommitInfo(tableId).getPartitionCommitInfo(partitionId) != null){
return true;
}
}
}
return false;
}
/**
* if the table is deleted between commit and publish version, then should ignore the partition
*

1
fe/fe-core/src/main/jflex/sql_scanner.flex
View File

@ -185,6 +185,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("follower", new Integer(SqlParserSymbols.KW_FOLLOWER));
keywordMap.put("following", new Integer(SqlParserSymbols.KW_FOLLOWING));
keywordMap.put("for", new Integer(SqlParserSymbols.KW_FOR));
keywordMap.put("force", new Integer(SqlParserSymbols.KW_FORCE));
keywordMap.put("format", new Integer(SqlParserSymbols.KW_FORMAT));
keywordMap.put("free", new Integer(SqlParserSymbols.KW_FREE));
keywordMap.put("from", new Integer(SqlParserSymbols.KW_FROM));

6
fe/fe-core/src/test/java/org/apache/doris/analysis/AlterTableStmtTest.java
View File

@ -62,7 +62,7 @@ public class AlterTableStmtTest {
}
@Test
public void testNormal() throws AnalysisException, UserException {
public void testNormal() throws UserException {
List<AlterClause> ops = Lists.newArrayList();
ops.add(new DropColumnClause("col1", "", null));
ops.add(new DropColumnClause("col2", "", null));
@ -89,7 +89,7 @@ public class AlterTableStmtTest {
}
@Test(expected = AnalysisException.class)
public void testNoTable() throws AnalysisException, UserException {
public void testNoTable() throws UserException {
List<AlterClause> ops = Lists.newArrayList();
ops.add(new DropColumnClause("col1", "", null));
AlterTableStmt stmt = new AlterTableStmt(null, ops);
@ -99,7 +99,7 @@ public class AlterTableStmtTest {
}
@Test(expected = AnalysisException.class)
public void testNoClause() throws AnalysisException, UserException {
public void testNoClause() throws UserException {
List<AlterClause> ops = Lists.newArrayList();
AlterTableStmt stmt = new AlterTableStmt(new TableName("testDb", "testTbl"), ops);
stmt.analyze(analyzer);

6
fe/fe-core/src/test/java/org/apache/doris/analysis/DropDbStmtTest.java
View File

@ -46,7 +46,7 @@ public class DropDbStmtTest {
@Test
public void testNormal() throws UserException, AnalysisException {
DropDbStmt stmt = new DropDbStmt(false, "test");
DropDbStmt stmt = new DropDbStmt(false, "test", true);
stmt.analyze(analyzer);
Assert.assertEquals("testCluster:test", stmt.getDbName());
@ -55,7 +55,7 @@ public class DropDbStmtTest {
@Test(expected = AnalysisException.class)
public void testFailed() throws UserException, AnalysisException {
DropDbStmt stmt = new DropDbStmt(false, "");
DropDbStmt stmt = new DropDbStmt(false, "", true);
stmt.analyze(analyzer);
Assert.fail("no exception");
@ -63,7 +63,7 @@ public class DropDbStmtTest {
@Test(expected = AnalysisException.class)
public void testNoPriv() throws UserException, AnalysisException {
DropDbStmt stmt = new DropDbStmt(false, "");
DropDbStmt stmt = new DropDbStmt(false, "", true);
stmt.analyze(AccessTestUtil.fetchBlockAnalyzer());
Assert.fail("no exception");

1
fe/fe-core/src/test/java/org/apache/doris/analysis/DropMaterializedViewStmtTest.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.analysis;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.mysql.privilege.PrivPredicate;

8
fe/fe-core/src/test/java/org/apache/doris/analysis/DropTableStmtTest.java
View File

@ -66,7 +66,7 @@ public class DropTableStmtTest {
@Test
public void testNormal() throws UserException, AnalysisException {
DropTableStmt stmt = new DropTableStmt(false, tbl);
DropTableStmt stmt = new DropTableStmt(false, tbl, true);
stmt.analyze(analyzer);
Assert.assertEquals("testCluster:db1", stmt.getDbName());
Assert.assertEquals("table1", stmt.getTableName());
@ -75,7 +75,7 @@ public class DropTableStmtTest {
@Test
public void testDefaultNormal() throws UserException, AnalysisException {
DropTableStmt stmt = new DropTableStmt(false, noDbTbl);
DropTableStmt stmt = new DropTableStmt(false, noDbTbl, true);
stmt.analyze(analyzer);
Assert.assertEquals("testCluster:testDb", stmt.getDbName());
Assert.assertEquals("table1", stmt.getTableName());
@ -84,14 +84,14 @@ public class DropTableStmtTest {
@Test(expected = AnalysisException.class)
public void testNoDbFail() throws UserException, AnalysisException {
DropTableStmt stmt = new DropTableStmt(false, noDbTbl);
DropTableStmt stmt = new DropTableStmt(false, noDbTbl, true);
stmt.analyze(noDbAnalyzer);
Assert.fail("No Exception throws.");
}
@Test(expected = AnalysisException.class)
public void testNoTableFail() throws UserException, AnalysisException {
DropTableStmt stmt = new DropTableStmt(false, new TableName("db1", ""));
DropTableStmt stmt = new DropTableStmt(false, new TableName("db1", ""), true);
stmt.analyze(noDbAnalyzer);
Assert.fail("No Exception throws.");
}

75
fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java Normal file
View File

@ -0,0 +1,75 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.persist;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.meta.MetaContext;
import org.junit.Assert;
import org.junit.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
public class DropDbInfoTest {
@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeMetaVersion.VERSION_89);
metaContext.setThreadLocalInfo();
// 1. Write objects to file
File file = new File("./dropDbInfo");
file.createNewFile();
DataOutputStream dos = new DataOutputStream(new FileOutputStream(file));
DropDbInfo info1 = new DropDbInfo();
info1.write(dos);
DropDbInfo info2 = new DropDbInfo("test_db", true);
info2.write(dos);
dos.flush();
dos.close();
// 2. Read objects from file
DataInputStream dis = new DataInputStream(new FileInputStream(file));
DropDbInfo rInfo1 = DropDbInfo.read(dis);
Assert.assertTrue(rInfo1.equals(info1));
DropDbInfo rInfo2 = DropDbInfo.read(dis);
Assert.assertTrue(rInfo2.equals(info2));
Assert.assertEquals("test_db", rInfo2.getDbName());
Assert.assertTrue(rInfo2.isForceDrop());
Assert.assertTrue(rInfo2.equals(rInfo2));
Assert.assertFalse(rInfo2.equals(this));
Assert.assertFalse(info2.equals(new DropDbInfo("test_db1", true)));
Assert.assertFalse(info2.equals(new DropDbInfo("test_db", false)));
Assert.assertTrue(info2.equals(new DropDbInfo("test_db", true)));
// 3. delete files
dis.close();
file.delete();
}
}

16
fe/fe-core/src/test/java/org/apache/doris/persist/DropInfoTest.java
View File

@ -17,6 +17,8 @@
package org.apache.doris.persist;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.meta.MetaContext;
import org.junit.Assert;
import org.junit.Test;
@ -29,6 +31,10 @@ import java.io.FileOutputStream;
public class DropInfoTest {
@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeMetaVersion.VERSION_89);
metaContext.setThreadLocalInfo();
// 1. Write objects to file
File file = new File("./dropInfo");
file.createNewFile();
@ -37,7 +43,7 @@ public class DropInfoTest {
DropInfo info1 = new DropInfo();
info1.write(dos);
DropInfo info2 = new DropInfo(1, 2, -1);
DropInfo info2 = new DropInfo(1, 2, -1, true);
info2.write(dos);
dos.flush();
@ -54,12 +60,14 @@ public class DropInfoTest {
Assert.assertEquals(1, rInfo2.getDbId());
Assert.assertEquals(2, rInfo2.getTableId());
Assert.assertTrue(rInfo2.isForceDrop());
Assert.assertTrue(rInfo2.equals(rInfo2));
Assert.assertFalse(rInfo2.equals(this));
Assert.assertFalse(info2.equals(new DropInfo(0, 2, -1L)));
Assert.assertFalse(info2.equals(new DropInfo(1, 0, -1L)));
Assert.assertTrue(info2.equals(new DropInfo(1, 2, -1L)));
Assert.assertFalse(info2.equals(new DropInfo(0, 2, -1L, true)));
Assert.assertFalse(info2.equals(new DropInfo(1, 0, -1L, true)));
Assert.assertFalse(info2.equals(new DropInfo(1, 2, -1L, false)));
Assert.assertTrue(info2.equals(new DropInfo(1, 2, -1L, true)));
// 3. delete files
dis.close();

73
fe/fe-core/src/test/java/org/apache/doris/persist/DropPartitionInfoTest.java Normal file
View File

@ -0,0 +1,73 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.persist;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.meta.MetaContext;
import org.junit.Assert;
import org.junit.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
public class DropPartitionInfoTest {
@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeMetaVersion.VERSION_89);
metaContext.setThreadLocalInfo();
// 1. Write objects to file
File file = new File("./dropPartitionInfo");
file.createNewFile();
DataOutputStream dos = new DataOutputStream(new FileOutputStream(file));
DropPartitionInfo info1 = new DropPartitionInfo(1L, 2L, "test_partition", false, true);
info1.write(dos);
dos.flush();
dos.close();
// 2. Read objects from file
DataInputStream dis = new DataInputStream(new FileInputStream(file));
DropPartitionInfo rInfo1 = DropPartitionInfo.read(dis);
Assert.assertEquals(Long.valueOf(1L), rInfo1.getDbId());
Assert.assertEquals(Long.valueOf(2L), rInfo1.getTableId());
Assert.assertEquals("test_partition", rInfo1.getPartitionName());
Assert.assertFalse(rInfo1.isTempPartition());
Assert.assertTrue(rInfo1.isForceDrop());
Assert.assertTrue(rInfo1.equals(info1));
Assert.assertFalse(rInfo1.equals(this));
Assert.assertFalse(info1.equals(new DropPartitionInfo(-1L, 2L, "test_partition", false, true)));
Assert.assertFalse(info1.equals(new DropPartitionInfo(1L, -2L, "test_partition", false, true)));
Assert.assertFalse(info1.equals(new DropPartitionInfo(1L, 2L, "test_partition1", false, true)));
Assert.assertFalse(info1.equals(new DropPartitionInfo(1L, 2L, "test_partition", true, true)));
Assert.assertFalse(info1.equals(new DropPartitionInfo(1L, 2L, "test_partition", false, false)));
Assert.assertTrue(info1.equals(new DropPartitionInfo(1L, 2L, "test_partition", false, true)));
// 3. delete files
dis.close();
file.delete();
}
}