From 9d72d1bb8726d3f67caf679c180d1abec5864d2e Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Mon, 18 May 2020 14:53:59 +0800 Subject: [PATCH] [Refactor] Refactor some redundant code && Replace some UT by UtFrameUtils This CL have no logic changed, just do some code refactor and use new UtFrameWork to replace some old UT. NOTICE(#3622): This is a "revert of revert pull request". This pr is mainly used to synthesize the PRs whose commits were scattered and submitted due to the wrong merge method into a complete single commit. --- .../java/org/apache/doris/alter/AlterJob.java | 18 +- .../org/apache/doris/backup/RestoreJob.java | 175 ++--- .../org/apache/doris/catalog/Database.java | 2 +- .../doris/catalog/MaterializedIndex.java | 8 +- .../org/apache/doris/catalog/OlapTable.java | 6 +- .../org/apache/doris/catalog/Partition.java | 7 +- .../apache/doris/catalog/PartitionInfo.java | 8 +- .../org/apache/doris/catalog/Replica.java | 1 + .../java/org/apache/doris/catalog/Tablet.java | 19 +- .../org/apache/doris/catalog/TabletMeta.java | 12 +- .../doris/clone/BackendLoadStatistic.java | 25 +- .../doris/clone/RootPathLoadStatistic.java | 8 +- .../apache/doris/clone/TabletScheduler.java | 89 ++- .../proc/ClusterLoadStatisticProcDir.java | 6 +- .../doris/system/BackendHbResponse.java | 1 + .../apache/doris/system/BrokerHbResponse.java | 1 + .../catalog/DynamicPartitionTableTest.java | 688 +++++++----------- .../doris/common/PropertyAnalyzerTest.java | 10 +- 18 files changed, 428 insertions(+), 656 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterJob.java b/fe/src/main/java/org/apache/doris/alter/AlterJob.java index ab927cc18f..7b10fd5da0 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterJob.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterJob.java @@ -189,18 +189,14 @@ public abstract class AlterJob implements Writable { return false; } else if (!backend.isAlive()) { long currentTime = System.currentTimeMillis(); - if (backend.getLastUpdateMs() > 0 - && currentTime - backend.getLastUpdateMs() > Config.max_backend_down_time_second * 1000) { - // this backend is done for a long time and not restart automatically. - // we consider it as dead - return false; - } - return true; - } else if (backend.isDecommissioned()) { - return false; + // If this backend is done for a long time and not restart automatically. + // we consider it as dead and return false. + return backend.getLastUpdateMs() <= 0 + || currentTime - backend.getLastUpdateMs() <= Config.max_backend_down_time_second * 1000; + } else { + return !backend.isDecommissioned(); } - - return true; + } public static AlterJob read(DataInput in) throws IOException { diff --git a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java index ef576b4572..335ac17228 100644 --- a/fe/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -172,11 +172,7 @@ public class RestoreJob extends AbstractJob { } public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { - Preconditions.checkState(task.getJobId() == jobId); - Preconditions.checkState(task.getDbId() == dbId); - - if (request.getTask_status().getStatus_code() != TStatusCode.OK) { - taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + if (checkTaskStatus(task, task.getJobId(), request)) { return false; } @@ -198,19 +194,15 @@ public class RestoreJob extends AbstractJob { taskErrMsg.remove(task.getSignature()); Preconditions.checkState(task.getTabletId() == removedTabletId, removedTabletId); LOG.debug("get finished snapshot info: {}, unfinished tasks num: {}, remove result: {}. {}", - info, unfinishedSignatureToId.size(), this); + info, unfinishedSignatureToId.size(), this, removedTabletId); return true; } - return false; } - public synchronized boolean finishTabletDownloadTask(DownloadTask task, TFinishTaskRequest request) { - Preconditions.checkState(task.getJobId() == jobId); - Preconditions.checkState(task.getDbId() == dbId); - if (request.getTask_status().getStatus_code() != TStatusCode.OK) { - taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + public synchronized boolean finishTabletDownloadTask(DownloadTask task, TFinishTaskRequest request) { + if (checkTaskStatus(task, task.getJobId(), request)) { return false; } @@ -237,11 +229,7 @@ public class RestoreJob extends AbstractJob { } public synchronized boolean finishDirMoveTask(DirMoveTask task, TFinishTaskRequest request) { - Preconditions.checkState(task.getJobId() == jobId); - Preconditions.checkState(task.getDbId() == dbId); - - if (request.getTask_status().getStatus_code() != TStatusCode.OK) { - taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + if (checkTaskStatus(task, task.getJobId(), request)) { return false; } @@ -256,6 +244,17 @@ public class RestoreJob extends AbstractJob { return true; } + private boolean checkTaskStatus(AgentTask task, long jobId, TFinishTaskRequest request) { + Preconditions.checkState(jobId == this.jobId); + Preconditions.checkState(dbId == task.getDbId()); + + if (request.getTask_status().getStatus_code() != TStatusCode.OK) { + taskErrMsg.put(task.getSignature(), Joiner.on(",").join(request.getTask_status().getError_msgs())); + return true; + } + return false; + } + @Override public synchronized void replayRun() { LOG.info("replay run restore job: {}", this); @@ -342,7 +341,7 @@ public class RestoreJob extends AbstractJob { } } - /* + /** * return true if some restored objs have been dropped. */ private void checkIfNeedCancel() { @@ -353,6 +352,7 @@ public class RestoreJob extends AbstractJob { Database db = catalog.getDb(dbId); if (db == null) { status = new Status(ErrCode.NOT_FOUND, "database " + dbId + " has been dropped"); + return; } db.readLock(); @@ -381,7 +381,7 @@ public class RestoreJob extends AbstractJob { } } - /* + /** * Restore rules as follow: * A. Table already exist * A1. Partition already exist, generate file mapping @@ -505,19 +505,9 @@ public class RestoreJob extends AbstractJob { Range remoteRange = remoteRangePartInfo.getRange(backupPartInfo.id); if (localRange.equals(remoteRange)) { // Same partition, same range - if (localRangePartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { - status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name - + " in table " + localTbl.getName() - + " has different replication num '" - + localRangePartInfo.getReplicationNum(localPartition.getId()) - + "' with partition in repository, which is " + restoreReplicationNum); + if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, tblInfo)) { return; } - genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, - true /* overwrite when commit */); - restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), - Pair.create(backupPartInfo.version, - backupPartInfo.versionHash)); } else { // Same partition name, different range status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name @@ -527,21 +517,9 @@ public class RestoreJob extends AbstractJob { } } else { // If this is a single partitioned table. - if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { - status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name - + " in table " + localTbl.getName() - + " has different replication num '" - + localPartInfo.getReplicationNum(localPartition.getId()) - + "' with partition in repository, which is " + restoreReplicationNum); + if (genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl, backupPartInfo, tblInfo)) { return; } - - // No need to check range, just generate file mapping - genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, - true /* overwrite when commit */); - restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), - Pair.create(backupPartInfo.version, - backupPartInfo.versionHash)); } } else { // partitions does not exist @@ -616,65 +594,20 @@ public class RestoreJob extends AbstractJob { BackupPartitionInfo backupPartitionInfo = jobInfo.getTableInfo(entry.first).getPartInfo(restorePart.getName()); - Set bfColumns = localTbl.getCopiedBfColumns(); - double bfFpp = localTbl.getBfFpp(); - for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { - MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId()); - TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(), - restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); - for (Tablet restoreTablet : restoredIdx.getTablets()) { - Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); - for (Replica restoreReplica : restoreTablet.getReplicas()) { - Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); - CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), dbId, - localTbl.getId(), restorePart.getId(), restoredIdx.getId(), - restoreTablet.getId(), indexMeta.getShortKeyColumnCount(), - indexMeta.getSchemaHash(), restoreReplica.getVersion(), - restoreReplica.getVersionHash(), indexMeta.getKeysType(), TStorageType.COLUMN, - TStorageMedium.HDD /* all restored replicas will be saved to HDD */, - indexMeta.getSchema(), bfColumns, bfFpp, null, - localTbl.getCopiedIndexes(), - localTbl.isInMemory()); - task.setInRestoreMode(true); - batchTask.addTask(task); - } - } - } + createReplicas(db, batchTask, localTbl, restorePart); genFileMapping(localTbl, restorePart, remoteTbl.getId(), backupPartitionInfo, - allowLoad ? false : true /* if allow load, do not overwrite when commit */); + !allowLoad /* if allow load, do not overwrite when commit */); } // generate create replica task for all restored tables for (OlapTable restoreTbl : restoredTbls) { for (Partition restorePart : restoreTbl.getPartitions()) { - Set bfColumns = restoreTbl.getCopiedBfColumns(); - double bfFpp = restoreTbl.getBfFpp(); - for (MaterializedIndex index : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { - MaterializedIndexMeta indexMeta = restoreTbl.getIndexMetaByIndexId(index.getId()); - TabletMeta tabletMeta = new TabletMeta(db.getId(), restoreTbl.getId(), restorePart.getId(), - index.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); - for (Tablet tablet : index.getTablets()) { - Catalog.getCurrentInvertedIndex().addTablet(tablet.getId(), tabletMeta); - for (Replica replica : tablet.getReplicas()) { - Catalog.getCurrentInvertedIndex().addReplica(tablet.getId(), replica); - CreateReplicaTask task = new CreateReplicaTask(replica.getBackendId(), dbId, - restoreTbl.getId(), restorePart.getId(), index.getId(), tablet.getId(), - indexMeta.getShortKeyColumnCount(), indexMeta.getSchemaHash(), - replica.getVersion(), replica.getVersionHash(), - indexMeta.getKeysType(), TStorageType.COLUMN, TStorageMedium.HDD, - indexMeta.getSchema(), bfColumns, bfFpp, null, - restoreTbl.getCopiedIndexes(), - restoreTbl.isInMemory()); - task.setInRestoreMode(true); - batchTask.addTask(task); - } - } - } + createReplicas(db, batchTask, restoreTbl, restorePart); BackupTableInfo backupTableInfo = jobInfo.getTableInfo(restoreTbl.getName()); genFileMapping(restoreTbl, restorePart, backupTableInfo.id, backupTableInfo.getPartInfo(restorePart.getName()), - allowLoad ? false : true /* if allow load, do not overwrite when commit */); + !allowLoad /* if allow load, do not overwrite when commit */); } // set restored table's new name after all 'genFileMapping' restoreTbl.setName(jobInfo.getAliasByOriginNameIfSet(restoreTbl.getName())); @@ -689,7 +622,7 @@ public class RestoreJob extends AbstractJob { if (batchTask.getTaskNum() > 0) { MarkedCountDownLatch latch = new MarkedCountDownLatch(batchTask.getTaskNum()); for (AgentTask task : batchTask.getAllTasks()) { - latch.addMark(((CreateReplicaTask) task).getBackendId(), ((CreateReplicaTask) task).getTabletId()); + latch.addMark(task.getBackendId(), task.getTabletId()); ((CreateReplicaTask) task).setLatch(latch); AgentTaskQueue.addTask(task); } @@ -806,7 +739,54 @@ public class RestoreJob extends AbstractJob { // No log here, PENDING state restore job will redo this method LOG.info("finished to prepare meta and send snapshot tasks, num: {}. {}", batchTask.getTaskNum(), this); - return; + } + + private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo localPartInfo, Partition localPartition, Table localTbl, + BackupPartitionInfo backupPartInfo, BackupTableInfo tblInfo) { + if (localPartInfo.getReplicationNum(localPartition.getId()) != restoreReplicationNum) { + status = new Status(ErrCode.COMMON_ERROR, "Partition " + backupPartInfo.name + + " in table " + localTbl.getName() + + " has different replication num '" + + localPartInfo.getReplicationNum(localPartition.getId()) + + "' with partition in repository, which is " + restoreReplicationNum); + return true; + } + + // No need to check range, just generate file mapping + OlapTable localOlapTbl = (OlapTable) localTbl; + genFileMapping(localOlapTbl, localPartition, tblInfo.id, backupPartInfo, + true /* overwrite when commit */); + restoredVersionInfo.put(localOlapTbl.getId(), localPartition.getId(), + Pair.create(backupPartInfo.version, + backupPartInfo.versionHash)); + return false; + } + + private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable localTbl, Partition restorePart) { + Set bfColumns = localTbl.getCopiedBfColumns(); + double bfFpp = localTbl.getBfFpp(); + for (MaterializedIndex restoredIdx : restorePart.getMaterializedIndices(IndexExtState.VISIBLE)) { + MaterializedIndexMeta indexMeta = localTbl.getIndexMetaByIndexId(restoredIdx.getId()); + TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(), + restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD); + for (Tablet restoreTablet : restoredIdx.getTablets()) { + Catalog.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); + for (Replica restoreReplica : restoreTablet.getReplicas()) { + Catalog.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); + CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendId(), dbId, + localTbl.getId(), restorePart.getId(), restoredIdx.getId(), + restoreTablet.getId(), indexMeta.getShortKeyColumnCount(), + indexMeta.getSchemaHash(), restoreReplica.getVersion(), + restoreReplica.getVersionHash(), indexMeta.getKeysType(), TStorageType.COLUMN, + TStorageMedium.HDD /* all restored replicas will be saved to HDD */, + indexMeta.getSchema(), bfColumns, bfFpp, null, + localTbl.getCopiedIndexes(), + localTbl.isInMemory()); + task.setInRestoreMode(true); + batchTask.addTask(task); + } + } + } } // reset remote partition. @@ -825,11 +805,11 @@ public class RestoreJob extends AbstractJob { // indexes Map localIdxNameToId = localTbl.getIndexNameToId(); - for (String localidxName : localIdxNameToId.keySet()) { + for (String localIdxName : localIdxNameToId.keySet()) { // set ids of indexes in remote partition to the local index ids - long remoteIdxId = remoteTbl.getIndexIdByName(localidxName); + long remoteIdxId = remoteTbl.getIndexIdByName(localIdxName); MaterializedIndex remoteIdx = remotePart.getIndex(remoteIdxId); - long localIdxId = localIdxNameToId.get(localidxName); + long localIdxId = localIdxNameToId.get(localIdxName); remoteIdx.setIdForRestore(localIdxId); if (localIdxId != localTbl.getBaseIndexId()) { // not base table, reset @@ -1510,6 +1490,7 @@ public class RestoreJob extends AbstractJob { } } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); diff --git a/fe/src/main/java/org/apache/doris/catalog/Database.java b/fe/src/main/java/org/apache/doris/catalog/Database.java index fa53e14792..0d46c7a160 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/src/main/java/org/apache/doris/catalog/Database.java @@ -113,7 +113,7 @@ public class Database extends MetaObject implements Writable { } this.rwLock = new ReentrantReadWriteLock(true); this.idToTable = new ConcurrentHashMap<>(); - this.nameToTable = new HashMap(); + this.nameToTable = new HashMap<>(); this.dataQuotaBytes = FeConstants.default_db_data_quota_bytes; this.replicaQuotaSize = FeConstants.default_db_replica_quota_size; this.dbState = DbState.NORMAL; diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 6f8abd9bfe..a921758d5c 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -76,8 +76,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP public MaterializedIndex() { this.state = IndexState.NORMAL; - this.idToTablets = new HashMap(); - this.tablets = new ArrayList(); + this.idToTablets = new HashMap<>(); + this.tablets = new ArrayList<>(); } public MaterializedIndex(long id, IndexState state) { @@ -88,8 +88,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP this.state = IndexState.NORMAL; } - this.idToTablets = new HashMap(); - this.tablets = new ArrayList(); + this.idToTablets = new HashMap<>(); + this.tablets = new ArrayList<>(); this.rowCount = 0; diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index fe252dd50b..f307a5f593 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -643,6 +643,7 @@ public class OlapTable extends Table { */ // get partition by name, not including temp partitions + @Override public Partition getPartition(String partitionName) { return getPartition(partitionName, false); } @@ -732,6 +733,7 @@ public class OlapTable extends Table { return false; } + @Override public TTableDescriptor toThrift() { TOlapTable tOlapTable = new TOlapTable(getName()); TTableDescriptor tTableDescriptor = new TTableDescriptor(id, TTableType.OLAP_TABLE, @@ -930,6 +932,7 @@ public class OlapTable extends Table { tempPartitions.write(out); } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); @@ -947,7 +950,7 @@ public class OlapTable extends Table { if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_75) { // schema int colCount = in.readInt(); - List schema = new LinkedList(); + List schema = new LinkedList<>(); for (int j = 0; j < colCount; j++) { Column column = Column.read(in); schema.add(column); @@ -1073,6 +1076,7 @@ public class OlapTable extends Table { rebuildFullSchema(); } + @Override public boolean equals(Table table) { if (this == table) { return true; diff --git a/fe/src/main/java/org/apache/doris/catalog/Partition.java b/fe/src/main/java/org/apache/doris/catalog/Partition.java index 447555aa36..0730bed5b8 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/src/main/java/org/apache/doris/catalog/Partition.java @@ -67,13 +67,13 @@ public class Partition extends MetaObject implements Writable { private PartitionState state; @SerializedName(value = "baseIndex") private MaterializedIndex baseIndex; - /* + /** * Visible rollup indexes are indexes which are visible to user. * User can do query on them, show them in related 'show' stmt. */ @SerializedName(value = "idToVisibleRollupIndex") private Map idToVisibleRollupIndex = Maps.newHashMap(); - /* + /** * Shadow indexes are indexes which are not visible to user. * Query will not run on these shadow indexes, and user can not see them neither. * But load process will load data into these shadow indexes. @@ -81,7 +81,7 @@ public class Partition extends MetaObject implements Writable { @SerializedName(value = "idToShadowIndex") private Map idToShadowIndex = Maps.newHashMap(); - /* + /** * committed version(hash): after txn is committed, set committed version(hash) * visible version(hash): after txn is published, set visible version * next version(hash): next version is set after finished committing, it should equals to committed version + 1 @@ -354,6 +354,7 @@ public class Partition extends MetaObject implements Writable { distributionInfo.write(out); } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); diff --git a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 1359f0c11d..f41f595f22 100644 --- a/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -51,15 +51,15 @@ public class PartitionInfo implements Writable { protected Map idToInMemory; public PartitionInfo() { - this.idToDataProperty = new HashMap(); - this.idToReplicationNum = new HashMap(); + this.idToDataProperty = new HashMap<>(); + this.idToReplicationNum = new HashMap<>(); this.idToInMemory = new HashMap<>(); } public PartitionInfo(PartitionType type) { this.type = type; - this.idToDataProperty = new HashMap(); - this.idToReplicationNum = new HashMap(); + this.idToDataProperty = new HashMap<>(); + this.idToReplicationNum = new HashMap<>(); this.idToInMemory = new HashMap<>(); } diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index ad762926c1..c9cab33710 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -505,6 +505,7 @@ public class Replica implements Writable { return replica; } + @Override public boolean equals(Object obj) { if (this == obj) { return true; diff --git a/fe/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/src/main/java/org/apache/doris/catalog/Tablet.java index 1b1cb358b9..d8eb69144e 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/src/main/java/org/apache/doris/catalog/Tablet.java @@ -39,7 +39,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -80,18 +79,18 @@ public class Tablet extends MetaObject implements Writable { private long lastStatusCheckTime = -1; public Tablet() { - this(0L, new ArrayList()); + this(0L, new ArrayList<>()); } public Tablet(long tabletId) { - this(tabletId, new ArrayList()); + this(tabletId, new ArrayList<>()); } public Tablet(long tabletId, List replicas) { this.id = tabletId; this.replicas = replicas; if (this.replicas == null) { - this.replicas = new ArrayList(); + this.replicas = new ArrayList<>(); } checkedVersion = -1L; @@ -300,13 +299,15 @@ public class Tablet extends MetaObject implements Writable { public static void sortReplicaByVersionDesc(List replicas) { // sort replicas by version. higher version in the tops - Collections.sort(replicas, Replica.VERSION_DESC_COMPARATOR); + replicas.sort(Replica.VERSION_DESC_COMPARATOR); } + @Override public String toString() { return "tabletId=" + this.id; } + @Override public void write(DataOutput out) throws IOException { super.write(out); @@ -321,6 +322,7 @@ public class Tablet extends MetaObject implements Writable { out.writeLong(checkedVersionHash); out.writeBoolean(isConsistent); } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); @@ -346,6 +348,7 @@ public class Tablet extends MetaObject implements Writable { return tablet; } + @Override public boolean equals(Object obj) { if (this == obj) { return true; @@ -392,7 +395,7 @@ public class Tablet extends MetaObject implements Writable { return dataSize; } - /* + /** * A replica is healthy only if * 1. the backend is available * 2. replica version is caught up, and last failed version is -1 @@ -502,7 +505,7 @@ public class Tablet extends MetaObject implements Writable { return Pair.create(TabletStatus.HEALTHY, TabletSchedCtx.Priority.NORMAL); } - /* + /** * Check colocate table's tablet health * 1. Mismatch: * backends set: 1,2,3 @@ -551,7 +554,7 @@ public class Tablet extends MetaObject implements Writable { return TabletStatus.HEALTHY; } - /* + /** * check if this tablet is ready to be repaired, based on priority. * VERY_HIGH: repair immediately * HIGH: delay Config.tablet_repair_delay_factor_second * 1; diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java index 4cb2b44b97..0c695ca14d 100644 --- a/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java +++ b/fe/src/main/java/org/apache/doris/catalog/TabletMeta.java @@ -137,12 +137,12 @@ public class TabletMeta { lock.readLock().lock(); try { StringBuilder sb = new StringBuilder(); - sb.append("dbId=" + dbId); - sb.append(" tableId=" + tableId); - sb.append(" partitionId=" + partitionId); - sb.append(" indexId=" + indexId); - sb.append(" oldSchemaHash=" + oldSchemaHash); - sb.append(" newSchemaHash=" + newSchemaHash); + sb.append("dbId=").append(dbId); + sb.append(" tableId=").append(tableId); + sb.append(" partitionId=").append(partitionId); + sb.append(" indexId=").append(indexId); + sb.append(" oldSchemaHash=").append(oldSchemaHash); + sb.append(" newSchemaHash=").append(newSchemaHash); return sb.toString(); } finally { diff --git a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java index 5479a16038..5dd6e60f4b 100644 --- a/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java +++ b/fe/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java @@ -55,13 +55,7 @@ public class BackendLoadStatistic { public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) { double score1 = o1.getLoadScore(medium); double score2 = o2.getLoadScore(medium); - if (score1 > score2) { - return 1; - } else if (score1 == score2) { - return 0; - } else { - return -1; - } + return Double.compare(score1, score2); } } @@ -70,14 +64,7 @@ public class BackendLoadStatistic { public int compare(BackendLoadStatistic o1, BackendLoadStatistic o2) { Double score1 = o1.getMixLoadScore(); Double score2 = o2.getMixLoadScore(); - - if (score1 > score2) { - return 1; - } else if (score1 == score2) { - return 0; - } else { - return -1; - } + return score1.compareTo(score2); } } @@ -180,11 +167,7 @@ public class BackendLoadStatistic { throw new LoadBalanceException("backend " + beId + " does not exist"); } - if (!be.isAvailable()) { - isAvailable = false; - } else { - isAvailable = true; - } + isAvailable = be.isAvailable(); ImmutableMap disks = be.getDisks(); for (DiskInfo diskInfo : disks.values()) { @@ -333,7 +316,7 @@ public class BackendLoadStatistic { return false; } - /* + /** * Classify the paths into 'low', 'mid' and 'high', * and skip offline path, and path with different storage medium */ diff --git a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java index 2b5fee92cc..3aeb4069ab 100644 --- a/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java +++ b/fe/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java @@ -115,13 +115,7 @@ public class RootPathLoadStatistic implements Comparable public int compareTo(RootPathLoadStatistic o) { double myPercent = getUsedPercent(); double otherPercent = o.getUsedPercent(); - if (myPercent < otherPercent) { - return -1; - } else if (myPercent > otherPercent) { - return 1; - } else { - return 0; - } + return Double.compare(myPercent, otherPercent); } @Override diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java index 036fe6e82a..a127255d43 100644 --- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.ColocateTableIndex; import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DiskInfo; import org.apache.doris.catalog.DiskInfo.DiskState; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; @@ -60,7 +61,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.PriorityQueue; @@ -68,7 +68,7 @@ import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; -/* +/** * TabletScheduler saved the tablets produced by TabletChecker and try to schedule them. * It also try to balance the cluster load. * @@ -170,7 +170,7 @@ public class TabletScheduler extends MasterDaemon { if (backends.containsKey(beId)) { List pathHashes = backends.get(beId).getDisks().values().stream() .filter(v -> v.getState()==DiskState.ONLINE) - .map(v -> v.getPathHash()).collect(Collectors.toList()); + .map(DiskInfo::getPathHash).collect(Collectors.toList()); backendsWorkingSlots.get(beId).updatePaths(pathHashes); } else { deletedBeIds.add(beId); @@ -186,7 +186,7 @@ public class TabletScheduler extends MasterDaemon { // add new backends for (Backend be : backends.values()) { if (!backendsWorkingSlots.containsKey(be.getId())) { - List pathHashes = be.getDisks().values().stream().map(v -> v.getPathHash()).collect(Collectors.toList()); + List pathHashes = be.getDisks().values().stream().map(DiskInfo::getPathHash).collect(Collectors.toList()); PathSlot slot = new PathSlot(pathHashes, Config.schedule_slot_num_per_path); backendsWorkingSlots.put(be.getId(), slot); LOG.info("add new backend {} with slots num: {}", be.getId(), be.getDisks().size()); @@ -200,7 +200,7 @@ public class TabletScheduler extends MasterDaemon { return backendsWorkingSlots; } - /* + /** * add a ready-to-be-scheduled tablet to pendingTablets, if it has not being added before. * if force is true, do not check if tablet is already added before. */ @@ -227,7 +227,7 @@ public class TabletScheduler extends MasterDaemon { return allTabletIds.contains(tabletId); } - /* + /** * Iterate current tablets, change their priority to VERY_HIGH if necessary. */ public synchronized void changeTabletsPriorityToVeryHigh(long dbId, long tblId, List partitionIds) { @@ -242,10 +242,10 @@ public class TabletScheduler extends MasterDaemon { pendingTablets = newPendingTablets; } - /* + /** * TabletScheduler will run as a daemon thread at a very short interval(default 5 sec) - * Firstly, it will try to update cluster load statistic and check if priority need to be adjuested. - * Than, it will schedule the tablets in pendingTablets. + * Firstly, it will try to update cluster load statistic and check if priority need to be adjusted. + * Then, it will schedule the tablets in pendingTablets. * Thirdly, it will check the current running tasks. * Finally, it try to balance the cluster if possible. * @@ -287,7 +287,7 @@ public class TabletScheduler extends MasterDaemon { lastStatUpdateTime = System.currentTimeMillis(); } - /* + /** * Here is the only place we update the cluster load statistic info. * We will not update this info dynamically along with the clone job's running. * Although it will cause a little bit inaccurate, but is within a controllable range, @@ -311,13 +311,13 @@ public class TabletScheduler extends MasterDaemon { return statisticMap; } - /* + /** * adjust priorities of all tablet infos */ private synchronized void adjustPriorities() { int size = pendingTablets.size(); int changedNum = 0; - TabletSchedCtx tabletCtx = null; + TabletSchedCtx tabletCtx; for (int i = 0; i < size; i++) { tabletCtx = pendingTablets.poll(); if (tabletCtx == null) { @@ -333,7 +333,7 @@ public class TabletScheduler extends MasterDaemon { LOG.info("adjust priority for all tablets. changed: {}, total: {}", changedNum, size); } - /* + /** * get at most BATCH_NUM tablets from queue, and try to schedule them. * After handle, the tablet info should be * 1. in runningTablets with state RUNNING, if being scheduled success. @@ -423,7 +423,7 @@ public class TabletScheduler extends MasterDaemon { runningTablets.put(tabletCtx.getTabletId(), tabletCtx); } - /* + /** * we take the tablet out of the runningTablets and than handle it, * avoid other threads see it. * Whoever takes this tablet, make sure to put it to the schedHistory or back to runningTablets. @@ -432,7 +432,7 @@ public class TabletScheduler extends MasterDaemon { return runningTablets.remove(tabletId); } - /* + /** * Try to schedule a single tablet. */ private void scheduleTablet(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException { @@ -448,7 +448,7 @@ public class TabletScheduler extends MasterDaemon { throw new SchedException(Status.UNRECOVERABLE, "db does not exist"); } - Pair statusPair = null; + Pair statusPair; db.writeLock(); try { OlapTable tbl = (OlapTable) db.getTable(tabletCtx.getTblId()); @@ -586,7 +586,7 @@ public class TabletScheduler extends MasterDaemon { } } - /* + /** * Replica is missing, which means there is no enough alive replicas. * So we need to find a destination backend to clone a new replica as possible as we can. * 1. find an available path in a backend as destination: @@ -615,7 +615,7 @@ public class TabletScheduler extends MasterDaemon { batchTask.addTask(tabletCtx.createCloneReplicaAndTask()); } - /* + /** * Replica version is incomplete, which means this replica is missing some version, * and need to be cloned from a healthy replica, in-place. * @@ -649,7 +649,7 @@ public class TabletScheduler extends MasterDaemon { handleReplicaMissing(tabletCtx, batchTask); } - /* + /** * replica is redundant, which means there are more replicas than we expected, which need to be dropped. * we just drop one redundant replica at a time, for safety reason. * choosing a replica to drop base on following priority: @@ -843,7 +843,7 @@ public class TabletScheduler extends MasterDaemon { return false; } - /* + /** * Just delete replica which does not located in colocate backends set. * return true if delete one replica, otherwise, return false. */ @@ -920,7 +920,7 @@ public class TabletScheduler extends MasterDaemon { LOG.info("send delete replica task for tablet {} in backend {}", tabletId, backendId); } - /* + /** * Cluster migration, which means the tablet has enough healthy replicas, * but some replicas are not in right cluster. * It is just same as 'replica missing'. @@ -933,7 +933,7 @@ public class TabletScheduler extends MasterDaemon { handleReplicaMissing(tabletCtx, batchTask); } - /* + /** * Replicas of colocate table's tablet does not locate on right backends set. * backends set: 1,2,3 * tablet replicas: 1,2,5 @@ -960,7 +960,7 @@ public class TabletScheduler extends MasterDaemon { batchTask.addTask(tabletCtx.createCloneReplicaAndTask()); } - /* + /** * Try to select some alternative tablets for balance. Add them to pendingTablets with priority LOW, * and waiting to be scheduled. */ @@ -984,7 +984,7 @@ public class TabletScheduler extends MasterDaemon { } } - /* + /** * Try to create a balance task for a tablet. */ private void doBalance(TabletSchedCtx tabletCtx, AgentBatchTask batchTask) throws SchedException { @@ -1005,8 +1005,7 @@ public class TabletScheduler extends MasterDaemon { // get all available paths which this tablet can fit in. // beStatistics is sorted by mix load score in ascend order, so select from first to last. List allFitPaths = Lists.newArrayList(); - for (int i = 0; i < beStatistics.size(); i++) { - BackendLoadStatistic bes = beStatistics.get(i); + for (BackendLoadStatistic bes : beStatistics) { if (!bes.isAvailable()) { continue; } @@ -1071,7 +1070,7 @@ public class TabletScheduler extends MasterDaemon { throw new SchedException(Status.SCHEDULE_FAILED, "unable to find dest path which can be fit in"); } - /* + /** * For some reason, a tablet info failed to be scheduled this time, * So we dynamically change its priority and add back to queue, waiting for next round. */ @@ -1127,7 +1126,7 @@ public class TabletScheduler extends MasterDaemon { return total; } - /* + /** * return true if we want to remove the clone task from AgentTaskQueue */ public boolean finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) { @@ -1174,7 +1173,7 @@ public class TabletScheduler extends MasterDaemon { return true; } - /* + /** * Gather the running statistic of the task. * It will be evaluated for future strategy. * This should only be called when the tablet is down with state FINISHED. @@ -1208,7 +1207,7 @@ public class TabletScheduler extends MasterDaemon { lastSlotAdjustTime = System.currentTimeMillis(); } - /* + /** * handle tablets which are running. * We should finished the task if * 1. Tablet is already healthy @@ -1291,7 +1290,7 @@ public class TabletScheduler extends MasterDaemon { + runningTablets.values().stream().filter(t -> t.getType() == Type.BALANCE).count(); } - /* + /** * PathSlot keeps track of slot num per path of a Backend. * Each path on a Backend has several slot. * If a path's available slot num become 0, no task should be assigned to this path. @@ -1309,13 +1308,7 @@ public class TabletScheduler extends MasterDaemon { // update the path public synchronized void updatePaths(List paths) { // delete non exist path - Iterator> iter = pathSlots.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry entry = iter.next(); - if (!paths.contains(entry.getKey())) { - iter.remove(); - } - } + pathSlots.entrySet().removeIf(entry -> !paths.contains(entry.getKey())); // add new path for (Long pathHash : paths) { @@ -1339,7 +1332,7 @@ public class TabletScheduler extends MasterDaemon { } } - /* + /** * Update the statistic of specified path */ public synchronized void updateStatistic(long pathHash, long copySize, long copyTimeMs) { @@ -1351,7 +1344,7 @@ public class TabletScheduler extends MasterDaemon { slot.totalCopyTimeMs += copyTimeMs; } - /* + /** * If the specified 'pathHash' has available slot, decrease the slot number and return this path hash */ public synchronized long takeSlot(long pathHash) throws SchedException { @@ -1400,7 +1393,7 @@ public class TabletScheduler extends MasterDaemon { return total; } - /* + /** * get path whose balance slot num is larger than 0 */ public synchronized Set getAvailPathsForBalance() { @@ -1423,15 +1416,15 @@ public class TabletScheduler extends MasterDaemon { public synchronized List> getSlotInfo(long beId) { List> results = Lists.newArrayList(); - pathSlots.entrySet().stream().forEach(t -> { - t.getValue().rectify(); + pathSlots.forEach((key, value) -> { + value.rectify(); List result = Lists.newArrayList(); result.add(String.valueOf(beId)); - result.add(String.valueOf(t.getKey())); - result.add(String.valueOf(t.getValue().available)); - result.add(String.valueOf(t.getValue().total)); - result.add(String.valueOf(t.getValue().balanceSlot)); - result.add(String.valueOf(t.getValue().getAvgRate())); + result.add(String.valueOf(key)); + result.add(String.valueOf(value.available)); + result.add(String.valueOf(value.total)); + result.add(String.valueOf(value.balanceSlot)); + result.add(String.valueOf(value.getAvgRate())); results.add(result); }); return results; diff --git a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java index 47121af608..df0c182482 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/ClusterLoadStatisticProcDir.java @@ -50,11 +50,9 @@ public class ClusterLoadStatisticProcDir implements ProcDirInterface { statMap = Catalog.getCurrentCatalog().getTabletScheduler().getStatisticMap(); - statMap.values().stream().forEach(t -> { + statMap.values().forEach(t -> { List> statistics = t.getClusterStatistic(medium); - statistics.stream().forEach(v -> { - result.addRow(v); - }); + statistics.forEach(result::addRow); }); return result; diff --git a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java index 1559a4abe3..8b0c39ca29 100644 --- a/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java +++ b/fe/src/main/java/org/apache/doris/system/BackendHbResponse.java @@ -90,6 +90,7 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable { out.writeInt(brpcPort); } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); beId = in.readLong(); diff --git a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java index dda679a3cf..e6ae49b104 100644 --- a/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java +++ b/fe/src/main/java/org/apache/doris/system/BrokerHbResponse.java @@ -81,6 +81,7 @@ public class BrokerHbResponse extends HeartbeatResponse implements Writable { out.writeInt(port); } + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); name = Text.readString(in); diff --git a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 70ce4c06e9..b744cf2c22 100644 --- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -17,484 +17,294 @@ package org.apache.doris.catalog; -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ColumnDef; +import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; -import org.apache.doris.analysis.HashDistributionDesc; -import org.apache.doris.analysis.KeysDesc; -import org.apache.doris.analysis.PartitionKeyDesc; -import org.apache.doris.analysis.PartitionValue; -import org.apache.doris.analysis.RangePartitionDesc; -import org.apache.doris.analysis.SingleRangePartitionDesc; -import org.apache.doris.analysis.TableName; -import org.apache.doris.analysis.TypeDef; import org.apache.doris.common.DdlException; -import org.apache.doris.common.UserException; -import org.apache.doris.mysql.privilege.PaloAuth; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.persist.EditLog; +import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.system.SystemInfoService; -import org.apache.doris.task.AgentBatchTask; -import com.google.common.collect.Lists; - -import org.junit.Before; +import org.apache.doris.utframe.UtFrameUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import mockit.Expectations; -import mockit.Injectable; -import mockit.Mock; -import mockit.MockUp; +import java.util.UUID; public class DynamicPartitionTableTest { - private TableName dbTableName; - private String dbName = "testDb"; - private String tableName = "testTable"; - private String clusterName = "default"; - private List beIds = Lists.newArrayList(); - private List columnNames = Lists.newArrayList(); - private List columnDefs = Lists.newArrayList(); + private static String runningDir = "fe/mocked/DynamicPartitionTableTest/" + UUID.randomUUID().toString() + "/"; - private Catalog catalog = Catalog.getInstance(); - private Database db = new Database(); - private Analyzer analyzer; - - private Map properties; - private List singleRangePartitionDescs; - - @Injectable - ConnectContext connectContext; + private static ConnectContext connectContext; @Rule - public ExpectedException expectedEx = ExpectedException.none(); + public ExpectedException expectedException = ExpectedException.none(); - @Before - public void setUp() throws Exception { - dbTableName = new TableName(dbName, tableName); + @BeforeClass + public static void beforeClass() throws Exception { + FeConstants.default_scheduler_interval_millisecond = 1000; + FeConstants.runningUnitTest = true; - beIds.add(1L); - beIds.add(2L); - beIds.add(3L); + UtFrameUtils.createMinDorisCluster(runningDir); - columnNames.add("key1"); - columnNames.add("key2"); - columnNames.add("key3"); + // create connect context + connectContext = UtFrameUtils.createDefaultCtx(); + // create database + String createDbStmtStr = "create database test;"; + CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext); + Catalog.getCurrentCatalog().createDb(createDbStmt); + } - columnDefs.add(new ColumnDef("key1", new TypeDef(ScalarType.createType(PrimitiveType.INT)))); - columnDefs.add(new ColumnDef("key2", new TypeDef(ScalarType.createType(PrimitiveType.INT)))); - columnDefs.add(new ColumnDef("key3", new TypeDef(ScalarType.createVarchar(10)))); + @AfterClass + public static void TearDown() { + UtFrameUtils.cleanDorisFeDir(runningDir); + } - analyzer = new Analyzer(catalog, connectContext); - - properties = new HashMap<>(); - properties.put(DynamicPartitionProperty.ENABLE, "true"); - properties.put(DynamicPartitionProperty.PREFIX, "p"); - properties.put(DynamicPartitionProperty.TIME_UNIT, "day"); - properties.put(DynamicPartitionProperty.START, "-3"); - properties.put(DynamicPartitionProperty.END, "3"); - properties.put(DynamicPartitionProperty.BUCKETS, "30"); - - singleRangePartitionDescs = new LinkedList<>(); - singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"))), null)); - - new MockUp() { - @Mock - void run() { - return; - } - }; - - new MockUp() { - @Mock - boolean await(long timeout, TimeUnit unit) { - return true; - } - }; - - new Expectations(analyzer, catalog) {{ - analyzer.getClusterName(); - minTimes = 0; - result = clusterName; - }}; - - dbTableName.analyze(analyzer); + private static void createTable(String sql) throws Exception { + CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); + Catalog.getCurrentCatalog().createTable(createTableStmt); } @Test - public void testNormal(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - catalog.createTable(stmt); + public void testNormal() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + createTable(createOlapTblStmt); } @Test - public void testMissPrefix(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - properties.remove(DynamicPartitionProperty.PREFIX); - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Must assign dynamic_partition.prefix properties"); - - catalog.createTable(stmt); + public void testMissPrefix() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_prefix` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + expectedException.expect(DdlException.class); + expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.prefix properties"); + createTable(createOlapTblStmt); } @Test - public void testMissTimeUnit(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - properties.remove(DynamicPartitionProperty.TIME_UNIT); - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Must assign dynamic_partition.time_unit properties"); - - catalog.createTable(stmt); + public void testMissTimeUnit() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_time_unit` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + expectedException.expect(DdlException.class); + expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.time_unit properties"); + createTable(createOlapTblStmt); } @Test - public void testMissSTART(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; + public void testMissStart() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_start` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + createTable(createOlapTblStmt); + } - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; + @Test + public void testMissEnd() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_end` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + expectedException.expect(DdlException.class); + expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.end properties"); + createTable(createOlapTblStmt); + } - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - properties.remove(DynamicPartitionProperty.START); - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - catalog.createTable(stmt); + @Test + public void testMissBuckets() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\"\n" + + ");"; + expectedException.expect(DdlException.class); + expectedException.expectMessage("errCode = 2, detailMessage = Must assign dynamic_partition.buckets properties"); + createTable(createOlapTblStmt); } @Test - public void testMissEnd(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - properties.remove(DynamicPartitionProperty.END); - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Must assign dynamic_partition.end properties"); - - catalog.createTable(stmt); + public void testNotAllowed() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + expectedException.expect(DdlException.class); + expectedException.expectMessage("errCode = 2, detailMessage = Only support dynamic partition properties on range partition table"); + createTable(createOlapTblStmt); } @Test - public void testMissBuckets(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - properties.remove(DynamicPartitionProperty.BUCKETS); - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Must assign dynamic_partition.buckets properties"); - - catalog.createTable(stmt); - } - - @Test - public void testNotAllowed(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), null, - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Only support dynamic partition properties on range partition table"); - - catalog.createTable(stmt); - } - - @Test - public void testNotAllowedInMultiPartitions(@Injectable SystemInfoService systemInfoService, - @Injectable PaloAuth paloAuth, - @Injectable EditLog editLog) throws UserException { - new Expectations(catalog) { - { - catalog.getDb(dbTableName.getDb()); - minTimes = 0; - result = db; - - Catalog.getCurrentSystemInfo(); - minTimes = 0; - result = systemInfoService; - - systemInfoService.checkClusterCapacity(anyString); - minTimes = 0; - systemInfoService.seqChooseBackendIds(anyInt, true, true, anyString); - minTimes = 0; - result = beIds; - - catalog.getAuth(); - minTimes = 0; - result = paloAuth; - paloAuth.checkTblPriv((ConnectContext) any, anyString, anyString, PrivPredicate.CREATE); - minTimes = 0; - result = true; - - catalog.getEditLog(); - minTimes = 0; - result = editLog; - } - }; - - List rangePartitionDescs = new LinkedList<>(); - rangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", - new PartitionKeyDesc(Lists.newArrayList(new PartitionValue("-128"), new PartitionValue("100"))), null)); - - CreateTableStmt stmt = new CreateTableStmt(false, false, dbTableName, columnDefs, "olap", - new KeysDesc(KeysType.AGG_KEYS, columnNames), - new RangePartitionDesc(Lists.newArrayList("key1", "key2"), singleRangePartitionDescs), - new HashDistributionDesc(1, Lists.newArrayList("key1")), properties, null, ""); - stmt.analyze(analyzer); - - expectedEx.expect(DdlException.class); - expectedEx.expectMessage("Dynamic partition only support single-column range partition"); - - catalog.createTable(stmt); + public void testNotAllowedInMultiPartitions() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal` (\n" + + " `k1` date NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\",\n" + + " `k3` smallint NULL COMMENT \"\",\n" + + " `v1` varchar(2048) NULL COMMENT \"\",\n" + + " `v2` datetime NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE (k1, k2)\n" + + "(\n" + + "PARTITION p1 VALUES LESS THAN (\"2014-01-01\", \"100\"),\n" + + "PARTITION p2 VALUES LESS THAN (\"2014-06-01\", \"200\"),\n" + + "PARTITION p3 VALUES LESS THAN (\"2014-12-01\", \"300\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.time_unit\" = \"day\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\"\n" + + ");"; + expectedException.expect(DdlException.class); + expectedException.expectMessage("errCode = 2, detailMessage = Dynamic partition only support single-column range partition"); + createTable(createOlapTblStmt); } } diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 81c883ef30..2f88ecdf17 100644 --- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -17,12 +17,15 @@ package org.apache.doris.common; +import org.apache.doris.analysis.DateLiteral; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.thrift.TStorageMedium; import com.google.common.collect.Lists; @@ -122,11 +125,14 @@ public class PropertyAnalyzerTest { @Test public void testStorageMedium() throws AnalysisException { long tomorrowTs = System.currentTimeMillis() / 1000 + 86400; + String tomorrowTimeStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000); Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, "SSD"); - properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(tomorrowTs * 1000)); + properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COLDOWN_TIME, tomorrowTimeStr); DataProperty dataProperty = PropertyAnalyzer.analyzeDataProperty(properties, new DataProperty(TStorageMedium.SSD)); - Assert.assertEquals(tomorrowTs, dataProperty.getCooldownTimeMs() / 1000); + // avoid UT fail because time zone different + DateLiteral dateLiteral = new DateLiteral(tomorrowTimeStr, Type.DATETIME); + Assert.assertEquals(dateLiteral.unixTimestamp(TimeUtils.getTimeZone()), dataProperty.getCooldownTimeMs()); } }