database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Support determine isPreviousLoadFinished for some alter jobs in table level (#3196)

Browse Source 
This PR is to reduce the time cost for waiting transactions to be completed in same db by filter the running transactions in table level.

NOTICE: Update FE meta version to 79
This commit is contained in:
caiconghui
2020-03-27 07:16:23 -05:00
committed by GitHub
parent 0462607d8d
commit 32c4fc691c
19 changed files with 108 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.load.loadv2;
import com.google.common.collect.Lists;
import org.apache.doris.analysis.LoadStmt;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.AnalysisException;
@ -106,7 +107,7 @@ public class LoadJobTest {
LoadJob loadJob = new BrokerLoadJob();
new Expectations() {
{
globalTransactionMgr.beginTransaction(anyLong, anyString, (TUniqueId) any, anyString,
globalTransactionMgr.beginTransaction(anyLong, Lists.newArrayList(), anyString, (TUniqueId) any, anyString,
(TransactionState.LoadJobSourceType) any, anyLong, anyLong);
minTimes = 0;
result = 1;

24
fe/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java
View File

@ -106,7 +106,7 @@ public class GlobalTransactionMgrTest {
public void testBeginTransaction() throws LabelAlreadyUsedException, AnalysisException,
BeginTransactionException, DuplicatedRequestException {
FakeCatalog.setCatalog(masterCatalog);
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -124,7 +124,7 @@ public class GlobalTransactionMgrTest {
FakeCatalog.setCatalog(masterCatalog);
long transactionId = 0;
try {
transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -141,7 +141,7 @@ public class GlobalTransactionMgrTest {
assertEquals(transactionSource, transactionState.getCoordinator());
try {
transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -154,7 +154,7 @@ public class GlobalTransactionMgrTest {
@Test
public void testCommitTransaction1() throws UserException {
FakeCatalog.setCatalog(masterCatalog);
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -195,7 +195,7 @@ public class GlobalTransactionMgrTest {
public void testCommitTransactionWithOneFailed() throws UserException {
TransactionState transactionState = null;
FakeCatalog.setCatalog(masterCatalog);
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -217,7 +217,7 @@ public class GlobalTransactionMgrTest {
FakeCatalog.setCatalog(masterCatalog);
// commit another transaction with 1,3 success
long transactionId2 = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId2 = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable2,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -320,7 +320,7 @@ public class GlobalTransactionMgrTest {
partitionIdToOffset);
Deencapsulation.setField(routineLoadTaskInfo, "txnId", 1L);
routineLoadTaskInfoList.add(routineLoadTaskInfo);
TransactionState transactionState = new TransactionState(1L, 1L, "label", null,
TransactionState transactionState = new TransactionState(1L, Lists.newArrayList(1L), 1L, "label", null,
LoadJobSourceType.ROUTINE_LOAD_TASK, "be1", routineLoadJob.getId(),
Config.stream_load_default_timeout_second);
transactionState.setTransactionStatus(TransactionStatus.PREPARE);
@ -387,7 +387,7 @@ public class GlobalTransactionMgrTest {
partitionIdToOffset);
Deencapsulation.setField(routineLoadTaskInfo, "txnId", 1L);
routineLoadTaskInfoList.add(routineLoadTaskInfo);
TransactionState transactionState = new TransactionState(1L, 1L, "label", null,
TransactionState transactionState = new TransactionState(1L, Lists.newArrayList(1L), 1L, "label", null,
LoadJobSourceType.ROUTINE_LOAD_TASK, "be1", routineLoadJob.getId(),
Config.stream_load_default_timeout_second);
transactionState.setTransactionStatus(TransactionStatus.PREPARE);
@ -431,7 +431,7 @@ public class GlobalTransactionMgrTest {
}
public void testFinishTransaction() throws UserException {
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -477,7 +477,7 @@ public class GlobalTransactionMgrTest {
.getPartition(CatalogTestUtil.testPartition1);
Tablet tablet = testPartition.getIndex(CatalogTestUtil.testIndexId1).getTablet(CatalogTestUtil.testTabletId1);
FakeCatalog.setCatalog(masterCatalog);
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -531,7 +531,7 @@ public class GlobalTransactionMgrTest {
FakeCatalog.setCatalog(masterCatalog);
// commit another transaction with 1,3 success
long transactionId2 = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId2 = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable2,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);
@ -603,7 +603,7 @@ public class GlobalTransactionMgrTest {
public void testDeleteTransaction() throws LabelAlreadyUsedException,
AnalysisException, BeginTransactionException, DuplicatedRequestException {
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1,
long transactionId = masterTransMgr.beginTransaction(CatalogTestUtil.testDbId1, Lists.newArrayList(CatalogTestUtil.testTableId1),
CatalogTestUtil.testTxnLable1,
transactionSource,
LoadJobSourceType.FRONTEND, Config.stream_load_default_timeout_second);