database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Alter] Allow submitting alter jobs when table is unstable

Browse Source 
Alter job will wait table to be stable before running.
This commit is contained in:
WingC
2020-01-18 22:56:37 +08:00
committed by Mingyu Chen
parent ae018043b0
commit 92d8f6ae78
8 changed files with 274 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
View File

@ -58,6 +58,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
public class RollupJobV2Test {
private static FakeTransactionIDGenerator fakeTransactionIDGenerator;
private static GlobalTransactionMgr masterTransMgr;
@ -69,11 +71,13 @@ public class RollupJobV2Test {
private static Analyzer analyzer;
private static AddRollupClause clause;
FakeEditLog fakeEditLog;
private FakeCatalog fakeCatalog;
private FakeEditLog fakeEditLog;
@Before
public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException, SecurityException, AnalysisException {
fakeCatalog = new FakeCatalog();
fakeEditLog = new FakeEditLog();
fakeTransactionIDGenerator = new FakeTransactionIDGenerator();
masterCatalog = CatalogTestUtil.createTestCatalog();
@ -110,7 +114,9 @@ public class RollupJobV2Test {
@Test
public void testAddSchemaChange() throws UserException {
fakeCatalog = new FakeCatalog();
fakeEditLog = new FakeEditLog();
FakeCatalog.setCatalog(masterCatalog);
MaterializedViewHandler materializedViewHandler = Catalog.getInstance().getRollupHandler();
ArrayList<AlterClause> alterClauses = new ArrayList<>();
alterClauses.add(clause);
@ -125,7 +131,9 @@ public class RollupJobV2Test {
// start a schema change, then finished
@Test
public void testSchemaChange1() throws Exception {
fakeCatalog = new FakeCatalog();
fakeEditLog = new FakeEditLog();
FakeCatalog.setCatalog(masterCatalog);
MaterializedViewHandler materializedViewHandler = Catalog.getInstance().getRollupHandler();
// add a rollup job
@ -225,4 +233,82 @@ public class RollupJobV2Test {
*/
}
@Test
public void testSchemaChangeWhileTabletNotStable() throws Exception {
fakeCatalog = new FakeCatalog();
fakeEditLog = new FakeEditLog();
FakeCatalog.setCatalog(masterCatalog);
MaterializedViewHandler materializedViewHandler = Catalog.getInstance().getRollupHandler();
// add a rollup job
ArrayList<AlterClause> alterClauses = new ArrayList<>();
alterClauses.add(clause);
Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1);
OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1);
Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1);
materializedViewHandler.process(alterClauses, db.getClusterName(), db, olapTable);
Map<Long, AlterJobV2> alterJobsV2 = materializedViewHandler.getAlterJobsV2();
Assert.assertEquals(1, alterJobsV2.size());
RollupJobV2 rollupJob = (RollupJobV2) alterJobsV2.values().stream().findAny().get();
MaterializedIndex baseIndex = testPartition.getBaseIndex();
assertEquals(MaterializedIndex.IndexState.NORMAL, baseIndex.getState());
assertEquals(Partition.PartitionState.NORMAL, testPartition.getState());
assertEquals(OlapTableState.ROLLUP, olapTable.getState());
Tablet baseTablet = baseIndex.getTablets().get(0);
List<Replica> replicas = baseTablet.getReplicas();
Replica replica1 = replicas.get(0);
Replica replica2 = replicas.get(1);
Replica replica3 = replicas.get(2);
assertEquals(CatalogTestUtil.testStartVersion, replica1.getVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica2.getVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica3.getVersion());
assertEquals(-1, replica1.getLastFailedVersion());
assertEquals(-1, replica2.getLastFailedVersion());
assertEquals(-1, replica3.getLastFailedVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica1.getLastSuccessVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica2.getLastSuccessVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica3.getLastSuccessVersion());
// runPendingJob
replica1.setState(Replica.ReplicaState.DECOMMISSION);
materializedViewHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.PENDING, rollupJob.getJobState());
// table is stable, runPendingJob again
replica1.setState(Replica.ReplicaState.NORMAL);
materializedViewHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.WAITING_TXN, rollupJob.getJobState());
Assert.assertEquals(2, testPartition.getMaterializedIndices(IndexExtState.ALL).size());
Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.VISIBLE).size());
Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.SHADOW).size());
// runWaitingTxnJob
materializedViewHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.RUNNING, rollupJob.getJobState());
// runWaitingTxnJob, task not finished
materializedViewHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.RUNNING, rollupJob.getJobState());
// finish all tasks
List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
Assert.assertEquals(3, tasks.size());
for (AgentTask agentTask : tasks) {
agentTask.setFinished(true);
}
MaterializedIndex shadowIndex = testPartition.getMaterializedIndices(IndexExtState.SHADOW).get(0);
for (Tablet shadowTablet : shadowIndex.getTablets()) {
for (Replica shadowReplica : shadowTablet.getReplicas()) {
shadowReplica.updateVersionInfo(testPartition.getVisibleVersion(),
testPartition.getVisibleVersionHash(), shadowReplica.getDataSize(),
shadowReplica.getRowCount());
}
}
materializedViewHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.FINISHED, rollupJob.getJobState());
}
}

82
fe/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
View File

@ -206,6 +206,88 @@ public class SchemaChangeJobV2Test {
Assert.assertEquals(JobState.FINISHED, schemaChangeJob.getJobState());
}
@Test
public void testSchemaChangeWhileTabletNotStable() throws Exception {
fakeCatalog = new FakeCatalog();
fakeEditLog = new FakeEditLog();
FakeCatalog.setCatalog(masterCatalog);
SchemaChangeHandler schemaChangeHandler = Catalog.getInstance().getSchemaChangeHandler();
// add a schema change job
ArrayList<AlterClause> alterClauses = new ArrayList<>();
alterClauses.add(addColumnClause);
Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1);
OlapTable olapTable = (OlapTable) db.getTable(CatalogTestUtil.testTableId1);
Partition testPartition = olapTable.getPartition(CatalogTestUtil.testPartitionId1);
schemaChangeHandler.process(alterClauses, "default_cluster", db, olapTable);
Map<Long, AlterJobV2> alterJobsV2 = schemaChangeHandler.getAlterJobsV2();
Assert.assertEquals(1, alterJobsV2.size());
SchemaChangeJobV2 schemaChangeJob = (SchemaChangeJobV2) alterJobsV2.values().stream().findAny().get();
MaterializedIndex baseIndex = testPartition.getBaseIndex();
assertEquals(IndexState.NORMAL, baseIndex.getState());
assertEquals(PartitionState.NORMAL, testPartition.getState());
assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState());
Tablet baseTablet = baseIndex.getTablets().get(0);
List<Replica> replicas = baseTablet.getReplicas();
Replica replica1 = replicas.get(0);
Replica replica2 = replicas.get(1);
Replica replica3 = replicas.get(2);
assertEquals(CatalogTestUtil.testStartVersion, replica1.getVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica2.getVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica3.getVersion());
assertEquals(-1, replica1.getLastFailedVersion());
assertEquals(-1, replica2.getLastFailedVersion());
assertEquals(-1, replica3.getLastFailedVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica1.getLastSuccessVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica2.getLastSuccessVersion());
assertEquals(CatalogTestUtil.testStartVersion, replica3.getLastSuccessVersion());
// runPendingJob
replica1.setState(Replica.ReplicaState.DECOMMISSION);
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.PENDING, schemaChangeJob.getJobState());
// table is stable runPendingJob again
replica1.setState(Replica.ReplicaState.NORMAL);
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.WAITING_TXN, schemaChangeJob.getJobState());
Assert.assertEquals(2, testPartition.getMaterializedIndices(IndexExtState.ALL).size());
Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.VISIBLE).size());
Assert.assertEquals(1, testPartition.getMaterializedIndices(IndexExtState.SHADOW).size());
// runWaitingTxnJob
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.RUNNING, schemaChangeJob.getJobState());
// runWaitingTxnJob, task not finished
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.RUNNING, schemaChangeJob.getJobState());
// runRunningJob
schemaChangeHandler.runAfterCatalogReady();
// task not finished, still running
Assert.assertEquals(JobState.RUNNING, schemaChangeJob.getJobState());
// finish alter tasks
List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
Assert.assertEquals(3, tasks.size());
for (AgentTask agentTask : tasks) {
agentTask.setFinished(true);
}
MaterializedIndex shadowIndex = testPartition.getMaterializedIndices(IndexExtState.SHADOW).get(0);
for (Tablet shadowTablet : shadowIndex.getTablets()) {
for (Replica shadowReplica : shadowTablet.getReplicas()) {
shadowReplica.updateVersionInfo(testPartition.getVisibleVersion(), testPartition.getVisibleVersionHash(), shadowReplica.getDataSize(), shadowReplica.getRowCount());
}
}
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(JobState.FINISHED, schemaChangeJob.getJobState());
}
@Test
public void testModifyDynamicPartitionNormal() throws UserException {
fakeCatalog = new FakeCatalog();

7
fe/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
View File

@ -89,8 +89,11 @@ public class CatalogTestUtil {
catalog.setEditLog(new EditLog("name"));
FakeCatalog.setCatalog(catalog);
Backend backend1 = createBackend(testBackendId1, "host1", 123, 124, 125);
Backend backend2 = createBackend(testBackendId2, "host1", 123, 124, 125);
Backend backend3 = createBackend(testBackendId3, "host1", 123, 124, 125);
Backend backend2 = createBackend(testBackendId2, "host2", 123, 124, 125);
Backend backend3 = createBackend(testBackendId3, "host3", 123, 124, 125);
backend1.setOwnerClusterName(SystemInfoService.DEFAULT_CLUSTER);
backend2.setOwnerClusterName(SystemInfoService.DEFAULT_CLUSTER);
backend3.setOwnerClusterName(SystemInfoService.DEFAULT_CLUSTER);
Catalog.getCurrentSystemInfo().addBackend(backend1);
Catalog.getCurrentSystemInfo().addBackend(backend2);
Catalog.getCurrentSystemInfo().addBackend(backend3);