database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](schema change) Light schema change support rollup (#11494)

Browse Source 
1. Move max colUniqueId from OlapTable to IndexMeta.
2. Add updateSlotUniqueId.
This commit is contained in:
Lei Zhang
2022-08-15 21:39:27 +08:00
committed by GitHub
parent 5104982614
commit d2d4423c88
13 changed files with 270 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View File

@ -504,10 +504,18 @@ public class MaterializedViewHandler extends AlterHandler {
newMVColumns.add(new Column(olapTable.getSequenceCol()));
}
// set MV column unique id to Column.COLUMN_UNIQUE_ID_INIT_VALUE support old unique id rule.
newMVColumns.stream().forEach(column -> {
column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE);
});
if (olapTable.getEnableLightSchemaChange()) {
int nextColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE + 1;
for (Column column : newMVColumns) {
column.setUniqueId(nextColUniqueId);
nextColUniqueId++;
}
} else {
newMVColumns.stream().forEach(column -> {
column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE);
});
}
LOG.debug("lightSchemaChange:{}, newMVColumns:{}", olapTable.getEnableLightSchemaChange(), newMVColumns);
return newMVColumns;
}
@ -549,25 +557,27 @@ public class MaterializedViewHandler extends AlterHandler {
for (Column column : olapTable.getSchemaByIndexId(baseIndexId, true)) {
baseColumnNameToColumn.put(column.getName(), column);
}
LOG.debug("baseSchema:{}", olapTable.getSchemaByIndexId(baseIndexId, true));
if (keysType.isAggregationFamily()) {
int keysNumOfRollup = 0;
for (String columnName : rollupColumnNames) {
Column oneColumn = baseColumnNameToColumn.get(columnName);
if (oneColumn == null) {
Column baseColumn = baseColumnNameToColumn.get(columnName);
if (baseColumn == null) {
throw new DdlException("Column[" + columnName + "] does not exist");
}
if (oneColumn.isKey() && meetValue) {
if (baseColumn.isKey() && meetValue) {
throw new DdlException("Invalid column order. value should be after key");
}
if (oneColumn.isKey()) {
if (baseColumn.isKey()) {
keysNumOfRollup += 1;
hasKey = true;
} else {
meetValue = true;
if (oneColumn.getAggregationType().isReplaceFamily()) {
if (baseColumn.getAggregationType().isReplaceFamily()) {
meetReplaceValue = true;
}
}
Column oneColumn = new Column(baseColumn);
rollupSchema.add(oneColumn);
}
@ -696,6 +706,19 @@ public class MaterializedViewHandler extends AlterHandler {
}
}
}
if (olapTable.getEnableLightSchemaChange()) {
int nextColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE + 1;
for (Column column : rollupSchema) {
column.setUniqueId(nextColUniqueId);
nextColUniqueId++;
}
} else {
rollupSchema.stream().forEach(column -> {
column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE);
});
}
LOG.debug("lightSchemaChange:{}, rollupSchema:{}, baseSchema:{}",
olapTable.getEnableLightSchemaChange(), rollupSchema, olapTable.getSchemaByIndexId(baseIndexId, true));
return rollupSchema;
}

15
fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
View File

@ -319,7 +319,8 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
}
tbl.setIndexMeta(rollupIndexId, rollupIndexName, rollupSchema, 0 /* init schema version */,
rollupSchemaHash, rollupShortKeyColumnCount, TStorageType.COLUMN, rollupKeysType, origStmt);
rollupSchemaHash, rollupShortKeyColumnCount, TStorageType.COLUMN,
rollupKeysType, origStmt);
tbl.rebuildFullSchema();
}
@ -532,6 +533,18 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
}
partition.visualiseShadowIndex(rollupIndexId, false);
}
//update max column unique id
int maxColUniqueId = tbl.getIndexMetaByIndexId(rollupIndexId).getMaxColUniqueId();
for (Column column : tbl.getIndexMetaByIndexId(rollupIndexId).getSchema(true)) {
if (column.getUniqueId() > maxColUniqueId) {
maxColUniqueId = column.getUniqueId();
}
}
tbl.getIndexMetaByIndexId(rollupIndexId).setMaxColUniqueId(maxColUniqueId);
LOG.debug("rollupIndexId:{}, maxColUniqueId:{}, indexIdToSchema:{}", rollupIndexId, maxColUniqueId,
tbl.getIndexIdToSchema(true));
tbl.rebuildFullSchema();
}

209
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -134,12 +134,13 @@ public class SchemaChangeHandler extends AlterHandler {
* @param alterClause
* @param olapTable
* @param indexSchemaMap
* @param colUniqueIdSupplier for multi add columns clause, we need stash middle state of maxColUniqueId
* @param colUniqueIdSupplierMap for multi add columns clause, we need stash middle state of maxColUniqueId
* @return true: can light schema change, false: cannot light schema change
* @throws DdlException
*/
private boolean processAddColumn(AddColumnClause alterClause, OlapTable olapTable,
Map<Long, LinkedList<Column>> indexSchemaMap, IntSupplier colUniqueIdSupplier) throws DdlException {
Map<Long, LinkedList<Column>> indexSchemaMap,
Map<Long, IntSupplier> colUniqueIdSupplierMap) throws DdlException {
Column column = alterClause.getColumn();
ColumnPosition columnPos = alterClause.getColPos();
String targetIndexName = alterClause.getRollupName();
@ -156,13 +157,8 @@ public class SchemaChangeHandler extends AlterHandler {
Set<String> newColNameSet = Sets.newHashSet(column.getName());
//only new table generate ColUniqueId, exist table do not.
if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) {
column.setUniqueId(colUniqueIdSupplier.getAsInt());
}
return addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, indexSchemaMap,
newColNameSet, false);
newColNameSet, false, colUniqueIdSupplierMap);
}
private void processAddColumn(AddColumnClause alterClause, Table externalTable, List<Column> newSchema)
@ -192,13 +188,13 @@ public class SchemaChangeHandler extends AlterHandler {
* @param olapTable
* @param indexSchemaMap
* @param ignoreSameColumn
* @param colUniqueIdSupplier for multi add columns clause, we need stash middle state of maxColUniqueId
* @param colUniqueIdSupplierMap for multi add columns clause, we need stash middle state of maxColUniqueId
* @return true: can light schema change, false: cannot light schema change
* @throws DdlException
*/
public boolean processAddColumns(AddColumnsClause alterClause, OlapTable olapTable,
Map<Long, LinkedList<Column>> indexSchemaMap, boolean ignoreSameColumn, IntSupplier colUniqueIdSupplier)
throws DdlException {
Map<Long, LinkedList<Column>> indexSchemaMap, boolean ignoreSameColumn,
Map<Long, IntSupplier> colUniqueIdSupplierMap) throws DdlException {
List<Column> columns = alterClause.getColumns();
String targetIndexName = alterClause.getRollupName();
checkIndexExists(olapTable, targetIndexName);
@ -217,22 +213,15 @@ public class SchemaChangeHandler extends AlterHandler {
targetIndexId = olapTable.getIndexIdByName(targetIndexName);
}
//for new table calculate column unique id
if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) {
for (Column column : columns) {
column.setUniqueId(colUniqueIdSupplier.getAsInt());
}
}
boolean ligthSchemaChange = true;
boolean lightSchemaChange = true;
for (Column column : columns) {
boolean result = addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, indexSchemaMap,
newColNameSet, ignoreSameColumn);
newColNameSet, ignoreSameColumn, colUniqueIdSupplierMap);
if (!result) {
ligthSchemaChange = false;
lightSchemaChange = false;
}
}
return ligthSchemaChange;
return lightSchemaChange;
}
private void processDropColumn(DropColumnClause alterClause, Table externalTable, List<Column> newSchema)
@ -273,19 +262,20 @@ public class SchemaChangeHandler extends AlterHandler {
private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTable,
Map<Long, LinkedList<Column>> indexSchemaMap, List<Index> indexes) throws DdlException {
boolean ligthSchemaChange = false;
if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) {
//assume can light schema change.
ligthSchemaChange = true;
}
String dropColName = alterClause.getColName();
String targetIndexName = alterClause.getRollupName();
checkIndexExists(olapTable, targetIndexName);
String baseIndexName = olapTable.getName();
checkAssignedTargetIndexName(baseIndexName, targetIndexName);
long baseIndexId = olapTable.getBaseIndexId();
long targetIndexId = -1L;
if (targetIndexName != null) {
targetIndexId = olapTable.getIndexIdByName(targetIndexName);
}
boolean lightSchemaChange = olapTable.getEnableLightSchemaChange();
/*
* UNIQUE:
* Can not drop any key column.
@ -293,12 +283,11 @@ public class SchemaChangeHandler extends AlterHandler {
* Can not drp any key column is has value with REPLACE method
*/
if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
boolean isKey = false;
for (Column column : baseSchema) {
if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) {
ligthSchemaChange = false;
lightSchemaChange = false;
isKey = true;
break;
}
@ -311,14 +300,13 @@ public class SchemaChangeHandler extends AlterHandler {
} else if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
if (null == targetIndexName) {
// drop column in base table
long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
boolean isKey = false;
boolean hasReplaceColumn = false;
for (Column column : baseSchema) {
if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) {
isKey = true;
ligthSchemaChange = false;
lightSchemaChange = false;
} else if (AggregateType.REPLACE == column.getAggregationType()
|| AggregateType.REPLACE_IF_NOT_NULL == column.getAggregationType()) {
hasReplaceColumn = true;
@ -330,7 +318,6 @@ public class SchemaChangeHandler extends AlterHandler {
}
} else {
// drop column in rollup and base index
long targetIndexId = olapTable.getIndexIdByName(targetIndexName);
// find column
List<Column> targetIndexSchema = indexSchemaMap.get(targetIndexId);
boolean isKey = false;
@ -338,7 +325,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Column column : targetIndexSchema) {
if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) {
isKey = true;
ligthSchemaChange = false;
lightSchemaChange = false;
} else if (AggregateType.REPLACE == column.getAggregationType()
|| AggregateType.REPLACE_IF_NOT_NULL == column.getAggregationType()) {
hasReplaceColumn = true;
@ -350,11 +337,10 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
} else if (KeysType.DUP_KEYS == olapTable.getKeysType()) {
long baseIndexId = olapTable.getBaseIndexId();
List<Column> baseSchema = indexSchemaMap.get(baseIndexId);
for (Column column : baseSchema) {
if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) {
ligthSchemaChange = false;
lightSchemaChange = false;
break;
}
}
@ -371,7 +357,6 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
long baseIndexId = olapTable.getBaseIndexId();
if (targetIndexName == null) {
// if not specify rollup index, column should be dropped from both base and rollup indexes.
List<Long> indexIds = new ArrayList<Long>();
@ -401,16 +386,13 @@ public class SchemaChangeHandler extends AlterHandler {
while (iter.hasNext()) {
Column column = iter.next();
if (column.getName().equalsIgnoreCase(dropColName)) {
ligthSchemaChange = false;
iter.remove();
break;
}
}
} // end for index names
} else {
ligthSchemaChange = false;
// if specify rollup index, only drop column from specified rollup index
long targetIndexId = olapTable.getIndexIdByName(targetIndexName);
// find column
List<Column> targetIndexSchema = indexSchemaMap.get(targetIndexId);
boolean found = false;
@ -420,6 +402,9 @@ public class SchemaChangeHandler extends AlterHandler {
if (column.getName().equalsIgnoreCase(dropColName)) {
iter.remove();
found = true;
if (column.isKey()) {
lightSchemaChange = false;
}
break;
}
}
@ -427,7 +412,7 @@ public class SchemaChangeHandler extends AlterHandler {
throw new DdlException("Column does not exists: " + dropColName);
}
}
return ligthSchemaChange;
return lightSchemaChange;
}
// User can modify column type and column position
@ -837,24 +822,19 @@ public class SchemaChangeHandler extends AlterHandler {
* @param indexSchemaMap Modified schema will be saved in 'indexSchemaMap'
* @param newColNameSet
* @param ignoreSameColumn
* @param colUniqueIdSupplierMap
* @return true: can light schema change, false: cannot
* @throws DdlException
*/
private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos,
long targetIndexId, long baseIndexId, Map<Long, LinkedList<Column>> indexSchemaMap,
Set<String> newColNameSet, boolean ignoreSameColumn) throws DdlException {
Set<String> newColNameSet, boolean ignoreSameColumn,
Map<Long, IntSupplier> colUniqueIdSupplierMap) throws DdlException {
//only new table generate ColUniqueId, exist table do not.
boolean ligthSchemaChange = olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE;
boolean lightSchemaChange = olapTable.getEnableLightSchemaChange();
String newColName = newColumn.getName();
//make sure olapTable has locked
if (newColumn.getUniqueId() > Integer.MAX_VALUE) {
throw new DdlException("schema change add column times overflow: " + newColName);
}
LOG.debug("table: {}, newColumn: {}, uniqueId: {}", olapTable.getName(), newColumn.getName(),
newColumn.getUniqueId());
// check the validation of aggregation method on column.
// also fill the default aggregation method if not specified.
if (KeysType.AGG_KEYS == olapTable.getKeysType()) {
@ -909,7 +889,8 @@ public class SchemaChangeHandler extends AlterHandler {
//type key column do not allow light schema change.
if (newColumn.isKey()) {
ligthSchemaChange = false;
LOG.debug("newColumn: {}, isKey()==true", newColumn);
lightSchemaChange = false;
}
// check if the new column already exist in base schema.
@ -959,62 +940,94 @@ public class SchemaChangeHandler extends AlterHandler {
for (Map.Entry<Long, LinkedList<Column>> entry : indexSchemaMap.entrySet()) {
modIndexSchema = entry.getValue();
boolean isBaseIdex = entry.getKey() == baseIndexId;
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, isBaseIdex);
IntSupplier colUniqueIdSupplier = colUniqueIdSupplierMap.get(entry.getKey());
int newColumnUniqueId = olapTable.getEnableLightSchemaChange()
? colUniqueIdSupplier.getAsInt() : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
isBaseIdex, newColumnUniqueId);
}
} else {
// 1. add to base table
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
IntSupplier baseIndexColUniqueIdSupplier = colUniqueIdSupplierMap.get(baseIndexId);
int baseIndexNewColumnUniqueId = olapTable.getEnableLightSchemaChange()
? baseIndexColUniqueIdSupplier.getAsInt()
: Column.COLUMN_UNIQUE_ID_INIT_VALUE;
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
true, baseIndexNewColumnUniqueId);
if (targetIndexId == -1L) {
return ligthSchemaChange;
return lightSchemaChange;
}
// 2. add to rollup
ligthSchemaChange = false;
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
IntSupplier targetIndexColUniqueIdSupplier = colUniqueIdSupplierMap.get(targetIndexId);
int rollUpNewColumnUniqueId = olapTable.getEnableLightSchemaChange()
? targetIndexColUniqueIdSupplier.getAsInt()
: Column.COLUMN_UNIQUE_ID_INIT_VALUE;
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
false, rollUpNewColumnUniqueId);
}
} else if (KeysType.DUP_KEYS == olapTable.getKeysType()) {
//get baseIndexColUniqueIdSupplier
IntSupplier baseIndexColUniqueIdSupplier = colUniqueIdSupplierMap.get(baseIndexId);
int baseIndexNewColumnUniqueId = olapTable.getEnableLightSchemaChange()
? baseIndexColUniqueIdSupplier.getAsInt()
: Column.COLUMN_UNIQUE_ID_INIT_VALUE;
if (targetIndexId == -1L) {
// add to base index
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
true, baseIndexNewColumnUniqueId);
// no specified target index. return
return ligthSchemaChange;
return lightSchemaChange;
} else {
// add to rollup index
ligthSchemaChange = false;
List<Column> modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
IntSupplier targetIndexColUniqueIdSupplier = colUniqueIdSupplierMap.get(targetIndexId);
int rollUpNewColumnUniqueId = olapTable.getEnableLightSchemaChange()
? targetIndexColUniqueIdSupplier.getAsInt() : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
false, rollUpNewColumnUniqueId);
if (newColumn.isKey()) {
/*
* if add column in rollup is key,
* then put the column in base table as the last key column
*/
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, null, newColNameSet, true);
checkAndAddColumn(modIndexSchema, newColumn, null, newColNameSet,
true, baseIndexNewColumnUniqueId);
} else {
modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
true, baseIndexNewColumnUniqueId);
}
}
} else {
// check if has default value. this should be done in Analyze phase
// 1. add to base index first
IntSupplier baseIndexColUniqueIdSupplier = colUniqueIdSupplierMap.get(baseIndexId);
int baseIndexNewColumnUniqueId = olapTable.getEnableLightSchemaChange()
? baseIndexColUniqueIdSupplier.getAsInt() : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
List<Column> modIndexSchema = indexSchemaMap.get(baseIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet,
true, baseIndexNewColumnUniqueId);
if (targetIndexId == -1L) {
// no specified target index. return
return ligthSchemaChange;
return lightSchemaChange;
}
ligthSchemaChange = false;
// 2. add to rollup index
IntSupplier targetIndexColUniqueIdSupplier = colUniqueIdSupplierMap.get(targetIndexId);
int rollUpNewColumnUniqueId = olapTable.getEnableLightSchemaChange()
? targetIndexColUniqueIdSupplier.getAsInt() : Column.COLUMN_UNIQUE_ID_INIT_VALUE;
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false, rollUpNewColumnUniqueId);
}
return ligthSchemaChange;
return lightSchemaChange;
}
/*
@ -1027,7 +1040,7 @@ public class SchemaChangeHandler extends AlterHandler {
* So that k1 will be added to base index 'twice', and we just ignore this repeat adding.
*/
private void checkAndAddColumn(List<Column> modIndexSchema, Column newColumn, ColumnPosition columnPos,
Set<String> newColNameSet, boolean isBaseIndex) throws DdlException {
Set<String> newColNameSet, boolean isBaseIndex, int newColumnUniqueId) throws DdlException {
int posIndex = -1;
int lastVisibleIdx = -1;
String newColName = newColumn.getName();
@ -1077,6 +1090,7 @@ public class SchemaChangeHandler extends AlterHandler {
hasPos = true;
}
newColumn.setUniqueId(newColumnUniqueId);
if (hasPos) {
modIndexSchema.add(posIndex + 1, newColumn);
} else if (newColumn.isKey()) {
@ -1089,6 +1103,7 @@ public class SchemaChangeHandler extends AlterHandler {
// value
modIndexSchema.add(newColumn);
}
LOG.debug("newColumn setUniqueId({}), modIndexSchema:{}", newColumnUniqueId, modIndexSchema);
}
private void checkIndexExists(OlapTable olapTable, String targetIndexName) throws DdlException {
@ -1595,22 +1610,34 @@ public class SchemaChangeHandler extends AlterHandler {
try {
//alterClauses can or cannot light schema change
boolean lightSchemaChange = true;
//for multi add colmuns clauses
IntSupplier colUniqueIdSupplier = new IntSupplier() {
public int pendingMaxColUniqueId = olapTable.getMaxColUniqueId();
@Override
public int getAsInt() {
pendingMaxColUniqueId++;
return pendingMaxColUniqueId;
}
};
// index id -> index schema
Map<Long, LinkedList<Column>> indexSchemaMap = new HashMap<>();
//for multi add colmuns clauses
//index id -> index col_unique_id supplier
Map<Long, IntSupplier> colUniqueIdSupplierMap = new HashMap<>();
for (Map.Entry<Long, List<Column>> entry : olapTable.getIndexIdToSchema(true).entrySet()) {
indexSchemaMap.put(entry.getKey(), new LinkedList<>(entry.getValue()));
IntSupplier colUniqueIdSupplier = null;
if (olapTable.getEnableLightSchemaChange()) {
colUniqueIdSupplier = new IntSupplier() {
public int pendingMaxColUniqueId = olapTable
.getIndexMetaByIndexId(entry.getKey()).getMaxColUniqueId();
public long indexId = entry.getKey();
@Override
public int getAsInt() {
pendingMaxColUniqueId++;
LOG.debug("index id:{}, pendingMaxColUniqueId:{}", indexId, pendingMaxColUniqueId);
return pendingMaxColUniqueId;
}
};
}
colUniqueIdSupplierMap.put(entry.getKey(), colUniqueIdSupplier);
}
LOG.debug("in process indexSchemaMap:{}", indexSchemaMap);
List<Index> newIndexes = olapTable.getCopiedIndexes();
Map<String, String> propertyMap = new HashMap<>();
for (AlterClause alterClause : alterClauses) {
@ -1682,14 +1709,14 @@ public class SchemaChangeHandler extends AlterHandler {
if (alterClause instanceof AddColumnClause) {
// add column
boolean clauseCanLigthSchemaChange = processAddColumn((AddColumnClause) alterClause, olapTable,
indexSchemaMap, colUniqueIdSupplier);
indexSchemaMap, colUniqueIdSupplierMap);
if (clauseCanLigthSchemaChange == false) {
lightSchemaChange = false;
}
} else if (alterClause instanceof AddColumnsClause) {
// add columns
boolean clauseCanLigthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable,
indexSchemaMap, false, colUniqueIdSupplier);
indexSchemaMap, false, colUniqueIdSupplierMap);
if (clauseCanLigthSchemaChange == false) {
lightSchemaChange = false;
}
@ -1727,8 +1754,8 @@ public class SchemaChangeHandler extends AlterHandler {
}
} // end for alter clauses
LOG.debug("processAddColumns, table: {}({}), lightSchemaChange: {}", olapTable.getName(), olapTable.getId(),
lightSchemaChange);
LOG.debug("table: {}({}), lightSchemaChange: {}, indexSchemaMap:{}", olapTable.getName(), olapTable.getId(),
lightSchemaChange, indexSchemaMap);
if (lightSchemaChange) {
long jobId = Env.getCurrentEnv().getNextId();
@ -2233,19 +2260,19 @@ public class SchemaChangeHandler extends AlterHandler {
int currentSchemaVersion = currentIndexMeta.getSchemaVersion();
int newSchemaVersion = currentSchemaVersion + 1;
currentIndexMeta.setSchemaVersion(newSchemaVersion);
//update max column unique id
int maxColUniqueId = currentIndexMeta.getMaxColUniqueId();
for (Column column : indexSchema) {
if (column.getUniqueId() > maxColUniqueId) {
maxColUniqueId = column.getUniqueId();
}
}
currentIndexMeta.setMaxColUniqueId(maxColUniqueId);
}
olapTable.setIndexes(indexes);
olapTable.rebuildFullSchema();
//update max column unique id
int maxColUniqueId = olapTable.getMaxColUniqueId();
for (Column column : indexSchemaMap.get(olapTable.getBaseIndexId())) {
if (column.getUniqueId() > maxColUniqueId) {
maxColUniqueId = column.getUniqueId();
}
}
olapTable.setMaxColUniqueId(maxColUniqueId);
if (!isReplay) {
TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(db.getId(), olapTable.getId(),
indexSchemaMap, indexes, jobId);

23
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -352,7 +352,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion,
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash,
indexShortKeyMap.get(shadowIdxId), TStorageType.COLUMN,
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)));
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)));
}
tbl.rebuildFullSchema();
@ -486,7 +486,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
// and the job will be in RUNNING state forever.
Database db = Env.getCurrentInternalCatalog()
.getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
OlapTable tbl;
try {
tbl = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP);
@ -620,6 +619,16 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
long originIdxId = entry.getValue();
String shadowIdxName = tbl.getIndexNameById(shadowIdxId);
String originIdxName = tbl.getIndexNameById(originIdxId);
int maxColUniqueId = tbl.getIndexMetaByIndexId(originIdxId).getMaxColUniqueId();
for (Column column : indexSchemaMap.get(shadowIdxId)) {
if (column.getUniqueId() > maxColUniqueId) {
maxColUniqueId = column.getUniqueId();
}
}
tbl.getIndexMetaByIndexId(shadowIdxId).setMaxColUniqueId(maxColUniqueId);
LOG.debug("originIdxId:{}, shadowIdxId:{}, maxColUniqueId:{}, indexSchema:{}",
originIdxId, shadowIdxId, maxColUniqueId, indexSchemaMap.get(shadowIdxId));
tbl.deleteIndexInfo(originIdxName);
// the shadow index name is '__doris_shadow_xxx', rename it to origin name 'xxx'
// this will also remove the prefix of columns
@ -648,16 +657,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.setStorageFormat(storageFormat);
}
// update max column unique id
int maxColUniqueId = tbl.getMaxColUniqueId();
for (Column column : tbl.getFullSchema()) {
if (column.getUniqueId() > maxColUniqueId) {
maxColUniqueId = column.getUniqueId();
}
}
tbl.setMaxColUniqueId(maxColUniqueId);
LOG.debug("fullSchema:{}, maxColUniqueId:{}", tbl.getFullSchema(), maxColUniqueId);
tbl.setState(OlapTableState.NORMAL);
}

1
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -551,7 +551,6 @@ public class Column implements Writable {
if (StringUtils.isNotBlank(comment)) {
sb.append(" COMMENT '").append(getComment(true)).append("'");
}
return sb.toString();
}

6
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -2897,9 +2897,9 @@ public class Env {
}
// show lightSchemaChange only when it is set true
if (olapTable.getUseLightSchemaChange()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE).append("\" = \"");
sb.append(olapTable.getUseLightSchemaChange()).append("\"");
if (olapTable.getEnableLightSchemaChange()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE).append("\" = \"");
sb.append(olapTable.getEnableLightSchemaChange()).append("\"");
}
// storage policy

33
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
View File

@ -33,6 +33,8 @@ import org.apache.doris.thrift.TStorageType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
@ -59,6 +61,12 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
private KeysType keysType;
@SerializedName(value = "defineStmt")
private OriginStatement defineStmt;
//for light schema change
@SerializedName(value = "maxColUniqueId")
private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
private static final Logger LOG = LogManager.getLogger(MaterializedIndexMeta.class);
public MaterializedIndexMeta(long indexId, List<Column> schema, int schemaVersion, int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType keysType, OriginStatement defineStmt) {
@ -179,6 +187,9 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
if (indexMeta.keysType != this.keysType) {
return false;
}
if (maxColUniqueId != maxColUniqueId) {
return false;
}
return true;
}
@ -212,4 +223,26 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
}
}
//take care: only use when creating MaterializedIndexMeta's schema.
public int incAndGetMaxColUniqueId() {
this.maxColUniqueId++;
return this.maxColUniqueId;
}
public int getMaxColUniqueId() {
return this.maxColUniqueId;
}
public void setMaxColUniqueId(int maxColUniqueId) {
this.maxColUniqueId = maxColUniqueId;
}
public void initSchemaColumnUniqueId() {
maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
this.schema.stream().forEach(column -> {
column.setUniqueId(incAndGetMaxColUniqueId());
LOG.debug("indexId: {}, column:{}, uniqueId:{}",
indexId, column, column.getUniqueId());
});
}
}

43
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -38,7 +38,6 @@ import org.apache.doris.clone.TabletScheduler;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.DeepCopy;
@ -142,8 +141,6 @@ public class OlapTable extends Table {
private TableProperty tableProperty;
private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
public OlapTable() {
// for persist
super(TableType.OLAP);
@ -194,20 +191,6 @@ public class OlapTable extends Table {
return this.tableProperty;
}
//take care: only use at create olap table.
public int incAndGetMaxColUniqueId() {
this.maxColUniqueId++;
return this.maxColUniqueId;
}
public int getMaxColUniqueId() {
return this.maxColUniqueId;
}
public void setMaxColUniqueId(int maxColUniqueId) {
this.maxColUniqueId = maxColUniqueId;
}
public boolean dynamicPartitionExists() {
return tableProperty != null
&& tableProperty.getDynamicPartitionProperty() != null
@ -324,6 +307,7 @@ public class OlapTable extends Table {
MaterializedIndexMeta indexMeta = new MaterializedIndexMeta(indexId, schema, schemaVersion,
schemaHash, shortKeyColumnCount, storageType, keysType, origStmt);
indexIdToMeta.put(indexId, indexMeta);
indexNameToId.put(indexName, indexId);
}
@ -1161,7 +1145,6 @@ public class OlapTable extends Table {
}
tempPartitions.write(out);
out.writeInt(maxColUniqueId);
}
@Override
@ -1254,9 +1237,6 @@ public class OlapTable extends Table {
}
tempPartitions.unsetPartitionInfo();
if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_112) {
maxColUniqueId = in.readInt();
}
// In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV.
// After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns.
// So, here we need to rebuild the fullSchema to ensure the correctness of the properties.
@ -1562,7 +1542,7 @@ public class OlapTable extends Table {
tableProperty.buildInMemory();
}
public Boolean getUseLightSchemaChange() {
public boolean getEnableLightSchemaChange() {
if (tableProperty != null) {
return tableProperty.getUseSchemaLightChange();
}
@ -1570,13 +1550,13 @@ public class OlapTable extends Table {
return false;
}
public void setUseLightSchemaChange(boolean useLightSchemaChange) {
public void setEnableLightSchemaChange(boolean enableLightSchemaChange) {
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());
}
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE,
Boolean.valueOf(useLightSchemaChange).toString());
tableProperty.buildUseLightSchemaChange();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE,
Boolean.valueOf(enableLightSchemaChange).toString());
tableProperty.buildEnableLightSchemaChange();
}
public void setStoragePolicy(String storagePolicy) {
@ -1879,4 +1859,15 @@ public class OlapTable extends Table {
}
tableProperty.buildReplicaAllocation();
}
//for light schema change
public void initSchemaColumnUniqueId() {
if (!getEnableLightSchemaChange()) {
return;
}
for (MaterializedIndexMeta indexMeta : indexIdToMeta.values()) {
indexMeta.initSchemaColumnUniqueId();
}
}
}

14
fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View File

@ -73,7 +73,7 @@ public class TableProperty implements Writable {
private TCompressionType compressionType = TCompressionType.LZ4F;
private Boolean useSchemaLightChange;
private boolean enableLightSchemaChange = false;
private DataSortInfo dataSortInfo = new DataSortInfo();
@ -146,9 +146,9 @@ public class TableProperty implements Writable {
return this;
}
public TableProperty buildUseLightSchemaChange() {
useSchemaLightChange = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE, "false"));
public TableProperty buildEnableLightSchemaChange() {
enableLightSchemaChange = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE, "false"));
return this;
}
@ -261,8 +261,8 @@ public class TableProperty implements Writable {
return compressionType;
}
public Boolean getUseSchemaLightChange() {
return useSchemaLightChange;
public boolean getUseSchemaLightChange() {
return enableLightSchemaChange;
}
public void setEnableUniqueKeyMergeOnWrite(boolean enable) {
@ -301,7 +301,7 @@ public class TableProperty implements Writable {
.buildRemoteStoragePolicy()
.buildCompressionType()
.buildStoragePolicy()
.buildUseLightSchemaChange();
.buildEnableLightSchemaChange();
if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_105) {
// get replica num from property map and create replica allocation
String repNum = tableProperty.properties.remove(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM);

8
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -79,7 +79,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_TIMEOUT = "timeout";
public static final String PROPERTIES_COMPRESSION = "compression";
public static final String PROPERTIES_USE_LIGHT_SCHEMA_CHANGE = "light_schema_change";
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE = "distribution_type";
public static final String PROPERTIES_SEND_CLEAR_ALTER_TASK = "send_clear_alter_tasks";
@ -457,18 +457,18 @@ public class PropertyAnalyzer {
if (properties == null || properties.isEmpty()) {
return false;
}
String value = properties.get(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
String value = properties.get(PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE);
// set light schema change false by default
if (null == value) {
return false;
}
properties.remove(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE);
properties.remove(PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE);
if (value.equalsIgnoreCase("true")) {
return true;
} else if (value.equalsIgnoreCase("false")) {
return false;
}
throw new AnalysisException(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE
throw new AnalysisException(PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE
+ " must be `true` or `false`");
}

18
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
View File

@ -1727,23 +1727,14 @@ public class InternalDataSource implements DataSourceIf<Database> {
Map<String, String> properties = stmt.getProperties();
// get use light schema change
Boolean useLightSchemaChange = false;
Boolean enableLightSchemaChange = false;
try {
useLightSchemaChange = PropertyAnalyzer.analyzeUseLightSchemaChange(properties);
enableLightSchemaChange = PropertyAnalyzer.analyzeUseLightSchemaChange(properties);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
// use light schema change optimization
olapTable.setUseLightSchemaChange(useLightSchemaChange);
if (useLightSchemaChange) {
for (Column column : baseSchema) {
column.setUniqueId(olapTable.incAndGetMaxColUniqueId());
LOG.debug("table: {}, newColumn: {}, uniqueId: {}", olapTable.getName(), column.getName(),
column.getUniqueId());
}
} else {
LOG.debug("table: {} doesn't use light schema change", olapTable.getName());
}
olapTable.setEnableLightSchemaChange(enableLightSchemaChange);
// get storage format
TStorageFormat storageFormat = TStorageFormat.V2; // default is segment v2
@ -1923,6 +1914,9 @@ public class InternalDataSource implements DataSourceIf<Database> {
throw new DdlException(e.getMessage());
}
olapTable.initSchemaColumnUniqueId();
olapTable.rebuildFullSchema();
// analyze version info
Long versionInfo = null;
try {

21
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -323,6 +323,7 @@ public class OlapScanNode extends ScanNode {
if (update) {
this.selectedIndexId = selectedIndexId;
updateSlotUniqueId();
setIsPreAggregation(isPreAggregation, reasonOfDisable);
updateColumnType();
if (LOG.isDebugEnabled()) {
@ -368,6 +369,25 @@ public class OlapScanNode extends ScanNode {
}
}
/**
* In some situation, we need use mv col unique id , because mv col unique and
* base col unique id is different.
* For example: select count(*) from table (table has a mv named mv1)
* if Optimizer deceide use mv1, we need updateSlotUniqueId.
*/
private void updateSlotUniqueId() {
if (!olapTable.getEnableLightSchemaChange() || selectedIndexId == olapTable.getBaseIndexId()) {
return;
}
MaterializedIndexMeta meta = olapTable.getIndexMetaByIndexId(selectedIndexId);
for (SlotDescriptor slotDescriptor : desc.getSlots()) {
Column baseColumn = slotDescriptor.getColumn();
Column mvColumn = meta.getColumnByName(baseColumn.getName());
slotDescriptor.setColumn(mvColumn);
}
LOG.debug("updateSlotUniqueId() slots: {}", desc.getSlots());
}
public OlapTable getOlapTable() {
return olapTable;
}
@ -696,6 +716,7 @@ public class OlapScanNode extends ScanNode {
}
final RollupSelector rollupSelector = new RollupSelector(analyzer, desc, olapTable);
selectedIndexId = rollupSelector.selectBestRollup(selectedPartitionIds, conjuncts, isPreAggregation);
updateSlotUniqueId();
LOG.debug("select best roll up cost: {} ms, best index id: {}",
(System.currentTimeMillis() - start), selectedIndexId);
}

6
fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
View File

@ -123,6 +123,8 @@ public class SchemaChangeHandlerTest extends TestWithFeService {
Assertions.assertEquals(baseIndexName, tbl.getName());
MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId());
Assertions.assertNotNull(indexMeta);
//col_unique_id 0-9
Assertions.assertEquals(9, indexMeta.getMaxColUniqueId());
} finally {
tbl.readUnlock();
}
@ -228,7 +230,7 @@ public class SchemaChangeHandlerTest extends TestWithFeService {
Assertions.assertEquals(baseIndexName, tbl.getName());
MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId());
Assertions.assertNotNull(indexMeta);
Assertions.assertEquals(12, tbl.getMaxColUniqueId());
Assertions.assertEquals(12, indexMeta.getMaxColUniqueId());
} finally {
tbl.readUnlock();
}
@ -365,7 +367,7 @@ public class SchemaChangeHandlerTest extends TestWithFeService {
try {
Deencapsulation.invoke(schemaChangeHandler, "addColumnInternal", olapTable, newColumn, columnPosition,
new Long(2), new Long(1), Maps.newHashMap(), Sets.newHashSet(), false);
new Long(2), new Long(1), Maps.newHashMap(), Sets.newHashSet(), false, Maps.newHashMap());
Assert.fail();
} catch (Exception e) {
System.out.println(e.getMessage());