database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Pick 2.1](inverted index) support inverted index format v2 (#30145) (#32418)

Browse Source
This commit is contained in:
airborne12
2024-03-19 08:11:33 +08:00
committed by GitHub
parent 711c0cd55c
commit ecadb60bcd
77 changed files with 4414 additions and 1006 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -92,6 +92,7 @@ import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.ClearAlterTask;
import org.apache.doris.task.UpdateTabletMetaInfoTask;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTaskType;
@ -1312,6 +1313,9 @@ public class SchemaChangeHandler extends AlterHandler {
TStorageFormat storageFormat = PropertyAnalyzer.analyzeStorageFormat(propertyMap);
TInvertedIndexStorageFormat invertedIndexStorageFormat =
PropertyAnalyzer.analyzeInvertedIndexStorageFormat(propertyMap);
// begin checking each table
// ATTN: DO NOT change any meta in this loop
long tableId = olapTable.getId();
@ -1377,6 +1381,10 @@ public class SchemaChangeHandler extends AlterHandler {
if (olapTable.getStorageFormat() != TStorageFormat.V2) {
needAlter = true;
}
} else if (invertedIndexStorageFormat == TInvertedIndexStorageFormat.V2) {
if (olapTable.getInvertedIndexStorageFormat() != TInvertedIndexStorageFormat.V2) {
needAlter = true;
}
}
if (!needAlter) {

2
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java
View File

@ -37,6 +37,7 @@ public class ShowTableStmt extends ShowStmt {
private static final String NAME_COL_PREFIX = "Tables_in_";
private static final String TYPE_COL = "Table_type";
private static final String STORAGE_FORMAT_COL = "Storage_format";
private static final String INVERTED_INDEX_STORAGE_FORMAT_COL = "Inverted_index_storage_format";
private String db;
private String catalog;
private boolean isVerbose;
@ -164,6 +165,7 @@ public class ShowTableStmt extends ShowStmt {
if (isVerbose) {
builder.addColumn(new Column(TYPE_COL, ScalarType.createVarchar(20)));
builder.addColumn(new Column(STORAGE_FORMAT_COL, ScalarType.createVarchar(20)));
builder.addColumn(new Column(INVERTED_INDEX_STORAGE_FORMAT_COL, ScalarType.createVarchar(20)));
}
return builder.build();
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -3347,6 +3347,10 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT).append("\" = \"");
sb.append(olapTable.getStorageFormat()).append("\"");
// inverted index storage type
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT).append("\" = \"");
sb.append(olapTable.getInvertedIndexStorageFormat()).append("\"");
// compression type
if (olapTable.getCompressionType() != TCompressionType.LZ4F) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_COMPRESSION).append("\" = \"");

15
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -65,6 +65,7 @@ import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TFetchOption;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TOlapTable;
import org.apache.doris.thrift.TPrimitiveType;
import org.apache.doris.thrift.TSortType;
@ -2319,6 +2320,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
tableProperty.buildStorageFormat();
}
public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat invertedIndexStorageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
invertedIndexStorageFormat.name());
tableProperty.buildInvertedIndexStorageFormat();
}
public TStorageFormat getStorageFormat() {
if (tableProperty == null) {
return TStorageFormat.DEFAULT;
@ -2326,6 +2334,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
return tableProperty.getStorageFormat();
}
public TInvertedIndexStorageFormat getInvertedIndexStorageFormat() {
if (tableProperty == null) {
return TInvertedIndexStorageFormat.V2;
}
return tableProperty.getInvertedIndexStorageFormat();
}
public TCompressionType getCompressionType() {
if (tableProperty == null) {
return TCompressionType.LZ4F;

14
fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@ -77,6 +78,8 @@ public class TableProperty implements Writable {
*/
private TStorageFormat storageFormat = TStorageFormat.DEFAULT;
private TInvertedIndexStorageFormat invertedIndexStorageFormat = TInvertedIndexStorageFormat.DEFAULT;
private TCompressionType compressionType = TCompressionType.LZ4F;
private boolean enableLightSchemaChange = false;
@ -416,6 +419,13 @@ public class TableProperty implements Writable {
return this;
}
public TableProperty buildInvertedIndexStorageFormat() {
invertedIndexStorageFormat = TInvertedIndexStorageFormat.valueOf(properties.getOrDefault(
PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT,
TInvertedIndexStorageFormat.DEFAULT.name()));
return this;
}
public void modifyTableProperties(Map<String, String> modifyProperties) {
properties.putAll(modifyProperties);
removeDuplicateReplicaNumProperty();
@ -479,6 +489,10 @@ public class TableProperty implements Writable {
return storageFormat;
}
public TInvertedIndexStorageFormat getInvertedIndexStorageFormat() {
return invertedIndexStorageFormat;
}
public DataSortInfo getDataSortInfo() {
return dataSortInfo;
}

32
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -40,6 +40,7 @@ import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TSortType;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@ -99,6 +100,8 @@ public class PropertyAnalyzer {
*/
public static final String PROPERTIES_STORAGE_FORMAT = "storage_format";
public static final String PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT = "inverted_index_storage_format";
public static final String PROPERTIES_INMEMORY = "in_memory";
// _auto_bucket can only set in create table stmt rewrite bucket and can not be changed
@ -873,6 +876,35 @@ public class PropertyAnalyzer {
}
}
public static TInvertedIndexStorageFormat analyzeInvertedIndexStorageFormat(Map<String, String> properties)
throws AnalysisException {
String invertedIndexStorageFormat = "";
if (properties != null && properties.containsKey(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT)) {
invertedIndexStorageFormat = properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
} else {
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
return TInvertedIndexStorageFormat.V1;
} else {
return TInvertedIndexStorageFormat.V2;
}
}
if (invertedIndexStorageFormat.equalsIgnoreCase("v1")) {
return TInvertedIndexStorageFormat.V1;
} else if (invertedIndexStorageFormat.equalsIgnoreCase("v2")) {
return TInvertedIndexStorageFormat.V2;
} else if (invertedIndexStorageFormat.equalsIgnoreCase("default")) {
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
return TInvertedIndexStorageFormat.V1;
} else {
return TInvertedIndexStorageFormat.V2;
}
} else {
throw new AnalysisException("unknown inverted index storage format: " + invertedIndexStorageFormat);
}
}
// analyze common boolean properties, such as "in_memory" = "false"
public static boolean analyzeBooleanProp(Map<String, String> properties, String propKey, boolean defaultVal) {
if (properties != null && properties.containsKey(propKey)) {

11
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -160,6 +160,7 @@ import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.CreateReplicaTask;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@ -1832,7 +1833,6 @@ public class InternalCatalog implements CatalogIf<Database> {
BinlogConfig binlogConfig,
boolean isStorageMediumSpecified, List<Integer> clusterKeyIndexes)
throws DdlException {
// create base index first.
Preconditions.checkArgument(tbl.getBaseIndexId() != -1);
MaterializedIndex baseIndex = new MaterializedIndex(tbl.getBaseIndexId(), IndexState.NORMAL);
@ -1912,6 +1912,7 @@ public class InternalCatalog implements CatalogIf<Database> {
tbl.storeRowColumn(), binlogConfig);
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat());
task.setClusterKeyIndexes(clusterKeyIndexes);
batchTask.addTask(task);
// add to AgentTaskQueue for handling finish report.
@ -2232,6 +2233,14 @@ public class InternalCatalog implements CatalogIf<Database> {
}
olapTable.setStorageFormat(storageFormat);
TInvertedIndexStorageFormat invertedIndexStorageFormat;
try {
invertedIndexStorageFormat = PropertyAnalyzer.analyzeInvertedIndexStorageFormat(properties);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
olapTable.setInvertedIndexStorageFormat(invertedIndexStorageFormat);
// get compression type
TCompressionType compressionType = TCompressionType.LZ4;
try {

5
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -873,10 +873,13 @@ public class ShowExecutor {
}
if (showTableStmt.isVerbose()) {
String storageFormat = "NONE";
String invertedIndexStorageFormat = "NONE";
if (tbl instanceof OlapTable) {
storageFormat = ((OlapTable) tbl).getStorageFormat().toString();
invertedIndexStorageFormat = ((OlapTable) tbl).getInvertedIndexStorageFormat().toString();
}
rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(), storageFormat));
rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(), storageFormat,
invertedIndexStorageFormat));
} else {
rows.add(Lists.newArrayList(tbl.getName()));
}

10
fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.policy.PolicyTypeEnum;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TCreateTabletReq;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TOlapTableIndex;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageFormat;
@ -89,6 +90,8 @@ public class CreateReplicaTask extends AgentTask {
// TODO should unify the naming of v1(alpha rowset), v2(beta rowset), it is very confused to read code
private TStorageFormat storageFormat = TStorageFormat.V2;
private TInvertedIndexStorageFormat invertedIndexStorageFormat = TInvertedIndexStorageFormat.V1;
// true if this task is created by recover request(See comment of Config.recover_with_empty_tablet)
private boolean isRecoverTask = false;
@ -229,6 +232,10 @@ public class CreateReplicaTask extends AgentTask {
this.storageFormat = storageFormat;
}
public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat invertedIndexStorageFormat) {
this.invertedIndexStorageFormat = invertedIndexStorageFormat;
}
public void setClusterKeyIndexes(List<Integer> clusterKeyIndexes) {
this.clusterKeyIndexes = clusterKeyIndexes;
}
@ -325,6 +332,9 @@ public class CreateReplicaTask extends AgentTask {
createTabletReq.setStorageFormat(storageFormat);
}
if (invertedIndexStorageFormat != null) {
createTabletReq.setInvertedIndexStorageFormat(invertedIndexStorageFormat);
}
createTabletReq.setTabletType(tabletType);
createTabletReq.setCompressionType(compressionType);
createTabletReq.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite);