database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[FIX] Pick array inverted index bugfix (#35837)

Browse Source 
here with some array with inverted index bugfix:
see also: 
https://github.com/apache/doris/pull/34766
https://github.com/apache/doris/pull/35086
https://github.com/apache/doris/pull/34683
https://github.com/apache/doris/pull/34076
This commit is contained in:
amory
2024-06-06 09:54:14 +08:00
committed by GitHub
parent efe17245b0
commit b5a35b9cef
28 changed files with 5936 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@ -237,6 +238,9 @@ public class IndexDef {
}
if (indexType == IndexType.INVERTED) {
if (!Config.enable_create_inverted_index_for_array && colType.isArrayType()) {
throw new AnalysisException("inverted index does not support array type column:" + indexColName);
}
InvertedIndexUtil.checkInvertedIndexParser(indexColName, colType, properties);
} else if (indexType == IndexType.NGRAM_BF) {
if (colType != PrimitiveType.CHAR && colType != PrimitiveType.VARCHAR

61
fe/fe-core/src/main/java/org/apache/doris/analysis/MatchPredicate.java
View File

@ -49,13 +49,7 @@ public class MatchPredicate extends Predicate {
MATCH_PHRASE("MATCH_PHRASE", "match_phrase", TExprOpcode.MATCH_PHRASE),
MATCH_PHRASE_PREFIX("MATCH_PHRASE_PREFIX", "match_phrase_prefix", TExprOpcode.MATCH_PHRASE_PREFIX),
MATCH_REGEXP("MATCH_REGEXP", "match_regexp", TExprOpcode.MATCH_REGEXP),
MATCH_PHRASE_EDGE("MATCH_PHRASE_EDGE", "match_phrase_edge", TExprOpcode.MATCH_PHRASE_EDGE),
MATCH_ELEMENT_EQ("MATCH_ELEMENT_EQ", "match_element_eq", TExprOpcode.MATCH_ELEMENT_EQ),
MATCH_ELEMENT_LT("MATCH_ELEMENT_LT", "match_element_lt", TExprOpcode.MATCH_ELEMENT_LT),
MATCH_ELEMENT_GT("MATCH_ELEMENT_GT", "match_element_gt", TExprOpcode.MATCH_ELEMENT_GT),
MATCH_ELEMENT_LE("MATCH_ELEMENT_LE", "match_element_le", TExprOpcode.MATCH_ELEMENT_LE),
MATCH_ELEMENT_GE("MATCH_ELEMENT_GE", "match_element_ge", TExprOpcode.MATCH_ELEMENT_GE);
MATCH_PHRASE_EDGE("MATCH_PHRASE_EDGE", "match_phrase_edge", TExprOpcode.MATCH_PHRASE_EDGE);
private final String description;
private final String name;
@ -86,34 +80,6 @@ public class MatchPredicate extends Predicate {
public static void initBuiltins(FunctionSet functionSet) {
String symbolNotUsed = "symbol_not_used";
for (Type t : Type.getNumericDateTimeTypes()) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(
Operator.MATCH_ELEMENT_EQ.getName(),
symbolNotUsed,
Lists.<Type>newArrayList(new ArrayType(t), t),
Type.BOOLEAN));
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(
Operator.MATCH_ELEMENT_LT.getName(),
symbolNotUsed,
Lists.<Type>newArrayList(new ArrayType(t), t),
Type.BOOLEAN));
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(
Operator.MATCH_ELEMENT_GT.getName(),
symbolNotUsed,
Lists.<Type>newArrayList(new ArrayType(t), t),
Type.BOOLEAN));
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(
Operator.MATCH_ELEMENT_LE.getName(),
symbolNotUsed,
Lists.<Type>newArrayList(new ArrayType(t), t),
Type.BOOLEAN));
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(
Operator.MATCH_ELEMENT_GE.getName(),
symbolNotUsed,
Lists.<Type>newArrayList(new ArrayType(t), t),
Type.BOOLEAN));
}
for (Type t : Type.getStringTypes()) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(
Operator.MATCH_ANY.getName(),
@ -198,14 +164,6 @@ public class MatchPredicate extends Predicate {
invertedIndexParserMode = InvertedIndexUtil.INVERTED_INDEX_PARSER_FINE_GRANULARITY;
}
public Boolean isMatchElement(Operator op) {
return Objects.equals(op.getName(), Operator.MATCH_ELEMENT_EQ.getName())
|| Objects.equals(op.getName(), Operator.MATCH_ELEMENT_LT.getName())
|| Objects.equals(op.getName(), Operator.MATCH_ELEMENT_GT.getName())
|| Objects.equals(op.getName(), Operator.MATCH_ELEMENT_LE.getName())
|| Objects.equals(op.getName(), Operator.MATCH_ELEMENT_GE.getName());
}
protected MatchPredicate(MatchPredicate other) {
super(other);
op = other.op;
@ -267,17 +225,10 @@ public class MatchPredicate extends Predicate {
@Override
public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
super.analyzeImpl(analyzer);
if (isMatchElement(op) && !getChild(0).getType().isArrayType()) {
throw new AnalysisException(
"left operand of " + op.toString() + " must be Array: " + toSql());
}
if (getChild(0).getType().isObjectStored()) {
throw new AnalysisException(
"left operand of " + op.toString() + " must not be Bitmap or HLL: " + toSql());
}
if (!isMatchElement(op) && !getChild(1).getType().isStringType() && !getChild(1).getType().isNull()) {
throw new AnalysisException("right operand of " + op.toString() + " must be of type STRING: " + toSql());
}
if (!getChild(0).getType().isStringType() && !getChild(0).getType().isArrayType()
&& !getChild(0).getType().isVariantType()) {
@ -293,16 +244,6 @@ public class MatchPredicate extends Predicate {
}
Expr e1 = getChild(0);
Expr e2 = getChild(1);
// Here we cast match_element_xxx value type from string to array item type.
// Because be need to know the actual TExprNodeType when doing Expr Literal transform
if (isMatchElement(op) && e1.type.isArrayType()) {
Type itemType = ((ArrayType) e1.type).getItemType();
try {
setChild(1, e2.castTo(itemType));
} catch (NumberFormatException nfe) {
throw new AnalysisException("Invalid number format literal: " + e2.getStringValue());
}
}
// CAST variant to right expr type
if (e1.type.isVariantType()) {

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
@ -123,6 +124,9 @@ public class IndexDefinition {
}
if (indexType == IndexType.INVERTED) {
if (!Config.enable_create_inverted_index_for_array && colType.isArrayType()) {
throw new AnalysisException("inverted index does not support array type column: " + indexColName);
}
try {
InvertedIndexUtil.checkInvertedIndexParser(indexColName,
colType.toCatalogDataType().getPrimitiveType(), properties);