Add long text type STRING, with a maximum length of 2GB. Usage is similar to varchar, and there is no guarantee for the performance of storing extremely long data (#6391)

This commit is contained in:
Zhengguo Yang
2021-08-18 09:05:40 +08:00
committed by GitHub
parent 2f90aaab8e
commit 8738ce380b
106 changed files with 1353 additions and 433 deletions

View File

@ -10,7 +10,7 @@
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// "AS IS" BASIS, WITHOUT WARRANTIES ORF CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
@ -4311,7 +4311,7 @@ type ::=
| KW_BITMAP
{: RESULT = Type.BITMAP; :}
| KW_STRING
{: RESULT = ScalarType.createVarcharType(-1); :}
{: RESULT = ScalarType.createStringType(); :}
| KW_VARCHAR LPAREN INTEGER_LITERAL:len RPAREN
{: ScalarType type = ScalarType.createVarcharType(len.intValue());
type.setAssignedStrLenInColDefinition();

View File

@ -33,7 +33,7 @@ public enum AlterOpType {
// table property
MODIFY_TABLE_PROPERTY,
MODIFY_TABLE_PROPERTY_SYNC, // Some operations are performed synchronously, so we distinguish them by suffix _SYNC
// others operation, such as add/drop backend. currently we do not care about them
// others operation, such as add/drop backend. currently, we do not care about them
ALTER_OTHER,
ENABLE_FEATURE,
REPLACE_TABLE,

View File

@ -303,6 +303,11 @@ public class BinaryPredicate extends Predicate implements Writable {
if (t1 == PrimitiveType.VARCHAR && t2 == PrimitiveType.VARCHAR) {
return Type.VARCHAR;
}
if (t1 == PrimitiveType.STRING && t2 == PrimitiveType.STRING
|| t1 == PrimitiveType.STRING && t2 == PrimitiveType.VARCHAR
|| t1 == PrimitiveType.VARCHAR && t2 == PrimitiveType.STRING) {
return Type.STRING;
}
if (t1 == PrimitiveType.BIGINT && t2 == PrimitiveType.BIGINT) {
return Type.getAssignmentCompatibleType(getChild(0).getType(), getChild(1).getType(), false);
}
@ -322,14 +327,14 @@ public class BinaryPredicate extends Predicate implements Writable {
// When int column compares with string, Mysql will convert string to int.
// So it is also compatible with Mysql.
if (t1 == PrimitiveType.BIGINT && t2 == PrimitiveType.VARCHAR) {
if (t1 == PrimitiveType.BIGINT && (t2 == PrimitiveType.VARCHAR || t2 ==PrimitiveType.STRING)) {
Expr rightChild = getChild(1);
Long parsedLong = Type.tryParseToLong(rightChild);
if(parsedLong != null) {
return Type.BIGINT;
}
}
if (t1 == PrimitiveType.VARCHAR && t2 == PrimitiveType.BIGINT) {
if ((t1 == PrimitiveType.VARCHAR || t1 ==PrimitiveType.STRING) && t2 == PrimitiveType.BIGINT) {
Expr leftChild = getChild(0);
Long parsedLong = Type.tryParseToLong(leftChild);
if(parsedLong != null) {

View File

@ -279,6 +279,7 @@ public class ColumnDef {
case CHAR:
case VARCHAR:
case HLL:
case STRING:
if (defaultValue.length() > scalarType.getLength()) {
throw new AnalysisException("Default value is too long: " + defaultValue);
}

View File

@ -232,9 +232,10 @@ public class OutFileClause {
break;
case CHAR:
case VARCHAR:
case STRING:
case DECIMALV2:
if (!type.equals("byte_array")) {
throw new AnalysisException("project field type is CHAR/VARCHAR/DECIMAL, should use byte_array, " +
throw new AnalysisException("project field type is CHAR/VARCHAR/STRING/DECIMAL, should use byte_array, " +
"but the definition type of column " + i + " is " + type);
}
break;
@ -272,6 +273,7 @@ public class OutFileClause {
break;
case CHAR:
case VARCHAR:
case STRING:
case DECIMALV2:
column.add("byte_array");
break;

View File

@ -131,10 +131,10 @@ public class TypeDef implements ParseNode {
String name;
int maxLen;
if (type == PrimitiveType.VARCHAR) {
name = "Varchar";
name = "VARCHAR";
maxLen = ScalarType.MAX_VARCHAR_LENGTH;
} else if (type == PrimitiveType.CHAR) {
name = "Char";
name = "CHAR";
maxLen = ScalarType.MAX_CHAR_LENGTH;
} else {
Preconditions.checkState(false);

View File

@ -49,6 +49,7 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.FLOAT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.INT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
@ -56,6 +57,7 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.FLOAT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
@ -63,23 +65,29 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DATE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.FLOAT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.FLOAT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.DOUBLE.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.DOUBLE.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.CHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.TINYINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.SMALLINT.ordinal()] = true;
@ -89,6 +97,7 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.FLOAT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.DATE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.TINYINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.SMALLINT.ordinal()] = true;
@ -98,6 +107,7 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.FLOAT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.DATE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.DECIMALV2.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

View File

@ -491,6 +491,7 @@ public class Function implements Writable {
case CHAR:
case HLL:
case BITMAP:
case STRING:
return "string_val";
case DATE:
case DATETIME:
@ -528,6 +529,7 @@ public class Function implements Writable {
case CHAR:
case HLL:
case BITMAP:
case STRING:
return "StringVal";
case DATE:
case DATETIME:

View File

@ -57,6 +57,7 @@ public enum PrimitiveType {
ARRAY("ARRAY", 24, TPrimitiveType.ARRAY),
MAP("MAP", 24, TPrimitiveType.MAP),
STRUCT("MAP", 24, TPrimitiveType.STRUCT),
STRING("STRING", 16, TPrimitiveType.STRING),
// Unsupported scalar types.
BINARY("BINARY", -1, TPrimitiveType.BINARY);
@ -64,6 +65,7 @@ public enum PrimitiveType {
private static final int DATE_INDEX_LEN = 3;
private static final int DATETIME_INDEX_LEN = 8;
private static final int VARCHAR_INDEX_LEN = 20;
private static final int STRING_INDEX_LEN = 20;
private static final int DECIMAL_INDEX_LEN = 12;
private static ImmutableSetMultimap<PrimitiveType, PrimitiveType> implicitCastMap;
@ -83,6 +85,7 @@ public enum PrimitiveType {
builder.put(NULL_TYPE, DECIMALV2);
builder.put(NULL_TYPE, CHAR);
builder.put(NULL_TYPE, VARCHAR);
builder.put(NULL_TYPE, STRING);
builder.put(NULL_TYPE, BITMAP);
builder.put(NULL_TYPE, TIME);
// Boolean
@ -98,6 +101,7 @@ public enum PrimitiveType {
builder.put(BOOLEAN, DATETIME);
builder.put(BOOLEAN, DECIMALV2);
builder.put(BOOLEAN, VARCHAR);
builder.put(BOOLEAN, STRING);
// Tinyint
builder.put(TINYINT, BOOLEAN);
builder.put(TINYINT, TINYINT);
@ -111,6 +115,7 @@ public enum PrimitiveType {
builder.put(TINYINT, DATETIME);
builder.put(TINYINT, DECIMALV2);
builder.put(TINYINT, VARCHAR);
builder.put(TINYINT, STRING);
// Smallint
builder.put(SMALLINT, BOOLEAN);
builder.put(SMALLINT, TINYINT);
@ -124,6 +129,7 @@ public enum PrimitiveType {
builder.put(SMALLINT, DATETIME);
builder.put(SMALLINT, DECIMALV2);
builder.put(SMALLINT, VARCHAR);
builder.put(SMALLINT, STRING);
// Int
builder.put(INT, BOOLEAN);
builder.put(INT, TINYINT);
@ -137,6 +143,7 @@ public enum PrimitiveType {
builder.put(INT, DATETIME);
builder.put(INT, DECIMALV2);
builder.put(INT, VARCHAR);
builder.put(INT, STRING);
// Bigint
builder.put(BIGINT, BOOLEAN);
builder.put(BIGINT, TINYINT);
@ -150,6 +157,7 @@ public enum PrimitiveType {
builder.put(BIGINT, DATETIME);
builder.put(BIGINT, DECIMALV2);
builder.put(BIGINT, VARCHAR);
builder.put(BIGINT, STRING);
// Largeint
builder.put(LARGEINT, BOOLEAN);
builder.put(LARGEINT, TINYINT);
@ -163,6 +171,7 @@ public enum PrimitiveType {
builder.put(LARGEINT, DATETIME);
builder.put(LARGEINT, DECIMALV2);
builder.put(LARGEINT, VARCHAR);
builder.put(LARGEINT, STRING);
// Float
builder.put(FLOAT, BOOLEAN);
builder.put(FLOAT, TINYINT);
@ -176,6 +185,7 @@ public enum PrimitiveType {
builder.put(FLOAT, DATETIME);
builder.put(FLOAT, DECIMALV2);
builder.put(FLOAT, VARCHAR);
builder.put(FLOAT, STRING);
// Double
builder.put(DOUBLE, BOOLEAN);
builder.put(DOUBLE, TINYINT);
@ -189,6 +199,7 @@ public enum PrimitiveType {
builder.put(DOUBLE, DATETIME);
builder.put(DOUBLE, DECIMALV2);
builder.put(DOUBLE, VARCHAR);
builder.put(DOUBLE, STRING);
// Date
builder.put(DATE, BOOLEAN);
builder.put(DATE, TINYINT);
@ -202,6 +213,7 @@ public enum PrimitiveType {
builder.put(DATE, DATETIME);
builder.put(DATE, DECIMALV2);
builder.put(DATE, VARCHAR);
builder.put(DATE, STRING);
// Datetime
builder.put(DATETIME, BOOLEAN);
builder.put(DATETIME, TINYINT);
@ -215,9 +227,11 @@ public enum PrimitiveType {
builder.put(DATETIME, DATETIME);
builder.put(DATETIME, DECIMALV2);
builder.put(DATETIME, VARCHAR);
builder.put(DATETIME, STRING);
// Char
builder.put(CHAR, CHAR);
builder.put(CHAR, VARCHAR);
builder.put(CHAR, STRING);
// Varchar
builder.put(VARCHAR, BOOLEAN);
builder.put(VARCHAR, TINYINT);
@ -231,9 +245,27 @@ public enum PrimitiveType {
builder.put(VARCHAR, DATETIME);
builder.put(VARCHAR, DECIMALV2);
builder.put(VARCHAR, VARCHAR);
builder.put(VARCHAR, STRING);
builder.put(VARCHAR, HLL);
builder.put(VARCHAR, BITMAP);
// Varchar
builder.put(STRING, BOOLEAN);
builder.put(STRING, TINYINT);
builder.put(STRING, SMALLINT);
builder.put(STRING, INT);
builder.put(STRING, BIGINT);
builder.put(STRING, LARGEINT);
builder.put(STRING, FLOAT);
builder.put(STRING, DOUBLE);
builder.put(STRING, DATE);
builder.put(STRING, DATETIME);
builder.put(STRING, DECIMALV2);
builder.put(STRING, VARCHAR);
builder.put(STRING, STRING);
builder.put(STRING, HLL);
builder.put(STRING, BITMAP);
// DecimalV2
builder.put(DECIMALV2, BOOLEAN);
builder.put(DECIMALV2, TINYINT);
@ -245,14 +277,17 @@ public enum PrimitiveType {
builder.put(DECIMALV2, DOUBLE);
builder.put(DECIMALV2, DECIMALV2);
builder.put(DECIMALV2, VARCHAR);
builder.put(DECIMALV2, STRING);
// HLL
builder.put(HLL, HLL);
builder.put(HLL, VARCHAR);
builder.put(HLL, STRING);
// BITMAP
builder.put(BITMAP, BITMAP);
builder.put(BITMAP, VARCHAR);
builder.put(BITMAP, STRING);
//TIME
builder.put(TIME, TIME);
@ -294,6 +329,7 @@ public enum PrimitiveType {
supportedTypes.add(FLOAT);
supportedTypes.add(DOUBLE);
supportedTypes.add(VARCHAR);
supportedTypes.add(STRING);
supportedTypes.add(HLL);
supportedTypes.add(CHAR);
supportedTypes.add(DATE);
@ -336,7 +372,7 @@ public enum PrimitiveType {
private static PrimitiveType[][] compatibilityMatrix;
static {
compatibilityMatrix = new PrimitiveType[BINARY.ordinal() + 1][BINARY.ordinal() + 1];
compatibilityMatrix = new PrimitiveType[PrimitiveType.values().length][PrimitiveType.values().length];
// NULL_TYPE is compatible with any type and results in the non-null type.
compatibilityMatrix[NULL_TYPE.ordinal()][NULL_TYPE.ordinal()] = NULL_TYPE;
@ -352,6 +388,7 @@ public enum PrimitiveType {
compatibilityMatrix[NULL_TYPE.ordinal()][DATETIME.ordinal()] = DATETIME;
compatibilityMatrix[NULL_TYPE.ordinal()][CHAR.ordinal()] = CHAR;
compatibilityMatrix[NULL_TYPE.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[NULL_TYPE.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[NULL_TYPE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[NULL_TYPE.ordinal()][TIME.ordinal()] = TIME;
compatibilityMatrix[NULL_TYPE.ordinal()][BITMAP.ordinal()] = BITMAP;
@ -368,6 +405,7 @@ public enum PrimitiveType {
compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[BOOLEAN.ordinal()][TIME.ordinal()] = TIME;
@ -382,6 +420,7 @@ public enum PrimitiveType {
compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[TINYINT.ordinal()][TIME.ordinal()] = TIME;
@ -395,6 +434,7 @@ public enum PrimitiveType {
compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[SMALLINT.ordinal()][TIME.ordinal()] = TIME;
@ -407,6 +447,7 @@ public enum PrimitiveType {
compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[INT.ordinal()][TIME.ordinal()] = TIME;
@ -418,6 +459,7 @@ public enum PrimitiveType {
compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[BIGINT.ordinal()][TIME.ordinal()] = TIME;
@ -428,6 +470,7 @@ public enum PrimitiveType {
compatibilityMatrix[LARGEINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[LARGEINT.ordinal()][TIME.ordinal()] = TIME;
@ -437,6 +480,7 @@ public enum PrimitiveType {
compatibilityMatrix[FLOAT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[FLOAT.ordinal()][TIME.ordinal()] = TIME;
@ -445,6 +489,7 @@ public enum PrimitiveType {
compatibilityMatrix[DOUBLE.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[DOUBLE.ordinal()][TIME.ordinal()] = TIME;
@ -452,24 +497,32 @@ public enum PrimitiveType {
compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = DATETIME;
compatibilityMatrix[DATE.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][TIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][DATETIME.ordinal()] = DATETIME;
compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][TIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][CHAR.ordinal()] = CHAR;
compatibilityMatrix[CHAR.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[CHAR.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][TIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[VARCHAR.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][TIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[STRING.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][TIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[DECIMALV2.ordinal()][TIME.ordinal()] = INVALID_TYPE;
@ -514,8 +567,6 @@ public enum PrimitiveType {
public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) {
switch (tPrimitiveType) {
case INVALID_TYPE:
return INVALID_TYPE;
case NULL_TYPE:
return NULL_TYPE;
case BOOLEAN:
@ -546,6 +597,8 @@ public enum PrimitiveType {
return TIME;
case VARCHAR:
return VARCHAR;
case STRING:
return STRING;
case CHAR:
return CHAR;
case HLL:
@ -651,11 +704,11 @@ public enum PrimitiveType {
}
public boolean isStringType() {
return (this == VARCHAR || this == CHAR || this == HLL);
return (this == VARCHAR || this == CHAR || this == HLL || this == STRING);
}
public boolean isCharFamily() {
return (this == VARCHAR || this == CHAR);
return (this == VARCHAR || this == CHAR || this == STRING);
}
public boolean isIntegerType() {
@ -709,6 +762,8 @@ public enum PrimitiveType {
case CHAR:
// char index size is length
return -1;
case STRING:
return STRING_INDEX_LEN;
case DECIMALV2:
return DECIMAL_INDEX_LEN;
default:

View File

@ -161,6 +161,7 @@ public class ScalarFunction extends Function {
case VARCHAR:
case HLL:
case BITMAP:
case STRING:
beFn += "_string_val";
break;
case DATE:

View File

@ -51,6 +51,10 @@ public class ScalarType extends Type {
// Longest supported VARCHAR and CHAR, chosen to match Hive.
public static final int MAX_VARCHAR_LENGTH = 65533;
// 2GB - 4 4bytes for storage string length
public static final int MAX_STRING_LENGTH = 2147483643;
public static final int MAX_CHAR_LENGTH = 255;
// HLL DEFAULT LENGTH 2^14(registers) + 1(type)
@ -62,7 +66,6 @@ public class ScalarType extends Type {
// Hive, mysql, sql server standard.
public static final int MAX_PRECISION = 38;
public static final int MAX_SCALE = MAX_PRECISION;
@SerializedName(value = "type")
private final PrimitiveType type;
@ -92,6 +95,8 @@ public class ScalarType extends Type {
return createCharType(len);
case VARCHAR:
return createVarcharType(len);
case STRING:
return createStringType();
case DECIMALV2:
return createDecimalV2Type(precision, scale);
default:
@ -123,6 +128,8 @@ public class ScalarType extends Type {
return CHAR;
case VARCHAR:
return createVarcharType();
case STRING:
return createStringType();
case HLL:
return createHllType();
case BITMAP:
@ -168,6 +175,8 @@ public class ScalarType extends Type {
return CHAR;
case "VARCHAR":
return createVarcharType();
case "STRING":
return createStringType();
case "HLL":
return createHllType();
case "BITMAP":
@ -233,6 +242,13 @@ public class ScalarType extends Type {
return type;
}
public static ScalarType createStringType() {
// length checked in analysis
ScalarType type = new ScalarType(PrimitiveType.STRING);
type.len = -1;
return type;
}
public static ScalarType createVarchar(int len) {
// length checked in analysis
ScalarType type = new ScalarType(PrimitiveType.VARCHAR);
@ -267,6 +283,8 @@ public class ScalarType extends Type {
return "VARCHAR(*)";
}
return "VARCHAR(" + len + ")";
} else if (type == PrimitiveType.STRING) {
return "STRING";
}
return type.toString();
}
@ -301,6 +319,7 @@ public class ScalarType extends Type {
case DATE:
case DATETIME:
case HLL:
case STRING:
case BITMAP:
stringBuilder.append(type.toString().toLowerCase());
break;
@ -323,42 +342,27 @@ public class ScalarType extends Type {
public void toThrift(TTypeDesc container) {
TTypeNode node = new TTypeNode();
container.types.add(node);
node.setType(TTypeNodeType.SCALAR);
TScalarType scalarType = new TScalarType();
scalarType.setType(type.toThrift());
switch(type) {
case VARCHAR:
case CHAR:
case HLL: {
node.setType(TTypeNodeType.SCALAR);
TScalarType scalarType = new TScalarType();
scalarType.setType(type.toThrift());
case HLL:
case STRING: {
scalarType.setLen(len);
node.setScalarType(scalarType);
break;
}
case DECIMALV2: {
node.setType(TTypeNodeType.SCALAR);
TScalarType scalarType = new TScalarType();
scalarType.setType(type.toThrift());
scalarType.setScale(scale);
scalarType.setPrecision(precision);
node.setScalarType(scalarType);
break;
}
default: {
node.setType(TTypeNodeType.SCALAR);
TScalarType scalarType = new TScalarType();
scalarType.setType(type.toThrift());
node.setScalarType(scalarType);
default:
break;
}
}
}
public static Type[] toColumnType(PrimitiveType[] types) {
Type result[] = new Type[types.length];
for (int i = 0; i < types.length; ++i) {
result[i] = createType(types[i]);
}
return result;
node.setScalarType(scalarType);
}
public int decimalPrecision() {
@ -599,6 +603,9 @@ public class ScalarType extends Type {
}
if (t1.isStringType() || t2.isStringType()) {
if (t1.type == PrimitiveType.STRING || t2.type == PrimitiveType.STRING) {
return createStringType();
}
return createVarcharType(Math.max(t1.len, t2.len));
}
@ -668,6 +675,8 @@ public class ScalarType extends Type {
return 16385;
case BITMAP:
return 1024; // this is a estimated value
case STRING:
return 1024;
default:
return 0;
}

View File

@ -38,7 +38,6 @@ import java.util.Map;
public class SchemaTable extends Table {
private final static int FN_REFLEN = 512;
private final static int NAME_CHAR_LEN = 64;
private final static int MAX_FIELD_VARCHARLENGTH = 65535;
private final static int MY_CS_NAME_SIZE = 32;
private final static int GRANTEE_len = 81;
private final static int PRIVILEGE_TYPE_LEN = 64;
@ -49,11 +48,6 @@ public class SchemaTable extends Table {
super(id, name, type, baseSchema);
}
protected SchemaTable(long id, String name, SchemaTableType type) {
super(TableType.SCHEMA);
schemaTableType = type;
}
@Override
public void write(DataOutput out) throws IOException {
throw new UnsupportedOperationException("Do not allow to write SchemaTable to image.");

View File

@ -65,6 +65,7 @@ public abstract class Type {
public static final ScalarType DATE = new ScalarType(PrimitiveType.DATE);
public static final ScalarType DATETIME = new ScalarType(PrimitiveType.DATETIME);
public static final ScalarType TIME = new ScalarType(PrimitiveType.TIME);
public static final ScalarType STRING = new ScalarType(PrimitiveType.STRING);
public static final ScalarType DEFAULT_DECIMALV2 = (ScalarType)
ScalarType.createDecimalV2Type(ScalarType.DEFAULT_PRECISION,
ScalarType.DEFAULT_SCALE);
@ -117,6 +118,8 @@ public abstract class Type {
supportedTypes.add(DATETIME);
supportedTypes.add(DECIMALV2);
supportedTypes.add(TIME);
supportedTypes.add(STRING);
}
public static ArrayList<ScalarType> getIntegerTypes() {
@ -177,7 +180,9 @@ public abstract class Type {
public boolean isWildcardChar() { return false; }
public boolean isStringType() {
return isScalarType(PrimitiveType.VARCHAR) || isScalarType(PrimitiveType.CHAR);
return isScalarType(PrimitiveType.VARCHAR)
|| isScalarType(PrimitiveType.CHAR)
|| isScalarType(PrimitiveType.STRING);
}
// only metric types have the following constraint:
@ -487,6 +492,8 @@ public abstract class Type {
return Type.CHAR;
case VARCHAR:
return Type.VARCHAR;
case STRING:
return Type.STRING;
case HLL:
return Type.HLL;
case ARRAY:
@ -617,6 +624,7 @@ public abstract class Type {
switch (t.getPrimitiveType()) {
case CHAR:
case VARCHAR:
case STRING:
case HLL:
return t.getLength();
default:
@ -759,6 +767,8 @@ public abstract class Type {
compatibilityMatrix[BOOLEAN.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[BOOLEAN.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// TINYINT
compatibilityMatrix[TINYINT.ordinal()][SMALLINT.ordinal()] = PrimitiveType.SMALLINT;
@ -776,6 +786,7 @@ public abstract class Type {
compatibilityMatrix[TINYINT.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[TINYINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// SMALLINT
compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
@ -792,6 +803,7 @@ public abstract class Type {
compatibilityMatrix[SMALLINT.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[SMALLINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// INT
compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
@ -811,6 +823,7 @@ public abstract class Type {
compatibilityMatrix[INT.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[INT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// BIGINT
// 64 bit integer does not fit in mantissa of double or float.
@ -831,6 +844,7 @@ public abstract class Type {
compatibilityMatrix[BIGINT.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[BIGINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// LARGEINT
compatibilityMatrix[LARGEINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.DOUBLE;
@ -843,6 +857,7 @@ public abstract class Type {
compatibilityMatrix[LARGEINT.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[LARGEINT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// FLOAT
compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
@ -854,6 +869,7 @@ public abstract class Type {
compatibilityMatrix[FLOAT.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[FLOAT.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// DOUBLE
compatibilityMatrix[DOUBLE.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE;
@ -864,6 +880,7 @@ public abstract class Type {
compatibilityMatrix[DOUBLE.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][TIME.ordinal()] = PrimitiveType.DOUBLE;
compatibilityMatrix[DOUBLE.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// DATE
compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIME;
@ -873,6 +890,7 @@ public abstract class Type {
compatibilityMatrix[DATE.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// DATETIME
compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE;
@ -881,7 +899,8 @@ public abstract class Type {
compatibilityMatrix[DATETIME.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// We can convert some but not all string values to timestamps.
// CHAR
compatibilityMatrix[CHAR.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR;
@ -889,24 +908,40 @@ public abstract class Type {
compatibilityMatrix[CHAR.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING;
// VARCHAR
compatibilityMatrix[VARCHAR.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING;
//String
compatibilityMatrix[STRING.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
// DECIMALV2
compatibilityMatrix[DECIMALV2.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// HLL
compatibilityMatrix[HLL.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// BITMAP
compatibilityMatrix[BITMAP.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE;
// TIME
compatibilityMatrix[TIME.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE;
// Check all of the necessary entries that should be filled.
// ignore binary
@ -929,7 +964,6 @@ public abstract class Type {
}
}
public Type getResultType() {
switch (this.getPrimitiveType()) {
case BOOLEAN:
@ -953,6 +987,8 @@ public abstract class Type {
return VARCHAR;
case DECIMALV2:
return DECIMALV2;
case STRING:
return STRING;
default:
return INVALID;
@ -1033,6 +1069,7 @@ public abstract class Type {
case TIME:
case CHAR:
case VARCHAR:
case STRING:
case HLL:
return Type.DOUBLE;
case DECIMALV2:

View File

@ -68,6 +68,7 @@ public class Util {
TYPE_STRING_MAP.put(PrimitiveType.DATETIME, "datetime");
TYPE_STRING_MAP.put(PrimitiveType.CHAR, "char(%d)");
TYPE_STRING_MAP.put(PrimitiveType.VARCHAR, "varchar(%d)");
TYPE_STRING_MAP.put(PrimitiveType.STRING, "string");
TYPE_STRING_MAP.put(PrimitiveType.DECIMALV2, "decimal(%d,%d)");
TYPE_STRING_MAP.put(PrimitiveType.HLL, "varchar(%d)");
TYPE_STRING_MAP.put(PrimitiveType.BOOLEAN, "bool");

View File

@ -86,6 +86,7 @@ public class HttpServer extends SpringBootServletInitializer {
// To avoid some unexpected behavior.
System.setProperty("spring.devtools.restart.enabled", "false");
System.setProperty("spring.http.multipart.location", PaloFe.DORIS_HOME_DIR);
System.setProperty("spring.banner.image.location", "doris-logo.png");
properties.put("logging.config", Config.custom_config_dir + "/" + SpringLog4j2Config.SPRING_LOG_XML_FILE);
new SpringApplicationBuilder()
.sources(HttpServer.class)

View File

@ -524,7 +524,7 @@ public class OlapScanNode extends ScanNode {
scanBackendIds.add(backend.getId());
}
if (tabletIsNull) {
throw new UserException(tabletId + "have no alive replicas");
throw new UserException("tablet: " + tabletId + " have no alive replicas.");
}
TScanRange scanRange = new TScanRange();
scanRange.setPaloScanRange(paloRange);

View File

@ -174,6 +174,7 @@ public class PartitionRange {
case DECIMALV2:
case CHAR:
case VARCHAR:
case STRING:
case LARGEINT:
LOG.info("PartitionCache not support such key type {}", type.toSql());
return false;

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 KiB