diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java index cd48b0afd9..0b7e9ab22b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java @@ -60,6 +60,10 @@ public enum MysqlColType { private static final Map CODE_MAP = new HashMap<>(); + public static final int MYSQL_CODE_MASK = 0xFF; + + public static final int UNSIGNED_MASK = 0x8000; + static { for (MysqlColType type : MysqlColType.values()) { CODE_MAP.put(type.code, type); @@ -89,7 +93,12 @@ public enum MysqlColType { } public static MysqlColType fromCode(int code) { - return CODE_MAP.get(code); + // Use the lower 8 bits of the code. + return CODE_MAP.get(code & MYSQL_CODE_MASK); + } + + public static boolean isUnsigned(int code) { + return (code & MysqlColType.UNSIGNED_MASK) != 0; } public String getJdbcColumnTypeName() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 8920e61d23..cbc34e8791 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -1807,7 +1807,7 @@ public class DateLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { int len = getParmLen(data); if (type.getPrimitiveType() == PrimitiveType.DATE) { if (len >= 4) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java index 278d1d80d5..888fb870ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java @@ -409,7 +409,7 @@ public class DecimalLiteral extends NumericLiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { int len = getParmLen(data); BigDecimal v = null; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index ef0493d8e0..bddc8e103c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -271,7 +271,7 @@ public class FloatLiteral extends NumericLiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { if (type.getPrimitiveType() == PrimitiveType.FLOAT) { value = data.getFloat(); return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java index c3a76443a4..b252695752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; +import org.apache.doris.common.util.ByteBufferUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -379,19 +380,19 @@ public class IntLiteral extends NumericLiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { switch (type.getPrimitiveType()) { case TINYINT: value = data.get(); break; case SMALLINT: - value = data.getChar(); + value = !isUnsigned ? data.getChar() : ByteBufferUtil.getUnsignedByte(data); break; case INT: - value = data.getInt(); + value = !isUnsigned ? data.getInt() : ByteBufferUtil.getUnsignedShort(data); break; case BIGINT: - value = data.getLong(); + value = !isUnsigned ? data.getLong() : ByteBufferUtil.getUnsignedInt(data); break; default: Preconditions.checkState(false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 613e96e0fb..2ee87a6323 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -20,6 +20,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; @@ -343,54 +344,57 @@ public abstract class LiteralExpr extends Expr implements Comparable 0); - return LiteralExpr.getLiteralByMysqlType(mysqlTypeCode); + return LiteralExpr.getLiteralByMysqlType(mysqlTypeCode, isUnsigned()); } public static PlaceHolderExpr create(String value, Type type) throws AnalysisException { @@ -87,6 +88,10 @@ public class PlaceHolderExpr extends LiteralExpr { return lExpr.isMinValue(); } + public boolean isUnsigned() { + return MysqlColType.isUnsigned(mysqlTypeCode); + } + @Override public int compareLiteral(LiteralExpr expr) { return lExpr.compareLiteral(expr); @@ -130,6 +135,11 @@ public class PlaceHolderExpr extends LiteralExpr { return "?"; } + @Override + protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { + return this.lExpr.uncheckedCastTo(targetType); + } + // Swaps the sign of numeric literals. // Throws for non-numeric literals. public void swapSign() throws NotImplementedException { @@ -181,7 +191,7 @@ public class PlaceHolderExpr extends LiteralExpr { return "\"" + getStringValue() + "\""; } - public void setupParamFromBinary(ByteBuffer data) { - lExpr.setupParamFromBinary(data); + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { + lExpr.setupParamFromBinary(data, isUnsigned); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 40051fa911..36e980ba4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -339,7 +339,7 @@ public class StringLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { int strLen = getParmLen(data); if (strLen > data.remaining()) { strLen = data.remaining(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java new file mode 100644 index 0000000000..4ec8f01149 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import java.nio.ByteBuffer; + +public class ByteBufferUtil { + public static short getUnsignedByte(ByteBuffer buffer) { + return (short) (buffer.get() & 0xFF); + } + + public static int getUnsignedShort(ByteBuffer buffer) { + return buffer.getShort() & 0xFFFF; + } + + public static long getUnsignedInt(ByteBuffer buffer) { + return buffer.getInt() & 0xFFFFFFFFL; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java index 9ae054d82a..0432beadd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java @@ -33,15 +33,17 @@ import java.util.Optional; public class Placeholder extends Expression implements LeafExpression { private final PlaceholderId placeholderId; private final Optional mysqlColType; + private int mysqlTypeCode = -1; public Placeholder(PlaceholderId placeholderId) { this.placeholderId = placeholderId; this.mysqlColType = Optional.empty(); } - public Placeholder(PlaceholderId placeholderId, MysqlColType mysqlColType) { + public Placeholder(PlaceholderId placeholderId, int mysqlTypeCode) { this.placeholderId = placeholderId; - this.mysqlColType = Optional.of(mysqlColType); + this.mysqlColType = Optional.of(MysqlColType.fromCode(mysqlTypeCode)); + this.mysqlTypeCode = mysqlTypeCode; } public PlaceholderId getPlaceholderId() { @@ -68,8 +70,12 @@ public class Placeholder extends Expression implements LeafExpression { return NullType.INSTANCE; } - public Placeholder withNewMysqlColType(MysqlColType mysqlColType) { - return new Placeholder(getPlaceholderId(), mysqlColType); + public Placeholder withNewMysqlColType(int mysqlTypeCode) { + return new Placeholder(getPlaceholderId(), mysqlTypeCode); + } + + public boolean isUnsigned() { + return MysqlColType.isUnsigned(mysqlTypeCode); } public MysqlColType getMysqlColType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index d81206f710..4619e35d5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; +import org.apache.doris.common.util.ByteBufferUtil; import org.apache.doris.mysql.MysqlProto; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.UnboundException; @@ -448,42 +449,68 @@ public abstract class Literal extends Expression implements LeafExpression, Comp * Retrieves a Literal object based on the MySQL type and the data provided. * * @param mysqlType the MySQL type identifier + * @param isUnsigned true if it is an unsigned type * @param data the ByteBuffer containing the data * @return a Literal object corresponding to the MySQL type * @throws AnalysisException if the MySQL type is unsupported or if data conversion fails * @link .... */ - public static Literal getLiteralByMysqlType(MysqlColType mysqlType, ByteBuffer data) throws AnalysisException { + public static Literal getLiteralByMysqlType(MysqlColType mysqlType, boolean isUnsigned, ByteBuffer data) + throws AnalysisException { + Literal literal = null; + // If this is an unsigned numeric type, we convert it by using larger data types. For example, we can use + // small int to represent unsigned tiny int (0-255), big int to represent unsigned ints (0-2 ^ 32-1), + // and so on. switch (mysqlType) { case MYSQL_TYPE_TINY: - return new TinyIntLiteral(data.get()); + literal = !isUnsigned + ? new TinyIntLiteral(data.get()) : + new SmallIntLiteral(ByteBufferUtil.getUnsignedByte(data)); + break; case MYSQL_TYPE_SHORT: - return new SmallIntLiteral((short) data.getChar()); + literal = !isUnsigned + ? new SmallIntLiteral((short) data.getChar()) : + new IntegerLiteral(ByteBufferUtil.getUnsignedShort(data)); + break; case MYSQL_TYPE_LONG: - return new IntegerLiteral(data.getInt()); + literal = !isUnsigned + ? new IntegerLiteral(data.getInt()) : + new BigIntLiteral(ByteBufferUtil.getUnsignedInt(data)); + break; case MYSQL_TYPE_LONGLONG: - return new BigIntLiteral(data.getLong()); + literal = !isUnsigned + ? new BigIntLiteral(data.getLong()) : + new LargeIntLiteral(new BigInteger(Long.toUnsignedString(data.getLong()))); + break; case MYSQL_TYPE_FLOAT: - return new FloatLiteral(data.getFloat()); + literal = new FloatLiteral(data.getFloat()); + break; case MYSQL_TYPE_DOUBLE: - return new DoubleLiteral(data.getDouble()); + literal = new DoubleLiteral(data.getDouble()); + break; case MYSQL_TYPE_DECIMAL: case MYSQL_TYPE_NEWDECIMAL: - return handleDecimalLiteral(data); + literal = handleDecimalLiteral(data); + break; case MYSQL_TYPE_DATE: - return handleDateLiteral(data); + literal = handleDateLiteral(data); + break; case MYSQL_TYPE_DATETIME: case MYSQL_TYPE_TIMESTAMP: case MYSQL_TYPE_TIMESTAMP2: - return handleDateTimeLiteral(data); + literal = handleDateTimeLiteral(data); + break; case MYSQL_TYPE_STRING: case MYSQL_TYPE_VARSTRING: - return handleStringLiteral(data); + literal = handleStringLiteral(data); + break; case MYSQL_TYPE_VARCHAR: - return handleVarcharLiteral(data); + literal = handleVarcharLiteral(data); + break; default: throw new AnalysisException("Unsupported MySQL type: " + mysqlType); } + return literal; } private static Literal handleDecimalLiteral(ByteBuffer data) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 6c1507722d..fc19330268 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -120,7 +120,8 @@ public class MysqlConnectProcessor extends ConnectProcessor { continue; } LiteralExpr l = prepareStmt.placeholders().get(i).createLiteralFromType(); - l.setupParamFromBinary(packetBuf); + boolean isUnsigned = prepareStmt.placeholders().get(i).isUnsigned(); + l.setupParamFromBinary(packetBuf, isUnsigned); realValueExprs.add(l); } } @@ -167,8 +168,7 @@ public class MysqlConnectProcessor extends ConnectProcessor { LOG.debug("code {}", typeCode); // assign type to placeholders typedPlaceholders.add( - prepareCommand.getPlaceholders().get(i) - .withNewMysqlColType(MysqlColType.fromCode(typeCode))); + prepareCommand.getPlaceholders().get(i).withNewMysqlColType(typeCode)); } // rewrite with new prepared statment with type info in placeholders prepCtx.command = prepareCommand.withPlaceholders(typedPlaceholders); @@ -183,7 +183,8 @@ public class MysqlConnectProcessor extends ConnectProcessor { continue; } MysqlColType type = prepareCommand.getPlaceholders().get(i).getMysqlColType(); - Literal l = Literal.getLiteralByMysqlType(type, packetBuf); + boolean isUnsigned = prepareCommand.getPlaceholders().get(i).isUnsigned(); + Literal l = Literal.getLiteralByMysqlType(type, isUnsigned, packetBuf); statementContext.getIdToPlaceholderRealExpr().put(exprId, l); } }