database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](prepared statement) Handle unsigned numeric type in prepare statement (#36388)

Browse Source 
## Proposed changes

Issue Number: bp #36133

<!--Describe your changes.-->
This commit is contained in:
xy720
2024-06-18 19:33:12 +08:00
committed by GitHub
parent 8149b2b00d
commit 74162a1b7e
12 changed files with 171 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View File

@ -1807,7 +1807,7 @@ public class DateLiteral extends LiteralExpr {
}
@Override
public void setupParamFromBinary(ByteBuffer data) {
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
int len = getParmLen(data);
if (type.getPrimitiveType() == PrimitiveType.DATE) {
if (len >= 4) {

2
fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java
View File

@ -409,7 +409,7 @@ public class DecimalLiteral extends NumericLiteralExpr {
}
@Override
public void setupParamFromBinary(ByteBuffer data) {
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
int len = getParmLen(data);
BigDecimal v = null;
try {

2
fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
View File

@ -271,7 +271,7 @@ public class FloatLiteral extends NumericLiteralExpr {
}
@Override
public void setupParamFromBinary(ByteBuffer data) {
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
if (type.getPrimitiveType() == PrimitiveType.FLOAT) {
value = data.getFloat();
return;

9
fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.util.ByteBufferUtil;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@ -379,19 +380,19 @@ public class IntLiteral extends NumericLiteralExpr {
}
@Override
public void setupParamFromBinary(ByteBuffer data) {
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
switch (type.getPrimitiveType()) {
case TINYINT:
value = data.get();
break;
case SMALLINT:
value = data.getChar();
value = !isUnsigned ? data.getChar() : ByteBufferUtil.getUnsignedByte(data);
break;
case INT:
value = data.getInt();
value = !isUnsigned ? data.getInt() : ByteBufferUtil.getUnsignedShort(data);
break;
case BIGINT:
value = data.getLong();
value = !isUnsigned ? data.getLong() : ByteBufferUtil.getUnsignedInt(data);
break;
default:
Preconditions.checkState(false);

98
fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View File

@ -20,6 +20,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.MysqlColType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@ -343,54 +344,57 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
return getStringValue();
}
public static LiteralExpr getLiteralByMysqlType(int mysqlType) throws AnalysisException {
switch (mysqlType) {
// MYSQL_TYPE_TINY
case 1:
return LiteralExpr.create("0", Type.TINYINT);
// MYSQL_TYPE_SHORT
case 2:
return LiteralExpr.create("0", Type.SMALLINT);
// MYSQL_TYPE_LONG
case 3:
return LiteralExpr.create("0", Type.INT);
// MYSQL_TYPE_LONGLONG
case 8:
return LiteralExpr.create("0", Type.BIGINT);
// MYSQL_TYPE_FLOAT
case 4:
return LiteralExpr.create("0", Type.FLOAT);
// MYSQL_TYPE_DOUBLE
case 5:
return LiteralExpr.create("0", Type.DOUBLE);
// MYSQL_TYPE_DECIMAL
case 0:
// MYSQL_TYPE_NEWDECIMAL
case 246:
return LiteralExpr.create("0", Type.DECIMAL32);
// MYSQL_TYPE_TIME
case 11:
return LiteralExpr.create("", Type.TIME);
// MYSQL_TYPE_DATE
case 10:
return LiteralExpr.create("1970-01-01", Type.DATE);
// MYSQL_TYPE_DATETIME
case 12:
// MYSQL_TYPE_TIMESTAMP
case 7:
// MYSQL_TYPE_TIMESTAMP2
case 17:
return LiteralExpr.create("1970-01-01 00:00:00", Type.DATETIME);
// MYSQL_TYPE_STRING
case 254:
case 253:
return LiteralExpr.create("", Type.STRING);
// MYSQL_TYPE_VARCHAR
case 15:
return LiteralExpr.create("", Type.VARCHAR);
public static LiteralExpr getLiteralByMysqlType(int mysqlType, boolean isUnsigned) throws AnalysisException {
LiteralExpr literalExpr = null;
// If this is an unsigned numeric type, we convert it by using larger data types. For example, we can use
// small int to represent unsigned tiny int (0-255), big int to represent unsigned ints (0-2 ^ 32-1),
// and so on.
switch (mysqlType & MysqlColType.MYSQL_CODE_MASK) {
case 1: // MYSQL_TYPE_TINY
literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.TINYINT : Type.SMALLINT);
break;
case 2: // MYSQL_TYPE_SHORT
literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.SMALLINT : Type.INT);
break;
case 3: // MYSQL_TYPE_LONG
literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.INT : Type.BIGINT);
break;
case 8: // MYSQL_TYPE_LONGLONG
literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.BIGINT : Type.LARGEINT);
break;
case 4: // MYSQL_TYPE_FLOAT
literalExpr = LiteralExpr.create("0", Type.FLOAT);
break;
case 5: // MYSQL_TYPE_DOUBLE
literalExpr = LiteralExpr.create("0", Type.DOUBLE);
break;
case 0: // MYSQL_TYPE_DECIMAL
case 246: // MYSQL_TYPE_NEWDECIMAL
literalExpr = LiteralExpr.create("0", Type.DECIMAL32);
break;
case 11: // MYSQL_TYPE_TIME
literalExpr = LiteralExpr.create("", Type.TIME);
break;
case 10: // MYSQL_TYPE_DATE
literalExpr = LiteralExpr.create("1970-01-01", Type.DATE);
break;
case 12: // MYSQL_TYPE_DATETIME
case 7: // MYSQL_TYPE_TIMESTAMP
case 17: // MYSQL_TYPE_TIMESTAMP2
literalExpr = LiteralExpr.create("1970-01-01 00:00:00", Type.DATETIME);
break;
case 254: // MYSQL_TYPE_STRING
case 253: // MYSQL_TYPE_VAR_STRING
literalExpr = LiteralExpr.create("", Type.STRING);
break;
case 15: // MYSQL_TYPE_VARCHAR
literalExpr = LiteralExpr.create("", Type.VARCHAR);
break;
default:
return null;
throw new AnalysisException("Unsupported MySQL type: " + mysqlType);
}
return literalExpr;
}
@Override
@ -495,7 +499,7 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
// Parse from binary data, the format follows mysql binary protocal
// see https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html.
// Return next offset
public void setupParamFromBinary(ByteBuffer data) {
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
Preconditions.checkState(false,
"should implement this in derived class. " + this.type.toSql());
}

16
fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.MysqlColType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
@ -60,7 +61,7 @@ public class PlaceHolderExpr extends LiteralExpr {
public LiteralExpr createLiteralFromType() throws AnalysisException {
Preconditions.checkState(mysqlTypeCode > 0);
return LiteralExpr.getLiteralByMysqlType(mysqlTypeCode);
return LiteralExpr.getLiteralByMysqlType(mysqlTypeCode, isUnsigned());
}
public static PlaceHolderExpr create(String value, Type type) throws AnalysisException {
@ -87,6 +88,10 @@ public class PlaceHolderExpr extends LiteralExpr {
return lExpr.isMinValue();
}
public boolean isUnsigned() {
return MysqlColType.isUnsigned(mysqlTypeCode);
}
@Override
public int compareLiteral(LiteralExpr expr) {
return lExpr.compareLiteral(expr);
@ -130,6 +135,11 @@ public class PlaceHolderExpr extends LiteralExpr {
return "?";
}
@Override
protected Expr uncheckedCastTo(Type targetType) throws AnalysisException {
return this.lExpr.uncheckedCastTo(targetType);
}
// Swaps the sign of numeric literals.
// Throws for non-numeric literals.
public void swapSign() throws NotImplementedException {
@ -181,7 +191,7 @@ public class PlaceHolderExpr extends LiteralExpr {
return "\"" + getStringValue() + "\"";
}
public void setupParamFromBinary(ByteBuffer data) {
lExpr.setupParamFromBinary(data);
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
lExpr.setupParamFromBinary(data, isUnsigned);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
View File

@ -339,7 +339,7 @@ public class StringLiteral extends LiteralExpr {
}
@Override
public void setupParamFromBinary(ByteBuffer data) {
public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) {
int strLen = getParmLen(data);
if (strLen > data.remaining()) {
strLen = data.remaining();

34
fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java Normal file
View File

@ -0,0 +1,34 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.util;
import java.nio.ByteBuffer;
public class ByteBufferUtil {
public static short getUnsignedByte(ByteBuffer buffer) {
return (short) (buffer.get() & 0xFF);
}
public static int getUnsignedShort(ByteBuffer buffer) {
return buffer.getShort() & 0xFFFF;
}
public static long getUnsignedInt(ByteBuffer buffer) {
return buffer.getInt() & 0xFFFFFFFFL;
}
}

14
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java
View File

@ -33,15 +33,17 @@ import java.util.Optional;
public class Placeholder extends Expression implements LeafExpression {
private final PlaceholderId placeholderId;
private final Optional<MysqlColType> mysqlColType;
private int mysqlTypeCode = -1;
public Placeholder(PlaceholderId placeholderId) {
this.placeholderId = placeholderId;
this.mysqlColType = Optional.empty();
}
public Placeholder(PlaceholderId placeholderId, MysqlColType mysqlColType) {
public Placeholder(PlaceholderId placeholderId, int mysqlTypeCode) {
this.placeholderId = placeholderId;
this.mysqlColType = Optional.of(mysqlColType);
this.mysqlColType = Optional.of(MysqlColType.fromCode(mysqlTypeCode));
this.mysqlTypeCode = mysqlTypeCode;
}
public PlaceholderId getPlaceholderId() {
@ -68,8 +70,12 @@ public class Placeholder extends Expression implements LeafExpression {
return NullType.INSTANCE;
}
public Placeholder withNewMysqlColType(MysqlColType mysqlColType) {
return new Placeholder(getPlaceholderId(), mysqlColType);
public Placeholder withNewMysqlColType(int mysqlTypeCode) {
return new Placeholder(getPlaceholderId(), mysqlTypeCode);
}
public boolean isUnsigned() {
return MysqlColType.isUnsigned(mysqlTypeCode);
}
public MysqlColType getMysqlColType() {

51
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.catalog.MysqlColType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.ByteBufferUtil;
import org.apache.doris.mysql.MysqlProto;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.UnboundException;
@ -448,42 +449,68 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
* Retrieves a Literal object based on the MySQL type and the data provided.
*
* @param mysqlType the MySQL type identifier
* @param isUnsigned true if it is an unsigned type
* @param data the ByteBuffer containing the data
* @return a Literal object corresponding to the MySQL type
* @throws AnalysisException if the MySQL type is unsupported or if data conversion fails
* @link <a href="https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html">...</a>.
*/
public static Literal getLiteralByMysqlType(MysqlColType mysqlType, ByteBuffer data) throws AnalysisException {
public static Literal getLiteralByMysqlType(MysqlColType mysqlType, boolean isUnsigned, ByteBuffer data)
throws AnalysisException {
Literal literal = null;
// If this is an unsigned numeric type, we convert it by using larger data types. For example, we can use
// small int to represent unsigned tiny int (0-255), big int to represent unsigned ints (0-2 ^ 32-1),
// and so on.
switch (mysqlType) {
case MYSQL_TYPE_TINY:
return new TinyIntLiteral(data.get());
literal = !isUnsigned
? new TinyIntLiteral(data.get()) :
new SmallIntLiteral(ByteBufferUtil.getUnsignedByte(data));
break;
case MYSQL_TYPE_SHORT:
return new SmallIntLiteral((short) data.getChar());
literal = !isUnsigned
? new SmallIntLiteral((short) data.getChar()) :
new IntegerLiteral(ByteBufferUtil.getUnsignedShort(data));
break;
case MYSQL_TYPE_LONG:
return new IntegerLiteral(data.getInt());
literal = !isUnsigned
? new IntegerLiteral(data.getInt()) :
new BigIntLiteral(ByteBufferUtil.getUnsignedInt(data));
break;
case MYSQL_TYPE_LONGLONG:
return new BigIntLiteral(data.getLong());
literal = !isUnsigned
? new BigIntLiteral(data.getLong()) :
new LargeIntLiteral(new BigInteger(Long.toUnsignedString(data.getLong())));
break;
case MYSQL_TYPE_FLOAT:
return new FloatLiteral(data.getFloat());
literal = new FloatLiteral(data.getFloat());
break;
case MYSQL_TYPE_DOUBLE:
return new DoubleLiteral(data.getDouble());
literal = new DoubleLiteral(data.getDouble());
break;
case MYSQL_TYPE_DECIMAL:
case MYSQL_TYPE_NEWDECIMAL:
return handleDecimalLiteral(data);
literal = handleDecimalLiteral(data);
break;
case MYSQL_TYPE_DATE:
return handleDateLiteral(data);
literal = handleDateLiteral(data);
break;
case MYSQL_TYPE_DATETIME:
case MYSQL_TYPE_TIMESTAMP:
case MYSQL_TYPE_TIMESTAMP2:
return handleDateTimeLiteral(data);
literal = handleDateTimeLiteral(data);
break;
case MYSQL_TYPE_STRING:
case MYSQL_TYPE_VARSTRING:
return handleStringLiteral(data);
literal = handleStringLiteral(data);
break;
case MYSQL_TYPE_VARCHAR:
return handleVarcharLiteral(data);
literal = handleVarcharLiteral(data);
break;
default:
throw new AnalysisException("Unsupported MySQL type: " + mysqlType);
}
return literal;
}
private static Literal handleDecimalLiteral(ByteBuffer data) throws AnalysisException {

9
fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
View File

@ -120,7 +120,8 @@ public class MysqlConnectProcessor extends ConnectProcessor {
continue;
}
LiteralExpr l = prepareStmt.placeholders().get(i).createLiteralFromType();
l.setupParamFromBinary(packetBuf);
boolean isUnsigned = prepareStmt.placeholders().get(i).isUnsigned();
l.setupParamFromBinary(packetBuf, isUnsigned);
realValueExprs.add(l);
}
}
@ -167,8 +168,7 @@ public class MysqlConnectProcessor extends ConnectProcessor {
LOG.debug("code {}", typeCode);
// assign type to placeholders
typedPlaceholders.add(
prepareCommand.getPlaceholders().get(i)
.withNewMysqlColType(MysqlColType.fromCode(typeCode)));
prepareCommand.getPlaceholders().get(i).withNewMysqlColType(typeCode));
}
// rewrite with new prepared statment with type info in placeholders
prepCtx.command = prepareCommand.withPlaceholders(typedPlaceholders);
@ -183,7 +183,8 @@ public class MysqlConnectProcessor extends ConnectProcessor {
continue;
}
MysqlColType type = prepareCommand.getPlaceholders().get(i).getMysqlColType();
Literal l = Literal.getLiteralByMysqlType(type, packetBuf);
boolean isUnsigned = prepareCommand.getPlaceholders().get(i).isUnsigned();
Literal l = Literal.getLiteralByMysqlType(type, isUnsigned, packetBuf);
statementContext.getIdToPlaceholderRealExpr().put(exprId, l);
}
}