database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Prepared Statment) use fixed charset to init StringLiteral (#37084)

Browse Source 
picked from #36860
This commit is contained in:
lihangyu
2024-07-01 23:11:13 +08:00
committed by GitHub
parent e686e85f27
commit 6425ce8a89
5 changed files with 35 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
View File

@ -40,6 +40,7 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
public class StringLiteral extends LiteralExpr {
@ -346,7 +347,9 @@ public class StringLiteral extends LiteralExpr {
}
byte[] bytes = new byte[strLen];
data.get(bytes);
value = new String(bytes);
// ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in
// different environment
value = new String(bytes, StandardCharsets.UTF_8);
if (LOG.isDebugEnabled()) {
LOG.debug("parsed value '{}'", value);
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
View File

@ -45,6 +45,7 @@ import com.google.common.collect.ImmutableList;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
@ -582,7 +583,9 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
strLen = Math.min(strLen, data.remaining());
byte[] bytes = new byte[strLen];
data.get(bytes);
return new StringLiteral(new String(bytes));
// ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in
// different environment
return new StringLiteral(new String(bytes, StandardCharsets.UTF_8));
}
private static Literal handleVarcharLiteral(ByteBuffer data) {
@ -590,6 +593,8 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
strLen = Math.min(strLen, data.remaining());
byte[] bytes = new byte[strLen];
data.get(bytes);
return new VarcharLiteral(new String(bytes));
// ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in
// different environment
return new VarcharLiteral(new String(bytes, StandardCharsets.UTF_8));
}
}

4
fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
View File

@ -213,7 +213,9 @@ public class MysqlConnectProcessor extends ConnectProcessor {
// process COM_EXECUTE, parse binary row data
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html
private void handleExecute() {
// debugPacket();
if (LOG.isDebugEnabled()) {
debugPacket();
}
packetBuf = packetBuf.order(ByteOrder.LITTLE_ENDIAN);
// parse stmt_id, flags, params
int stmtId = packetBuf.getInt();

9
regression-test/data/prepared_stmt_p0/prepared_stmt.out
View File

@ -74,6 +74,12 @@ a
-- !select7 --
2 1 user1 \N 1111111 1111111
-- !select6_1 --
2 1 user1 \N 1234.1111 xxxlalala
-- !select7_1 --
2 1 user1 \N 1111111 1111111
-- !select8 --
1
1
@ -99,6 +105,3 @@ a
1236 100320.111390000 laa ddd Will we ignore LIMIT ?,? 2021-01-01 2020-01-01T12:36:38 2.7692 6022-01-01 [null]
1237 120939.111300000 a ddd Will we ignore LIMIT ?,? 2021-01-01 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"]
-- !select16 --
mytable1 CREATE TABLE `mytable1` (\n `siteid` INT NULL DEFAULT "10",\n `citycode` SMALLINT NULL,\n `username` VARCHAR(32) NULL DEFAULT "",\n `pv` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`siteid`, `citycode`, `username`)\nDISTRIBUTED BY HASH(`siteid`) BUCKETS 10\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);

16
regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy
View File

@ -57,6 +57,7 @@ suite("test_prepared_stmt", "nonConcurrent") {
qt_sql """select * from ${tableName} order by 1, 2, 3"""
qt_sql """select * from ${tableName} order by 1, 2, 3"""
sql "set global max_prepared_stmt_count = 10000"
def stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1"
assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement);
@ -103,6 +104,7 @@ suite("test_prepared_stmt", "nonConcurrent") {
sql """insert into mytable1 values(1,1,'user1',10);"""
sql """insert into mytable1 values(1,1,'user1',10);"""
sql "sync"
stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where citycode = ?) AS `SpotfireCustomQuery1` WHERE 1 = 1"
assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement);
stmt_read.setInt(1, 12345)
@ -147,6 +149,9 @@ suite("test_prepared_stmt", "nonConcurrent") {
qe_select5 stmt_read
sql """insert into mytable1 values(2,1,'user1',null);"""
// sql "set experimental_enable_nereids_planner = false"
stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where pv is null) AS `SpotfireCustomQuery1` WHERE 1 = 1"
assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement);
stmt_read.setString(1, "xxxlalala")
@ -155,7 +160,16 @@ suite("test_prepared_stmt", "nonConcurrent") {
stmt_read.setString(1, "1111111")
stmt_read.setString(2, "1111111")
qe_select7 stmt_read
stmt_read.close()
// stmt_read.close()
// sql "set experimental_enable_nereids_planner = true"
stmt_read.setString(1, "xxxlalala")
stmt_read.setDouble(2, 1234.1111)
qe_select6_1 stmt_read
stmt_read.setString(1, "1111111")
stmt_read.setString(2, "1111111")
qe_select7_1 stmt_read
stmt_read = prepareStatement "SELECT COUNT() from mytable1 WHERE citycode = ? GROUP BY siteid"
assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement);