database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](mysql-table) support utf8mb4 for mysql external table (#9402)

Browse Source 
This patch supports utf8mb4 for mysql external table.

if someone needs a mysql external table with utf8mb4 charset, but only support charset utf8 right now.

When create mysql external table, it can add an optional propertiy "charset" which can set character fom mysql connection, 
default value is "utf8". You can set "utf8mb4" instead of "utf8" when you need.
This commit is contained in:
xueweizhang
2022-05-11 09:39:23 +08:00
committed by GitHub
parent 092a12e983
commit 375c1bf5c0
18 changed files with 51 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
View File

@ -207,7 +207,8 @@ public class DescribeStmt extends ShowStmt {
mysqlTable.getUserName(),
mysqlTable.getPasswd(),
mysqlTable.getMysqlDatabaseName(),
mysqlTable.getMysqlTableName());
mysqlTable.getMysqlTableName(),
mysqlTable.getCharset());
totalRows.add(row);
} else {
ErrorReport.reportAnalysisException(ErrorCode.ERR_UNKNOWN_STORAGE_ENGINE, table.getType());

1
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -4290,6 +4290,7 @@ public class Catalog {
sb.append("\"port\" = \"").append(mysqlTable.getPort()).append("\",\n");
sb.append("\"user\" = \"").append(mysqlTable.getUserName()).append("\",\n");
sb.append("\"password\" = \"").append(hidePassword ? "" : mysqlTable.getPasswd()).append("\",\n");
sb.append("\"charset\" = \"").append(mysqlTable.getCharset()).append("\",\n");
} else {
sb.append("\"odbc_catalog_resource\" = \"").append(mysqlTable.getOdbcCatalogResourceName()).append("\",\n");
}

25
fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlTable.java
View File

@ -48,6 +48,7 @@ public class MysqlTable extends Table {
private static final String MYSQL_PASSWORD = "password";
private static final String MYSQL_DATABASE = "database";
private static final String MYSQL_TABLE = "table";
private static final String MYSQL_CHARSET = "charset";
private String odbcCatalogResourceName;
private String host;
@ -56,6 +57,7 @@ public class MysqlTable extends Table {
private String passwd;
private String mysqlDatabaseName;
private String mysqlTableName;
private String charset;
public MysqlTable() {
super(TableType.MYSQL);
@ -124,6 +126,15 @@ public class MysqlTable extends Table {
throw new DdlException("Password of MySQL table is null. "
+ "Please set proper resource or add properties('password'='xxxx') when create table");
}
charset = properties.get(MYSQL_CHARSET);
if (charset == null) {
charset = "utf8";
}
if (!charset.equalsIgnoreCase("utf8") && !charset.equalsIgnoreCase("utf8mb4")) {
throw new DdlException("Unknown character set of MySQL table. "
+ "Please set charset 'utf8' or 'utf8mb4', other charsets not be unsupported now.");
}
}
mysqlDatabaseName = properties.get(MYSQL_DATABASE);
@ -193,9 +204,16 @@ public class MysqlTable extends Table {
return mysqlTableName;
}
public String getCharset() {
if (charset != null) {
return charset;
}
return "utf8";
}
public TTableDescriptor toThrift() {
TMySQLTable tMySQLTable =
new TMySQLTable(getHost(), getPort(), getUserName(), getPasswd(), mysqlDatabaseName, mysqlTableName);
TMySQLTable tMySQLTable = new TMySQLTable(getHost(), getPort(), getUserName(), getPasswd(),
mysqlDatabaseName, mysqlTableName, getCharset());
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.MYSQL_TABLE,
fullSchema.size(), 0, getName(), "");
tTableDescriptor.setMysqlTable(tMySQLTable);
@ -213,6 +231,7 @@ public class MysqlTable extends Table {
sb.append(getPasswd());
sb.append(mysqlDatabaseName);
sb.append(mysqlTableName);
sb.append(getCharset());
String md5 = DigestUtils.md5Hex(sb.toString());
LOG.debug("get signature of mysql table {}: {}. signature string: {}", name, md5, sb.toString());
return md5;
@ -230,6 +249,7 @@ public class MysqlTable extends Table {
serializeMap.put(MYSQL_PASSWORD, passwd);
serializeMap.put(MYSQL_DATABASE, mysqlDatabaseName);
serializeMap.put(MYSQL_TABLE, mysqlTableName);
serializeMap.put(MYSQL_CHARSET, charset);
int size = (int) serializeMap.values().stream().filter(v -> {
return v != null;
@ -262,5 +282,6 @@ public class MysqlTable extends Table {
passwd = serializeMap.get(MYSQL_PASSWORD);
mysqlDatabaseName = serializeMap.get(MYSQL_DATABASE);
mysqlTableName = serializeMap.get(MYSQL_TABLE);
charset = serializeMap.get(MYSQL_CHARSET);
}
}

4
fe/fe-core/src/main/java/org/apache/doris/planner/MysqlTableSink.java
View File

@ -30,6 +30,7 @@ public class MysqlTableSink extends DataSink {
private final String passwd;
private final String db;
private final String tbl;
private final String charset;
public MysqlTableSink(MysqlTable mysqlTable) {
host = mysqlTable.getHost();
@ -38,6 +39,7 @@ public class MysqlTableSink extends DataSink {
passwd = mysqlTable.getPasswd();
db = mysqlTable.getMysqlDatabaseName();
tbl = mysqlTable.getMysqlTableName();
charset = mysqlTable.getCharset();
}
@Override
@ -52,7 +54,7 @@ public class MysqlTableSink extends DataSink {
protected TDataSink toThrift() {
TDataSink tDataSink = new TDataSink(TDataSinkType.MYSQL_TABLE_SINK);
tDataSink.setMysqlTableSink(new TMysqlTableSink(host, port, user, passwd, db, tbl));
tDataSink.setMysqlTableSink(new TMysqlTableSink(host, port, user, passwd, db, tbl, charset));
return tDataSink;
}