database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[chore](log opt) opt mysql load log (#26149)

Browse Source
This commit is contained in:
zhiqiang
2023-10-31 01:59:52 -05:00
committed by GitHub
parent 95c765f5fe
commit feefebe0da
1 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java
View File

@ -180,7 +180,7 @@ public class MysqlLoadManager {
boolean clientLocal = dataDesc.isClientLocal();
MySqlLoadContext loadContext = new MySqlLoadContext();
loadContextMap.put(loadId, loadContext);
LOG.info("execute MySqlLoadJob for id: {}.", loadId);
LOG.info("Executing mysql load with id: {}.", loadId);
try (final CloseableHttpClient httpclient = HttpClients.createDefault()) {
for (String file : filePaths) {
InputStreamEntity entity = getInputStreamEntity(context, clientLocal, file, loadId);
@ -193,7 +193,8 @@ public class MysqlLoadManager {
String errorUrl = Optional.ofNullable(result.get("ErrorURL"))
.map(JsonElement::getAsString).orElse("");
failedRecords.offer(new MySqlLoadFailRecord(loadId, errorUrl));
LOG.warn("Execute mysql data load failed with request: {} and response: {}", request, body);
LOG.warn("Execute mysql load failed with request: {} and response: {}, job id: {}",
request, body, loadId);
throw new LoadException(result.get("Message").getAsString() + " with load id " + loadId);
}
loadResult.incRecords(result.get("NumberLoadedRows").getAsLong());
@ -201,10 +202,10 @@ public class MysqlLoadManager {
}
}
} catch (Throwable t) {
LOG.warn("Execute mysql load {} failed", loadId, t);
LOG.warn("Execute mysql load {} failed, msg: {}", loadId, t);
// drain the data from client conn util empty packet received, otherwise the connection will be reset
if (clientLocal && loadContextMap.containsKey(loadId) && !loadContextMap.get(loadId).isFinished()) {
LOG.warn("not drained yet, try reading left data from client connection for load {}.", loadId);
LOG.warn("Not drained yet, try reading left data from client connection for load {}.", loadId);
ByteBuffer buffer = context.getMysqlChannel().fetchOnePacket();
// MySql client will send an empty packet when eof
while (buffer != null && buffer.limit() != 0) {
@ -219,6 +220,7 @@ public class MysqlLoadManager {
throw t;
}
} finally {
LOG.info("Mysql load job {} finished, loaded records: {}", loadId, loadResult.getRecords());
loadContextMap.remove(loadId);
}
return loadResult;