database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](export) fix ConcurrentModificationException in export (#25096)

Browse Source 
The session variable in export job should be copied from session variable in connection context.
Because both session variable in connection context and in export job may be modified at same time,
cause ConcurrentModificationException like:

2023-10-07 22:56:12,818 WARN (mysql-nio-pool-2|249) [ConnectProcessor.handleQueryException():396] Process one query failed because unknown reason:
java.util.ConcurrentModificationException: null
        at java.util.HashMap$HashIterator.nextNode(HashMap.java:1437) ~[?:1.8.0_131]
        at java.util.HashMap$KeyIterator.next(HashMap.java:1461) ~[?:1.8.0_131]
        at org.apache.doris.qe.VariableMgr.revertSessionValue(VariableMgr.java:238) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:474) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:438) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:353) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.dispatch(ConnectProcessor.java:501) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.qe.ConnectProcessor.processOnce(ConnectProcessor.java:752) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_131]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_131]
        at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_131]
This error is reported by external_table_p0/export/test_export_external_table.groovy
This commit is contained in:
Mingyu Chen
2023-10-08 11:33:27 +08:00
committed by GitHub
parent 6fe060b79e
commit b4e385e926
2 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
View File

@ -304,11 +304,11 @@ public class ExportCommand extends Command implements ForwardWithSync {
exportJob.setQualifiedUser(ctx.getQualifiedUser());
exportJob.setUserIdentity(ctx.getCurrentUserIdentity());
Optional<SessionVariable> optionalSessionVariable = Optional.ofNullable(
ConnectContext.get().getSessionVariable());
exportJob.setSessionVariables(optionalSessionVariable.orElse(VariableMgr.getDefaultSessionVariable()));
exportJob.setTimeoutSecond(optionalSessionVariable.orElse(VariableMgr.getDefaultSessionVariable())
.getQueryTimeoutS());
// Must copy session variable, because session variable may be changed during export job running.
SessionVariable clonedSessionVariable = VariableMgr.cloneSessionVariable(Optional.ofNullable(
ConnectContext.get().getSessionVariable()).orElse(VariableMgr.getDefaultSessionVariable()));
exportJob.setSessionVariables(clonedSessionVariable);
exportJob.setTimeoutSecond(clonedSessionVariable.getQueryTimeoutS());
// exportJob generate outfile sql
exportJob.generateOutfileLogicalPlans(RelationUtil.getQualifierName(ctx, this.nameParts));

6
fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
View File

@ -245,12 +245,16 @@ public class VariableMgr {
public static SessionVariable newSessionVariable() {
wlock.lock();
try {
return (SessionVariable) SerializationUtils.clone(defaultSessionVariable);
return cloneSessionVariable(defaultSessionVariable);
} finally {
wlock.unlock();
}
}
public static SessionVariable cloneSessionVariable(SessionVariable var) {
return SerializationUtils.clone(var);
}
// Check if this setVar can be set correctly
// Do not use ErrorReport.reportDdlException to throw exeception, it will set the query state in connection context.
// But in some case, we do not want to set the query state and need to ignore that error.