diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 52b448a5f6..01151dcf34 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -1849,9 +1849,10 @@ void PInternalServiceImpl::group_commit_insert(google::protobuf::RpcController*
             }
         }
         st.to_protobuf(response->mutable_status());
+        _exec_env->new_load_stream_mgr()->remove(load_id);
     });
-    _exec_env->new_load_stream_mgr()->remove(load_id);
     if (!ret) {
+        _exec_env->new_load_stream_mgr()->remove(load_id);
         offer_failed(response, done, _light_work_pool);
         return;
     }