This commit is contained in:
@ -85,6 +85,7 @@ suite("load_p2", "variant_type,p2"){
|
||||
|
||||
// Executor service for managing threads
|
||||
def executorService = Executors.newFixedThreadPool(numberOfThreads)
|
||||
def futures = []
|
||||
|
||||
try {
|
||||
def table_name = "github_events"
|
||||
@ -152,16 +153,25 @@ suite("load_p2", "variant_type,p2"){
|
||||
def fileName = year + "-" + month + "-" + day + "-" + hour + ".json"
|
||||
log.info("cuurent fileName: ${fileName}")
|
||||
// Submitting tasks to the executor service
|
||||
executorService.submit({
|
||||
futures << executorService.submit({
|
||||
log.info("Loading file: ${fileName}")
|
||||
s3load_paral_wait.call(table_name, "JSON", "regression/github_events_dataset/${fileName}", 3)
|
||||
} as Runnable)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shutdown executor service and wait for all tasks to complete
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS)
|
||||
|
||||
try {
|
||||
futures.each { future ->
|
||||
future.get()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw e.cause // throw original exception
|
||||
} finally {
|
||||
// Shutdown executor service and wait for all tasks to complete
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS)
|
||||
}
|
||||
|
||||
qt_sql("select count() from github_events")
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user