Allow user to ignore the broken disk (#2755)

Add a BE config `ignore_broken_disk`.

be/src/common/config.h

@@ -312,7 +312,7 @@ namespace config {
 
     CONF_Bool(disable_mem_pools, "false");
 
-    // Whether to allocate chunk using mmap. If you enable this, you'd better to 
+    // Whether to allocate chunk using mmap. If you enable this, you'd better to
     // increase vm.max_map_count's value whose default value is 65530.
     // you can do it as root via "sysctl -w vm.max_map_count=262144" or
     // "echo 262144 > /proc/sys/vm/max_map_count"
@@ -336,7 +336,7 @@ namespace config {
     CONF_Bool(enable_partitioned_hash_join, "false")
     CONF_Bool(enable_partitioned_aggregation, "false")
     CONF_Bool(enable_new_partitioned_aggregation, "true")
-    
+
     // for kudu
     // "The maximum size of the row batch queue, for Kudu scanners."
     CONF_Int32(kudu_max_row_batches, "0")
@@ -363,7 +363,7 @@ namespace config {
 
     // to open/close system metrics
     CONF_Bool(enable_system_metrics, "true");
-    
+
     CONF_Bool(enable_prefetch, "true");
 
     // Number of cores Doris will used, this will effect only when it's greater than 0.
@@ -376,6 +376,10 @@ namespace config {
     // check on all data spilled to disk during a query
     CONF_Bool(disk_spill_encryption, "false");
 
+    // When BE start, If there is a broken disk, BE process will exit by default.
+    // Otherwise, we will ignore the broken disk,
+    CONF_Bool(ignore_broken_disk, "false");
+
     // Writable scratch directories
     CONF_String(scratch_dirs, "/tmp");
 
@@ -450,7 +454,7 @@ namespace config {
     // max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row
     // default is 20, batch_size's defualt value is 1024 means 20 * 1024 rows will be cached
     CONF_Int32(max_memory_sink_batch_count, "20");
-    
+
     // This configuration is used for the context gc thread schedule period
     // note: unit is minute, default is 5min
     CONF_Int32(scan_context_gc_interval_min, "5");

be/src/service/doris_main.cpp

@@ -39,6 +39,7 @@
 #include "common/status.h"
 #include "codegen/llvm_codegen.h"
 #include "runtime/exec_env.h"
+#include "util/file_utils.h"
 #include "util/logging.h"
 #include "util/network_util.h"
 #include "util/thrift_util.h"
@@ -134,6 +135,26 @@ int main(int argc, char** argv) {
         exit(-1);
     }
 
+    auto it = paths.begin();
+    for (;it != paths.end();) {
+        if (!doris::FileUtils::check_exist(it->path)) {
+            if (doris::config::ignore_broken_disk) {
+                LOG(WARNING) << "opendir failed, path=" << it->path;
+                it = paths.erase(it);
+            } else {
+                LOG(FATAL) << "opendir failed, path=" << it->path;
+                exit(-1);
+            }
+        } else {
+            ++it;
+        }
+    }
+
+    if (paths.empty()) {
+        LOG(FATAL) << "All disks are broken, exit.";
+        exit(-1);
+    }
+
     doris::LlvmCodeGen::initialize_llvm();
 
     // initilize libcurl here to avoid concurrent initialization