improve error code when flashback operations have failed

mittest/logservice/test_ob_simple_log_flashback.cpp

@@ -171,6 +171,15 @@ TEST_F(TestObSimpleLogClusterFlashback, flashback_basic_func)
   // 3. do flashback
   flashback_srv = get_cluster()[0]->get_flashback_service();
   const int64_t TIMEOUT_US = 10 * 1000 * 1000;
+
+  // 4. test a follower blocknet
+  block_net(leader_idx1, (leader_idx1+1) % 3);
+  int64_t mode_version;
+  flashback_srv = get_cluster()[0]->get_flashback_service();
+  EXPECT_EQ(OB_OP_NOT_ALLOW, flashback_srv->flashback(MTL_ID(), flashback_scn, 5 * 1000 * 1000));
+  unblock_net(leader_idx1, (leader_idx1+1) % 3);
+
+  // 5. test basic flashback
   EXPECT_EQ(OB_SUCCESS, flashback_srv->flashback(MTL_ID(), flashback_scn, TIMEOUT_US));
   LSN new_log_tail1 = leader1.palf_handle_impl_->log_engine_.log_storage_.log_tail_;
   EXPECT_EQ(new_log_tail1, leader1.palf_handle_impl_->sw_.committed_end_lsn_);

src/logservice/ob_log_flashback_service.cpp

@@ -169,6 +169,22 @@ int ObLogFlashbackService::wait_all_ls_replicas_log_sync_(
         CLOG_LOG(WARN, "wait_all_ls_replicas_log_sync_ fail", COMMON_LOG_INFO, K(check_log_op_array));
       }
     }
+    // returns user-friendly error code
+    if (OB_TIMEOUT == ret) {
+      ret = OB_OP_NOT_ALLOW;
+      CheckLogOpArray failed_ls_ops;
+      for (int64_t i = 0; i < check_log_op_array.count(); i++) {
+        CheckLSLogSyncOperator &op = check_log_op_array.at(i);
+        int tmp_ret = OB_SUCCESS;
+        if (OB_SUCCESS == op.ret_) {
+        } else if (OB_SUCCESS != (tmp_ret = failed_ls_ops.push_back(op))) {
+          CLOG_LOG(WARN, "push_back failed", K(ret), K_(self), K(ls_id));
+        }
+      }
+      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "logs of some replicas may be not sync, check these replicas");
+      CLOG_LOG(WARN, "logs of some replicas may be not sync, check these replicas", K(ret),
+          K_(self), K(ls_id), K(failed_ls_ops));
+    }
   }
   #undef COMMON_LOG_INFO
   return ret;
@@ -246,6 +262,22 @@ int ObLogFlashbackService::do_flashback_(
         CLOG_LOG(INFO, "do_flashback_ fail", COMMON_LOG_INFO, K_(flashback_op_array));
       }
     }
+    // returns user-friendly error code
+    if (OB_TIMEOUT == ret) {
+      ret = OB_OP_NOT_ALLOW;
+      FlashbackOpArray failed_ls_ops;
+      for (int64_t i = 0; i < flashback_op_array_.count(); i++) {
+        ExecuteFlashbackOperator &op = flashback_op_array_.at(i);
+        int tmp_ret = OB_SUCCESS;
+        if (OB_SUCCESS == op.ret_) {
+        } else if (OB_SUCCESS != (tmp_ret = failed_ls_ops.push_back(op))) {
+          CLOG_LOG(WARN, "push_back failed", K(ret), K_(self), K(ls_id));
+        }
+      }
+      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "logs of some replicas may have not been flashbacked, check these replicas");
+      CLOG_LOG(WARN, "logs of some replicas may have not been flashbacked, check these replicas",
+          K(ret), K_(self), K(ls_id), K(failed_ls_ops));
+    }
   }
   #undef COMMON_LOG_INFO
   return ret;
@@ -433,6 +465,7 @@ int ObLogFlashbackService::CheckLSLogSyncOperator::switch_state()
       CLOG_LOG(WARN, "check_ls_log_sync eagain", K(ret), KPC(this));
     }
   }
+  ret_ = ret;
   return ret;
 }
 
@@ -475,6 +508,7 @@ int ObLogFlashbackService::ChangeAccessModeOperator::switch_state()
       CLOG_LOG(WARN, "change_access_mode eagain", K(ret), KPC(this));
     }
   }
+  ret_ = ret;
   return ret;
 }
 
@@ -516,6 +550,7 @@ int ObLogFlashbackService::ExecuteFlashbackOperator::switch_state()
       CLOG_LOG(WARN, "do_flashback eagain", K(ret), KPC(this));
     }
   }
+  ret_ = ret;
   return ret;
 }
 

src/logservice/ob_log_flashback_service.h

@@ -70,7 +70,8 @@ private:
           leader_(),
           flashback_scn_(),
           location_adapter_(NULL),
-          rpc_proxy_(NULL) { }
+          rpc_proxy_(NULL),
+          ret_(OB_NOT_INIT) { }
     BaseLSOperator(const uint64_t tenant_id,
                    const share::ObLSID &ls_id,
                    const common::ObAddr &self,
@@ -94,6 +95,7 @@ private:
       flashback_scn_.reset();
       location_adapter_ = NULL;
       rpc_proxy_ = NULL;
+      ret_ = OB_NOT_INIT;
     }
     bool is_valid() const {
       // leader may be invalid
@@ -105,7 +107,7 @@ private:
              OB_NOT_NULL(rpc_proxy_);
     }
     virtual int switch_state() = 0;
-    TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(leader), K_(flashback_scn));
+    TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(leader), K_(flashback_scn), "inner_ret", ret_);
   protected:
     int update_leader_();
     int get_leader_palf_stat_(palf::PalfStat &palf_stat);
@@ -118,6 +120,7 @@ private:
     share::SCN flashback_scn_;
     logservice::ObLocationAdapter *location_adapter_;
     obrpc::ObLogServiceRpcProxy *rpc_proxy_;
+    int ret_;
   };
 
   class CheckLSLogSyncOperator : public BaseLSOperator