From e1a8b451176edea6ca339f0e27ceaf6a47afcf8b Mon Sep 17 00:00:00 2001
From: congzhou2603 <zhoucong42@huawei.com>
Date: Tue, 19 Sep 2023 10:43:51 +0800
Subject: [PATCH] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=20=E4=BF=AE=E5=A4=8Dpo?=
 =?UTF-8?q?stmaster=E7=BA=BF=E7=A8=8B=E5=8D=A1=E5=9C=A8PM=5FWAIT=5FBACKEND?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../process/postmaster/postmaster.cpp         | 22 +++++++++++++++++++
 src/include/ddes/dms/ss_reform_common.h       |  2 ++
 2 files changed, 24 insertions(+)

diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp
index d8b0ff25e..706a785dc 100644
--- a/src/gausskernel/process/postmaster/postmaster.cpp
+++ b/src/gausskernel/process/postmaster/postmaster.cpp
@@ -486,6 +486,7 @@ static bool CheckSignalByFile(const char *filename, void *infoPtr, size_t infoSi
 
 int GaussDbThreadMain(knl_thread_arg* arg);
 const char* GetThreadName(knl_thread_role role);
+void SSOndemandProcExitIfStayWaitBackends();
 
 #ifdef EXEC_BACKEND
 
@@ -3976,6 +3977,10 @@ static int ServerLoop(void)
                             result = BackendStartup(port, isConnectHaPort);
                         }
 
+                        if (SS_CLUSTER_ONDEMAND_RECOVERY && SS_IN_REFORM &&
+                            result != STATUS_OK && pmState == PM_WAIT_BACKENDS) {
+                                SSOndemandProcExitIfStayWaitBackends();
+                        }
                         if (result != STATUS_OK) {
                             if (port->is_logic_conn) {
                                 gs_close_gsocket(&port->gs_sock);
@@ -15086,3 +15091,20 @@ void SSRestartFailoverPromote()
     pmState = PM_WAIT_BACKENDS;
     SShandle_promote_signal();
 }
+
+void SSOndemandProcExitIfStayWaitBackends() 
+{
+    int failTimes = 0;
+    while (failTimes < WAIT_PMSTATE_UPDATE_TRIES && pmState == PM_WAIT_BACKENDS) {
+        PostmasterStateMachine();
+        pg_usleep(REFORM_WAIT_LONG);
+        failTimes++;
+    }
+    if (pmState == PM_WAIT_BACKENDS) {
+        ereport(PANIC, (errmsg("Proc exit because pmState stay %s for %d times, "
+            "when reform failed and in ondemand recovery, "
+            "to avoid pmState being stuck in PM_WAIT_BACKENDS.", 
+            GetPMState(pmState), WAIT_PMSTATE_UPDATE_TRIES)));
+        proc_exit(1);
+    }
+}
\ No newline at end of file
diff --git a/src/include/ddes/dms/ss_reform_common.h b/src/include/ddes/dms/ss_reform_common.h
index 227d762e8..a11429d1d 100644
--- a/src/include/ddes/dms/ss_reform_common.h
+++ b/src/include/ddes/dms/ss_reform_common.h
@@ -31,6 +31,8 @@
 #define REFORM_WAIT_LONG 100000 /* 0.1 sec */
 #define WAIT_REFORM_CTRL_REFRESH_TRIES 1000
 
+#define WAIT_PMSTATE_UPDATE_TRIES 100
+
 #define REFORM_CTRL_VERSION 1
 
 typedef struct SSBroadcastCancelTrx {