"DC lost during wait" during online upgrade

crmsh/utils.py

@@ -957,14 +957,16 @@ def append_file(dest, src):
         return False
 
 
-def get_dc(peer=None):
+def get_dc(peer=None, show_rc=False):
     cmd = "crmadmin -D -t 1"
-    _, out, _ = sh.cluster_shell().get_rc_stdout_stderr_without_input(peer, cmd)
-    if not out:
-        return None
-    if not out.startswith("Designated"):
-        return None
-    return out.split()[-1]
+    rc, out, _ = sh.cluster_shell().get_rc_stdout_stderr_without_input(peer, cmd)
+    if not out or not out.startswith("Designated"):
+        out = None
+    else:
+        out = out.split()[-1]
+    if show_rc:
+        return (out, rc)
+    return out
 
 
 def wait4dc(what="", show_progress=True):
@@ -990,8 +992,21 @@ def wait4dc(what="", show_progress=True):
     There's no timeout, as we expect the DC to eventually becomes
     idle.
     '''
-    dc = get_dc()
-    if not dc:
+    def dc_waiter():
+        while True:
+            dc, rc = get_dc(show_rc=True)
+            if rc == 0:
+                return dc
+            if rc in (1, 102):
+                logger.warning("Could not connect to controller: Connection refused")
+                return None
+            if rc in (113, 124):
+                logger.warning("No reply received from controller before timeout")
+                continue
+            logger.warning("Unknown return code from crmadmin: %d", rc)
+            return None
+
+    if not dc_waiter:
         logger.warning("can't find DC")
         return False
     cmd = "crm_attribute -Gq -t crm_config -n crmd-transition-delay 2> /dev/null"
@@ -1007,7 +1022,7 @@ def wait4dc(what="", show_progress=True):
     max_sleep = 1.00
     sleep_time = init_sleep
     while True:
-        dc = get_dc()
+        dc = dc_waiter()
         if not dc:
             logger.warning("DC lost during wait")
             return False