guest-diag: retry the guest pin-scanner startup (a transient agent exec failure left a boot without pins and the client hung)

1 files changed, 25 insertions, 14 deletions

diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index fbe2ee5..a11bc84 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -54,23 +54,34 @@ def vexec(cmd, timeout=40):
         return ""
 
 
-def install_and_start():
+def _start_cmd():
     with open(GUEST_SCRIPT) as f:
         b64 = subprocess.run(["base64", "-w0"], stdin=f,
                              capture_output=True, text=True).stdout
-    cmd = ("echo '%s' | base64 -d > /tmp/guest_pin_stream.py; "
-           "if [ -f /tmp/guest_pin_stream.pid ]; then "
-           "p=$(cat /tmp/guest_pin_stream.pid 2>/dev/null); "
-           "[ -n \"$p\" ] && grep -q guest_pin_stream "
-           "/proc/$p/cmdline 2>/dev/null && sudo kill $p 2>/dev/null; "
-           "fi; "
-           "sudo rm -f /tmp/guest_pin_stream.pid; "
-           "sudo setsid nohup python3 /tmp/guest_pin_stream.py %d %d %s %s %s "
-           ">/tmp/guest_pin_stream.log 2>&1 & sleep 2; "
-           "sudo cat /tmp/guest_pin_stream.pid 2>/dev/null"
-           % (b64, 2, PORT, LO, HI, PAMAX))
-    out = vexec(cmd, 60)
-    return any(ch.isdigit() for ch in out)
+    return ("echo '%s' | base64 -d > /tmp/guest_pin_stream.py; "
+            "if [ -f /tmp/guest_pin_stream.pid ]; then "
+            "p=$(cat /tmp/guest_pin_stream.pid 2>/dev/null); "
+            "[ -n \"$p\" ] && grep -q guest_pin_stream "
+            "/proc/$p/cmdline 2>/dev/null && sudo kill $p 2>/dev/null; "
+            "fi; "
+            "sudo rm -f /tmp/guest_pin_stream.pid; "
+            "sudo setsid nohup python3 /tmp/guest_pin_stream.py %d %d %s %s %s "
+            ">/tmp/guest_pin_stream.log 2>&1 & sleep 2; "
+            "sudo cat /tmp/guest_pin_stream.pid 2>/dev/null"
+            % (b64, 2, PORT, LO, HI, PAMAX))
+
+
+def install_and_start():
+    # The guest-agent exec can fail transiently (agent busy just after boot,
+    # or a stale qga client); retry a few times instead of giving up, because
+    # a boot without the scanner streams no pins and the client then hangs.
+    for attempt in range(5):
+        out = vexec(_start_cmd(), 60)
+        if any(ch.isdigit() for ch in out):
+            return True
+        log("WARN: scanner start attempt %d failed; retrying" % (attempt + 1))
+        time.sleep(5)
+    return False
 
 
 def inject(va, pa, ln):