guest-diag: semfix counter-pin retry backoff + shorter exec timeout (stops guest-agent starvation)

1 files changed, 15 insertions, 6 deletions

diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py
index 09cc5f8..fba465d 100755
--- a/tools/guest-diag/semfix_uvmt.py
+++ b/tools/guest-diag/semfix_uvmt.py
@@ -48,6 +48,9 @@ TRACE_EVERY = int(os.environ.get("UVMT_TRACE_EVERY", "5"))
 # stream (pin_gr_stream.py) already supplies the push-buffer pages, skip it so
 # the tracking-sem poller stays live during the client's deadline window.
 SKIP_PB = os.environ.get("SEMFIX_SKIP_PB", "0") == "1"
+# Seconds between counter-page pin attempts (the guest-agent exec blocks for
+# its full timeout when the client saturates the guest).
+COUNTER_RETRY_S = float(os.environ.get("SEMFIX_COUNTER_RETRY", "10"))
 
 
 class Stream:
@@ -220,7 +223,7 @@ def pin_counter_page(pid, pinned_pids):
            % (pid, COUNTER_LO, COUNTER_HI))
     try:
         r = subprocess.run([VMEXEC, "exec", cmd], capture_output=True,
-                           text=True, timeout=60)
+                           text=True, timeout=25)
     except Exception:
         return False
     n = 0
@@ -252,6 +255,7 @@ def main():
     last_seen_t = {}
     dseen = set()
     counter_pids = set()
+    counter_next_try = {}
     last_pid = None
     stream = Stream(SEMSTREAM)
     loop = 0
@@ -386,16 +390,21 @@ def main():
 
         # Pin the completion-counter page of a newly started client, before its
         # first GR pushes decode their releases. The pid comes from the guest
-        # scanner; the guest-agent fallback is only a safety net.
+        # scanner; the guest-agent fallback is only a safety net. The exec can
+        # block for its whole timeout under client load, so retry on a slow
+        # cadence: hammering the agent here starves it and wedges the run.
         if last_pid is None and loop % TRACE_EVERY == 0:
             last_pid = cdt_pid()
             if last_pid:
                 log("counter: client pid %d (agent)" % last_pid)
         if last_pid and last_pid not in counter_pids:
-            try:
-                pin_counter_page(last_pid, counter_pids)
-            except Exception as e:
-                log("counter: pin failed: %s" % e)
+            now = time.time()
+            if now >= counter_next_try.get(last_pid, 0):
+                try:
+                    pin_counter_page(last_pid, counter_pids)
+                except Exception as e:
+                    log("counter: pin failed: %s" % e)
+                counter_next_try[last_pid] = now + COUNTER_RETRY_S
 
         time.sleep(interval)