From 035bd0e17ae0e5d3786bf250481883200c25feeb Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Tue, 29 Sep 2026 18:08:38 -0400 Subject: guest-diag: semfix counter-pin retry backoff + shorter exec timeout (stops guest-agent starvation) --- tools/guest-diag/semfix_uvmt.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py index 09cc5f8..fba465d 100755 --- a/tools/guest-diag/semfix_uvmt.py +++ b/tools/guest-diag/semfix_uvmt.py @@ -48,6 +48,9 @@ TRACE_EVERY = int(os.environ.get("UVMT_TRACE_EVERY", "5")) # stream (pin_gr_stream.py) already supplies the push-buffer pages, skip it so # the tracking-sem poller stays live during the client's deadline window. SKIP_PB = os.environ.get("SEMFIX_SKIP_PB", "0") == "1" +# Seconds between counter-page pin attempts (the guest-agent exec blocks for +# its full timeout when the client saturates the guest). +COUNTER_RETRY_S = float(os.environ.get("SEMFIX_COUNTER_RETRY", "10")) class Stream: @@ -220,7 +223,7 @@ def pin_counter_page(pid, pinned_pids): % (pid, COUNTER_LO, COUNTER_HI)) try: r = subprocess.run([VMEXEC, "exec", cmd], capture_output=True, - text=True, timeout=60) + text=True, timeout=25) except Exception: return False n = 0 @@ -252,6 +255,7 @@ def main(): last_seen_t = {} dseen = set() counter_pids = set() + counter_next_try = {} last_pid = None stream = Stream(SEMSTREAM) loop = 0 @@ -386,16 +390,21 @@ def main(): # Pin the completion-counter page of a newly started client, before its # first GR pushes decode their releases. The pid comes from the guest - # scanner; the guest-agent fallback is only a safety net. + # scanner; the guest-agent fallback is only a safety net. The exec can + # block for its whole timeout under client load, so retry on a slow + # cadence: hammering the agent here starves it and wedges the run. if last_pid is None and loop % TRACE_EVERY == 0: last_pid = cdt_pid() if last_pid: log("counter: client pid %d (agent)" % last_pid) if last_pid and last_pid not in counter_pids: - try: - pin_counter_page(last_pid, counter_pids) - except Exception as e: - log("counter: pin failed: %s" % e) + now = time.time() + if now >= counter_next_try.get(last_pid, 0): + try: + pin_counter_page(last_pid, counter_pids) + except Exception as e: + log("counter: pin failed: %s" % e) + counter_next_try[last_pid] = now + COUNTER_RETRY_S time.sleep(interval) -- cgit v1.2.3