guest-diag: single-instance scanner start + full-re-assert backoff

r148 ran THREE guest scanners (pin_gr_stream install_and_start retried
while a partially-failed first attempt had already started one, and the
pid-file kill missed it). Each scanner re-asserted the whole CE pair set
every 2 s; with a fully fragmented pair set (descending PAs, one page per
line) that is ~2 k debugfs writes/s per scanner, and the guest hit load 19
with clients wedged spinning in userspace.

- pin_gr_stream._start_cmd: kill EVERY existing scanner first (pattern
  assembled at runtime with printf so the command's own cmdline cannot
  self-match), then start exactly one.
- guest_pin_stream._emit_ce_mem_delta: the full re-assert keeps its 1.5 s
  cadence for the first STREAM_FULLREM_FAST_S seconds of a mem's life (the
  client-switch flush race), then backs off to STREAM_FULLREM_SLOW_S (10 s)
  as a safety net. Configurable via env.

2 files changed, 24 insertions, 10 deletions

diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index 7d07f55..e80751a 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -458,7 +458,9 @@ def _emit_ce_mem_delta(kc, st, out, fresh_map=False):
     nowm = time.monotonic()
     full = nowm >= st.get("full_next", 0.0)
     if full:
-        st["full_next"] = nowm + KC_FULLREM_S
+        age = nowm - st.get("resolved_at", nowm)
+        st["full_next"] = nowm + (KC_FULLREM_S if age < KC_FULLREM_FAST_S
+                                  else KC_FULLREM_SLOW_S)
     if fresh_map:
         for k in range(npages):
             kc._pa_cache.pop(cpu + (k + off) * 0x1000, None)
@@ -680,8 +682,15 @@ KC_REMAP_S = 30.0		# full re-translation sweep (catches remaps)
 # whole translatable pair set regardless of the delta, healing host-side
 # session-flush wipes (see _emit_ce_mem_delta). Must stay well under the
 # host's gsp_sema_ce_grace_ms (8 s) so a wiped-and-needed page is restored
-# inside the hold window.
-KC_FULLREM_S = 1.5
+# inside the hold window. The fast cadence applies to a mem's first
+# KC_FULLREM_FAST_S seconds (the flush race lives at the client switch);
+# after that the re-assert backs off to KC_FULLREM_SLOW_S - a fully
+# fragmented pair set (descending PAs) emits one page per line, and 4 k
+# lines every 2 s saturated a guest under multi-client load (r148: load 19,
+# three scanners from an install race, clients wedged).
+KC_FULLREM_S = float(os.environ.get("STREAM_FULLREM_S", "1.5"))
+KC_FULLREM_FAST_S = float(os.environ.get("STREAM_FULLREM_FAST_S", "45"))
+KC_FULLREM_SLOW_S = float(os.environ.get("STREAM_FULLREM_SLOW_S", "10.0"))
 
 
 def _chain_mems_at(kc, w):
@@ -870,7 +879,8 @@ def resolve_pb_chain(kc, sem, out):
     gpus, cpu, off, npages = params
     st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus,
           "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0,
-          "page_off": off, "npages": npages, "last": {}}
+          "page_off": off, "npages": npages, "last": {},
+          "resolved_at": time.monotonic()}
     # Fresh translation for the first sweep: the Kcore PA cache may hold
     # pairs from before this mem's vmap area was (re)populated.
     kc._pa_cache.clear()
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index c0a8c9d..2e45bbf 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -62,12 +62,16 @@ def _start_cmd():
                              capture_output=True, text=True).stdout
     prefixes = os.environ.get("CLIENT_PREFIXES", "cdt,va,victory")
     vma_every = os.environ.get("STREAM_VMA_EVERY", "1")
-    return ("echo '%s' | base64 -d > /tmp/guest_pin_stream.py; "
-            "if [ -f /tmp/guest_pin_stream.pid ]; then "
-            "p=$(cat /tmp/guest_pin_stream.pid 2>/dev/null); "
-            "[ -n \"$p\" ] && grep -q guest_pin_stream "
-            "/proc/$p/cmdline 2>/dev/null && sudo kill $p 2>/dev/null; "
-            "fi; "
+    # Kill EVERY existing scanner, not just the pid-file one: a retry after
+    # a partially-failed first attempt (pid file not written yet) used to
+    # leave the first scanner alive, and multiple scanners each re-assert
+    # the whole CE pair set - r148 ran three and drove the guest to load 19
+    # (clients wedged in userspace spins). The pattern is assembled at
+    # runtime via printf so this command's own cmdline (which contains the
+    # literal file path) cannot match the pkill.
+    return ("P=$(printf 'guest_pin_%%s.py' stream); "
+            "sudo pkill -f \"$P\" 2>/dev/null; sleep 1; "
+            "echo '%s' | base64 -d > /tmp/guest_pin_stream.py; "
             "sudo rm -f /tmp/guest_pin_stream.pid; "
             "sudo setsid nohup env CLIENT_PREFIXES='%s' "
             "STREAM_VMA_EVERY='%s' "