guest-diag: stream uvmt sems over vsock (no guest-agent race); dump channel last release VA; larger push log budget

4 files changed, 149 insertions, 23 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b29e833..9d116ad 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -2557,7 +2557,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		off += 4 + count * 4;
 	}
 
-	if (ch_h && svc.entry_log < 64) {
+	if (ch_h && svc.entry_log < 512) {
 		svc.entry_log++;
 		{
 			u8 tail[24];
@@ -4477,11 +4477,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu)
 			svc_read_ap(vgpu, ch->notifier_pa + 16 + 8,
 				    ch->notifier_as, &info1, 4);
 		}
-		pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
+		pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
 			ch->h_object, ch->engine_type, ch->gpfifo_va,
 			ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
 			ch->userd_ok, ch->last_get, ch->submissions,
-			ch->expected_sem_payload, ch->notifier_pa,
+			ch->expected_sem_payload, ch->last_rel_va,
+			ch->counter_pa,
+			ch->notifier_pa,
 			ch->notifier_as, info0, st0, info1, st1,
 			ch->notified);
 	}
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index bb3f99a..1ff0d3f 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -11,7 +11,9 @@
 #
 # Usage: sudo python3 guest_pin_stream.py [host-cid] [port] [lo-hex] [hi-hex] [pa-max-hex]
 import os
+import re
 import socket
+import subprocess
 import sys
 import time
 
@@ -24,6 +26,30 @@ INTERVAL = float(os.environ.get("STREAM_INTERVAL", "0.25"))
 BATCH = 4096
 REGION = 0x200000
 SPAN = int(os.environ.get("STREAM_SPAN", "0x10000"), 16)
+TRACE = ("/sys/kernel/tracing/trace"
+         if os.path.exists("/sys/kernel/tracing/trace")
+         else "/sys/kernel/debug/tracing/trace")
+SEM_EVERY = int(os.environ.get("STREAM_SEM_EVERY", "4"))
+
+
+def new_sems(seen):
+    """New tracking-sem kptrs (uvmt lines) in the trace tail.
+
+    Read in the guest so the host does not need a guest-agent round trip for
+    every semaphore discovery; the trace is only tailed while a client runs.
+    """
+    try:
+        out = subprocess.run(["tail", "-c", "262144", TRACE],
+                             capture_output=True, timeout=10).stdout
+    except Exception:
+        return []
+    found = []
+    for m in re.finditer(rb"sem=0x([0-9a-fA-F]+)", out):
+        v = int(m.group(1), 16)
+        if v not in seen:
+            seen.add(v)
+            found.append(v)
+    return found
 
 
 def cdt_pid():
@@ -82,18 +108,23 @@ def main():
         f.write('%d\n' % os.getpid())
     sock = None
     seen = {}
+    seen_sems = set()
     last_pid = None
     last_log = 0.0
+    loop = 0
     while True:
         pid = cdt_pid()
+        loop += 1
         if pid is None:
             last_pid = None
             seen = {}
+            seen_sems = set()
             time.sleep(INTERVAL)
             continue
         if pid != last_pid:
             last_pid = pid
             seen = {}
+            seen_sems = set()
             if sock is not None:
                 try:
                     sock.sendall(("#pid %d\n" % pid).encode())
@@ -109,28 +140,33 @@ def main():
         dt = time.time() - t0
         new = [(va, pa) for va, pa in cur.items() if seen.get(va) != pa]
         seen = cur
+        lines = []
+        if loop % SEM_EVERY == 0:
+            for sem in new_sems(seen_sems):
+                lines.append('#sem 0x%x\n' % sem)
         now = time.time()
         if now - last_log > 1.0:
             last_log = now
             try:
                 with open('/tmp/guest_pin_stream.log', 'a') as f:
-                    f.write('%s pid=%s present=%d new=%d scan=%.3fs\n'
-                            % (time.strftime('%T'), pid, len(cur), len(new), dt))
+                    f.write('%s pid=%s present=%d new=%d sem=%d scan=%.3fs\n'
+                            % (time.strftime('%T'), pid, len(cur), len(new),
+                               len(seen_sems), dt))
             except OSError:
                 pass
-        if not new:
+        if not new and not lines:
             time.sleep(INTERVAL)
             continue
-        new.sort()
-        lines = []
-        va0, pa0, n = new[0][0], new[0][1], 1
-        for va, pa in new[1:]:
-            if va == va0 + n * 0x1000 and pa == pa0 + n * 0x1000:
-                n += 1
-                continue
+        if new:
+            new.sort()
+            va0, pa0, n = new[0][0], new[0][1], 1
+            for va, pa in new[1:]:
+                if va == va0 + n * 0x1000 and pa == pa0 + n * 0x1000:
+                    n += 1
+                    continue
+                lines.append('0x%x,0x%x,%d\n' % (va0, pa0, n))
+                va0, pa0, n = va, pa, 1
             lines.append('0x%x,0x%x,%d\n' % (va0, pa0, n))
-            va0, pa0, n = va, pa, 1
-        lines.append('0x%x,0x%x,%d\n' % (va0, pa0, n))
         payload = ''.join(lines).encode()
         if sock is None:
             try:
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index c1ef7b2..44f8681 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -25,12 +25,23 @@ PORT = int(os.environ.get("PINWIN_PORT", "45678"))
 LO = os.environ.get("PINWIN_LO", "0x200400000")
 HI = os.environ.get("PINWIN_HI", "0x206000000")
 PAMAX = os.environ.get("PINWIN_PA_MAX", "0x180000000")
+SEMSTREAM = os.environ.get("SEM_STREAM_FILE", "/tmp/opencode/sem_stream.txt")
 
 
 def log(msg):
     print("[%s] %s" % (time.strftime("%H:%M:%S"), msg), flush=True)
 
 
+def _mirror(line):
+    """Append guest control lines (#pid/#sem) to the shared host file so the
+    single-QMP watcher can consume them without guest-agent round trips."""
+    try:
+        with open(SEMSTREAM, "a") as f:
+            f.write(line + "\n")
+    except OSError:
+        pass
+
+
 def vexec(cmd, timeout=40):
     try:
         r = subprocess.run([VMEXEC, "exec", "--timeout", str(timeout - 5), cmd],
@@ -115,6 +126,10 @@ def main():
                     m = re.search(r"(\d+)", text)
                     seen = {}
                     log("guest reports cdt pid %s" % (m.group(1) if m else "?"))
+                    _mirror(text)
+                    continue
+                if text.startswith("#sem"):
+                    _mirror(text)
                     continue
                 m = re.match(r"^(0x[0-9a-f]+),(0x[0-9a-f]+)(?:,(\d+))?$", text)
                 if m:
diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py
index 0d31394..fb0e92e 100755
--- a/tools/guest-diag/semfix_uvmt.py
+++ b/tools/guest-diag/semfix_uvmt.py
@@ -41,6 +41,53 @@ DLOG = os.environ.get("UVMT_DMESG", "1") == "1"
 VMEXEC = os.path.normpath(os.path.join(HERE, "..", "..", "scripts", "vgpu_vm.py"))
 COUNTER_LO = int(os.environ.get("COUNTER_LO", "0x20400f000"), 16)
 COUNTER_HI = int(os.environ.get("COUNTER_HI", "0x204010000"), 16)
+SEMSTREAM = os.environ.get("SEM_STREAM_FILE", "/tmp/opencode/sem_stream.txt")
+TRACE_EVERY = int(os.environ.get("UVMT_TRACE_EVERY", "5"))
+
+
+class Stream:
+    """Tail the guest scanner's control lines mirrored by pin_gr_stream.py.
+
+    The guest emits "#pid <n>" and "#sem 0x<k>" over vsock; the listener
+    appends them here. This replaces per-loop guest-agent trace reads, which
+    were too slow to catch short-lived tracking semaphores.
+    """
+
+    def __init__(self, path):
+        self.path = path
+        self.off = 0
+        self.sems = []
+        self.pids = []
+
+    def poll(self):
+        try:
+            with open(self.path) as f:
+                f.seek(0, os.SEEK_END)
+                size = f.tell()
+                if size < self.off:
+                    self.off = 0
+                f.seek(self.off)
+                data = f.read()
+                self.off = f.tell()
+        except OSError:
+            return [], []
+        new_sems, new_pids = [], []
+        for line in data.splitlines():
+            if line.startswith("#sem"):
+                m = re.search(r"0x([0-9a-fA-F]+)", line)
+                if m:
+                    v = int(m.group(1), 16)
+                    if v not in self.sems:
+                        self.sems.append(v)
+                        new_sems.append(v)
+            elif line.startswith("#pid"):
+                m = re.search(r"(\d+)", line)
+                if m:
+                    v = int(m.group(1))
+                    if v not in self.pids:
+                        self.pids.append(v)
+                        new_pids.append(v)
+        return new_sems, new_pids
 
 
 def log(msg):
@@ -191,6 +238,8 @@ def main():
     dseen = set()
     counter_pids = set()
     last_pid = None
+    stream = Stream(SEMSTREAM)
+    loop = 0
 
     def pin_range_once(gpu_vas, cpu, gpa, size=0):
         key = tuple(gpu_vas), cpu, size
@@ -200,7 +249,30 @@ def main():
         pin_pushbuffer(q, gpu_vas, cpu, gpa, size)
 
     while True:
-        sems, pbs, chans = guest_trace()
+        loop += 1
+        sems = set()
+        pbs = set()
+        chans = set()
+
+        new_sems, new_pids = stream.poll()
+        for s in new_sems:
+            log("stream: new uvmt sem 0x%x" % s)
+        for p in new_pids:
+            log("stream: client pid %d" % p)
+            last_pid = p
+        sems.update(stream.sems)
+
+        # The guest-agent trace read is slow under client load; use it only
+        # as a fallback for channel/pushbuffer discovery and to recover sems
+        # the scanner may have missed.
+        if loop % TRACE_EVERY == 0:
+            try:
+                t_sems, t_pbs, t_chans = guest_trace()
+            except Exception:
+                t_sems, t_pbs, t_chans = set(), set(), set()
+            sems.update(t_sems)
+            pbs.update(t_pbs)
+            chans.update(t_chans)
 
         # Poll every sem ever seen: the client's final wait may only appear in
         # the trace once, and the trace ring wraps while it spins.
@@ -293,14 +365,15 @@ def main():
             dmesg_releases(dseen)
 
         # Pin the completion-counter page of a newly started client, before its
-        # first GR pushes decode their releases.
-        pid = cdt_pid()
-        if pid and pid != last_pid:
-            last_pid = pid
-            log("counter: client pid %d" % pid)
-        if pid and pid not in counter_pids:
+        # first GR pushes decode their releases. The pid comes from the guest
+        # scanner; the guest-agent fallback is only a safety net.
+        if last_pid is None and loop % TRACE_EVERY == 0:
+            last_pid = cdt_pid()
+            if last_pid:
+                log("counter: client pid %d (agent)" % last_pid)
+        if last_pid and last_pid not in counter_pids:
             try:
-                pin_counter_page(pid, counter_pids)
+                pin_counter_page(last_pid, counter_pids)
             except Exception as e:
                 log("counter: pin failed: %s" % e)