summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/guest-diag/gl_frames.c237
-rw-r--r--tools/guest-diag/scrub_pin_qmp.py219
2 files changed, 444 insertions, 12 deletions
diff --git a/tools/guest-diag/gl_frames.c b/tools/guest-diag/gl_frames.c
new file mode 100644
index 0000000..d442d66
--- /dev/null
+++ b/tools/guest-diag/gl_frames.c
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * gl_frames.c - render animated frames with direct GL (GLX pbuffer) on the
+ * guest's NVIDIA vGPU and publish them into the ivshmem (kvmfr) BAR so the
+ * host can see what the GL client actually rendered.
+ *
+ * This is the GL sibling of tools/cuda-ab/frame_producer.cu: the transport
+ * (guest GPU -> guest sysmem -> ivshmem -> host frame_reader.py) is identical,
+ * but the pixels come from the direct-GL client channel instead of CUDA.
+ * The pbuffer avoids the X window draw path that wedges the X server.
+ *
+ * Layout at BAR offset 0x100000 (same as frame_producer.cu):
+ * struct frame_hdr { magic, version, width, height, stride, format, seq }
+ * then width*height*4 bytes of RGBA8.
+ *
+ * Usage: sudo ./gl_frames [frames] [width] [height] [fps]
+ * Prints: FRAME seq=<n> ms=<render+readback ms>
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <time.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <glob.h>
+#include <math.h>
+#include <X11/Xlib.h>
+#include <GL/glx.h>
+#include <GL/gl.h>
+
+#define FRAME_OFFSET 0x100000UL
+#define HDR_MAGIC 0x4b564d46u /* "FMVK" little-endian-ish */
+#define BAR_BYTES (128UL << 20)
+#define BAR_PATH_DEF "/sys/bus/pci/devices/0000:00:04.0/resource2"
+
+struct frame_hdr {
+ uint32_t magic;
+ uint32_t version;
+ uint32_t width;
+ uint32_t height;
+ uint32_t stride;
+ uint32_t format; /* 0 = RGBA8 */
+ uint32_t seq;
+ uint32_t reserved;
+ uint64_t timestamp_ns;
+};
+
+/*
+ * The ivshmem kvmfr device moves with the QEMU topology (it was 00:03.0 in an
+ * older launch, 00:04.0 in the current one). Discover the 128 MiB BAR instead
+ * of hardcoding a slot; GL_FRAMES_BAR overrides for debugging.
+ */
+static const char *find_bar(char *buf, size_t len)
+{
+ glob_t g;
+ size_t i;
+ const char *hit = NULL;
+
+ if (glob("/sys/bus/pci/devices/*/resource2", 0, NULL, &g) != 0)
+ return NULL;
+ for (i = 0; i < g.gl_pathc; i++) {
+ struct stat st;
+
+ if (stat(g.gl_pathv[i], &st) == 0 &&
+ (unsigned long long)st.st_size == BAR_BYTES) {
+ snprintf(buf, len, "%s", g.gl_pathv[i]);
+ hit = buf;
+ break;
+ }
+ }
+ globfree(&g);
+ return hit;
+}
+
+static uint64_t now_ns(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (uint64_t)ts.tv_sec * 1000000000ull + ts.tv_nsec;
+}
+
+static void draw_frame(int f, int w, int h)
+{
+ float a = (float)f * 0.15f;
+ int i;
+
+ glViewport(0, 0, w, h);
+ glClearColor(0.05f, 0.05f, 0.10f, 1.0f);
+ glClear(GL_COLOR_BUFFER_BIT);
+
+ glMatrixMode(GL_PROJECTION);
+ glLoadIdentity();
+ glOrtho(-1.0, 1.0, -1.0, 1.0, -1.0, 1.0);
+ glMatrixMode(GL_MODELVIEW);
+ glLoadIdentity();
+
+ for (i = 0; i < 8; i++) {
+ float t = a + (float)i * 0.78539816339f;
+
+ glBegin(GL_TRIANGLES);
+ glColor3f(1.0f, 0.2f + 0.09f * i, 0.1f);
+ glVertex2f(0.0f, 0.0f);
+ glColor3f(0.1f, 0.9f, 0.3f);
+ glVertex2f(0.75f * cosf(t), 0.75f * sinf(t));
+ glColor3f(0.2f, 0.3f, 1.0f);
+ glVertex2f(0.55f * cosf(t + 0.5f),
+ 0.55f * sinf(t + 0.5f));
+ glEnd();
+ }
+ glFinish();
+}
+
+int main(int argc, char **argv)
+{
+ int frames = argc > 1 ? atoi(argv[1]) : 30;
+ int w = argc > 2 ? atoi(argv[2]) : 640;
+ int h = argc > 3 ? atoi(argv[3]) : 480;
+ int fps = argc > 4 ? atoi(argv[4]) : 5;
+ static int attribs[] = {
+ GLX_DRAWABLE_TYPE, GLX_PBUFFER_BIT,
+ GLX_RENDER_TYPE, GLX_RGBA_BIT,
+ GLX_RED_SIZE, 8, GLX_GREEN_SIZE, 8, GLX_BLUE_SIZE, 8,
+ GLX_ALPHA_SIZE, 8,
+ None
+ };
+ GLXFBConfig *fb;
+ GLXPbuffer pb;
+ XVisualInfo *vi;
+ GLXContext ctx;
+ Display *d;
+ int scr, n = 0, fd, i;
+ unsigned char *bar, *px;
+ struct frame_hdr *hdr;
+ volatile uint32_t *seqp;
+
+ char found[256];
+ const char *bar_path = getenv("GL_FRAMES_BAR");
+
+ setvbuf(stdout, NULL, _IONBF, 0);
+ if (!bar_path)
+ bar_path = find_bar(found, sizeof(found));
+ if (!bar_path)
+ bar_path = BAR_PATH_DEF;
+ printf("BAR %s\n", bar_path);
+ fd = open(bar_path, O_RDWR | O_SYNC);
+ if (fd < 0) {
+ printf("OPEN_FAIL %s\n", bar_path);
+ return 1;
+ }
+ bar = (unsigned char *)mmap(NULL, BAR_BYTES, PROT_READ | PROT_WRITE,
+ MAP_SHARED, fd, 0);
+ if (bar == MAP_FAILED) {
+ printf("MMAP_FAIL\n");
+ return 1;
+ }
+ hdr = (struct frame_hdr *)(bar + FRAME_OFFSET);
+ seqp = &hdr->seq;
+
+ d = XOpenDisplay(NULL);
+ if (!d) {
+ printf("NODISPLAY\n");
+ return 2;
+ }
+ scr = DefaultScreen(d);
+ fb = glXChooseFBConfig(d, scr, attribs, &n);
+ if (!fb || n < 1) {
+ printf("NOFBCONFIG\n");
+ return 3;
+ }
+ pb = glXCreatePbuffer(d, fb[0], (int[]){
+ GLX_PBUFFER_WIDTH, w, GLX_PBUFFER_HEIGHT, h, None });
+ if (!pb) {
+ printf("NOPBUFFER\n");
+ return 4;
+ }
+ vi = glXGetVisualFromFBConfig(d, fb[0]);
+ ctx = glXCreateContext(d, vi, NULL, True);
+ if (!ctx) {
+ printf("NOCTX\n");
+ return 5;
+ }
+ if (!glXMakeCurrent(d, pb, ctx)) {
+ printf("MAKECURRENT_FAIL\n");
+ return 6;
+ }
+ printf("GL %s | %s\n", (const char *)glGetString(GL_VENDOR),
+ (const char *)glGetString(GL_RENDERER));
+
+ px = (unsigned char *)malloc((size_t)w * h * 4);
+ if (!px) {
+ printf("HOST_ALLOC_FAIL\n");
+ return 7;
+ }
+
+ hdr->magic = HDR_MAGIC;
+ hdr->version = 1;
+ hdr->width = (uint32_t)w;
+ hdr->height = (uint32_t)h;
+ hdr->stride = (uint32_t)w * 4;
+ hdr->format = 0;
+
+ for (i = 0; i < frames; i++) {
+ uint64_t t0 = now_ns();
+ uint32_t seq;
+
+ draw_frame(i, w, h);
+ glReadPixels(0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, px);
+ if (glGetError() != GL_NO_ERROR)
+ printf("GL_ERROR frame=%d\n", i);
+ memcpy(bar + FRAME_OFFSET + sizeof(struct frame_hdr), px,
+ (size_t)w * h * 4);
+ hdr->timestamp_ns = now_ns();
+ __sync_synchronize();
+ seq = (uint32_t)(i + 1);
+ *seqp = seq;
+ __sync_synchronize();
+ printf("FRAME seq=%u ms=%.1f px0=%02x%02x%02x%02x\n", seq,
+ (double)(now_ns() - t0) / 1e6,
+ px[0], px[1], px[2], px[3]);
+ if (fps > 0)
+ usleep((useconds_t)(1000000 / fps));
+ }
+ printf("FRAME_DONE frames=%d %dx%d\n", frames, w, h);
+
+ glXMakeCurrent(d, None, NULL);
+ glXDestroyContext(d, ctx);
+ glXDestroyPbuffer(d, pb);
+ XCloseDisplay(d);
+ free(px);
+ munmap(bar, BAR_BYTES);
+ close(fd);
+ return 0;
+}
diff --git a/tools/guest-diag/scrub_pin_qmp.py b/tools/guest-diag/scrub_pin_qmp.py
index a69f8eb..653bb5d 100644
--- a/tools/guest-diag/scrub_pin_qmp.py
+++ b/tools/guest-diag/scrub_pin_qmp.py
@@ -44,12 +44,32 @@ def log(msg):
print("[%s] %s" % (time.strftime("%H:%M:%S"), msg), flush=True)
-def load_qmp():
+QMP_PATH = "/tmp/qemu-qmp.sock"
+
+
+def load_qmp(wait=0.0):
+ """Connect to the QEMU monitor, retrying until it exists.
+
+ The systemd unit starts at host boot, before the VM exists, and cycle.sh
+ removes and re-creates the socket on every VM cycle. Wait for the socket
+ instead of exiting, so one long-lived scanner covers the whole session
+ (it must already be scanning when Xorg creates its DDX channel buffer -
+ that is the point of running it from boot)."""
spec = importlib.util.spec_from_file_location(
"semfix_watch", os.path.join(HERE, "semfix_watch.py"))
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
- return mod.Qmp("/tmp/qemu-qmp.sock")
+ deadline = time.time() + wait if wait else None
+ last = "socket missing"
+ while True:
+ if os.path.exists(QMP_PATH):
+ try:
+ return mod.Qmp(QMP_PATH)
+ except Exception as e:
+ last = e
+ if deadline is not None and time.time() >= deadline:
+ raise SystemExit("no QMP socket: %s" % last)
+ time.sleep(2)
def vmexec(cmd, timeout=25):
@@ -113,6 +133,22 @@ def ring_ok(q, ring_pa):
return True
+def wait_shape(q, hsem):
+ """A channel-buffer wait signature: the host semaphore reads a
+ plausible consumed count and the finish word is still zero.
+
+ r1749: the DDX's own channel buffer does not expose the CeUtils ring
+ layout at +0x64000 (its ring reads the semaphore word 0xfff), so the
+ ring test rejected exactly the page X was blocked on
+ (_nv015102rm timed wait on vmap+0x6c004). The consumed value with a
+ zero finish word is the wait the release pair exists to end."""
+ hv = phys_read64(q, hsem, n=4)
+ if len(hv) < 2:
+ return False
+ return 0 < hv[0] <= 0x20000 and hv[1] == 0 and hv[2] == 0 \
+ and hv[3] == 0
+
+
def register(fin_pa, hsem_pa):
req = "0x%x,0x%x" % (fin_pa, hsem_pa)
r = subprocess.run(["sudo", "tee", PARAM], input=req + "\n",
@@ -134,13 +170,148 @@ def scan(q):
continue
if not hsem or not ring:
continue
- if not ring_ok(q, ring):
- log("reject vmap %#x (ring shape)" % base)
- continue
+ # r1749/r1752: the ring-shape test rejected exactly the buffers
+ # the guest was blocked on, and the QMP physical reads are not
+ # stable enough to shape-test a moving page. The candidate set is
+ # already narrow (nv_vm_map_pages area of a channel-buffer size),
+ # so register every stable candidate; the service only authors a
+ # record the guest itself polls (and only while !submitted).
+ #
+ # r1760: in the single green cycle the strict "two identical reads"
+ # gate skipped the live DDX wait page (hsem_pa=0x103789000) for the
+ # WHOLE cycle -> no registration -> the Xorg DDX wedge the release
+ # exists to end. Retry the read a few times (the flakiness is often
+ # a single bad sample), and accept a page that reads wait-shaped
+ # even when the samples disagree: consumed count in (0, 0x20000]
+ # with a zero finish word is the exact signature of the wait this
+ # registration releases (wait_shape below).
+ stable = None
+ prev = None
+ last = []
+ for _ in range(6):
+ hv = phys_read64(q, hsem, n=2)
+ if len(hv) < 2:
+ continue
+ last = hv
+ if prev is not None and hv == prev:
+ stable = hv
+ break
+ prev = hv
+ if stable is None:
+ # r1760: the live page read hsem=0x20000064/fin=0x6001 while the
+ # DDX spun on it and every sample pair disagreed. The page the
+ # guest waits on IS a live channel buffer; a nonzero sema word
+ # at the channel-buffer sema offset is that signature. Register
+ # with a loud log; the service only authors records the guest
+ # polls and (for all-ones) only while no USERD submissions are
+ # observed, so a false registration is bounded.
+ if wait_shape(q, hsem) or (last and last[0]):
+ log("register vmap %#x (unstable but live: last=%s) "
+ "hsem_pa=%#x" % (base, [hex(v) for v in last], hsem))
+ else:
+ log("skip vmap %#x (unstable read) hsem_pa=%#x" %
+ (base, hsem))
+ continue
pages.append((hsem + 4, hsem))
+ wp = walk_page_cached(SCRUB_SEMA_VA)
+ if wp:
+ pages.append((wp + 4, wp))
return pages
+# r1762: the DDX scrub wait polls the CeUtils scrubber channel's sema/fin
+# page at the GPU VA 0x12006c000 (+4 = fin, the mem_scrub finish word), which
+# the service's kernel-VAS walk resolves (walk_probe) - NOT the guest-vmap
+# candidate pages this scanner used to register (0x128013000 was already
+# all-ones and the guest still spun; registering the walk-resolved
+# 0x12801b000 released the wait in ~23 ms). Register both: the vmap path
+# stays the fallback for boots where the walk is stale (r1712).
+SCRUB_SEMA_VA = 0x12006c000
+WALK_PARAM = "/sys/module/vgsp/parameters/walk_probe"
+_walk_cache = {"va": 0, "pa": 0, "at": 0.0}
+
+
+def host_walk_page(va):
+ """Resolve a GPU VA to a guest PA through the service's kernel-VAS walk.
+
+ walk_probe is a module parameter; the result lands in dmesg as the last
+ 'sema-walk-probe: ... discovered rc=0 pa=0x...' block."""
+ try:
+ r = subprocess.run(["sudo", "tee", WALK_PARAM],
+ input="0x%x\n" % va,
+ capture_output=True, text=True)
+ if r.returncode != 0:
+ return 0
+ out = subprocess.run(["sudo", "dmesg"], capture_output=True,
+ text=True).stdout
+ except Exception:
+ return 0
+ pa = 0
+ kern_pa = 0
+ vas_h = None
+ for line in out.splitlines():
+ if "sema-walk-probe: va=0x" in line:
+ pa = 0 # a newer block begins; only its result counts
+ kern_pa = 0
+ vas_h = None
+ elif "sema-walk-probe: vas[" in line:
+ try:
+ vas_h = int(line.split("h=0x", 1)[1].split()[0], 16)
+ except Exception:
+ vas_h = None
+ elif "sema-walk-probe:" in line and "rc=0 pa=0x" in line:
+ try:
+ p = int(line.rsplit("pa=0x", 1)[1].split()[0], 16)
+ except Exception:
+ p = 0
+ if "discovered" in line:
+ pa = p
+ elif vas_h == 0xc and p:
+ # r1764: the summary 'discovered' path can fail (rc=-2)
+ # while the per-VAS kernel row (h=0xc) still resolves the
+ # polled scrubber sema page (r1667-authoritative).
+ kern_pa = p
+ return pa or kern_pa
+
+
+def walk_page_cached(va):
+ now = time.time()
+ if _walk_cache["va"] == va and now - _walk_cache["at"] < 15:
+ return _walk_cache["pa"]
+ pa = host_walk_page(va)
+ if pa != _walk_cache["pa"] or not _walk_cache["at"]:
+ log("scrubber walk page va=%#x -> pa=%#x (fin=%#x)" %
+ (va, pa, pa + 4))
+ _walk_cache.update({"va": va, "pa": pa, "at": now})
+ return pa
+
+
+def module_generation():
+ """Identity of the current vgsp module + QEMU instance.
+
+ scrub_pin_qmp is long-lived: cycle.sh rmmod/insmods vgsp and relaunches
+ QEMU between rounds. Both reset the registration state the scanner
+ tracks implicitly, so a changed generation must force a full re-register
+ even when the guest's vmap addresses happen to repeat."""
+ mod = "?"
+ try:
+ with open("/proc/modules") as f:
+ for line in f:
+ if line.startswith("vgsp "):
+ f = line.split()
+ mod = f[0] + "@" + (f[5] if len(f) > 5 else "?")
+ break
+ except Exception:
+ pass
+ pid = "?"
+ try:
+ with open("/tmp/qemu-harness.pid") as f:
+ pid = f.read().strip()
+ except Exception:
+ pass
+ return (mod, pid)
+
+
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--duration", type=float, default=0, help="seconds (0 = forever)")
@@ -149,25 +320,49 @@ def main():
q = load_qmp()
cur = None
+ gen = None
+ last_refresh = 0.0
start = time.time()
while True:
if args.duration and time.time() - start >= args.duration:
break
+ g = module_generation()
+ if g != gen:
+ log("generation %s -> %s, re-registering" % (gen, g))
+ gen = g
+ cur = None
try:
pages = scan(q)
except Exception as e:
log("scan failed: %s" % e)
pages = []
key = tuple(sorted(pages))
+ now = time.time()
if key != cur:
- if cur is not None:
- clear()
+ if cur is not None and not key:
+ # r1743: a transiently-empty scan (the live buffer was only
+ # visible through a rejected alias for a pass) used to
+ # clear() and the release stopped, wedging the DDX wait the
+ # registration exists to release. Keep the live records;
+ # only a *different non-empty* set replaces them.
+ log("scan empty; keeping %d registration(s)" % len(cur))
+ else:
+ if cur is not None:
+ clear()
+ for fin, hsem in pages:
+ if register(fin, hsem):
+ log("registered fin=%#x hsem=%#x" % (fin, hsem))
+ else:
+ log("register failed fin=%#x hsem=%#x" % (fin, hsem))
+ cur = key
+ last_refresh = now
+ elif key and now - last_refresh > 30:
+ # Additive re-assert: the service reaps records on its own
+ # (bounded registry); re-registering is idempotent because
+ # records are keyed by hsem_pa.
for fin, hsem in pages:
- if register(fin, hsem):
- log("registered fin=%#x hsem=%#x" % (fin, hsem))
- else:
- log("register failed fin=%#x hsem=%#x" % (fin, hsem))
- cur = key
+ register(fin, hsem)
+ last_refresh = now
time.sleep(args.interval)
return 0
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-11 12:20:14 +0000