vgsp: direct-GL ring/push rescue, VAS/pdb scanners and PTIMER tick; r1780 cycle tooling

3 files changed, 376 insertions, 0 deletions

diff --git a/tools/guest-diag/cycle-r1780.sh b/tools/guest-diag/cycle-r1780.sh
new file mode 100755
index 0000000..23cf0d1
--- /dev/null
+++ b/tools/guest-diag/cycle-r1780.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2026 Open-IOV contributors
+# cycle-r1780.sh - durable vGPU cycle launcher for the r1780 campaign
+# (12 GiB file-backed guest RAM, kvmfr/kvmsr ivshmem, V31-class module).
+#
+# Usage: tools/guest-diag/cycle-r1780.sh [cycle-tag] [module.ko]
+# Env:   INSOMOD_PARAMS override args file (default params-r1780.txt)
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$HERE/../.." && pwd)"
+TAG="${1:-c27}"
+MODULE="${2:-$REPO/kernel/vgsp.ko}"
+ARGV_FILE="${INSMOD_PARAMS:-$HERE/params-r1780.txt}"
+BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
+DISK="${GUEST_DISK:-/home/rmcore/.local/libvf.io/kernel/linux.qcow2}"
+MDEV_PARENT="0000:01:00.0"
+MDEV_TYPE="nvidia-nvidia-vgpu-turing"
+RUNDIR="$HERE/run"
+QEMU_SCRIPT="$HERE/launch_qemu_r1780.run.sh"
+mkdir -p "$RUNDIR" "$BACKUP"
+
+echo "[cycle $TAG] stop QEMU"
+# Flush the guest before the forced teardown: a killed GL/X workload has
+# corrupted the ext4 root twice (c38 panic, c43 EIO), costing an offline fsck
+# cycle each time.
+if [ -S /tmp/qemu-guest-agent.sock ]; then
+    timeout 8 python3 - <<'PYEOF' >/dev/null 2>&1 || true
+import json, socket
+s = socket.socket(socket.AF_UNIX)
+s.settimeout(6)
+s.connect('/tmp/qemu-guest-agent.sock')
+for cmd in (["sync"], ["/bin/sh", "-c", "sync; echo 1 > /proc/sys/kernel/sysrq; echo o > /proc/sysrq-trigger"]):
+    try:
+        s.sendall(json.dumps({"execute": "guest-exec", "arguments": {"path": cmd[0], "arg": cmd[1:], "capture-output": True}}).encode() + b'\n')
+        s.recv(65536)
+    except Exception:
+        break
+s.close()
+PYEOF
+    sleep 2
+fi
+sudo pkill -f qemu-system-x86_64 2>/dev/null || true
+for i in $(seq 1 20); do pgrep -f qemu-system-x86_64 >/dev/null || break; sleep 0.5; done
+sudo pkill -9 -f qemu-system-x86_64 2>/dev/null || true
+sleep 1
+
+# Always repair the guest root after a forced teardown (cheap when clean,
+# prevents the recurring "guest-agent exec EIO / kill init" corruption class).
+DISK="/home/rmcore/.local/libvf.io/kernel/linux.qcow2"
+if [ -f "$DISK" ]; then
+    sudo modprobe nbd max_part=8 2>/dev/null || true
+    sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
+    if sudo qemu-nbd --connect=/dev/nbd0 "$DISK" -f qcow2 2>/dev/null; then
+        sleep 1
+        sudo e2fsck -fy /dev/nbd0p1 > "$RUNDIR/fsck-$TAG.log" 2>&1 || true
+        sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
+    fi
+fi
+
+echo "[cycle $TAG] remove mdevs"
+for m in /sys/bus/mdev/devices/*; do
+    [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null || true
+done
+
+echo "[cycle $TAG] unload vgsp"
+sudo rmmod vgsp 2>/dev/null || true
+for i in $(seq 1 20); do [ -d /sys/module/vgsp ] || break; sleep 0.5; done
+if [ -d /sys/module/vgsp ]; then
+    echo "FATAL: vgsp stuck loaded (see /proc/modules)"; exit 1
+fi
+
+echo "[cycle $TAG] load VFIO/mdev prerequisites"
+sudo modprobe vfio vfio_iommu_type1 vfio_pci mdev 2>/dev/null || true
+sudo modprobe vfio_iommu_type1 2>/dev/null || true
+
+echo "[cycle $TAG] stop nvidia-persistenced, power/control=on"
+sudo systemctl stop nvidia-persistenced 2>/dev/null || true
+echo on | sudo tee /sys/bus/pci/devices/0000:01:00.0/power/control >/dev/null
+echo on | sudo tee /sys/bus/pci/devices/0000:00:01.0/power/control >/dev/null 2>&1 || true
+
+echo "[cycle $TAG] insmod $(basename "$MODULE")"
+# shellcheck disable=SC2046
+sudo insmod "$MODULE" $(cat "$ARGV_FILE")
+
+U=$(uuidgen)
+echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null
+if [ ! -d "/sys/bus/mdev/devices/$U" ]; then echo "FATAL: mdev create failed"; exit 1; fi
+
+mkdir -p "/tmp/sockets/$U"
+sudo rm -f /tmp/qemu-qmp.sock /tmp/qemu-guest-agent.sock /tmp/qemu-harness.pid
+rm -f /dev/shm/vgpu-guest-ram 2>/dev/null || true
+touch /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
+chmod 660 /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
+
+QLOG="$BACKUP/${TAG}-qemu-session.txt"
+SL="$BACKUP/${TAG}-serial.log"
+cat > "$QEMU_SCRIPT" <<EOF
+#!/bin/bash
+exec /bin/qemu-system-x86_64 -D $QLOG -no-hpet \
+  -qmp unix:/tmp/qemu-qmp.sock,server,nowait \
+  -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 \
+  -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
+  -serial file:$SL -gdb tcp::1234 -display none \
+  -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr \
+  -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-$U,size=128M,share=yes \
+  -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr \
+  -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-$U,size=2M,share=yes \
+  -object memory-backend-file,id=pc.ram,size=12288M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on \
+  -uuid $U -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
+  -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on \
+  -rtc clock=host,base=localtime -m 12288 -smp cores=4,threads=1,sockets=1 \
+  -hda $DISK --enable-kvm \
+  -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 \
+  -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/$U,display=off \
+  -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 \
+  -net user -net nic -device vhost-vsock-pci,guest-cid=85770 \
+  -daemonize -pidfile /tmp/qemu-harness.pid
+EOF
+chmod +x "$QEMU_SCRIPT"
+
+echo "[cycle $TAG] launch QEMU (mdev $U)"
+sudo bash "$QEMU_SCRIPT" > "$RUNDIR/qemu-$TAG.log" 2>&1
+sleep 3
+if ! pgrep -f qemu-system-x86_64 >/dev/null; then
+    echo "FATAL: QEMU failed"; tail -5 "$RUNDIR/qemu-$TAG.log"; exit 1
+fi
+
+echo "[cycle $TAG] mdev $U"
+echo "$U" > "$RUNDIR/current-mdev"
+echo "$TAG" > "$RUNDIR/current-tag"
+
+for i in $(seq 1 90); do
+    if timeout 5 sudo python3 "$REPO/scripts/vgpu_vm.py" exec "echo CYCLE_UP" 2>/dev/null | grep -q CYCLE_UP; then
+        echo "[cycle $TAG] guest agent up"
+        exit 0
+    fi
+    sleep 2
+done
+echo "WARN: guest agent not reachable yet (serial: $SL)"
+exit 0
diff --git a/tools/guest-diag/gzf2_video.py b/tools/guest-diag/gzf2_video.py
new file mode 100644
index 0000000..aa88aa0
--- /dev/null
+++ b/tools/guest-diag/gzf2_video.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# gzf2_video.py - turn a GZCAPV01/GZF2 guest capture into a constant-rate mp4.
+#
+# Reads the append-only frame stream written by sdl_cap.c (LD_PRELOAD
+# SDL_GL_SwapWindow shim) or xgrab.c, resamples by CLOCK_MONOTONIC timestamps
+# (nearest frame), streams raw pixels straight into ffmpeg (no PPM
+# intermediates), and validates the encoded file with ffprobe.
+#
+# Format: [0..8) = "GZCAPV01", zero pad to 4096; then records:
+#   4B "GZF2" | u32 w | u32 h | u32 stride | u32 fmt | u64 ts_ns | u64 seq |
+#   stride*h bytes, top-down.  fmt 32 = RGBA8 (GL swap path), 33 = BGRA8
+#   (SDL_RenderReadPixels ARGB8888 / X surface).  A trailing
+#   "GZF2" + u64 0xffffffffffffffff marks a clean stop.
+#
+# Usage:
+#   gzf2_video.py CAPTURE OUT.mp4 [--fps 60] [--skip 5] [--duration 20]
+#                 [--encoder libx264|nvenc] [--crf 18]
+#                 [--pix-fmt rgba|bgra] [--info-only]
+import argparse
+import bisect
+import hashlib
+import json
+import os
+import struct
+import subprocess
+import sys
+
+MAGIC = b"GZCAPV01"
+SENTINEL = b"\xff" * 8
+
+
+def die(msg):
+    print(f"GZF2-ERROR {msg}", file=sys.stderr)
+    sys.exit(2)
+
+
+def index_capture(path):
+    size = os.path.getsize(path)
+    recs, clean, corrupt = [], False, 0
+    with open(path, "rb") as f:
+        hdr = f.read(4096)
+        if len(hdr) < 4096 or hdr[:8] != MAGIC:
+            die(f"{path}: not a GZCAPV01 file")
+        off = 4096
+        while off + 12 <= size:
+            f.seek(off)
+            head = f.read(36)
+            if len(head) < 12 or head[:4] != b"GZF2":
+                break
+            if head[4:12] == SENTINEL:
+                clean = True
+                break
+            if len(head) < 36:
+                break
+            w, h, stride, fmt = struct.unpack_from("<IIII", head, 4)
+            ts, seq = struct.unpack_from("<QQ", head, 20)
+            if not (0 < w <= 8192 and 0 < h <= 8192
+                    and 0 < stride <= 4 * 8192) or seq == 0xFFFFFFFFFFFFFFFF:
+                corrupt += 1
+                break
+            n = stride * h
+            if off + 36 + n > size:
+                corrupt += 1
+                break
+            f.seek(off + 36)
+            px = f.read(n)
+            if len(px) != n:
+                corrupt += 1
+                break
+            recs.append({
+                "off": off + 36, "ts": ts, "w": w, "h": h,
+                "stride": stride, "fmt": fmt, "size": n,
+                "md5": hashlib.md5(px).hexdigest(),
+            })
+            off += 36 + n
+    if not recs:
+        die(f"{path}: header only (no frames captured)")
+    return recs, clean, corrupt
+
+
+def sampled_black(px):
+    s = px[:65536]
+    return sum(s[0::4]) + sum(s[1::4]) + sum(s[2::4]) == 0
+
+
+def pick_frames(recs, fps, skip_s, duration_s):
+    t0 = recs[0]["ts"] + int(skip_s * 1e9)
+    cand = [r for r in recs if r["ts"] >= t0] or recs
+    ts = [r["ts"] for r in cand]
+    span = (ts[-1] - ts[0]) / 1e9
+    n = int(duration_s * fps) if duration_s else max(1, int(span * fps))
+    picks, held = [], 0
+    for i in range(n):
+        want = ts[0] + int(i * 1e9 / fps)
+        j = bisect.bisect_left(ts, want)
+        if j == 0:
+            k = 0
+        elif j >= len(cand):
+            k, held = len(cand) - 1, held + 1
+        else:
+            k = j if ts[j] - want < want - ts[j - 1] else j - 1
+        picks.append(cand[k])
+    return picks, span, held
+
+
+def encoder_args(name, crf):
+    if name == "nvenc":
+        return ["-c:v", "h264_nvenc", "-preset", "p6", "-rc", "vbr",
+                "-cq", str(crf), "-b:v", "12M"]
+    return ["-c:v", "libx264", "-preset", "slow", "-crf", str(crf)]
+
+
+def encode(cap, picks, args, encoder):
+    first = picks[0]
+    w, h, fmt = first["w"], first["h"], args.pix_fmt
+    if any(p["w"] != w or p["h"] != h for p in picks):
+        die("geometry changes inside the capture; resize before recording")
+    if args.pix_fmt == "auto":
+        fmt = "bgra" if first["fmt"] == 33 else "rgba"
+    cmd = ["ffmpeg", "-hide_banner", "-v", "error", "-y",
+           "-f", "rawvideo", "-pixel_format", fmt,
+           "-video_size", f"{w}x{h}", "-framerate", str(args.fps),
+           "-i", "pipe:0", "-an", *encoder_args(encoder, args.crf),
+           "-frames:v", str(len(picks)), "-pix_fmt", "yuv420p",
+           "-movflags", "+faststart", args.out]
+    print(f"GZF2-ENCODE {encoder} {w}x{h} {fmt} {len(picks)}f @ {args.fps}fps")
+    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+    last, last_px = None, None
+    try:
+        with open(cap, "rb") as f:
+            for p in picks:
+                if p["off"] != last:
+                    f.seek(p["off"])
+                    last_px = f.read(p["size"])
+                    if len(last_px) != p["size"]:
+                        raise IOError(f"short read at {p['off']}")
+                    last = p["off"]
+                if p["stride"] == w * 4:
+                    proc.stdin.write(last_px)
+                else:
+                    row = w * 4
+                    proc.stdin.write(b"".join(
+                        last_px[y * p["stride"]:y * p["stride"] + row]
+                        for y in range(h)))
+    except (BrokenPipeError, IOError) as e:
+        print(f"GZF2-ENCODE write failed: {e}", file=sys.stderr)
+    finally:
+        if proc.stdin:
+            try:
+                proc.stdin.close()
+            except BrokenPipeError:
+                pass
+    return proc.wait()
+
+
+def ffprobe(path):
+    r = subprocess.run(
+        ["ffprobe", "-v", "error", "-count_frames", "-select_streams", "v:0",
+         "-show_entries",
+         "stream=codec_name,width,height,r_frame_rate,nb_read_frames",
+         "-show_entries", "format=duration,size",
+         "-of", "json", path], capture_output=True, text=True)
+    if r.returncode:
+        die(f"ffprobe failed: {r.stderr.strip()}")
+    return json.loads(r.stdout)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("capture")
+    ap.add_argument("out")
+    ap.add_argument("--fps", type=int, default=60)
+    ap.add_argument("--skip", type=float, default=0.0,
+                    help="seconds of menu/load to drop from the start")
+    ap.add_argument("--duration", type=float, default=0.0,
+                    help="output seconds (0 = whole remaining span)")
+    ap.add_argument("--encoder", choices=["libx264", "nvenc"],
+                    default="libx264")
+    ap.add_argument("--crf", type=int, default=18)
+    ap.add_argument("--pix-fmt", choices=["auto", "rgba", "bgra"],
+                    default="auto", help="auto uses record fmt (33=BGRA)")
+    ap.add_argument("--info-only", action="store_true")
+    args = ap.parse_args()
+
+    recs, clean, corrupt = index_capture(args.capture)
+    uniq = len({r["md5"] for r in recs})
+    span = (recs[-1]["ts"] - recs[0]["ts"]) / 1e9
+    rate = len(recs) / span if span else 0.0
+    blacks = 0
+    with open(args.capture, "rb") as f:
+        for i, r in enumerate(recs):
+            if i % 60 == 0:
+                f.seek(r["off"])
+                blacks += sampled_black(f.read(min(r["size"], 65536)))
+    print(f"GZF2-CAPTURE frames={len(recs)} span={span:.2f}s rate={rate:.1f}/s "
+          f"unique={uniq} ({100.0 * uniq / len(recs):.1f}%) "
+          f"black_sampled={blacks} clean_end={clean} corrupt={corrupt} "
+          f"size={os.path.getsize(args.capture)}B "
+          f"geom={recs[0]['w']}x{recs[0]['h']} fmt={recs[0]['fmt']}")
+    if args.info_only:
+        return
+    picks, span, held = pick_frames(recs, args.fps, args.skip,
+                                    args.duration or None)
+    md5_by_off = {r["off"]: r["md5"] for r in recs}
+    pu = len({md5_by_off[p["off"]] for p in picks})
+    print(f"GZF2-RESAMPLE out={len(picks)}f span={span:.2f}s "
+          f"unique={pu}/{len(picks)} ({100.0 * pu / len(picks):.1f}%) "
+          f"held_last={held}")
+    if pu < len(picks) // 10:
+        print("GZF2-WARN mostly static/black output", file=sys.stderr)
+
+    rc = encode(args.capture, picks, args, args.encoder)
+    if rc != 0 and args.encoder == "nvenc":
+        print("GZF2-WARN nvenc failed; retrying libx264", file=sys.stderr)
+        rc = encode(args.capture, picks, args, "libx264")
+    if rc != 0:
+        die(f"ffmpeg exited {rc}")
+    info = ffprobe(args.out)
+    st = info["streams"][0]
+    print("GZF2-OUTPUT " + " ".join(
+        f"{k}={st.get(k)}" for k in
+        ("codec_name", "width", "height", "r_frame_rate", "nb_read_frames")))
+    print(f"GZF2-OUTPUT duration={info['format'].get('duration')} "
+          f"size={info['format'].get('size')}")
+    if int(st.get("nb_read_frames", 0)) != len(picks):
+        die("encoded frame count mismatch")
+    print("GZF2-OK", args.out)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tools/guest-diag/params-r1780.txt b/tools/guest-diag/params-r1780.txt
new file mode 100644
index 0000000..70985c7
--- /dev/null
+++ b/tools/guest-diag/params-r1780.txt
@@ -0,0 +1,2 @@
+gsp_phys_gpu=1 gsp_phys_hazard_ack=1 gsp_phys_boot=1 gsp_phys_nocat=1 gsp_phys_fw=nvidia/tu102/gsp/gsp-570.144.bin gsp_phys_guest_mem=1 gsp_phys_gr_forward=1 cuda_bridge=0 gsp_phys_rmcall_engine=1 gsp_phys_rmcall_golden=1 gsp_phys_rmcall_gr_promote=35 gsp_phys_rmcall_fwclient_mask=0xE3F gsp_phys_rmcall_vas_shared=1 gsp_phys_gr_nowrite_retries=0 gsp_phys_gr_retry_fast=1 gsp_phys_gr_first_poll_ms=2500 gsp_sema_winfill=1 gsp_sema_gr_swrel_gate=1 gsp_phys_rmcall_srv_ctxcommit=0 gsp_phys_rmcall_gr_qmdwin=1
+sema_dbg=1 sema_fake_rel=0