From 10c043a669fed5340eb953a17e341a35a4bce1b5 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 12 Nov 2026 14:55:25 -0500 Subject: vgsp: direct-GL ring/push rescue, VAS/pdb scanners and PTIMER tick; r1780 cycle tooling --- tools/guest-diag/cycle-r1780.sh | 142 +++++++++++++++++++++++ tools/guest-diag/gzf2_video.py | 232 ++++++++++++++++++++++++++++++++++++++ tools/guest-diag/params-r1780.txt | 2 + 3 files changed, 376 insertions(+) create mode 100755 tools/guest-diag/cycle-r1780.sh create mode 100644 tools/guest-diag/gzf2_video.py create mode 100644 tools/guest-diag/params-r1780.txt (limited to 'tools') diff --git a/tools/guest-diag/cycle-r1780.sh b/tools/guest-diag/cycle-r1780.sh new file mode 100755 index 0000000..23cf0d1 --- /dev/null +++ b/tools/guest-diag/cycle-r1780.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Open-IOV contributors +# cycle-r1780.sh - durable vGPU cycle launcher for the r1780 campaign +# (12 GiB file-backed guest RAM, kvmfr/kvmsr ivshmem, V31-class module). +# +# Usage: tools/guest-diag/cycle-r1780.sh [cycle-tag] [module.ko] +# Env: INSOMOD_PARAMS override args file (default params-r1780.txt) +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +TAG="${1:-c27}" +MODULE="${2:-$REPO/kernel/vgsp.ko}" +ARGV_FILE="${INSMOD_PARAMS:-$HERE/params-r1780.txt}" +BACKUP="$REPO/backups/evidence-20261002-victory/r1780" +DISK="${GUEST_DISK:-/home/rmcore/.local/libvf.io/kernel/linux.qcow2}" +MDEV_PARENT="0000:01:00.0" +MDEV_TYPE="nvidia-nvidia-vgpu-turing" +RUNDIR="$HERE/run" +QEMU_SCRIPT="$HERE/launch_qemu_r1780.run.sh" +mkdir -p "$RUNDIR" "$BACKUP" + +echo "[cycle $TAG] stop QEMU" +# Flush the guest before the forced teardown: a killed GL/X workload has +# corrupted the ext4 root twice (c38 panic, c43 EIO), costing an offline fsck +# cycle each time. +if [ -S /tmp/qemu-guest-agent.sock ]; then + timeout 8 python3 - <<'PYEOF' >/dev/null 2>&1 || true +import json, socket +s = socket.socket(socket.AF_UNIX) +s.settimeout(6) +s.connect('/tmp/qemu-guest-agent.sock') +for cmd in (["sync"], ["/bin/sh", "-c", "sync; echo 1 > /proc/sys/kernel/sysrq; echo o > /proc/sysrq-trigger"]): + try: + s.sendall(json.dumps({"execute": "guest-exec", "arguments": {"path": cmd[0], "arg": cmd[1:], "capture-output": True}}).encode() + b'\n') + s.recv(65536) + except Exception: + break +s.close() +PYEOF + sleep 2 +fi +sudo pkill -f qemu-system-x86_64 2>/dev/null || true +for i in $(seq 1 20); do pgrep -f qemu-system-x86_64 >/dev/null || break; sleep 0.5; done +sudo pkill -9 -f qemu-system-x86_64 2>/dev/null || true +sleep 1 + +# Always repair the guest root after a forced teardown (cheap when clean, +# prevents the recurring "guest-agent exec EIO / kill init" corruption class). +DISK="/home/rmcore/.local/libvf.io/kernel/linux.qcow2" +if [ -f "$DISK" ]; then + sudo modprobe nbd max_part=8 2>/dev/null || true + sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true + if sudo qemu-nbd --connect=/dev/nbd0 "$DISK" -f qcow2 2>/dev/null; then + sleep 1 + sudo e2fsck -fy /dev/nbd0p1 > "$RUNDIR/fsck-$TAG.log" 2>&1 || true + sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true + fi +fi + +echo "[cycle $TAG] remove mdevs" +for m in /sys/bus/mdev/devices/*; do + [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null || true +done + +echo "[cycle $TAG] unload vgsp" +sudo rmmod vgsp 2>/dev/null || true +for i in $(seq 1 20); do [ -d /sys/module/vgsp ] || break; sleep 0.5; done +if [ -d /sys/module/vgsp ]; then + echo "FATAL: vgsp stuck loaded (see /proc/modules)"; exit 1 +fi + +echo "[cycle $TAG] load VFIO/mdev prerequisites" +sudo modprobe vfio vfio_iommu_type1 vfio_pci mdev 2>/dev/null || true +sudo modprobe vfio_iommu_type1 2>/dev/null || true + +echo "[cycle $TAG] stop nvidia-persistenced, power/control=on" +sudo systemctl stop nvidia-persistenced 2>/dev/null || true +echo on | sudo tee /sys/bus/pci/devices/0000:01:00.0/power/control >/dev/null +echo on | sudo tee /sys/bus/pci/devices/0000:00:01.0/power/control >/dev/null 2>&1 || true + +echo "[cycle $TAG] insmod $(basename "$MODULE")" +# shellcheck disable=SC2046 +sudo insmod "$MODULE" $(cat "$ARGV_FILE") + +U=$(uuidgen) +echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null +if [ ! -d "/sys/bus/mdev/devices/$U" ]; then echo "FATAL: mdev create failed"; exit 1; fi + +mkdir -p "/tmp/sockets/$U" +sudo rm -f /tmp/qemu-qmp.sock /tmp/qemu-guest-agent.sock /tmp/qemu-harness.pid +rm -f /dev/shm/vgpu-guest-ram 2>/dev/null || true +touch /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true +chmod 660 /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true + +QLOG="$BACKUP/${TAG}-qemu-session.txt" +SL="$BACKUP/${TAG}-serial.log" +cat > "$QEMU_SCRIPT" < "$RUNDIR/qemu-$TAG.log" 2>&1 +sleep 3 +if ! pgrep -f qemu-system-x86_64 >/dev/null; then + echo "FATAL: QEMU failed"; tail -5 "$RUNDIR/qemu-$TAG.log"; exit 1 +fi + +echo "[cycle $TAG] mdev $U" +echo "$U" > "$RUNDIR/current-mdev" +echo "$TAG" > "$RUNDIR/current-tag" + +for i in $(seq 1 90); do + if timeout 5 sudo python3 "$REPO/scripts/vgpu_vm.py" exec "echo CYCLE_UP" 2>/dev/null | grep -q CYCLE_UP; then + echo "[cycle $TAG] guest agent up" + exit 0 + fi + sleep 2 +done +echo "WARN: guest agent not reachable yet (serial: $SL)" +exit 0 diff --git a/tools/guest-diag/gzf2_video.py b/tools/guest-diag/gzf2_video.py new file mode 100644 index 0000000..aa88aa0 --- /dev/null +++ b/tools/guest-diag/gzf2_video.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# gzf2_video.py - turn a GZCAPV01/GZF2 guest capture into a constant-rate mp4. +# +# Reads the append-only frame stream written by sdl_cap.c (LD_PRELOAD +# SDL_GL_SwapWindow shim) or xgrab.c, resamples by CLOCK_MONOTONIC timestamps +# (nearest frame), streams raw pixels straight into ffmpeg (no PPM +# intermediates), and validates the encoded file with ffprobe. +# +# Format: [0..8) = "GZCAPV01", zero pad to 4096; then records: +# 4B "GZF2" | u32 w | u32 h | u32 stride | u32 fmt | u64 ts_ns | u64 seq | +# stride*h bytes, top-down. fmt 32 = RGBA8 (GL swap path), 33 = BGRA8 +# (SDL_RenderReadPixels ARGB8888 / X surface). A trailing +# "GZF2" + u64 0xffffffffffffffff marks a clean stop. +# +# Usage: +# gzf2_video.py CAPTURE OUT.mp4 [--fps 60] [--skip 5] [--duration 20] +# [--encoder libx264|nvenc] [--crf 18] +# [--pix-fmt rgba|bgra] [--info-only] +import argparse +import bisect +import hashlib +import json +import os +import struct +import subprocess +import sys + +MAGIC = b"GZCAPV01" +SENTINEL = b"\xff" * 8 + + +def die(msg): + print(f"GZF2-ERROR {msg}", file=sys.stderr) + sys.exit(2) + + +def index_capture(path): + size = os.path.getsize(path) + recs, clean, corrupt = [], False, 0 + with open(path, "rb") as f: + hdr = f.read(4096) + if len(hdr) < 4096 or hdr[:8] != MAGIC: + die(f"{path}: not a GZCAPV01 file") + off = 4096 + while off + 12 <= size: + f.seek(off) + head = f.read(36) + if len(head) < 12 or head[:4] != b"GZF2": + break + if head[4:12] == SENTINEL: + clean = True + break + if len(head) < 36: + break + w, h, stride, fmt = struct.unpack_from(" size: + corrupt += 1 + break + f.seek(off + 36) + px = f.read(n) + if len(px) != n: + corrupt += 1 + break + recs.append({ + "off": off + 36, "ts": ts, "w": w, "h": h, + "stride": stride, "fmt": fmt, "size": n, + "md5": hashlib.md5(px).hexdigest(), + }) + off += 36 + n + if not recs: + die(f"{path}: header only (no frames captured)") + return recs, clean, corrupt + + +def sampled_black(px): + s = px[:65536] + return sum(s[0::4]) + sum(s[1::4]) + sum(s[2::4]) == 0 + + +def pick_frames(recs, fps, skip_s, duration_s): + t0 = recs[0]["ts"] + int(skip_s * 1e9) + cand = [r for r in recs if r["ts"] >= t0] or recs + ts = [r["ts"] for r in cand] + span = (ts[-1] - ts[0]) / 1e9 + n = int(duration_s * fps) if duration_s else max(1, int(span * fps)) + picks, held = [], 0 + for i in range(n): + want = ts[0] + int(i * 1e9 / fps) + j = bisect.bisect_left(ts, want) + if j == 0: + k = 0 + elif j >= len(cand): + k, held = len(cand) - 1, held + 1 + else: + k = j if ts[j] - want < want - ts[j - 1] else j - 1 + picks.append(cand[k]) + return picks, span, held + + +def encoder_args(name, crf): + if name == "nvenc": + return ["-c:v", "h264_nvenc", "-preset", "p6", "-rc", "vbr", + "-cq", str(crf), "-b:v", "12M"] + return ["-c:v", "libx264", "-preset", "slow", "-crf", str(crf)] + + +def encode(cap, picks, args, encoder): + first = picks[0] + w, h, fmt = first["w"], first["h"], args.pix_fmt + if any(p["w"] != w or p["h"] != h for p in picks): + die("geometry changes inside the capture; resize before recording") + if args.pix_fmt == "auto": + fmt = "bgra" if first["fmt"] == 33 else "rgba" + cmd = ["ffmpeg", "-hide_banner", "-v", "error", "-y", + "-f", "rawvideo", "-pixel_format", fmt, + "-video_size", f"{w}x{h}", "-framerate", str(args.fps), + "-i", "pipe:0", "-an", *encoder_args(encoder, args.crf), + "-frames:v", str(len(picks)), "-pix_fmt", "yuv420p", + "-movflags", "+faststart", args.out] + print(f"GZF2-ENCODE {encoder} {w}x{h} {fmt} {len(picks)}f @ {args.fps}fps") + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) + last, last_px = None, None + try: + with open(cap, "rb") as f: + for p in picks: + if p["off"] != last: + f.seek(p["off"]) + last_px = f.read(p["size"]) + if len(last_px) != p["size"]: + raise IOError(f"short read at {p['off']}") + last = p["off"] + if p["stride"] == w * 4: + proc.stdin.write(last_px) + else: + row = w * 4 + proc.stdin.write(b"".join( + last_px[y * p["stride"]:y * p["stride"] + row] + for y in range(h))) + except (BrokenPipeError, IOError) as e: + print(f"GZF2-ENCODE write failed: {e}", file=sys.stderr) + finally: + if proc.stdin: + try: + proc.stdin.close() + except BrokenPipeError: + pass + return proc.wait() + + +def ffprobe(path): + r = subprocess.run( + ["ffprobe", "-v", "error", "-count_frames", "-select_streams", "v:0", + "-show_entries", + "stream=codec_name,width,height,r_frame_rate,nb_read_frames", + "-show_entries", "format=duration,size", + "-of", "json", path], capture_output=True, text=True) + if r.returncode: + die(f"ffprobe failed: {r.stderr.strip()}") + return json.loads(r.stdout) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("capture") + ap.add_argument("out") + ap.add_argument("--fps", type=int, default=60) + ap.add_argument("--skip", type=float, default=0.0, + help="seconds of menu/load to drop from the start") + ap.add_argument("--duration", type=float, default=0.0, + help="output seconds (0 = whole remaining span)") + ap.add_argument("--encoder", choices=["libx264", "nvenc"], + default="libx264") + ap.add_argument("--crf", type=int, default=18) + ap.add_argument("--pix-fmt", choices=["auto", "rgba", "bgra"], + default="auto", help="auto uses record fmt (33=BGRA)") + ap.add_argument("--info-only", action="store_true") + args = ap.parse_args() + + recs, clean, corrupt = index_capture(args.capture) + uniq = len({r["md5"] for r in recs}) + span = (recs[-1]["ts"] - recs[0]["ts"]) / 1e9 + rate = len(recs) / span if span else 0.0 + blacks = 0 + with open(args.capture, "rb") as f: + for i, r in enumerate(recs): + if i % 60 == 0: + f.seek(r["off"]) + blacks += sampled_black(f.read(min(r["size"], 65536))) + print(f"GZF2-CAPTURE frames={len(recs)} span={span:.2f}s rate={rate:.1f}/s " + f"unique={uniq} ({100.0 * uniq / len(recs):.1f}%) " + f"black_sampled={blacks} clean_end={clean} corrupt={corrupt} " + f"size={os.path.getsize(args.capture)}B " + f"geom={recs[0]['w']}x{recs[0]['h']} fmt={recs[0]['fmt']}") + if args.info_only: + return + picks, span, held = pick_frames(recs, args.fps, args.skip, + args.duration or None) + md5_by_off = {r["off"]: r["md5"] for r in recs} + pu = len({md5_by_off[p["off"]] for p in picks}) + print(f"GZF2-RESAMPLE out={len(picks)}f span={span:.2f}s " + f"unique={pu}/{len(picks)} ({100.0 * pu / len(picks):.1f}%) " + f"held_last={held}") + if pu < len(picks) // 10: + print("GZF2-WARN mostly static/black output", file=sys.stderr) + + rc = encode(args.capture, picks, args, args.encoder) + if rc != 0 and args.encoder == "nvenc": + print("GZF2-WARN nvenc failed; retrying libx264", file=sys.stderr) + rc = encode(args.capture, picks, args, "libx264") + if rc != 0: + die(f"ffmpeg exited {rc}") + info = ffprobe(args.out) + st = info["streams"][0] + print("GZF2-OUTPUT " + " ".join( + f"{k}={st.get(k)}" for k in + ("codec_name", "width", "height", "r_frame_rate", "nb_read_frames"))) + print(f"GZF2-OUTPUT duration={info['format'].get('duration')} " + f"size={info['format'].get('size')}") + if int(st.get("nb_read_frames", 0)) != len(picks): + die("encoded frame count mismatch") + print("GZF2-OK", args.out) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/guest-diag/params-r1780.txt b/tools/guest-diag/params-r1780.txt new file mode 100644 index 0000000..70985c7 --- /dev/null +++ b/tools/guest-diag/params-r1780.txt @@ -0,0 +1,2 @@ +gsp_phys_gpu=1 gsp_phys_hazard_ack=1 gsp_phys_boot=1 gsp_phys_nocat=1 gsp_phys_fw=nvidia/tu102/gsp/gsp-570.144.bin gsp_phys_guest_mem=1 gsp_phys_gr_forward=1 cuda_bridge=0 gsp_phys_rmcall_engine=1 gsp_phys_rmcall_golden=1 gsp_phys_rmcall_gr_promote=35 gsp_phys_rmcall_fwclient_mask=0xE3F gsp_phys_rmcall_vas_shared=1 gsp_phys_gr_nowrite_retries=0 gsp_phys_gr_retry_fast=1 gsp_phys_gr_first_poll_ms=2500 gsp_sema_winfill=1 gsp_sema_gr_swrel_gate=1 gsp_phys_rmcall_srv_ctxcommit=0 gsp_phys_rmcall_gr_qmdwin=1 +sema_dbg=1 sema_fake_rel=0 -- cgit v1.2.3