From 10c043a669fed5340eb953a17e341a35a4bce1b5 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 12 Nov 2026 14:55:25 -0500 Subject: vgsp: direct-GL ring/push rescue, VAS/pdb scanners and PTIMER tick; r1780 cycle tooling --- tools/guest-diag/cycle-r1780.sh | 142 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 tools/guest-diag/cycle-r1780.sh (limited to 'tools/guest-diag/cycle-r1780.sh') diff --git a/tools/guest-diag/cycle-r1780.sh b/tools/guest-diag/cycle-r1780.sh new file mode 100755 index 0000000..23cf0d1 --- /dev/null +++ b/tools/guest-diag/cycle-r1780.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Open-IOV contributors +# cycle-r1780.sh - durable vGPU cycle launcher for the r1780 campaign +# (12 GiB file-backed guest RAM, kvmfr/kvmsr ivshmem, V31-class module). +# +# Usage: tools/guest-diag/cycle-r1780.sh [cycle-tag] [module.ko] +# Env: INSOMOD_PARAMS override args file (default params-r1780.txt) +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +TAG="${1:-c27}" +MODULE="${2:-$REPO/kernel/vgsp.ko}" +ARGV_FILE="${INSMOD_PARAMS:-$HERE/params-r1780.txt}" +BACKUP="$REPO/backups/evidence-20261002-victory/r1780" +DISK="${GUEST_DISK:-/home/rmcore/.local/libvf.io/kernel/linux.qcow2}" +MDEV_PARENT="0000:01:00.0" +MDEV_TYPE="nvidia-nvidia-vgpu-turing" +RUNDIR="$HERE/run" +QEMU_SCRIPT="$HERE/launch_qemu_r1780.run.sh" +mkdir -p "$RUNDIR" "$BACKUP" + +echo "[cycle $TAG] stop QEMU" +# Flush the guest before the forced teardown: a killed GL/X workload has +# corrupted the ext4 root twice (c38 panic, c43 EIO), costing an offline fsck +# cycle each time. +if [ -S /tmp/qemu-guest-agent.sock ]; then + timeout 8 python3 - <<'PYEOF' >/dev/null 2>&1 || true +import json, socket +s = socket.socket(socket.AF_UNIX) +s.settimeout(6) +s.connect('/tmp/qemu-guest-agent.sock') +for cmd in (["sync"], ["/bin/sh", "-c", "sync; echo 1 > /proc/sys/kernel/sysrq; echo o > /proc/sysrq-trigger"]): + try: + s.sendall(json.dumps({"execute": "guest-exec", "arguments": {"path": cmd[0], "arg": cmd[1:], "capture-output": True}}).encode() + b'\n') + s.recv(65536) + except Exception: + break +s.close() +PYEOF + sleep 2 +fi +sudo pkill -f qemu-system-x86_64 2>/dev/null || true +for i in $(seq 1 20); do pgrep -f qemu-system-x86_64 >/dev/null || break; sleep 0.5; done +sudo pkill -9 -f qemu-system-x86_64 2>/dev/null || true +sleep 1 + +# Always repair the guest root after a forced teardown (cheap when clean, +# prevents the recurring "guest-agent exec EIO / kill init" corruption class). +DISK="/home/rmcore/.local/libvf.io/kernel/linux.qcow2" +if [ -f "$DISK" ]; then + sudo modprobe nbd max_part=8 2>/dev/null || true + sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true + if sudo qemu-nbd --connect=/dev/nbd0 "$DISK" -f qcow2 2>/dev/null; then + sleep 1 + sudo e2fsck -fy /dev/nbd0p1 > "$RUNDIR/fsck-$TAG.log" 2>&1 || true + sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true + fi +fi + +echo "[cycle $TAG] remove mdevs" +for m in /sys/bus/mdev/devices/*; do + [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null || true +done + +echo "[cycle $TAG] unload vgsp" +sudo rmmod vgsp 2>/dev/null || true +for i in $(seq 1 20); do [ -d /sys/module/vgsp ] || break; sleep 0.5; done +if [ -d /sys/module/vgsp ]; then + echo "FATAL: vgsp stuck loaded (see /proc/modules)"; exit 1 +fi + +echo "[cycle $TAG] load VFIO/mdev prerequisites" +sudo modprobe vfio vfio_iommu_type1 vfio_pci mdev 2>/dev/null || true +sudo modprobe vfio_iommu_type1 2>/dev/null || true + +echo "[cycle $TAG] stop nvidia-persistenced, power/control=on" +sudo systemctl stop nvidia-persistenced 2>/dev/null || true +echo on | sudo tee /sys/bus/pci/devices/0000:01:00.0/power/control >/dev/null +echo on | sudo tee /sys/bus/pci/devices/0000:00:01.0/power/control >/dev/null 2>&1 || true + +echo "[cycle $TAG] insmod $(basename "$MODULE")" +# shellcheck disable=SC2046 +sudo insmod "$MODULE" $(cat "$ARGV_FILE") + +U=$(uuidgen) +echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null +if [ ! -d "/sys/bus/mdev/devices/$U" ]; then echo "FATAL: mdev create failed"; exit 1; fi + +mkdir -p "/tmp/sockets/$U" +sudo rm -f /tmp/qemu-qmp.sock /tmp/qemu-guest-agent.sock /tmp/qemu-harness.pid +rm -f /dev/shm/vgpu-guest-ram 2>/dev/null || true +touch /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true +chmod 660 /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true + +QLOG="$BACKUP/${TAG}-qemu-session.txt" +SL="$BACKUP/${TAG}-serial.log" +cat > "$QEMU_SCRIPT" < "$RUNDIR/qemu-$TAG.log" 2>&1 +sleep 3 +if ! pgrep -f qemu-system-x86_64 >/dev/null; then + echo "FATAL: QEMU failed"; tail -5 "$RUNDIR/qemu-$TAG.log"; exit 1 +fi + +echo "[cycle $TAG] mdev $U" +echo "$U" > "$RUNDIR/current-mdev" +echo "$TAG" > "$RUNDIR/current-tag" + +for i in $(seq 1 90); do + if timeout 5 sudo python3 "$REPO/scripts/vgpu_vm.py" exec "echo CYCLE_UP" 2>/dev/null | grep -q CYCLE_UP; then + echo "[cycle $TAG] guest agent up" + exit 0 + fi + sleep 2 +done +echo "WARN: guest agent not reachable yet (serial: $SL)" +exit 0 -- cgit v1.2.3