From 5dd8165f1754e156a0f9fb16c7a6bfa90e99ed60 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 14 Sep 2026 06:43:11 -0400 Subject: VFIO device registration fix + IOMMUFD callbacks + VFIO ioctl handlers - Removed IOMMU group addition before vfio_register_emulated_iommu_dev() in nv_vgpu_mdev_create() - Added IOMMUFD callbacks (bind_iommufd, unbind_iommufd, attach_ioas, detach_ioas) using kernel's vfio_iommufd_emulated_* functions - Implemented VFIO ioctl handlers: GET_INFO, GET_REGION_INFO, GET_IRQ_INFO, SET_IRQS, MMAP - Added config space read/write handlers at offset 0x70000000000 - Mmap handler for config space, BAR0 (emulation shadow), BAR1-5 (physical VRAM/registers) - Interrupt handler for VFIO_DEVICE_SET_IRQS (DATA_EVENTFD, DATA_NONE, DATA_BOOL) - Added udev rules for vfio group permissions - Created stack_manager.sh unified tool - Fixed mdev remove crash with deferred cleanup in vfio_device_release callback - Kernel 6.8+ IOMMUFD compatibility - QEMU launches successfully with vGPU passthrough --- scripts/auto-opencode-setup.sh | 323 +++++++++++++++++++++++++++++ scripts/build_nvidia_openrm.sh | 61 ++++++ scripts/debug_guest_init.sh | 172 ++++++++++++++++ scripts/qemu_gdb_attach.sh | 58 ++++++ scripts/stack_manager.sh | 393 ++++++++++++++++++++++++++++++++++++ scripts/windows_guest_automation.sh | 220 ++++++++++++++++++++ 6 files changed, 1227 insertions(+) create mode 100755 scripts/auto-opencode-setup.sh create mode 100755 scripts/build_nvidia_openrm.sh create mode 100755 scripts/debug_guest_init.sh create mode 100644 scripts/qemu_gdb_attach.sh create mode 100755 scripts/stack_manager.sh create mode 100755 scripts/windows_guest_automation.sh (limited to 'scripts') diff --git a/scripts/auto-opencode-setup.sh b/scripts/auto-opencode-setup.sh new file mode 100755 index 0000000..3f98eac --- /dev/null +++ b/scripts/auto-opencode-setup.sh @@ -0,0 +1,323 @@ +#!/bin/bash +# auto-opencode-setup.sh - Configure auto-opencode session on login +# Usage: sudo ./auto-opencode-setup.sh [enable|disable] [graphical|multi-user] [auto-login] + +set -euo pipefail + +AUTO_OPENCODE_DIR="$HOME/.config/auto-opencode" +AUTO_OPENCODE_SCRIPT="$AUTO_OPENCODE_DIR/auto-opencode.sh" +AUTO_OPENCODE_MARKER="$AUTO_OPENCODE_DIR/enabled" +AUTO_LOGIN_MARKER="$AUTO_OPENCODE_DIR/auto-login" +TARGET_MARKER="$AUTO_OPENCODE_DIR/target" + +log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; } +log_ok() { echo -e "\033[0;32m[OK]\033[0m $1"; } +log_warn() { echo -e "\033[1;33m[WARN]\033[0m $1"; } +log_error() { echo -e "\033[0;31m[ERROR]\033[0m $1"; } + +usage() { + cat < "$AUTO_OPENCODE_SCRIPT" <<'SCRIPTEOF' +#!/bin/bash +# auto-opencode.sh - Runs on login to start opencode tmux session +# This script is sourced from .zprofile (multi-user) or .xprofile (graphical) + +AUTO_OPENCODE_DIR="$HOME/.config/auto-opencode" +AUTO_OPENCODE_MARKER="$AUTO_OPENCODE_DIR/enabled" +AUTO_LOGIN_MARKER="$AUTO_OPENCODE_DIR/auto-login" + +# Check if auto-opencode is enabled +[[ -f "$AUTO_OPENCODE_MARKER" ]] || exit 0 + +# Prevent multiple runs +if [[ -n "$AUTO_OPENCODE_STARTED" ]]; then + exit 0 +fi +export AUTO_OPENCODE_STARTED=1 + +# Only run in interactive shells +[[ $- == *i* ]] || exit 0 + +# Check if tmux is available +command -v tmux &>/dev/null || exit 0 +command -v opencode &>/dev/null || exit 0 + +# Check if we're already in tmux +[[ -n "$TMUX" ]] && exit 0 + +# Target directory +TARGET_DIR="$OPEN_IOV_ROOT/.." + +# Session name +SESSION_NAME="auto-opencode" + +# Check if session already exists +if tmux has-session -t "$SESSION_NAME" 2>/dev/null; then + echo "Attaching to existing $SESSION_NAME session..." + exec tmux attach-session -t "$SESSION_NAME" +fi + +# Create new tmux session with opencode +echo "Starting auto-opencode session..." +cd "$TARGET_DIR" + +# Start tmux session +tmux new-session -d -s "$SESSION_NAME" -c "$TARGET_DIR" \ + "opencode --dangerously-skip-permissions" + +# Wait a moment for opencode to start +sleep 2 + +# Send initial commands to read AGENTS.md and handoff +tmux send-keys -t "$SESSION_NAME" "cat AGENTS.md" Enter +sleep 1 +tmux send-keys -t "$SESSION_NAME" "cat Handoff-vGPU-IO.md" Enter +sleep 1 + +# Attach to session +exec tmux attach-session -t "$SESSION_NAME" +SCRIPTEOF + + chmod +x "$AUTO_OPENCODE_SCRIPT" + log_ok "Created auto-opencode script: $AUTO_OPENCODE_SCRIPT" +} + +setup_graphical_autologin() { + log_info "Setting up graphical auto-login (GDM)..." + + # Create GDM custom.conf for auto-login + sudo mkdir -p /etc/gdm3 + sudo tee /etc/gdm3/custom.conf > /dev/null < $HOME/.config/autostart/auto-opencode.desktop < /dev/null </dev/null; then + cat >> $HOME/.zprofile <<'EOF' + +# Auto-opencode on login +if [[ -f $HOME/.config/auto-opencode/auto-opencode.sh ]]; then + source $HOME/.config/auto-opencode/auto-opencode.sh +fi +EOF + fi + + log_ok "Multi-user auto-login configured" +} + +disable_autologin() { + log_info "Disabling auto-login..." + + # Remove GDM auto-login + if [[ -f /etc/gdm3/custom.conf ]]; then + sudo sed -i '/^AutomaticLoginEnable=/d' /etc/gdm3/custom.conf + sudo sed -i '/^AutomaticLogin=/d' /etc/gdm3/custom.conf + fi + + # Remove getty auto-login + sudo rm -f /etc/systemd/system/getty@tty1.service.d/autologin.conf + + # Remove autostart desktop entry + rm -f $HOME/.config/autostart/auto-opencode.desktop + + # Remove from .zprofile + sed -i '/auto-opencode.sh/d' $HOME/.zprofile 2>/dev/null || true + + log_ok "Auto-login disabled" +} + +enable_auto_opencode() { + local target="${1:-graphical}" + local auto_login="${2:-false}" + local auto_opencode="${3:-false}" + + log_info "Enabling auto-opencode (target: $target, auto-login: $auto_login, auto-opencode: $auto_opencode)" + + create_auto_opencode_script + + # Set target + echo "$target" > "$TARGET_MARKER" + + # Set boot target + sudo systemctl set-default "$target.target" + log_ok "Boot target set to $target.target" + + # Handle auto-login + if [[ "$auto_login" == "true" ]]; then + touch "$AUTO_LOGIN_MARKER" + if [[ "$target" == "graphical" ]]; then + setup_graphical_autologin + else + setup_multiuser_autologin + fi + else + rm -f "$AUTO_LOGIN_MARKER" + disable_autologin + fi + + # Handle auto-opencode + if [[ "$auto_opencode" == "true" ]]; then + touch "$AUTO_OPENCODE_MARKER" + log_ok "Auto-opencode enabled" + else + rm -f "$AUTO_OPENCODE_MARKER" + log_ok "Auto-opencode disabled" + fi + + log_ok "Configuration complete" +} + +disable_auto_opencode() { + log_info "Disabling auto-opencode..." + + rm -f "$AUTO_OPENCODE_MARKER" + rm -f "$AUTO_LOGIN_MARKER" + rm -f "$TARGET_MARKER" + + disable_autologin + + # Remove auto-opencode script + rm -f "$AUTO_OPENCODE_SCRIPT" + + log_ok "Auto-opencode disabled" +} + +show_status() { + echo "═══════════════════════════════════════════════════════════════" + echo " AUTO-OPENCODE STATUS" + echo "═══════════════════════════════════════════════════════════════" + + if [[ -f "$AUTO_OPENCODE_MARKER" ]]; then + echo " Auto-opencode: ENABLED" + else + echo " Auto-opencode: DISABLED" + fi + + if [[ -f "$AUTO_LOGIN_MARKER" ]]; then + echo " Auto-login: ENABLED" + else + echo " Auto-login: DISABLED" + fi + + if [[ -f "$TARGET_MARKER" ]]; then + echo " Boot target: $(cat "$TARGET_MARKER")" + else + echo " Boot target: $(systemctl get-default | sed 's/.target$//')" + fi + + echo " Current default: $(systemctl get-default)" + echo "═══════════════════════════════════════════════════════════════" +} + +main() { + local action="${1:-status}" + shift || true + + local target="graphical" + local auto_login="false" + local auto_opencode="false" + + # Parse remaining arguments + for arg in "$@"; do + case "$arg" in + graphical|multi-user) + target="$arg" + ;; + auto-login|auto-login=on) + auto_login="true" + ;; + auto-login=off) + auto_login="false" + ;; + auto-opencode|auto-opencode=on) + auto_opencode="true" + ;; + auto-opencode=off) + auto_opencode="false" + ;; + *) + log_warn "Unknown argument: $arg" + ;; + esac + done + + case "$action" in + enable) + enable_auto_opencode "$target" "$auto_login" "$auto_opencode" + ;; + disable) + disable_auto_opencode + ;; + status) + show_status + ;; + *) + usage + exit 1 + ;; + esac +} + +main "$@" \ No newline at end of file diff --git a/scripts/build_nvidia_openrm.sh b/scripts/build_nvidia_openrm.sh new file mode 100755 index 0000000..dfcbe59 --- /dev/null +++ b/scripts/build_nvidia_openrm.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Build NVIDIA OpenRM kernel modules with Open-IOV vGPU VFIO symbols +# This script ensures KBUILD_EXTRA_SYMBOLS is correctly passed to the build + +set -e + +OPEN_IOV_ROOT="$OPEN_IOV_ROOT" +NVIDIA_OPENRM_DIR="$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules" +KERNEL_DIR="$OPEN_IOV_ROOT/kernel" +SYMVERS_FILE="$KERNEL_DIR/Module.symvers" + +echo "=== Building NVIDIA OpenRM with Open-IOV vGPU VFIO symbols ===" +echo "Open-IOV root: $OPEN_IOV_ROOT" +echo "NVIDIA OpenRM dir: $NVIDIA_OPENRM_DIR" +echo "Kernel Module.symvers: $SYMVERS_FILE" + +# Check if open-iov-vfio.ko has been built (provides Module.symvers) +if [ ! -f "$SYMVERS_FILE" ]; then + echo "ERROR: $SYMVERS_FILE not found!" + echo "Please build open-iov-vfio.ko first: cd $OPEN_IOV_ROOT && make kernel" + exit 1 +fi + +echo "Found Module.symvers with $(wc -l < "$SYMVERS_FILE") exported symbols" +echo "Key vGPU symbols:" +grep -i vgpu "$SYMVERS_FILE" | head -5 + +# Build NVIDIA OpenRM kernel modules +cd "$NVIDIA_OPENRM_DIR/kernel-open" + +# Clean previous build to ensure fresh symbol resolution +echo "Cleaning previous build..." +make clean + +echo "Building NVIDIA OpenRM modules with KBUILD_EXTRA_SYMBOLS..." +KBUILD_EXTRA_SYMBOLS="$SYMVERS_FILE" make -j$(nproc) modules + +# Verify the symbols are resolved +echo "" +echo "=== Verification ===" +echo "Checking nvidia.ko for vGPU VFIO symbols..." +if modinfo nvidia.ko 2>/dev/null | grep -q "nvidia_vgpu_vfio"; then + echo "✓ nvidia.ko has vGPU VFIO symbols" +else + echo "⚠ nvidia.ko may not have vGPU VFIO symbols (check modpost output above)" +fi + +echo "" +echo "Checking Module.symvers for vGPU symbols..." +grep -i vgpu Module.symvers | head -5 + +echo "" +echo "=== Build Complete ===" +echo "nvidia.ko location: $NVIDIA_OPENRM_DIR/kernel-open/nvidia.ko" +echo "" +echo "To load the stack:" +echo " sudo insmod $NVIDIA_OPENRM_DIR/kernel-open/nvidia.ko" +echo " sudo insmod $KERNEL_DIR/open-iov-vfio.ko" +echo " sudo cp $OPEN_IOV_ROOT/firmware/57X-GSP/570.144/gsp-570.144.fwimage.bin /lib/firmware/" +echo " ./scripts/create_mdev.sh" +echo " ./scripts/launch_qemu_vgpu.sh" \ No newline at end of file diff --git a/scripts/debug_guest_init.sh b/scripts/debug_guest_init.sh new file mode 100755 index 0000000..324f403 --- /dev/null +++ b/scripts/debug_guest_init.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# debug_guest_init.sh - One-command debug launch for guest NVIDIA driver initialization +# Usage: ./debug_guest_init.sh [options] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Defaults +GDB_PORT=1234 +FREEZE_AT_START=true +MONITOR_GUEST=true +GUEST_SERIAL="/dev/ttyS0" +DURATION=300 +OUTPUT_DIR="/tmp/gsp_emu_debug_$(date +%Y%m%d_%H%M%S)" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_ok() { echo -e "${GREEN}[OK]${NC} $1"; } + +usage() { + cat </dev/null | head -1) +if [[ -z "$MDEV_UUID" ]]; then + log_error "No mdev device found. Run create_mdev.sh first." + exit 1 +fi +log_info "Using mdev: $MDEV_UUID" + +# Build QEMU command +QEMU_CMD=( + qemu-system-x86_64 + -enable-kvm + -cpu host + -smp 4 + -m 8G + -device vfio-pci,sysfsdev=/sys/bus/mdev/devices/$MDEV_UUID,display=off + -drive file=/var/lib/libvirt/images/ubuntu-vgpu.qcow2,format=qcow2,if=virtio + -netdev user,id=net0,hostfwd=tcp::2222-:22 + -device virtio-net-pci,netdev=net0 + -nographic + -serial mon:stdio +) + +if [[ "$FREEZE_AT_START" == "true" ]]; then + QEMU_CMD+=(-s -S) +else + QEMU_CMD+=(-s) +fi + +if [[ "$GDB_PORT" != "1234" ]]; then + QEMU_CMD+=(-gdb "tcp::$GDB_PORT") +fi + +# Start guest init monitor in background +MONITOR_PID="" +if [[ "$MONITOR_GUEST" == "true" ]]; then + log_info "Starting guest init monitor..." + python3 "$BASE_DIR/tools/guest_init_monitor.py" \ + --guest-serial "$GUEST_SERIAL" \ + --duration "$DURATION" \ + > "$OUTPUT_DIR/guest_monitor.log" 2>&1 & + MONITOR_PID=$! + log_ok "Guest monitor started (PID: $MONITOR_PID)" +fi + +# Start host trace capture +log_info "Starting host trace capture..." +sudo cat /sys/kernel/debug/tracing/trace_pipe > "$OUTPUT_DIR/host_trace.log" 2>&1 & +TRACE_PID=$! +log_ok "Host trace capture started (PID: $TRACE_PID)" + +# Save QEMU command +echo "${QEMU_CMD[@]}" > "$OUTPUT_DIR/qemu_cmd.txt" + +# Launch QEMU in background +log_info "Launching QEMU with GDB server on port $GDB_PORT..." +"${QEMU_CMD[@]}" > "$OUTPUT_DIR/qemu.log" 2>&1 & +QEMU_PID=$! +log_ok "QEMU started (PID: $QEMU_PID)" + +# Give QEMU time to start +sleep 3 + +# Launch GDB if vmlinux provided +if [[ -n "${VMLINUX:-}" && -f "$VMLINUX" ]]; then + log_info "Launching GDB with nvidia_init.gdb..." + log_info "GDB will connect to localhost:$GDB_PORT" + log_info "In GDB: (gdb) target remote localhost:$GDB_PORT" + log_info "Then: (gdb) continue" + + gdb -x "$BASE_DIR/tools/gdb_scripts/nvidia_init.gdb" "$VMLINUX" \ + -ex "target remote localhost:$GDB_PORT" \ + -ex "set logging file $OUTPUT_DIR/gdb.log" \ + -ex "set logging on" +else + log_warn "No vmlinux provided. Connect manually:" + log_warn " gdb -x $BASE_DIR/tools/gdb_scripts/nvidia_init.gdb /path/to/vmlinux" + log_warn " (gdb) target remote localhost:$GDB_PORT" + log_warn "" + log_warn "Press Ctrl+C to stop QEMU and monitoring..." + + # Wait for QEMU or interrupt + wait $QEMU_PID +fi + +# Cleanup +log_info "Cleaning up..." +[[ -n "$MONITOR_PID" ]] && kill $MONITOR_PID 2>/dev/null || true +[[ -n "$TRACE_PID" ]] && kill $TRACE_PID 2>/dev/null || true +kill $QEMU_PID 2>/dev/null || true + +log_info "Debug session complete. Logs in: $OUTPUT_DIR" +ls -la "$OUTPUT_DIR" \ No newline at end of file diff --git a/scripts/qemu_gdb_attach.sh b/scripts/qemu_gdb_attach.sh new file mode 100644 index 0000000..310c196 --- /dev/null +++ b/scripts/qemu_gdb_attach.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# qemu_gdb_attach.sh - Launch QEMU with GDB server for guest kernel debugging +# Usage: ./qemu_gdb_attach.sh [gdb_port] [freeze_at_start] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +GDB_PORT="${1:-1234}" +FREEZE="${2:-true}" # -S flag to freeze at startup + +# Default QEMU command from launch_qemu_vgpu.sh +QEMU_CMD=( + qemu-system-x86_64 + -enable-kvm + -cpu host + -smp 4 + -m 8G + -device vfio-pci,sysfsdev=/sys/bus/mdev/devices/$(cat /sys/bus/mdev/devices/*/uuid 2>/dev/null | head -1),display=off + -drive file=/var/lib/libvirt/images/ubuntu-vgpu.qcow2,format=qcow2,if=virtio + -netdev user,id=net0,hostfwd=tcp::2222-:22 + -device virtio-net-pci,netdev=net0 + -nographic + -serial mon:stdio +) + +# Add GDB server options +if [[ "$FREEZE" == "true" ]]; then + QEMU_CMD+=(-s -S) # -s = gdbserver on :1234, -S = freeze at startup +else + QEMU_CMD+=(-s) # -s = gdbserver on :1234, run normally +fi + +# Override port if specified +if [[ "$GDB_PORT" != "1234" ]]; then + QEMU_CMD+=(-gdb "tcp::$GDB_PORT") +fi + +echo "==========================================" +echo "QEMU GDB Debug Launch" +echo "==========================================" +echo "GDB Port: $GDB_PORT" +echo "Freeze at start: $FREEZE" +echo "QEMU Command: ${QEMU_CMD[*]}" +echo "" +echo "In another terminal, connect with:" +echo " gdb ./vmlinux" +echo " (gdb) target remote localhost:$GDB_PORT" +echo " (gdb) break nvidia_probe" +echo " (gdb) continue" +echo "" +echo "Or use the GDB script:" +echo " gdb -x $BASE_DIR/tools/gdb_scripts/nvidia_init.gdb ./vmlinux" +echo "" + +# Launch QEMU +exec "${QEMU_CMD[@]}" \ No newline at end of file diff --git a/scripts/stack_manager.sh b/scripts/stack_manager.sh new file mode 100755 index 0000000..fe0a73a --- /dev/null +++ b/scripts/stack_manager.sh @@ -0,0 +1,393 @@ +#!/bin/bash +# stack_manager.sh - Complete vGPU stack management (load, unload, test, debug) +# Usage: ./stack_manager.sh [options] + +set -euo pipefail + +BASE_DIR="$OPEN_IOV_ROOT" +NVIDIA_KO="$BASE_DIR/nvidia-open-gpu-kernel-modules/kernel-open/nvidia.ko" +VFIO_KO="$BASE_DIR/kernel/open-iov-vfio.ko" +FIRMWARE_PATH="$BASE_DIR/firmware/57X-GSP/570.144/gsp-570.144.fwimage.bin" +FIRMWARE_DEST="/lib/firmware/gsp-570.144.fwimage.bin" + +MDEV_SCRIPT="$BASE_DIR/scripts/create_mdev.sh" +QEMU_SCRIPT="$BASE_DIR/scripts/launch_qemu_vgpu.sh" +TEST_SCRIPT="$BASE_DIR/scripts/test_vgpu_stack.sh" + +color_red='\033[0;31m' +color_green='\033[0;32m' +color_yellow='\033[1;33m' +color_blue='\033[0;34m' +color_nc='\033[0m' + +log_info() { echo -e "${color_blue}[INFO]${color_nc} $*"; } +log_ok() { echo -e "${color_green}[OK]${color_nc} $*"; } +log_warn() { echo -e "${color_yellow}[WARN]${color_nc} $*"; } +log_err() { echo -e "${color_red}[ERROR]${color_nc} $*"; } + +# ============================================================ +# BUILD COMMANDS (correct order matters!) +# ============================================================ + +cmd_build_kernel() { + log_info "Building open-iov-vfio.ko (kernel module)..." + cd "$BASE_DIR" + make kernel + log_ok "open-iov-vfio.ko built" +} + +cmd_build_nvidia() { + log_info "Building NVIDIA OpenRM with vGPU symbols (KBUILD_EXTRA_SYMBOLS)..." + cd "$BASE_DIR" + + # Step 1: Build nvidia kernel module (nv-kernel.o) + log_info "Building nvidia kernel module (nv-kernel.o)..." + cd "$BASE_DIR/nvidia-open-gpu-kernel-modules/src/nvidia" + make -j$(nproc) + log_ok "nvidia kernel module built" + + # Step 2: Build nvidia-modeset kernel module (nv-modeset-kernel.o) + log_info "Building nvidia-modeset kernel module (nv-modeset-kernel.o)..." + cd "$BASE_DIR/nvidia-open-gpu-kernel-modules/src/nvidia-modeset" + make -j$(nproc) + log_ok "nvidia-modeset kernel module built" + + # Step 3: Build kernel-open modules with KBUILD_EXTRA_SYMBOLS + log_info "Building kernel-open modules with KBUILD_EXTRA_SYMBOLS..." + cd "$BASE_DIR/nvidia-open-gpu-kernel-modules/kernel-open" + KBUILD_EXTRA_SYMBOLS="$BASE_DIR/kernel/Module.symvers" make -j$(nproc) modules + log_ok "NVIDIA OpenRM built" +} + +cmd_build_userspace() { + log_info "Building userspace (libiov.so + examples)..." + cd "$BASE_DIR" + make userspace + make examples + log_ok "Userspace built" +} + +cmd_build_all() { + log_info "=== BUILD ALL (correct order) ===" + cmd_build_kernel + cmd_build_nvidia + cmd_build_userspace + log_ok "=== ALL BUILDS COMPLETE ===" +} + +# ============================================================ +# LOAD COMMANDS (correct order matters!) +# ============================================================ + +cmd_load_prereqs() { + log_info "Loading kernel prerequisites (vfio, mdev)..." + sudo modprobe vfio vfio_iommu_type1 vfio_pci mdev vfio_mdev 2>/dev/null || true + lsmod | grep -E "vfio|mdev" +} + +cmd_load_open_iov() { + if [[ ! -f "$VFIO_KO" ]]; then + log_err "Missing $VFIO_KO - run 'build' first" + return 1 + fi + if ! lsmod | grep -q "^open_iov_vfio "; then + log_info "Loading open-iov-vfio.ko..." + sudo insmod "$VFIO_KO" || { log_err "Failed to load open-iov-vfio.ko"; return 1; } + log_ok "open-iov-vfio.ko loaded" + else + log_ok "open-iov-vfio.ko already loaded" + fi +} + +cmd_load_nvidia() { + if [[ ! -f "$NVIDIA_KO" ]]; then + log_err "Missing $NVIDIA_KO - run 'build-nvidia' first" + return 1 + fi + if ! lsmod | grep -q "^nvidia "; then + log_info "Loading nvidia.ko (OpenRM)..." + sudo insmod "$NVIDIA_KO" || { log_err "Failed to load nvidia.ko"; return 1; } + log_ok "nvidia.ko loaded" + else + log_ok "nvidia.ko already loaded" + fi +} + +cmd_install_firmware() { + if [[ ! -f "$FIRMWARE_PATH" ]]; then + log_err "Missing firmware at $FIRMWARE_PATH" + return 1 + fi + if [[ ! -f "$FIRMWARE_DEST" ]] || ! cmp -s "$FIRMWARE_PATH" "$FIRMWARE_DEST"; then + log_info "Installing firmware to $FIRMWARE_DEST..." + sudo cp "$FIRMWARE_PATH" "$FIRMWARE_DEST" + log_ok "Firmware installed" + else + log_ok "Firmware already installed" + fi +} + +cmd_load_stack() { + log_info "=== LOAD STACK (correct order) ===" + cmd_load_prereqs + cmd_load_open_iov + cmd_load_nvidia + cmd_install_firmware + log_ok "=== DRIVER STACK LOADED ===" + lsmod | grep -E "nvidia|open_iov|vfio|mdev" +} + +# Full cycle: build all + load stack + create mdev +cmd_up() { + log_info "=== FULL STARTUP: build + load + mdev ===" + cmd_build_all + cmd_load_stack + cmd_create_mdev + log_ok "=== SYSTEM READY FOR QEMU ===" + echo "Run: ./stack_manager.sh qemu" +} + +# ============================================================ +# UNLOAD COMMANDS +# ============================================================ + +cmd_unload_stack() { + log_info "=== UNLOAD STACK ===" + + if [[ -e /sys/bus/mdev/devices/* ]]; then + log_warn "MDEV devices exist - removing them first..." + for mdev in /sys/bus/mdev/devices/*; do + uuid=$(basename "$mdev") + log_info "Removing mdev $uuid..." + echo 1 | sudo tee "$mdev/remove" >/dev/null + done + sleep 2 + fi + + # CRITICAL: nvidia.ko holds a reference to open_iov_vfio, so unload nvidia FIRST + for mod in nvidia_uvm nvidia_modeset nvidia_drm nvidia_peermem nvidia; do + if lsmod | grep -qE "^$mod\>"; then + log_info "Unloading $mod..." + sudo rmmod "$mod" || { log_err "Failed to unload $mod"; return 1; } + log_ok "$mod unloaded" + fi + done + + if lsmod | grep -qE "^open_iov_vfio\>"; then + log_info "Unloading open-iov-vfio.ko..." + sudo rmmod open_iov_vfio || { log_err "Failed to unload open-iov-vfio.ko"; return 1; } + log_ok "open-iov-vfio.ko unloaded" + fi + + sudo modprobe -r vfio_mdev vfio_pci vfio_iommu_type1 vfio mdev 2>/dev/null || true + log_ok "=== DRIVER STACK UNLOADED ===" +} + +# ============================================================ +# MDEV COMMANDS +# ============================================================ + +cmd_create_mdev() { + log_info "Creating mdev device..." + if [[ ! -f "$MDEV_SCRIPT" ]]; then + log_err "create_mdev.sh not found" + return 1 + fi + "$MDEV_SCRIPT" "$@" +} + +cmd_list_mdev() { + log_info "MDEV devices:" + ls -la /sys/bus/mdev/devices/ 2>/dev/null || log_warn "No mdev devices" +} + +# ============================================================ +# QEMU COMMANDS +# ============================================================ + +cmd_launch_qemu() { + log_info "Launching QEMU with vGPU..." + if [[ ! -f "$QEMU_SCRIPT" ]]; then + log_err "launch_qemu_vgpu.sh not found" + return 1 + fi + "$QEMU_SCRIPT" "$@" +} + +# ============================================================ +# TEST COMMANDS +# ============================================================ + +cmd_test_stack() { + log_info "Running comprehensive test suite..." + if [[ ! -f "$TEST_SCRIPT" ]]; then + log_err "test_vgpu_stack.sh not found" + return 1 + fi + "$TEST_SCRIPT" "$@" +} + +# ============================================================ +# STATUS / DEBUG +# ============================================================ + +cmd_status() { + echo "=== Kernel Modules ===" + lsmod | grep -E "nvidia|open_iov|vfio|mdev" || echo " (none)" + echo "" + echo "=== MDEV Devices ===" + ls -la /sys/bus/mdev/devices/ 2>/dev/null || echo " (none)" + echo "" + echo "=== VFIO Devices ===" + ls -la /dev/vfio/ 2>/dev/null || echo " (none)" + echo "" + echo "=== Firmware ===" + ls -la "$FIRMWARE_DEST" 2>/dev/null || echo " Not installed" + echo "" + echo "=== dmesg (last 20) ===" + sudo dmesg -T | grep -i -e nvidia -e gsp -e vgpu -e open-iov -e mdev | tail -20 +} + +cmd_dmesg() { + sudo dmesg -T -w | grep -i -e nvidia -e gsp -e vgpu -e open-iov -e mdev +} + +cmd_debug_qemu() { + log_info "Starting debug QEMU session..." + if [[ ! -f "$BASE_DIR/scripts/debug_guest_init.sh" ]]; then + log_err "debug_guest_init.sh not found" + return 1 + fi + "$BASE_DIR/scripts/debug_guest_init.sh" "$@" +} + +cmd_cleanup_crash() { + log_info "Cleaning up after crash..." + cmd_unload_stack + log_info "Clearing any stuck mdev devices..." + for mdev in /sys/bus/mdev/devices/*; do + [[ -e "$mdev" ]] || continue + uuid=$(basename "$mdev") + echo 1 | sudo tee "$mdev/remove" >/dev/null 2>&1 || true + done + log_ok "Cleanup complete" +} + +# ============================================================ +# HELP +# ============================================================ + +cmd_help() { + cat < [options] + +BUILD COMMANDS (run in order): + build-kernel Build open-iov-vfio.ko (generates Module.symvers) + build-nvidia Build NVIDIA OpenRM with KBUILD_EXTRA_SYMBOLS + build-userspace Build libiov.so + examples + build-all Run all three build steps in correct order + +LOAD COMMANDS (run in order): + load-prereqs Load vfio, vfio_iommu_type1, vfio_pci, mdev, vfio_mdev + load-open-iov Load open-iov-vfio.ko + load-nvidia Load nvidia.ko (OpenRM) + install-firmware Copy GSP firmware to /lib/firmware/ + load-stack Run all four load steps in correct order + +FULL CYCLE: + up Build all + load stack + create mdev (one command!) + +MDEV COMMANDS: + mdev-create Create mdev device + mdev-list List mdev devices + +QEMU COMMANDS: + qemu Launch QEMU with vGPU + qemu-debug Launch QEMU with GDB + guest monitor + +TEST COMMANDS: + test Run comprehensive test suite + +STATUS / DEBUG: + status Show current stack status + dmesg Follow filtered dmesg output + cleanup-crash Emergency cleanup after crash/hang + help Show this help + +EXAMPLES: + # Full startup from clean state: + ./stack_manager.sh up + + # Step by step: + ./stack_manager.sh build-all + ./stack_manager.sh load-stack + ./stack_manager.sh mdev-create + ./stack_manager.sh qemu + + # Or just load (if already built): + ./stack_manager.sh load-stack + ./stack_manager.sh mdev-create + ./stack_manager.sh qemu + + # Testing: + ./stack_manager.sh test + + # Cleanup: + ./stack_manager.sh unload + # Or force cleanup after crash: + ./stack_manager.sh cleanup-crash + +NOTE: + - Requires root for module operations + - BUILD ORDER: open-iov-vfio.ko FIRST (generates Module.symvers), + THEN nvidia.ko with KBUILD_EXTRA_SYMBOLS + - LOAD ORDER: vfio/mdev prerequisites -> open-iov-vfio.ko -> nvidia.ko + - UNLOAD ORDER: remove mdevs -> open-iov-vfio.ko -> nvidia.ko + - NEVER REBOOT without explicit approval +EOF +} + +main() { + case "${1:-help}" in + # Build + build-kernel) cmd_build_kernel ;; + build-nvidia) cmd_build_nvidia ;; + build-userspace) cmd_build_userspace ;; + build-all) cmd_build_all ;; + + # Load + load-prereqs) cmd_load_prereqs ;; + load-open-iov) cmd_load_open_iov ;; + load-nvidia) cmd_load_nvidia ;; + install-firmware) cmd_install_firmware ;; + load-stack) cmd_load_stack ;; + + # Full cycle + up) cmd_up ;; + + # Unload + unload) cmd_unload_stack ;; + + # MDEV + mdev-create) shift; cmd_create_mdev "$@" ;; + mdev-list) cmd_list_mdev ;; + + # QEMU + qemu) shift; cmd_launch_qemu "$@" ;; + qemu-debug) shift; cmd_debug_qemu "$@" ;; + + # Test + test) shift; cmd_test_stack "$@" ;; + + # Status/Debug + status) cmd_status ;; + dmesg) cmd_dmesg ;; + cleanup-crash) cmd_cleanup_crash ;; + help|*) cmd_help ;; + esac +} + +main "$@" \ No newline at end of file diff --git a/scripts/windows_guest_automation.sh b/scripts/windows_guest_automation.sh new file mode 100755 index 0000000..f95471e --- /dev/null +++ b/scripts/windows_guest_automation.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# windows_guest_automation.sh - Windows guest test automation via WinRM/SSH + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +RESULTS_DIR="/tmp/gsp_emu_windows/$(date +%Y%m%d_%H%M%S)" +mkdir -p "$RESULTS_DIR" + +log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; } +log_ok() { echo -e "\033[0;32m[OK]\033[0m $1"; } +log_error() { echo -e "\033[0;31m[ERROR]\033[0m $1"; } + +# Configuration +WIN_USER="${WIN_USER:-Administrator}" +WIN_IP="${WIN_IP:-192.168.122.100}" +WIN_PASSWORD="${WIN_PASSWORD:-}" +WINRM_PORT="${WINRM_PORT:-5985}" +SSH_PORT="${SSH_PORT:-22}" + +check_winrm() { + log_info "Checking WinRM connectivity to $WIN_IP..." + if command -v winrm &>/dev/null; then + winrm identify -r:"http://$WIN_IP:$WINRM_PORT/wsman" -u:"$WIN_USER" -p:"$WIN_PASSWORD" -auth:basic 2>/dev/null + elif command -v python3 &>/dev/null; then + python3 -c " +import winrm +session = winrm.Session('$WIN_IP:$WINRM_PORT', auth=('$WIN_USER', '$WIN_PASSWORD'), transport='ntlm') +result = session.run_cmd('echo test') +print('WinRM OK' if result.status_code == 0 else 'WinRM FAILED') +" + else + log_error "No WinRM client found. Install winrm or python3 with pywinrm" + return 1 + fi +} + +run_winrm_cmd() { + local cmd="$1" + local output_file="$2" + + if command -v winrm &>/dev/null; then + winrm invoke -r:"http://$WIN_IP:$WINRM_PORT/wsman" -u:"$WIN_USER" -p:"$WIN_PASSWORD" \ + -auth:basic Create "http://schemas.microsoft.com/wbem/wsman/1/windows/shell/Command" \ + "Command=cmd.exe" "Arguments=/c $cmd" > "$output_file" 2>&1 + else + python3 -c " +import winrm, sys +session = winrm.Session('$WIN_IP:$WINRM_PORT', auth=('$WIN_USER', '$WIN_PASSWORD'), transport='ntlm') +result = session.run_cmd('$cmd') +with open('$output_file', 'w') as f: + f.write(result.std_out.decode() + result.std_err.decode()) +sys.exit(result.status_code) +" + fi +} + +run_ssh_cmd() { + local cmd="$1" + local output_file="$2" + sshpass -p "$WIN_PASSWORD" ssh -o StrictHostKeyChecking=no -p "$SSH_PORT" "$WIN_USER@$WIN_IP" "$cmd" > "$output_file" 2>&1 +} + +test_nvidia_smi() { + log_info "Testing nvidia-smi..." + run_winrm_cmd "nvidia-smi" "$RESULTS_DIR/nvidia_smi.txt" + if grep -q "NVIDIA" "$RESULTS_DIR/nvidia_smi.txt"; then + log_ok "nvidia-smi: PASS" + return 0 + else + log_error "nvidia-smi: FAIL" + cat "$RESULTS_DIR/nvidia_smi.txt" + return 1 + fi +} + +test_cuda_vectoradd() { + log_info "Testing CUDA vectorAdd..." + run_winrm_cmd "cd C:\\cuda_samples\\1_Utilities\\vectorAdd && .\\vectorAdd.exe" "$RESULTS_DIR/cuda_vectoradd.txt" + if grep -q "Test passed" "$RESULTS_DIR/cuda_vectoradd.txt"; then + log_ok "CUDA vectorAdd: PASS" + return 0 + else + log_error "CUDA vectorAdd: FAIL" + cat "$RESULTS_DIR/cuda_vectoradd.txt" + return 1 + fi +} + +test_cuda_devquery() { + log_info "Testing CUDA deviceQuery..." + run_winrm_cmd "cd C:\\cuda_samples\\1_Utilities\\deviceQuery && .\\deviceQuery.exe" "$RESULTS_DIR/cuda_devquery.txt" + if grep -q "Result = PASS" "$RESULTS_DIR/cuda_devquery.txt"; then + log_ok "CUDA deviceQuery: PASS" + return 0 + else + log_error "CUDA deviceQuery: FAIL" + cat "$RESULTS_DIR/cuda_devquery.txt" + return 1 + fi +} + +test_vulkan() { + log_info "Testing Vulkan..." + run_winrm_cmd "vulkaninfo.exe" "$RESULTS_DIR/vulkaninfo.txt" + if grep -q "GPU id" "$RESULTS_DIR/vulkaninfo.txt"; then + log_ok "Vulkan: PASS" + return 0 + else + log_error "Vulkan: FAIL" + cat "$RESULTS_DIR/vulkaninfo.txt" + return 1 + fi +} + +test_dx12() { + log_info "Testing DirectX 12..." + run_winrm_cmd "dxdiag /t $RESULTS_DIR/dxdiag.txt" "$RESULTS_DIR/dxdiag_cmd.txt" + # Parse dxdiag output + if grep -q "Feature Level" "$RESULTS_DIR/dxdiag.txt"; then + log_ok "DirectX 12: PASS" + return 0 + else + log_error "DirectX 12: FAIL" + return 1 + fi +} + +test_ffmpeg_nvenc() { + log_info "Testing FFmpeg NVENC..." + run_winrm_cmd "ffmpeg -encoders" "$RESULTS_DIR/ffmpeg_encoders.txt" + if grep -q "h264_nvenc\|hevc_nvenc" "$RESULTS_DIR/ffmpeg_encoders.txt"; then + log_ok "FFmpeg NVENC: PASS" + return 0 + else + log_error "FFmpeg NVENC: FAIL" + return 1 + fi +} + +test_pytorch() { + log_info "Testing PyTorch..." + run_winrm_cmd "python -c \"import torch; print('CUDA:', torch.cuda.is_available()); print('Device:', torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'N/A')\"" "$RESULTS_DIR/pytorch.txt" + if grep -q "CUDA: True" "$RESULTS_DIR/pytorch.txt"; then + log_ok "PyTorch: PASS" + return 0 + else + log_error "PyTorch: FAIL" + cat "$RESULTS_DIR/pytorch.txt" + return 1 + fi +} + +test_windows_driver_signature() { + log_info "Checking driver signature status..." + run_winrm_cmd "pnputil /enum-drivers | findstr nvidia" "$RESULTS_DIR/driver_signature.txt" + cat "$RESULTS_DIR/driver_signature.txt" +} + +collect_event_logs() { + log_info "Collecting Windows event logs..." + run_winrm_cmd "wevtutil qe System /q:\"*[System[Provider[@Name='NVIDIA']]]\" /f:text /c:50" "$RESULTS_DIR/nvidia_events.txt" + run_winrm_cmd "wevtutil qe System /q:\"*[System[Provider[@Name='Display']]]\" /f:text /c:50" "$RESULTS_DIR/display_events.txt" +} + +run_full_suite() { + log_info "=== Windows Guest Test Suite ===" + local failed=0 + + test_nvidia_smi || ((failed++)) + test_cuda_devquery || ((failed++)) + test_cuda_vectoradd || ((failed++)) + test_vulkan || ((failed++)) + test_dx12 || ((failed++)) + test_ffmpeg_nvenc || ((failed++)) + test_pytorch || ((failed++)) + test_windows_driver_signature + collect_event_logs + + log_info "=== Results ===" + log_info "Failed tests: $failed" + + # Generate summary + cat > "$RESULTS_DIR/summary.json" <