From 68cdd7728a7cc6715bbbff7decfe2b11c3bdf558 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- tools/guest_init_monitor.py | 207 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 tools/guest_init_monitor.py (limited to 'tools/guest_init_monitor.py') diff --git a/tools/guest_init_monitor.py b/tools/guest_init_monitor.py new file mode 100644 index 0000000..bc34a08 --- /dev/null +++ b/tools/guest_init_monitor.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +guest_init_monitor.py - Monitor guest kernel initialization for NVIDIA driver +Correlates host tracepoints with guest dmesg/serial output +""" + +import sys +import os +import re +import time +import subprocess +import threading +import json +from datetime import datetime +from collections import deque +from pathlib import Path + +class GuestInitMonitor: + def __init__(self, guest_serial="/dev/ttyS0", host_trace_pipe="/sys/kernel/debug/tracing/trace_pipe"): + self.guest_serial = guest_serial + self.host_trace_pipe = host_trace_pipe + self.guest_log = deque(maxlen=1000) + self.host_events = deque(maxlen=1000) + self.failure_patterns = self._load_failure_patterns() + self.running = False + + def _load_failure_patterns(self): + return { + "pci_probe_fail": [ + r"nvidia: probe of .* failed with error -?\d+", + r"pci .*: can't claim BAR \d+", + r"pci .*: BAR \d+: no space for", + ], + "gsp_firmware_fail": [ + r"GSP.*firmware.*load.*failed", + r"GSP.*bootstrap.*failed", + r"GSP.*RPC.*timeout", + r"nv_gsp_firmware_load.*error", + ], + "msix_fail": [ + r"MSI-X.*failed", + r"pci_enable_msix.*failed", + r"request_threaded_irq.*failed", + ], + "bar_map_fail": [ + r"ioremap.*failed", + r"pci_iomap.*failed", + r"BAR.*mapping.*failed", + ], + "dma_fail": [ + r"dma_map.*failed", + r"IOMMU.*fault", + r"DMAR.*error", + ], + "vgpu_vfio_fail": [ + r"vGPU VFIO.*failed", + r"nvidia_vgpu_vfio_get_ops.*failed", + r"mdev.*register.*failed", + ], + } + + def start(self): + self.running = True + self.guest_thread = threading.Thread(target=self._monitor_guest_serial, daemon=True) + self.host_thread = threading.Thread(target=self._monitor_host_traces, daemon=True) + self.guest_thread.start() + self.host_thread.start() + print(f"[{datetime.now()}] Guest init monitor started") + print(f" Guest serial: {self.guest_serial}") + print(f" Host trace pipe: {self.host_trace_pipe}") + + def stop(self): + self.running = False + if hasattr(self, 'guest_thread'): + self.guest_thread.join(timeout=2) + if hasattr(self, 'host_thread'): + self.host_thread.join(timeout=2) + print(f"[{datetime.now()}] Guest init monitor stopped") + + def _monitor_guest_serial(self): + """Monitor guest kernel log via serial console""" + try: + # Try to open serial device or use virsh console + proc = subprocess.Popen( + ["timeout", "300", "cat", self.guest_serial], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + for line in proc.stdout: + if not self.running: + break + self._process_guest_line(line.strip()) + except Exception as e: + print(f"[{datetime.now()}] Guest serial monitor error: {e}") + + def _monitor_host_traces(self): + """Monitor host kernel tracepoints""" + try: + proc = subprocess.Popen( + ["cat", self.host_trace_pipe], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + for line in proc.stdout: + if not self.running: + break + self._process_host_trace(line.strip()) + except Exception as e: + print(f"[{datetime.now()}] Host trace monitor error: {e}") + + def _process_guest_line(self, line): + """Process a line from guest serial console""" + timestamp = datetime.now().isoformat() + entry = {"timestamp": timestamp, "source": "guest", "line": line} + self.guest_log.append(entry) + + # Check for failure patterns + failure = self._classify_failure(line) + if failure: + self._report_failure(failure, line, "guest") + + # Print important lines + if any(kw in line.lower() for kw in ["nvidia", "gsp", "vfio", "mdev", "error", "fail", "timeout"]): + print(f"[GUEST] {line}") + + def _process_host_trace(self, line): + """Process a line from host trace_pipe""" + timestamp = datetime.now().isoformat() + entry = {"timestamp": timestamp, "source": "host_trace", "line": line} + self.host_events.append(entry) + + # Look for GSP emulation tracepoints + if "gsp_emu" in line: + print(f"[HOST TRACE] {line}") + # Correlate with recent guest logs + self._correlate_events(line) + + def _classify_failure(self, line): + """Classify failure type based on patterns""" + for failure_type, patterns in self.failure_patterns.items(): + for pattern in patterns: + if re.search(pattern, line, re.IGNORECASE): + return failure_type + return None + + def _report_failure(self, failure_type, line, source): + """Report a detected failure""" + report = { + "timestamp": datetime.now().isoformat(), + "failure_type": failure_type, + "source": source, + "line": line, + "recent_guest_logs": list(self.guest_log)[-20:], + "recent_host_events": list(self.host_events)[-20:], + } + print(f"\n{'='*60}") + print(f"FAILURE DETECTED: {failure_type}") + print(f"Source: {source}") + print(f"Line: {line}") + print(f"{'='*60}\n") + + # Save failure report + report_dir = Path("/tmp/gsp_emu_failures") + report_dir.mkdir(exist_ok=True) + report_file = report_dir / f"failure_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + with open(report_file, 'w') as f: + json.dump(report, f, indent=2) + print(f"Failure report saved: {report_file}") + + def _correlate_events(self, host_event): + """Correlate host tracepoint with recent guest activity""" + # Look for matching sequence IDs, RPC types, etc. + pass + + def get_summary(self): + """Get monitoring summary""" + return { + "guest_log_entries": len(self.guest_log), + "host_events": len(self.host_events), + "recent_guest": list(self.guest_log)[-10:], + "recent_host": list(self.host_events)[-10:], + } + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Monitor guest NVIDIA driver initialization") + parser.add_argument("--guest-serial", default="/dev/ttyS0", help="Guest serial device") + parser.add_argument("--host-trace", default="/sys/kernel/debug/tracing/trace_pipe", help="Host trace pipe") + parser.add_argument("--duration", type=int, default=300, help="Monitor duration (seconds)") + args = parser.parse_args() + + monitor = GuestInitMonitor(args.guest_serial, args.host_trace) + monitor.start() + + try: + time.sleep(args.duration) + except KeyboardInterrupt: + pass + finally: + monitor.stop() + summary = monitor.get_summary() + print(json.dumps(summary, indent=2)) + +if __name__ == "__main__": + main() \ No newline at end of file -- cgit v1.2.3