Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 312 insertions, 0 deletions

diff --git a/tools/failure_analyzer.py b/tools/failure_analyzer.py
new file mode 100644
index 0000000..7ffb776
--- /dev/null
+++ b/tools/failure_analyzer.py
@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""
+failure_analyzer.py - Analyze and classify guest initialization failures
+Correlates host tracepoints, guest logs, and QEMU state
+"""
+
+import sys
+import os
+import re
+import json
+import argparse
+from datetime import datetime
+from pathlib import Path
+from collections import defaultdict
+from typing import Dict, List, Optional, Any
+
+class FailureAnalyzer:
+    def __init__(self):
+        self.failure_signatures = self._load_signatures()
+        self.recommendations = self._load_recommendations()
+    
+    def _load_signatures(self) -> Dict:
+        return {
+            "PCI_PROBE_FAILURE": {
+                "guest_patterns": [
+                    r"nvidia: probe of .* failed with error -?\d+",
+                    r"pci .*: can't claim BAR \d+",
+                    r"pci .*: BAR \d+: no space for \[mem size",
+                    r"pci .*: BAR \d+: failed to assign",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_rpc_received.*rpc_type=0x1.*status=-?\d+",
+                    r"gsp_emu_fallback_used.*rpc_type=0x1",
+                ],
+                "severity": "critical",
+                "category": "pci_config"
+            },
+            "GSP_FIRMWARE_LOAD_FAILURE": {
+                "guest_patterns": [
+                    r"GSP.*firmware.*load.*failed",
+                    r"GSP.*bootstrap.*failed",
+                    r"nv_gsp_firmware_load.*error",
+                    r"GSP.*RPC.*timeout",
+                    r"GSP.*communication.*failed",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_firmware_analysis.*matched=0",
+                    r"gsp_emu_dispatcher_select.*emulation=1",
+                    r"gsp_emu_rpc_received.*rpc_type=0x1.*seq_id=1",
+                    r"gsp_emu_rpc_processed.*status=-110",
+                ],
+                "severity": "critical",
+                "category": "gsp_firmware"
+            },
+            "MSIX_SETUP_FAILURE": {
+                "guest_patterns": [
+                    r"MSI-X.*failed",
+                    r"pci_enable_msix.*failed",
+                    r"request_threaded_irq.*failed",
+                    r"irq.*allocation.*failed",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_msix_injected.*vector=0",
+                    r"gsp_emu_rpc_processed.*rpc_type=0x7.*status=-?\d+",
+                ],
+                "severity": "high",
+                "category": "interrupts"
+            },
+            "BAR_MAPPING_FAILURE": {
+                "guest_patterns": [
+                    r"ioremap.*failed",
+                    r"pci_iomap.*failed",
+                    r"BAR.*mapping.*failed",
+                    r"VFIO.*region.*mmap.*failed",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_shared_mem_access.*is_write=1",
+                    r"gsp_emu_bar_emulation.*",
+                ],
+                "severity": "high",
+                "category": "memory_mapping"
+            },
+            "DMA_MAPPING_FAILURE": {
+                "guest_patterns": [
+                    r"dma_map.*failed",
+                    r"IOMMU.*fault",
+                    r"DMAR.*error",
+                    r"iommu.*domain.*failed",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_rpc_processed.*rpc_type=0x5.*status=-?\d+",
+                    r"gsp_emu_fallback_used.*rpc_type=0x5",
+                ],
+                "severity": "high",
+                "category": "dma"
+            },
+            "VGPU_VFIO_FAILURE": {
+                "guest_patterns": [
+                    r"vGPU VFIO.*failed",
+                    r"nvidia_vgpu_vfio_get_ops.*failed",
+                    r"mdev.*register.*failed",
+                    r"VFIO.*device.*not found",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_thread_state.*state=0",
+                ],
+                "severity": "critical",
+                "category": "vfio"
+            },
+            "GSP_RPC_TIMEOUT": {
+                "guest_patterns": [
+                    r"GSP.*RPC.*timeout",
+                    r"doorbell.*timeout",
+                    r"GSP.*response.*timeout",
+                ],
+                "host_trace_patterns": [
+                    r"gsp_emu_doorbell_poll.*",
+                    r"gsp_emu_rpc_received.*",
+                    r"gsp_emu_rpc_processed.*latency_ns=\d{9,}",
+                ],
+                "severity": "medium",
+                "category": "rpc_timeout"
+            },
+        }
+    
+    def _load_recommendations(self) -> Dict:
+        return {
+            "PCI_PROBE_FAILURE": [
+                "Check PCI config space emulation in open-iov-vfio.ko (main.c)",
+                "Verify BAR sizes match physical GPU (16MB/256MB/32MB/32MB)",
+                "Check MSI-X capability pointer (0x70) and table/PBA offsets",
+                "Ensure mdev parent registered with NVIDIA host driver",
+            ],
+            "GSP_FIRMWARE_LOAD_FAILURE": [
+                "Verify firmware file exists at /lib/firmware/nvidia/tu102/gsp/",
+                "Check firmware CRC32/SHA256 matches known signature",
+                "Ensure GSP emulation thread started (gsp_emu_thread_state)",
+                "Check doorbell register offset (BAR0+0x110C00)",
+                "Verify shared memory mapping (BAR0+0x1000, 4KB)",
+            ],
+            "MSIX_SETUP_FAILURE": [
+                "Verify MSI-X table at BAR0+0x2000 (32 entries × 16 bytes)",
+                "Verify MSI-X PBA at BAR0+0x3000 (64-bit)",
+                "Check eventfd signaling for interrupt injection",
+                "Ensure guest MSI-X enable bit set in Message Control",
+            ],
+            "BAR_MAPPING_FAILURE": [
+                "Check VFIO region mmap handlers (bar_emulation.c)",
+                "Verify BAR0 includes doorbell (0x110C00) and shared mem (0x1000)",
+                "Check MSI-X region (VFIO_PCI_MSIX_REGION_INDEX)",
+                "Ensure WC (write-combine) mapping for BAR0",
+            ],
+            "DMA_MAPPING_FAILURE": [
+                "Verify IOMMU domain attached to VFIO group",
+                "Check DMA mapping in memory.c (nv_vgpu_dma_map)",
+                "Ensure physical pages pinned before mapping",
+                "Check for IOMMU faults in dmesg",
+            ],
+            "VGPU_VFIO_FAILURE": [
+                "Verify nvidia.ko loaded before open-iov-vfio.ko",
+                "Check KBUILD_EXTRA_SYMBOLS points to nvidia.ko Module.symvers",
+                "Verify nvidia_vgpu_vfio_get_ops exported by nvidia.ko",
+                "Check mdev parent registration callback",
+            ],
+            "GSP_RPC_TIMEOUT": [
+                "Check GSP emulation thread CPU usage (should be polling)",
+                "Verify doorbell polling adaptive (busy-wait 100us then usleep)",
+                "Check shared memory coherence (cache flush/invalidate)",
+                "Trace RPC sequence IDs for gaps/duplicates",
+            ],
+        }
+    
+    def analyze(self, guest_log_file: str, host_trace_file: str, qemu_log_file: str = None) -> Dict:
+        """Analyze failure from log files"""
+        
+        guest_logs = self._read_logs(guest_log_file)
+        host_traces = self._read_logs(host_trace_file)
+        qemu_logs = self._read_logs(qemu_log_file) if qemu_log_file else []
+        
+        # Classify failure
+        failure_type = self._classify_failure(guest_logs, host_traces)
+        
+        # Get matching evidence
+        evidence = self._collect_evidence(failure_type, guest_logs, host_traces)
+        
+        # Generate report
+        report = {
+            "timestamp": datetime.now().isoformat(),
+            "failure_type": failure_type,
+            "severity": self.failure_signatures.get(failure_type, {}).get("severity", "unknown"),
+            "category": self.failure_signatures.get(failure_type, {}).get("category", "unknown"),
+            "evidence": evidence,
+            "recommendations": self.recommendations.get(failure_type, []),
+            "guest_log_summary": self._summarize_logs(guest_logs),
+            "host_trace_summary": self._summarize_logs(host_traces),
+        }
+        
+        return report
+    
+    def _read_logs(self, filepath: str) -> List[str]:
+        if not filepath or not os.path.exists(filepath):
+            return []
+        with open(filepath, 'r') as f:
+            return [line.strip() for line in f.readlines()]
+    
+    def _classify_failure(self, guest_logs: List[str], host_traces: List[str]) -> str:
+        """Classify failure based on patterns"""
+        scores = defaultdict(int)
+        
+        for failure_type, sig in self.failure_signatures.items():
+            # Check guest patterns
+            for pattern in sig.get("guest_patterns", []):
+                for line in guest_logs:
+                    if re.search(pattern, line, re.IGNORECASE):
+                        scores[failure_type] += 3
+            
+            # Check host trace patterns
+            for pattern in sig.get("host_trace_patterns", []):
+                for line in host_traces:
+                    if re.search(pattern, line):
+                        scores[failure_type] += 2
+        
+        if not scores:
+            return "UNKNOWN"
+        
+        # Return highest scoring failure type
+        return max(scores, key=scores.get)
+    
+    def _collect_evidence(self, failure_type: str, guest_logs: List[str], host_traces: List[str]) -> Dict:
+        """Collect matching evidence for failure type"""
+        sig = self.failure_signatures.get(failure_type, {})
+        evidence = {"guest": [], "host": []}
+        
+        for pattern in sig.get("guest_patterns", []):
+            for line in guest_logs:
+                if re.search(pattern, line, re.IGNORECASE):
+                    evidence["guest"].append({"pattern": pattern, "match": line})
+        
+        for pattern in sig.get("host_trace_patterns", []):
+            for line in host_traces:
+                if re.search(pattern, line):
+                    evidence["host"].append({"pattern": pattern, "match": line})
+        
+        return evidence
+    
+    def _summarize_logs(self, logs: List[str]) -> Dict:
+        """Summarize log content"""
+        return {
+            "total_lines": len(logs),
+            "error_lines": sum(1 for l in logs if "error" in l.lower() or "fail" in l.lower()),
+            "nvidia_lines": sum(1 for l in logs if "nvidia" in l.lower()),
+            "gsp_lines": sum(1 for l in logs if "gsp" in l.lower()),
+            "last_20": logs[-20:] if logs else [],
+        }
+    
+    def print_report(self, report: Dict):
+        """Print formatted failure report"""
+        print("\n" + "="*70)
+        print(f"FAILURE ANALYSIS REPORT")
+        print("="*70)
+        print(f"Timestamp:     {report['timestamp']}")
+        print(f"Failure Type:  {report['failure_type']}")
+        print(f"Severity:      {report['severity']}")
+        print(f"Category:      {report['category']}")
+        print()
+        
+        print("EVIDENCE:")
+        print("-"*70)
+        for src in ["guest", "host"]:
+            ev = report['evidence'].get(src, [])
+            if ev:
+                print(f"  {src.upper()} ({len(ev)} matches):")
+                for e in ev[:5]:
+                    print(f"    Pattern: {e['pattern']}")
+                    print(f"    Match:   {e['match'][:120]}")
+                    print()
+        
+        print("RECOMMENDATIONS:")
+        print("-"*70)
+        for i, rec in enumerate(report['recommendations'], 1):
+            print(f"  {i}. {rec}")
+        print()
+        
+        print("LOG SUMMARIES:")
+        print("-"*70)
+        for src in ["guest", "host"]:
+            summ = report.get(f'{src}_log_summary', {})
+            print(f"  {src.upper()}: {summ.get('total_lines', 0)} lines, "
+                  f"{summ.get('error_lines', 0)} errors, "
+                  f"{summ.get('nvidia_lines', 0)} nvidia, "
+                  f"{summ.get('gsp_lines', 0)} gsp")
+        print("="*70)
+
+def main():
+    parser = argparse.ArgumentParser(description="Analyze guest NVIDIA driver initialization failures")
+    parser.add_argument("--guest-log", required=True, help="Guest serial log file")
+    parser.add_argument("--host-trace", required=True, help="Host trace_pipe log file")
+    parser.add_argument("--qemu-log", help="QEMU log file (optional)")
+    parser.add_argument("--output", help="Output JSON report file")
+    args = parser.parse_args()
+    
+    analyzer = FailureAnalyzer()
+    report = analyzer.analyze(args.guest_log, args.host_trace, args.qemu_log)
+    analyzer.print_report(report)
+    
+    if args.output:
+        with open(args.output, 'w') as f:
+            json.dump(report, f, indent=2)
+        print(f"\nReport saved to: {args.output}")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file