diff options
29 files changed, 729 insertions, 477 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4fed5a..f3f71a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,19 @@ -# GSP vGPU-IO CI/CD Pipeline +# Open-IOV CI/CD Pipeline -name: GSP vGPU-IO CI +name: Open-IOV CI on: push: - branches: [main, develop] - paths: - - 'open-iov/**' + branches: [main, master, develop] pull_request: - branches: [main, develop] - paths: - - 'open-iov/**' + branches: [main, master, develop] env: KERNEL_VERSION: ${{ matrix.kernel }} GCC_VERSION: ${{ matrix.gcc }} + GSP_VERSION: '570.144' + FW_RAW_BASE: https://raw.githubusercontent.com/NVIDIA/linux-firmware/2237b99e29d09bebffdb706ae05005d9026fd7e4/nvidia/tu102/gsp + FW_MD5: 13ce12a392a31eeb79a2e6ebd3a4a93a jobs: # ============================================================ @@ -39,12 +38,10 @@ jobs: gcc: '13' kernel_package: 'linux-headers-6.8.0-48-generic' timeout-minutes: 30 - + steps: - name: Checkout repository uses: actions/checkout@v4 - with: - path: open-iov - name: Install kernel headers run: | @@ -56,19 +53,28 @@ jobs: sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ matrix.gcc }} 100 gcc --version - - name: Build driver + - name: Fetch GSP firmware (linux-firmware) run: | - cd open-iov/driver - make KDIR=/lib/modules/$(uname -r)/build + mkdir -p firmware/57X-GSP/570.144 + curl -fSL --retry 3 -o firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.bin \ + "${{ env.FW_RAW_BASE }}/gsp-${{ env.GSP_VERSION }}.bin" + objcopy -O binary --only-section=.fwimage \ + firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.bin \ + firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.fwimage.bin + echo "${{ env.FW_MD5 }} firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.fwimage.bin" | md5sum -c - + + - name: Build host kernel module + run: make kernel + + - name: Build legacy guest driver + run: make -C driver KDIR=/lib/modules/$(uname -r)/build - name: Build kernel tests - run: | - cd open-iov/kernel/tests - make KDIR=/lib/modules/$(uname -r)/build + run: make -C kernel/tests KDIR=/lib/modules/$(uname -r)/build - name: Run comprehensive test module run: | - cd open-iov/kernel/tests + cd kernel/tests sudo insmod gsp_vgpu_comprehensive_test.ko sleep 2 dmesg -T | tail -50 @@ -77,7 +83,7 @@ jobs: - name: Run KUnit tests if: matrix.kernel == '6.6' && matrix.gcc == '12' run: | - cd open-iov/kernel/tests + cd kernel/tests sudo insmod gsp_vgpu_kunit_test.ko sleep 2 dmesg -T | grep -E "(kunit|KUNIT|PASS|FAIL)" | tail -50 @@ -85,85 +91,52 @@ jobs: - name: Run mock tests run: | - cd open-iov/kernel/tests + cd kernel/tests sudo insmod gsp_vgpu_mock_test.ko sleep 2 dmesg -T | grep -E "(PASS|FAIL|Test Results)" | tail -50 sudo rmmod gsp_vgpu_mock_test # ============================================================ - # Firmware Validation + # Firmware Validation (fetched from linux-firmware.git) # ============================================================ firmware-validation: name: Firmware Validation runs-on: ubuntu-latest timeout-minutes: 15 - + steps: - name: Checkout repository uses: actions/checkout@v4 - with: - path: open-iov - - name: Verify firmware exists + - name: Fetch and extract firmware run: | - ls -la open-iov/firmware/570-GSP/gsp-570.144.fwimage.bin - + mkdir -p firmware/57X-GSP/570.144 + curl -fSL --retry 3 -o firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.bin \ + "${{ env.FW_RAW_BASE }}/gsp-${{ env.GSP_VERSION }}.bin" + objcopy -O binary --only-section=.fwimage \ + firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.bin \ + firmware/57X-GSP/570.144/gsp-${{ env.GSP_VERSION }}.fwimage.bin + - name: Verify firmware MD5 run: | - cd open-iov/firmware/570-GSP - expected="13ce12a392a31eeb79a2e6ebd3a4a93a" - actual=$(md5sum gsp-570.144.fwimage.bin | cut -d' ' -f1) - echo "Expected: $expected" + cd firmware/57X-GSP/570.144 + actual=$(md5sum gsp-${{ env.GSP_VERSION }}.fwimage.bin | cut -d' ' -f1) + echo "Expected: ${{ env.FW_MD5 }}" echo "Actual: $actual" - if [ "$expected" != "$actual" ]; then - echo "MD5 mismatch!" - exit 1 - fi - echo "MD5 verified ✓" + [ "$actual" = "${{ env.FW_MD5 }}" ] + echo "MD5 verified" - name: Verify firmware size run: | - cd open-iov/firmware/570-GSP - size=$(stat -c%s gsp-570.144.fwimage.bin) - expected=29622272 # 28.5 MB + cd firmware/57X-GSP/570.144 + size=$(stat -c%s gsp-${{ env.GSP_VERSION }}.fwimage.bin) echo "Size: $size bytes" if [ "$size" -lt 28000000 ] || [ "$size" -gt 31000000 ]; then echo "Unexpected firmware size!" exit 1 fi - echo "Size verified ✓" - - - name: Verify decompiled output - run: | - cd open-iov/firmware/570-GSP/decompiled - count=$(ls *.c | wc -l) - echo "Decompiled files: $count" - if [ "$count" -lt 80 ]; then - echo "Expected at least 80 decompiled files!" - exit 1 - fi - echo "Decompilation verified ✓" - - - name: Verify key addresses in decompiled output - run: | - cd open-iov/firmware/570-GSP/decompiled - # Check critical functions exist - for func in \ - "FUN_ram_057a1ac0" \ - "FUN_ram_04cb8188" \ - "FUN_ram_04f00b0c" \ - "FUN_ram_04f00c78" \ - "FUN_ram_057e94ec" \ - "FUN_ram_051efbf4" \ - "FUN_ram_059be7f4" \ - "entry_ram_04009000"; do - if [ ! -f "${func}_ram_*.c" ]; then - echo "Missing: $func" - exit 1 - fi - done - echo "Key functions verified ✓" + echo "Size verified" # ============================================================ # Documentation Validation @@ -172,16 +145,14 @@ jobs: name: Documentation Validation runs-on: ubuntu-latest timeout-minutes: 10 - + steps: - name: Checkout repository uses: actions/checkout@v4 - with: - path: open-iov - name: Check wiki files exist run: | - cd open-iov/wiki + cd wiki required=( "GSP-570.144W-Firmware-Analysis.md" "RPC-Message-Format.md" @@ -204,8 +175,7 @@ jobs: - name: Validate markdown syntax run: | - cd open-iov/wiki - # Check for basic markdown structure + cd wiki for f in *.md; do if ! head -1 "$f" | grep -q "^#"; then echo "Warning: $f missing H1 header" @@ -214,8 +184,7 @@ jobs: - name: Check cross-references run: | - cd open-iov/wiki - # Verify internal links resolve + cd wiki for f in *.md; do grep -o '\[.*\](.*\.md)' "$f" | while read link; do target=$(echo "$link" | sed -n 's/.*(\(.*\))/\1/p') @@ -233,33 +202,24 @@ jobs: name: Static Analysis runs-on: ubuntu-latest timeout-minutes: 20 - + steps: - name: Checkout repository uses: actions/checkout@v4 - with: - path: open-iov - - name: Install checkpatch.pl + - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y linux-headers-$(uname -r) - # checkpatch.pl comes with kernel headers + sudo apt-get install -y linux-headers-$(uname -r) cppcheck - - name: Run checkpatch on driver + - name: Run checkpatch on host kernel module run: | - cd open-iov/driver + cd kernel find . -name "*.c" -o -name "*.h" | xargs /usr/src/linux-headers-$(uname -r)/scripts/checkpatch.pl --no-tree --strict || true - - name: Run sparse - run: | - cd open-iov/driver - make C=1 CHECK="/usr/bin/sparse" 2>&1 | head -100 || true - - name: Run cppcheck run: | - sudo apt-get install -y cppcheck - cd open-iov/driver + cd kernel cppcheck --enable=all --std=c11 --force -I include . 2>&1 | head -100 || true # ============================================================ @@ -269,21 +229,20 @@ jobs: name: Userspace Build Test runs-on: ubuntu-latest timeout-minutes: 10 - + steps: - name: Checkout repository uses: actions/checkout@v4 - with: - path: open-iov + + - name: Build userspace + run: make userspace examples - name: Build userspace tests - run: | - cd open-iov/userspace/tests - make + run: make -C userspace/tests - name: Verify binary run: | - cd open-iov/userspace/tests + cd userspace/tests ./test_vgpu_userspace 2>&1 | head -20 || true # Expected to fail without driver, but should run @@ -295,51 +254,16 @@ jobs: runs-on: [self-hosted, gpu, nvidia] if: github.event_name == 'workflow_dispatch' timeout-minutes: 60 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - path: open-iov - - - name: Run full integration test - run: | - cd open-iov - sudo ./tools/test/run_integration_tests.sh all - # ============================================================ - # Release Artifacts - # ============================================================ - release: - name: Create Release - needs: [kernel-test, firmware-validation, docs-validation, static-analysis, userspace-build] - if: github.event_name == 'release' && github.event.action == 'published' - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - name: Checkout repository uses: actions/checkout@v4 - with: - path: open-iov - - name: Build release artifacts - run: | - cd open-iov - # Create release package - mkdir -p release - cp firmware/570-GSP/gsp-570.144.fwimage.bin release/ - cp -r wiki release/ - cp -r driver release/ - cp -r kernel/tests release/ - cp -r userspace/tests release/ - cp README.md release/ - tar -czf gsp-vgpu-io-${{ github.event.release.tag_name }}.tar.gz release/ - - - name: Upload release assets - uses: softprops/action-gh-release@v1 - with: - files: open-iov/gsp-vgpu-io-${{ github.event.release.tag_name }}.tar.gz + - name: Full install + run: ./install.sh --no-load + + - name: Run full integration test + run: sudo ./tools/test/run_integration_tests.sh all # ============================================================ # Workflow Permissions @@ -347,4 +271,4 @@ jobs: permissions: contents: read checks: write - pull-requests: write
\ No newline at end of file + pull-requests: write @@ -13,6 +13,8 @@ # Kernel / kbuild artifacts # ---------------------------------------------------------------------------- *.o +*.o.d +.*.o.d *.ko *.mod *.mod.c @@ -72,6 +74,9 @@ tools/ghidra/**/*.lock~ # Compiled helper binaries (build from adjacent .c source instead) tools/debug/nv2080_ctrl +# Prebuilt management tool binaries (not rebuilt from this tree) +/userspace/bin/ + # ---------------------------------------------------------------------------- # Local QEMU-wrapper / VM-management layer (NOT part of the reproducible core) # Bring your own wrapper: libvirt, /, plain qemu, etc. @@ -1,193 +1,119 @@ # Open-IOV -**Open-IOV** is an open source reimplementation of NVIDIA vGPU (Virtual GPU) technology. It provides: +**Open-IOV** is an open source reimplementation of NVIDIA vGPU (Virtual GPU) +technology. It lets an **unmodified NVIDIA guest driver** run inside a VM with +full GPU functionality, without a proprietary GRID license, by emulating the +GSP (GPU System Processor) RPC interface on the host. -- **open-iov-vfio.ko** - Kernel VFIO mediated device driver for vGPU -- **open-iov.so** - Userspace library for vGPU management (replaces libvgpu.so) -- **GSP Firmware Analysis** - Complete decompilation of NVIDIA GSP 570.144W (TU102/TU104) -- **RPC Interface** - 12 RPC types: ALLOC, FREE, MAP, UNMAP, PIN, UNPIN, SCHEDULE, YIELD, NOTIFY, BAR_TRAP, MIGRATE, DEBUG +- **open-iov-vfio.ko** — host VFIO mediated-device (mdev) kernel driver with a + host-side GSP emulation engine (all 12 RPC types: ALLOC, FREE, MAP, UNMAP, + PIN, UNPIN, SCHEDULE, YIELD, NOTIFY, BAR_TRAP, MIGRATE, DEBUG) +- **libiov.so** — userspace management library (`libvgpu.so` compat symlinks) +- **GSP firmware analysis** — complete decompilation of NVIDIA GSP 570.144 + (RV64IMAC, TU102/TU104): RPC dispatchers, jump tables, WPR, BAR trap, + migration protocol, entry/boot sequence -## Architecture +The guest VM runs the **stock, unmodified** `nvidia.ko`. All Open-IOV code +lives on the host. -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Open-IOV Stack │ -├─────────────────────────────────────────────────────────────────┤ -│ Userspace │ -│ ├── open-iov.so (libopen-iov) │ -│ ├── examples/ (demo applications) │ -│ └── tests/ (unit/integration tests) │ -├─────────────────────────────────────────────────────────────────┤ -│ Kernel │ -│ ├── open-iov-vfio.ko (VFIO mdev driver) │ -│ │ ├── mdev parent registration │ -│ │ ├── VF lifecycle (create/destroy) │ -│ │ ├── BAR emulation (BAR1, ForceBarAccess, Aperture) │ -│ │ ├── GSP RPC communication (doorbell, rings) │ -│ │ ├── WPR management (write/read protect) │ -│ │ ├── Memory management (alloc, map, pin) │ -│ │ └── Live migration (4-phase protocol) │ -│ └── debugfs interface (/sys/kernel/debug/open-iov/) │ -├─────────────────────────────────────────────────────────────────┤ -│ Firmware (GSP 570.144W - TU102/TU104) │ -│ ├── Primary Dispatcher 0x057A1AC0 (32 sub-commands) │ -│ ├── Secondary Dispatcher 0x04ED39DC (141KB) │ -│ ├── BAR Trap Handler 0x04CB8188 (40KB) │ -│ ├── WPR Handlers 0x04F00B0C, 0x04F00C78 (62KB each) │ -│ ├── Migration Tables 0x04179ED8, 0x04179FBC (dual 16-entry)│ -│ └── Entry Point 0x04009000 │ -└─────────────────────────────────────────────────────────────────┘ -``` +## Current state -## Quick Start +See **[STATE.md](STATE.md)** for the live project state: what works, current +limitations, recently fixed bugs, and known open issues. -### Build Everything +## Quick start (fresh Ubuntu 22.04/24.04 + supported NVIDIA GPU) ```bash +git clone https://git.vgpu.io/open-iov.git cd open-iov -make # Builds kernel module, userspace lib, examples -``` - -### Build NVIDIA OpenRM with vGPU VFIO Symbols - -**Critical**: The NVIDIA OpenRM `nvidia.ko` must be built with `KBUILD_EXTRA_SYMBOLS` pointing to `open-iov-vfio.ko`'s `Module.symvers` to resolve the vGPU VFIO interface symbols (`nvidia_vgpu_vfio_probe`, `nvidia_vgpu_vfio_remove`, etc.). - -```bash -# Build Open-IOV kernel module first (generates Module.symvers) -cd open-iov -make kernel - -# Build NVIDIA OpenRM with vGPU VFIO symbols (use helper script) -./scripts/build_nvidia_openrm.sh - -# Or manually: -cd nvidia-open-gpu-kernel-modules/kernel-open -KBUILD_EXTRA_SYMBOLS=$OPEN_IOV_ROOT/kernel/Module.symvers make -j$(nproc) modules +./install.sh ``` -### Install Kernel Module - -```bash -# Copy firmware -sudo cp firmware/57X-GSP/570.144/gsp-570.144.fwimage.bin /lib/firmware/ +`install.sh` reproduces the entire setup: -# Load NVIDIA OpenRM host driver (exports vGPU VFIO symbols) -sudo insmod nvidia-open-gpu-kernel-modules/kernel-open/nvidia.ko +| Stage | What it does | +|-----------|--------------| +| `deps` | apt build/runtime dependencies | +| `nvidia` | clone NVIDIA open-gpu-kernel-modules @ `610.57.04`, apply `patches/nvidia/` | +| `firmware`| fetch GSP 570.144 from linux-firmware.git, extract `.fwimage`, verify MD5 | +| `build` | `open-iov-vfio.ko` → `nvidia.ko` (KBUILD_EXTRA_SYMBOLS) → userspace | +| `system` | VFIO/mdev module persistence, hugepages, IOMMU/blacklist checks | +| `load` | load the driver stack (via `scripts/stack_manager.sh`) | +| `mdev` | create a vGPU mdev device | -# Load Open-IOV host VFIO driver (with GSP emulation engine) -sudo insmod kernel/open-iov-vfio.ko +Run individual stages with `./install.sh deps nvidia`, or skip the root-requiring +tail with `./install.sh --no-load`. -# Or load guest driver with emulation (legacy path) -sudo insmod driver/gsp-vgpu.ko firmware_name=gsp-570.144.fwimage.bin - -# Verify -dmesg -T | grep -E "gsp-vgpu|open-iov|nvidia" -ls -la /dev/gsp-vgpu /dev/open-iov -``` - -### Run Example +### Day-to-day operations ```bash -sudo LD_LIBRARY_PATH=userspace/lib ./userspace/examples/example_vgpu /dev/gsp-vgpu +scripts/stack_manager.sh up # build all + load stack + create mdev +scripts/stack_manager.sh status # modules, mdevs, vfio, firmware, dmesg +scripts/stack_manager.sh qemu # launch reference raw-QEMU guest +scripts/stack_manager.sh unload # correct-order teardown +scripts/stack_manager.sh test # test suite ``` -## Emulation Mode - -Open-IOV includes **emulation mode** that allows operation with unmodified consumer/OpenRM GSP firmware by emulating missing vGPU RPC handlers (BAR_TRAP, WPR, etc.) in userspace and kernel. +(`stack_manager.sh` auto-detects its location; override with `OPEN_IOV_ROOT`.) -### How It Works +## Using your own VM stack (libvirt, , ...) -| Component | Emulated In | RPC Types Emulated | -|-----------|-------------|-------------------| -| **Kernel** | `gsp-vgpu.ko` | BAR_TRAP (0x0A), WPR (MAP/UNMAP for WPR), ALLOC, PIN, MAP | -| **Userspace** | `libiov.so` | BAR config/force/save/restore, WPR protect/unprotect/timeout | +Open-IOV is **hypervisor-management neutral**. It provides the vGPU itself +(an mdev device + VFIO group); how you attach it to a VM is up to you: -When emulation mode is enabled: -- BAR aperture configuration is handled locally (no GSP RPC) -- WPR regions are tracked in kernel/userspace memory -- Memory allocation returns synthetic handles/VAs -- Memory pinning returns synthetic DMA addresses -- BAR trap handling is simulated +- **raw QEMU**: `scripts/launch_qemu_vgpu.sh` (reference launcher), or + `-device vfio-pci,sysfsdev=/sys/bus/mdev/devices/<UUID>` +- **libvirt**: `<hostdev mode='subsystem' type='mdev' model='vfio-pci'>` + with the UUID printed by `scripts/create_mdev.sh` +- **anything else** that can pass a VFIO mdev to QEMU -### Auto-Detection (Default) +No Open-IOV component depends on a specific wrapper. -**Emulation mode is ON by default** and auto-detects GSP capabilities: - -1. On driver load, sends a test `BAR_TRAP` RPC to GSP -2. If GSP responds successfully → vGPU handlers present → **disables emulation** -3. If GSP times out/errors → vGPU handlers missing → **enables emulation** +## Repository layout ``` -dmesg output: - gsp-vgpu: Auto-detecting GSP vGPU RPC handler support... - gsp-vgpu: GSP lacks vGPU RPC handlers (ret=-110) - ENABLING emulation mode - # or - gsp-vgpu: GSP has vGPU RPC handlers - DISABLING emulation mode +kernel/ open-iov-vfio.ko host driver (mdev, VFIO, GSP emulation, + firmware analyzer, dynamic RPC dispatcher, arch/version layers) +userspace/ libiov.so management library, examples, tests +driver/ legacy guest driver (being removed in the host-only refactor) +firmware/ GSP analysis: scripts, docs, address maps (blobs NOT in git - + fetched by install.sh from linux-firmware.git) +patches/nvidia/ patch series applied to NVIDIA open-gpu-kernel-modules + (base tag 610.57.04; re-brands version to 570.144 to match + the GSP firmware ABI) +config/ declarative hardware (turing/ampere/blackwell) and + driver-version (550-580) YAML configs +scripts/ stack_manager.sh + helpers (wrapper-neutral) +tools/ debugging/profiling tools, Ghidra analysis scripts +docs/ architecture and implementation docs +wiki/ technical wiki (GSP firmware analysis, RPC format, ...) ``` -### Override Flags - -Force emulation mode behavior via module parameters: +**Not tracked in git** (by design): GSP firmware blobs, decompiled firmware +output, the NVIDIA kernel tree (cloned+patched by `install.sh`), VM disk +images/ISOs, Ghidra project databases, and any local VM-wrapper stacks. -| Parameter | Values | Description | -|-----------|--------|-------------| -| `emulation_mode` | `0`/`1` | Current state (read-only, shows auto-detected value) | -| `emulation_override` | `0`/`1` | **Override auto-detection**: `1`=force emulation ON, `0`=force emulation OFF | - -```bash -# Force emulation ON (for testing with GRID GSP) -sudo insmod driver/gsp-vgpu.ko emulation_override=1 - -# Force emulation OFF (requires GRID host driver with vGPU GSP) -sudo insmod driver/gsp-vgpu.ko emulation_override=0 - -# Default: auto-detect (emulation_mode=1, emulation_override=0) -sudo insmod driver/gsp-vgpu.ko -``` - -### GSP Blob Handling Options - -The vGPU RPC handlers (at 0x04CB8188, 0x04F00B0C, etc.) must be present in GSP firmware. Options: - -| Option | Description | Pros | Cons | -|--------|-------------|------|------| -| **GRID Host Driver** | Use NVIDIA GRID vGPU KVM driver (535+/550+) to boot GSP with vGPU firmware | Official, complete vGPU support | Requires NVIDIA Enterprise license | -| **OpenRM GSP** | Use GSP firmware from NVIDIA open kernel modules (OpenRM) | Open source, no license | Consumer GSP lacks vGPU RPC handlers | -| **Custom GSP Blob** | Inject vGPU handlers into GSP firmware | Full control | Complex, requires firmware signing | -| **Userspace Emulation** | Emulate vGPU RPC handlers in userspace/kernel | Works with any GSP, no license | Performance, completeness | - -**Recommended**: Use GRID host driver on host, Open-IOV guest driver in VM. For development, OpenRM GSP + emulation mode (auto-enabled). - -## GPC (GPU Partitioning Controller) Interfaces - -Open-IOV implements GPC-compatible interfaces for GPU partitioning: +## Requirements -- **VF Creation/Deletion** - `nv_vgpu_create_callback`, `nv_vgpu_delete_callback` -- **BAR Management** - `nv_vgpu_bar_info_callback` -- **Memory Operations** - Alloc, map, pin via GSP RPC -- **Migration** - 4-phase live migration (QUERY→PREPARE→TRANSFER→COMMIT→CLEANUP) -- **Scheduling** - Work submission, yield, priority +- Linux kernel 6.8+ (tested on 6.8.0-139-generic), IOMMU enabled + (`intel_iommu=on` / `amd_iommu=on`) +- NVIDIA GPU: TU102/TU104 validated (RTX 2080/2080 Super); Ampere/Blackwell + config scaffolding in place +- No nvidia module blacklist on the kernel cmdline ## Documentation -- [Firmware Analysis](wiki/GSP-570.144W-Firmware-Analysis.md) -- [RPC Message Format](wiki/RPC-Message-Format.md) -- [BAR Emulation](wiki/BAR-Emulation-Specification.md) -- [WPR Management](wiki/WPR-Management.md) -- [Migration Protocol](wiki/Migration-Protocol.md) -- [Integration Guide](wiki/vGPU-IO-Integration-Guide.md) - -## Requirements - -- Linux kernel 5.15+ (6.8+ tested) -- NVIDIA GPU: TU102/TU104 (RTX 2080/2080 Super) -- For full vGPU: GRID KVM driver 535+ or 550+ -- Build: gcc, make, kernel headers +- [STATE.md](STATE.md) — current state, limitations, bug tracker +- [Portable architecture](docs/PORTABLE_ARCHITECTURE.md) +- [Host-only GSP emulation design](docs/refactored-architecture.md) +- [Implementation TODO](docs/refactored-implementation-to-do.md) +- [Firmware analysis](wiki/GSP-570.144W-Firmware-Analysis.md) — + [RPC format](wiki/RPC-Message-Format.md) — + [BAR emulation](wiki/BAR-Emulation-Specification.md) — + [Migration protocol](wiki/Migration-Protocol.md) — + [Address reference](wiki/Address-Reference.md) ## License -GPL v2 (kernel), LGPL (userspace library) - -## Contributing - -See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
\ No newline at end of file +GPL v2 (kernel components), LGPL (userspace library). +NVIDIA firmware remains property of NVIDIA and is not distributed here. diff --git a/STATE.md b/STATE.md new file mode 100644 index 0000000..2621bbd --- /dev/null +++ b/STATE.md @@ -0,0 +1,77 @@ +# Open-IOV Project State + +*Last updated: 2026-09-21* + +## Mission + +Run **unmodified NVIDIA guest drivers** in VMs with full GPU functionality +(CUDA, graphics, video, AI/ML) on consumer/datacenter GPUs, without +proprietary GRID licenses — via host-side GSP emulation in +`open-iov-vfio.ko`. + +## Current milestone + +| Phase | Status | +|-------|--------| +| GSP 570.144 firmware decompilation (132 files, 12 RPC types) | ✅ Complete | +| Refactor 1: GSP emulation engine (per-mdev thread, doorbell, shared mem, MSI-X) | ✅ Complete | +| Refactor 2: PCI config space + BAR + MSI-X emulation | ✅ Complete | +| Refactor 3: All 12 RPC types with emulation fallbacks | ✅ Complete | +| Refactor 4: Portable architecture (fw analyzer, dynamic dispatcher, arch/version layers, YAML configs) | ✅ Complete | +| Refactor 5: Guest driver integration (mdev creation, VFIO registration) | ✅ Unblocked, in test | +| Refactor 6-11: CUDA / graphics / benchmarks / migration / Windows / hardening | ⏳ Pending | + +**Right now**: host stack (nvidia.ko 610.57.04-base + open-iov-vfio.ko) loads +cleanly, mdev device creates and removes cleanly, IOMMU group assignment +works, `/dev/vfio/<n>` appears. Guest VM (Ubuntu 24.04) is installed; guest +NVIDIA driver installation is in progress. Next: guest `nvidia.ko` probe +against the emulated vGPU, then `nvidia-smi`, then CUDA validation. + +## Recently solved bugs + +| Bug | Root cause | Fix | +|-----|-----------|-----| +| Module unload hang (`open_iov_vfio` stuck "Unloading") | Legacy `nv_vgpu_gsp_init()` booting physical GSP firmware, conflicting with host nvidia.ko; unload blocked in `cancel_work_sync` | Skip legacy GSP boot, use `gsp_emu_init()`; thread-stop via completion with 5s timeout; `cancel_work_sync` before `destroy_workqueue` | +| mdev remove kernel panic (`vfio_device_group_unregister+0xa`) | NULL deref on VFIO group unregister | Defensive cleanup ordering in `nv_vgpu_mdev_remove()` | +| mdev creation `EINVAL` (-22) | Double IOMMU group handling conflicted with `vfio_register_emulated_iommu_dev()` | Let VFIO create its own group; IOMMUFD emulated callbacks wired | +| `Unknown symbol nvidia_vgpu_vfio_probe` | NVIDIA OpenRM built without `KBUILD_EXTRA_SYMBOLS` | Build order automated in `stack_manager.sh` (kernel → nvidia → userspace) | +| `vfio 1: No such file or directory` | VFIO device registration is NOT automatic for mdev | `vfio_register_emulated_iommu_dev()` in probe, unregister in remove (Intel kvmgt pattern) | +| Guest-visible PCI config space wrong | BAR types/offsets, subsystem ID offsets, MSI-X table size | Config space rewritten to match physical TU104 (BAR0 16MB 32-bit non-prefetch, BAR1 256MB 64-bit prefetch VRAM, BAR3 32MB; 32 MSI-X vectors; subsystem IDs at 0x2C/0x2E) | + +## Known open bugs / limitations + +- Guest `nvidia.ko` end-to-end probe not yet validated (in progress) +- Kernel test modules (`kernel/tests/`) need API updates to match final + driver headers (low priority) +- WPR callback registration lacks kernel eventfd support + (`TODO` in driver/wpr) +- Migration WPR state save/restore incomplete +- BAR `force_access` path not fully wired (unused-function warning) +- Kernel `SCHEDULE` emulation returns success without real work submission + (userspace emulation works) +- Async `NOTIFY` events not emulated in kernel (no eventfd/poll yet) +- `open-iov-vfio.ko` requires `nvidia.ko` loaded first (`MODULE_SOFTDEP`) +- Single GPU validated so far: TU104 (RTX 2080 SUPER Mobile), GSP 570.144, + kernel 6.8.0-139-generic +- Live migration protocol implemented (4-phase) but not yet validated +- Windows guest not yet validated (automation scaffolding present) + +## Reproducibility notes + +- NVIDIA base: `open-gpu-kernel-modules` tag **610.57.04** + patch series + in `patches/nvidia/` (re-brands version string to 570.144 to match the + GSP firmware ABI; adds the vGPU VFIO interface headers + symbol exports). +- GSP firmware: `nvidia/tu102/gsp/gsp-570.144.bin` from + [linux-firmware.git](https://github.com/NVIDIA/linux-firmware) + (pin `2237b99e`), `.fwimage` section extracted with `objcopy`, + MD5 `13ce12a392a31eeb79a2e6ebd3a4a93a`. `install.sh` does this. +- Firmware blobs, decompiled output, NVIDIA tree, VM images, Ghidra DBs and + local VM wrappers are intentionally **not** tracked in git. +- Fresh-system bring-up: `./install.sh` (see README.md). + +## Hardware/software reference environment + +- Host GPU: NVIDIA RTX 2080 SUPER Mobile (TU104, PCI `10de:1ed3`) +- Host OS: Ubuntu, kernel 6.8.0-139-generic +- Guest OS: Ubuntu 24.04.5 LTS (QEMU/KVM, q35, OVMF) +- GSP firmware: 570.144 (RV64IMAC, base `0x04000000`) diff --git a/docs/DEVELOPMENT_WORKFLOW.md b/docs/DEVELOPMENT_WORKFLOW.md index 97aa267..2263af6 100644 --- a/docs/DEVELOPMENT_WORKFLOW.md +++ b/docs/DEVELOPMENT_WORKFLOW.md @@ -10,7 +10,7 @@ This document describes the complete development workflow, tooling, and shell in ### Primary Workspace ```bash -~/Git/open-iov/ # Main monorepo (git + LFS) +$OPEN_IOV_ROOT/ # Main monorepo (git + LFS) ``` ### Key Directories @@ -36,47 +36,47 @@ Add to `~/.zshrc` and run `source ~/.zshrc`: ```bash # Project navigation -alias oiv="cd ~/Git/open-iov" -alias oivk="cd ~/Git/open-iov/kernel" -alias oivu="cd ~/Git/open-iov/userspace" -alias oivf="cd ~/Git/open-iov/firmware/57X-GSP/570.144" -alias oivd="cd ~/Git/open-iov/docs" -alias oivs="cd ~/Git/open-iov/scripts" -alias oivt="cd ~/Git/open-iov/tools" +alias oiv="cd $OPEN_IOV_ROOT" +alias oivk="cd $OPEN_IOV_ROOT/kernel" +alias oivu="cd $OPEN_IOV_ROOT/userspace" +alias oivf="cd $OPEN_IOV_ROOT/firmware/57X-GSP/570.144" +alias oivd="cd $OPEN_IOV_ROOT/docs" +alias oivs="cd $OPEN_IOV_ROOT/scripts" +alias oivt="cd $OPEN_IOV_ROOT/tools" # Build commands -alias oiv-build="cd ~/Git/open-iov && make -j$(nproc)" -alias oiv-build-kernel="cd ~/Git/open-iov && make kernel" -alias oiv-build-userspace="cd ~/Git/open-iov && make userspace" -alias oiv-clean="cd ~/Git/open-iov && make clean" +alias oiv-build="cd $OPEN_IOV_ROOT && make -j$(nproc)" +alias oiv-build-kernel="cd $OPEN_IOV_ROOT && make kernel" +alias oiv-build-userspace="cd $OPEN_IOV_ROOT && make userspace" +alias oiv-clean="cd $OPEN_IOV_ROOT && make clean" # Module loading (requires reboot first!) -alias oiv-load="~/Git/open-iov/scripts/load_vgpu_stack.sh" -alias oiv-unload="~/Git/open-iov/scripts/unload_vgpu_stack.sh" -alias oiv-test="~/Git/open-iov/scripts/test_vgpu_stack.sh" -alias oiv-test-gsp="~/Git/open-iov/scripts/test_gsp_emu.sh" -alias oiv-test-guest="~/Git/open-iov/scripts/test_guest_driver.sh" -alias oiv-test-cuda="~/Git/open-iov/scripts/test_cuda_workloads.sh" -alias oiv-test-graphics="~/Git/open-iov/scripts/test_graphics_workloads.sh" -alias oiv-test-windows="~/Git/open-iov/scripts/test_windows_guest.sh" +alias oiv-load="$OPEN_IOV_ROOT/scripts/load_vgpu_stack.sh" +alias oiv-unload="$OPEN_IOV_ROOT/scripts/unload_vgpu_stack.sh" +alias oiv-test="$OPEN_IOV_ROOT/scripts/test_vgpu_stack.sh" +alias oiv-test-gsp="$OPEN_IOV_ROOT/scripts/test_gsp_emu.sh" +alias oiv-test-guest="$OPEN_IOV_ROOT/scripts/test_guest_driver.sh" +alias oiv-test-cuda="$OPEN_IOV_ROOT/scripts/test_cuda_workloads.sh" +alias oiv-test-graphics="$OPEN_IOV_ROOT/scripts/test_graphics_workloads.sh" +alias oiv-test-windows="$OPEN_IOV_ROOT/scripts/test_windows_guest.sh" # mdev management -alias oiv-mdev-create="~/Git/open-iov/scripts/create_mdev.sh" +alias oiv-mdev-create="$OPEN_IOV_ROOT/scripts/create_mdev.sh" alias oiv-mdev-list="ls -la /sys/bus/mdev/devices/ 2>/dev/null || echo 'No mdev devices'" # QEMU -alias oiv-qemu="~/Git/open-iov/scripts/launch_qemu_vgpu.sh" -alias oiv-qemu-debug="~/Git/open-iov/scripts/debug_guest_init.sh" +alias oiv-qemu="$OPEN_IOV_ROOT/scripts/launch_qemu_vgpu.sh" +alias oiv-qemu-debug="$OPEN_IOV_ROOT/scripts/debug_guest_init.sh" # Firmware analysis -alias oiv-ghidra="cd ~/Git/open-iov/tools/ghidra && ./ghidraRun" -alias oiv-fw-analyze="cd ~/Git/open-iov && python3 tools/firmware_analyzer.py" +alias oiv-ghidra="cd $OPEN_IOV_ROOT/tools/ghidra && ./ghidraRun" +alias oiv-fw-analyze="cd $OPEN_IOV_ROOT && python3 tools/firmware_analyzer.py" # Debugging tools (NEW) -alias oiv-guest-monitor="python3 ~/Git/open-iov/tools/guest_init_monitor.py" -alias oiv-failure-analyzer="python3 ~/Git/open-iov/tools/failure_analyzer.py" -alias oiv-benchmark="~/Git/open-iov/tools/benchmark_infrastructure.sh" -alias oiv-windows-test="~/Git/open-iov/scripts/windows_guest_automation.sh" +alias oiv-guest-monitor="python3 $OPEN_IOV_ROOT/tools/guest_init_monitor.py" +alias oiv-failure-analyzer="python3 $OPEN_IOV_ROOT/tools/failure_analyzer.py" +alias oiv-benchmark="$OPEN_IOV_ROOT/tools/benchmark_infrastructure.sh" +alias oiv-windows-test="$OPEN_IOV_ROOT/scripts/windows_guest_automation.sh" # Kernel debugging alias oiv-dmesg="dmesg -T | grep -i -e nvidia -e gsp -e vgpu -e open-iov -e mdev" @@ -85,12 +85,12 @@ alias oiv-dmesg-errors="dmesg -T | grep -i -e nvidia -e gsp -e vgpu -e open-iov alias oiv-trace="sudo cat /sys/kernel/debug/tracing/trace_pipe | grep gsp_emu" # Git -alias oiv-git="cd ~/Git/open-iov && git status" -alias oiv-git-log="cd ~/Git/open-iov && git log --oneline -20" -alias oiv-git-diff="cd ~/Git/open-iov && git diff" +alias oiv-git="cd $OPEN_IOV_ROOT && git status" +alias oiv-git-log="cd $OPEN_IOV_ROOT && git log --oneline -20" +alias oiv-git-diff="cd $OPEN_IOV_ROOT && git diff" # Restore point -alias oiv-restore="cd ~/Git/open-iov && git checkout f9ed5d8 -- . && git clean -fd" +alias oiv-restore="cd $OPEN_IOV_ROOT && git checkout f9ed5d8 -- . && git clean -fd" # Help oiv-help() { @@ -178,7 +178,7 @@ oiv-dmesg-follow # Live kernel logs ### Full Build ```bash -cd ~/Git/open-iov +cd $OPEN_IOV_ROOT make # Builds kernel + userspace + examples # OR oiv-build @@ -229,13 +229,13 @@ oiv-clean ### Load Order (Critical) ```bash # 1. NVIDIA OpenRM host driver (exports vGPU VFIO symbols) -sudo insmod ~/Git/open-iov/nvidia-open-gpu-kernel-modules/kernel-open/nvidia.ko +sudo insmod $OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open/nvidia.ko # 2. Open-IOV VFIO mdev driver (depends on nvidia.ko symbols) -sudo insmod ~/Git/open-iov/kernel/open-iov-vfio.ko +sudo insmod $OPEN_IOV_ROOT/kernel/open-iov-vfio.ko # 3. Guest vGPU driver (standalone, works without above) -sudo insmod ~/Git/open-iov/driver/gsp-vgpu.ko firmware_name=gsp-570.144.fwimage.bin +sudo insmod $OPEN_IOV_ROOT/driver/gsp-vgpu.ko firmware_name=gsp-570.144.fwimage.bin ``` ### Using Helper Script @@ -338,7 +338,7 @@ vncviewer localhost:1 ### Inside Guest VM ```bash # Run setup script (installs NVIDIA driver, firmware, loads module) -sudo ~/Git/open-iov/scripts/setup_guest_driver.sh +sudo $OPEN_IOV_ROOT/scripts/setup_guest_driver.sh # Or manual: # 1. Install NVIDIA driver (OpenRM 570.144) @@ -408,18 +408,18 @@ ffmpeg -hwaccel cuda -i test.mp4 -f null - ```bash # 1. Download firmware from NVIDIA OpenRM release # 2. Place in firmware dir -cp new-gsp.bin ~/Git/open-iov/firmware/570-GSP/gsp-XXX.fwimage.bin +cp new-gsp.bin $OPEN_IOV_ROOT/firmware/570-GSP/gsp-XXX.fwimage.bin # 3. Run Ghidra headless analysis -cd ~/Git/open-iov/tools/ghidra +cd $OPEN_IOV_ROOT/tools/ghidra ./support/analyzeHeadless /tmp/ghidra_project "gsp_XXX" \ - -import ~/Git/open-iov/firmware/570-GSP/gsp-XXX.fwimage.bin \ + -import $OPEN_IOV_ROOT/firmware/570-GSP/gsp-XXX.fwimage.bin \ -processor "RISCV:RV64:default" \ -postScript DecompileFunctions.java \ -deleteProject # 4. Run firmware analyzer (auto-discovers addresses) -cd ~/Git/open-iov +cd $OPEN_IOV_ROOT python3 tools/firmware_analyzer.py firmware/570-GSP/gsp-XXX.fwimage.bin > analysis_XXX.json # 5. Compare with previous version @@ -565,7 +565,7 @@ oiv-restore # git checkout f9ed5d8 -- . && git clean -fd ### Branch for Refactor ```bash -cd ~/Git/open-iov +cd $OPEN_IOV_ROOT git checkout -b refactor/host-gsp-emulation # Implement changes per docs/refactored-implementation-to-do.md ``` @@ -606,11 +606,11 @@ When you create new scripts or workflows, add them to `~/.zshrc`: ```bash # Example: Add new test command -alias oiv-test-cuda="~/Git/open-iov/scripts/test_cuda_workloads.sh" -alias oiv-test-gfx="~/Git/open-iov/scripts/test_graphics_workloads.sh" +alias oiv-test-cuda="$OPEN_IOV_ROOT/scripts/test_cuda_workloads.sh" +alias oiv-test-gfx="$OPEN_IOV_ROOT/scripts/test_graphics_workloads.sh" # Example: Add firmware analysis shortcut -alias oiv-fw-new="cd ~/Git/open-iov && python3 tools/firmware_analyzer.py firmware/57X-GSP/570.144/gsp-XXX.fwimage.bin" +alias oiv-fw-new="cd $OPEN_IOV_ROOT && python3 tools/firmware_analyzer.py firmware/57X-GSP/570.144/gsp-XXX.fwimage.bin" # Reload source ~/.zshrc @@ -656,5 +656,5 @@ Update `oiv-help()` function in `~/.zshrc` to include new commands. --- *Development Workflow Guide Version: 1.0* -*Repository: ~/Git/open-iov* +*Repository: $OPEN_IOV_ROOT* *Last Updated: $(date)*
\ No newline at end of file diff --git a/driver/Module.symvers b/driver/Module.symvers deleted file mode 100644 index e69de29..0000000 --- a/driver/Module.symvers +++ /dev/null diff --git a/firmware/57X-GSP/570.144/analyze_firmware.py b/firmware/57X-GSP/570.144/analyze_firmware.py index ecc6c65..8a61673 100644 --- a/firmware/57X-GSP/570.144/analyze_firmware.py +++ b/firmware/57X-GSP/570.144/analyze_firmware.py @@ -6,10 +6,11 @@ Comprehensive GSP firmware analysis using call targets and pattern matching. import struct import json import sys +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -CALL_TARGETS_FILE = "$OPEN_IOV_ROOT/firmware/570-GSP/all_call_targets.txt" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +CALL_TARGETS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "all_call_targets.txt") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 EXEC_SECTIONS = [ diff --git a/firmware/57X-GSP/570.144/decompile_functions.py b/firmware/57X-GSP/570.144/decompile_functions.py index 4a9bc27..6e27d05 100644 --- a/firmware/57X-GSP/570.144/decompile_functions.py +++ b/firmware/57X-GSP/570.144/decompile_functions.py @@ -6,9 +6,10 @@ Uses our decoded instruction stream to generate pseudo-C. import struct import json +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Key functions to decompile diff --git a/firmware/57X-GSP/570.144/decompile_handlers.py b/firmware/57X-GSP/570.144/decompile_handlers.py index d6a1926..e2e258f 100644 --- a/firmware/57X-GSP/570.144/decompile_handlers.py +++ b/firmware/57X-GSP/570.144/decompile_handlers.py @@ -5,9 +5,10 @@ Decompile secondary dispatcher RPC handlers to pseudo-C. import struct import json +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Secondary dispatcher handler entry points diff --git a/firmware/57X-GSP/570.144/decompile_jump_table_handlers.py b/firmware/57X-GSP/570.144/decompile_jump_table_handlers.py index 46a7d4b..0038729 100644 --- a/firmware/57X-GSP/570.144/decompile_jump_table_handlers.py +++ b/firmware/57X-GSP/570.144/decompile_jump_table_handlers.py @@ -4,9 +4,10 @@ Decompile all jump table handlers (MAP, UNMAP, PIN, UNPIN, SCHEDULE, YIELD, NOTI """ import struct +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Jump table handlers from 0x059bfeb0 diff --git a/firmware/57X-GSP/570.144/find_functions.py b/firmware/57X-GSP/570.144/find_functions.py index 52ad25e..b9f3232 100644 --- a/firmware/57X-GSP/570.144/find_functions.py +++ b/firmware/57X-GSP/570.144/find_functions.py @@ -11,9 +11,10 @@ RISC-V typical prologue: import struct import json import sys +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Load address from memory_blocks.json # RISC-V instruction patterns (compressed and uncompressed) diff --git a/firmware/57X-GSP/570.144/find_functions_final.py b/firmware/57X-GSP/570.144/find_functions_final.py index 6e7ac0f..aa073f7 100644 --- a/firmware/57X-GSP/570.144/find_functions_final.py +++ b/firmware/57X-GSP/570.144/find_functions_final.py @@ -8,9 +8,10 @@ Find GSP firmware functions using the actual calling convention: import struct import json +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Executable sections diff --git a/firmware/57X-GSP/570.144/find_functions_v2.py b/firmware/57X-GSP/570.144/find_functions_v2.py index 12c344c..df9506d 100644 --- a/firmware/57X-GSP/570.144/find_functions_v2.py +++ b/firmware/57X-GSP/570.144/find_functions_v2.py @@ -6,9 +6,10 @@ Searches for multiple prologue patterns. import struct import json +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 def read_binary(path): diff --git a/firmware/57X-GSP/570.144/find_prologues.py b/firmware/57X-GSP/570.144/find_prologues.py index a4dc2d2..21f55d1 100644 --- a/firmware/57X-GSP/570.144/find_prologues.py +++ b/firmware/57X-GSP/570.144/find_prologues.py @@ -6,9 +6,10 @@ Only searches in executable code sections. import struct import json +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Executable sections from memory_blocks.json diff --git a/firmware/57X-GSP/570.144/find_prologues_v2.py b/firmware/57X-GSP/570.144/find_prologues_v2.py index 9d01305..a9de131 100644 --- a/firmware/57X-GSP/570.144/find_prologues_v2.py +++ b/firmware/57X-GSP/570.144/find_prologues_v2.py @@ -6,9 +6,10 @@ Handles the actual prologue patterns found. import struct import json +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" -OUTPUT_DIR = "$OPEN_IOV_ROOT/firmware/570-GSP" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") +OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_ADDR = 0x04000000 # Executable sections from memory_blocks.json (virtual addresses) diff --git a/firmware/57X-GSP/570.144/trace_all_subsystems.py b/firmware/57X-GSP/570.144/trace_all_subsystems.py index dd5cc56..e004791 100644 --- a/firmware/57X-GSP/570.144/trace_all_subsystems.py +++ b/firmware/57X-GSP/570.144/trace_all_subsystems.py @@ -4,8 +4,9 @@ Trace all subsystems: WPR, Migration, Primary Dispatcher sub-commands """ import struct +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") BASE_ADDR = 0x04000000 def read_binary(path): diff --git a/firmware/57X-GSP/570.144/trace_dispatchers.py b/firmware/57X-GSP/570.144/trace_dispatchers.py index 100a8a3..c5a066d 100644 --- a/firmware/57X-GSP/570.144/trace_dispatchers.py +++ b/firmware/57X-GSP/570.144/trace_dispatchers.py @@ -4,8 +4,9 @@ Trace dispatcher switch cases to find specific RPC handler functions. """ import struct +import os -FWIMAGE_PATH = "$OPEN_IOV_ROOT/firmware/570-GSP/gsp-570.144.fwimage.bin" +FWIMAGE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "gsp-570.144.fwimage.bin") BASE_ADDR = 0x04000000 def read_binary(path): diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..cb71ee8 --- /dev/null +++ b/install.sh @@ -0,0 +1,260 @@ +#!/bin/bash +# ============================================================================ +# Open-IOV install.sh - Reproduce the complete vGPU stack on a fresh Ubuntu +# +# This script takes a plain Ubuntu 22.04/24.04 machine with a supported +# NVIDIA GPU (Turing TU102/TU104 validated) to a running vGPU stack: +# +# 1. deps Install build/runtime dependencies (apt) +# 2. nvidia Clone NVIDIA open-gpu-kernel-modules @ pinned tag + patch +# 3. firmware Fetch GSP firmware from linux-firmware.git + extract fwimage +# 4. build Build open-iov-vfio.ko -> nvidia.ko -> userspace (in order) +# 5. system Host config: VFIO modules, hugepages, IOMMU/blacklist check +# 6. load Load the driver stack (root) +# 7. mdev Create a vGPU mdev device (root) +# +# After install.sh completes, attach the mdev to ANY QEMU wrapper you like +# (libvirt <hostdev>, , or raw QEMU -device vfio-pci,sysfsdev=...). +# A reference raw-QEMU launcher is provided: scripts/launch_qemu_vgpu.sh +# +# Usage: +# ./install.sh # full install (all stages) +# ./install.sh deps nvidia # run only selected stages +# ./install.sh --no-load # everything except load+mdev (no root needed +# # until the 'system' stage) +# +# Environment overrides: +# NVIDIA_TAG NVIDIA open-gpu-kernel-modules tag (default 610.57.04) +# GSP_VERSION GSP firmware version (default 570.144) +# FW_MD5 Expected fwimage MD5 (default: see below) +# SKIP_FIRMWARE=1 Skip firmware stage +# ============================================================================ + +set -euo pipefail + +OPEN_IOV_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PATCH_DIR="$OPEN_IOV_ROOT/patches/nvidia" +NVIDIA_DIR="$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules" +NVIDIA_REPO="https://github.com/NVIDIA/open-gpu-kernel-modules.git" +NVIDIA_TAG="${NVIDIA_TAG:-610.57.04}" + +GSP_VERSION="${GSP_VERSION:-570.144}" +# linux-firmware.git: nvidia/tu104/gsp is a symlink to nvidia/tu102/gsp +FW_GIT_REPO="https://github.com/NVIDIA/linux-firmware.git" +FW_GIT_COMMIT="2237b99e29d09bebffdb706ae05005d9026fd7e4" # known-good pin +FW_RAW_BASE="https://raw.githubusercontent.com/NVIDIA/linux-firmware/${FW_GIT_COMMIT}/nvidia/tu102/gsp" +FW_DIR="$OPEN_IOV_ROOT/firmware/57X-GSP/570.144" +FW_MD5="${FW_MD5:-13ce12a392a31eeb79a2e6ebd3a4a93a}" + +NR_HUGEPAGES="${NR_HUGEPAGES:-8192}" + +color_red=$'\033[0;31m'; color_green=$'\033[0;32m' +color_yellow=$'\033[1;33m'; color_blue=$'\033[0;34m'; color_nc=$'\033[0m' +log() { echo "${color_blue}[install]${color_nc} $*"; } +ok() { echo "${color_green}[ ok ]${color_nc} $*"; } +warn() { echo "${color_yellow}[ warn ]${color_nc} $*"; } +die() { echo "${color_red}[ FAIL ]${color_nc} $*" >&2; exit 1; } + +need_sudo() { + if [[ $EUID -ne 0 ]]; then + command -v sudo >/dev/null || die "stage requires root and sudo is missing" + echo sudo + fi +} + +# ---------------------------------------------------------------------------- +stage_deps() { + log "Installing dependencies (apt)..." + local SUDO; SUDO=$(need_sudo) + $SUDO apt-get update + $SUDO apt-get install -y --no-install-recommends \ + build-essential gcc make bc flex bison libssl-dev libelf-dev \ + linux-headers-$(uname -r) \ + git ca-certificates curl wget \ + python3 python3-pip \ + binutils uuid-runtime pkg-config \ + qemu-system-x86 ovmf \ + pciutils kmod + ok "dependencies installed" +} + +# ---------------------------------------------------------------------------- +stage_nvidia() { + if [[ -d "$NVIDIA_DIR/.git" ]]; then + ok "NVIDIA tree already present ($NVIDIA_DIR)" + else + log "Cloning NVIDIA open-gpu-kernel-modules @ $NVIDIA_TAG ..." + git clone --depth 1 --branch "$NVIDIA_TAG" "$NVIDIA_REPO" "$NVIDIA_DIR" + fi + + log "Applying Open-IOV patches from $PATCH_DIR ..." + cd "$NVIDIA_DIR" + local applied=0 p + for p in "$PATCH_DIR"/*.patch; do + [[ -e "$p" ]] || die "no patches found in $PATCH_DIR" + if git apply --check "$p" 2>/dev/null; then + git apply "$p" + applied=$((applied+1)) + log " applied $(basename "$p")" + elif git apply --reverse --check "$p" 2>/dev/null; then + ok " $(basename "$p") already applied" + else + die "patch does not apply cleanly: $p (expected pristine tag $NVIDIA_TAG)" + fi + done + ok "NVIDIA tree patched ($NVIDIA_TAG + $applied patch(es))" +} + +# ---------------------------------------------------------------------------- +stage_firmware() { + [[ "${SKIP_FIRMWARE:-0}" == "1" ]] && { warn "firmware stage skipped"; return; } + mkdir -p "$FW_DIR" + local raw_bin="$FW_DIR/gsp-${GSP_VERSION}.bin" + local fwimage="$FW_DIR/gsp-${GSP_VERSION}.fwimage.bin" + + if [[ -f "$fwimage" ]] && echo "$FW_MD5 $fwimage" | md5sum -c --status 2>/dev/null; then + ok "fwimage already present and verified" + else + log "Fetching GSP $GSP_VERSION firmware (linux-firmware, nvidia/tu102/gsp)..." + if ! curl -fSL --retry 3 -o "$raw_bin" "$FW_RAW_BASE/gsp-${GSP_VERSION}.bin"; then + warn "raw download failed; falling back to sparse git clone" + local tmp; tmp=$(mktemp -d) + git clone --depth 1 --filter=blob:none --sparse "$FW_GIT_REPO" "$tmp/lf" + git -C "$tmp/lf" sparse-checkout set nvidia/tu102/gsp + cp "$tmp/lf/nvidia/tu102/gsp/gsp-${GSP_VERSION}.bin" "$raw_bin" + rm -rf "$tmp" + fi + + log "Extracting .fwimage section -> $(basename "$fwimage")" + objcopy -O binary --only-section=.fwimage "$raw_bin" "$fwimage" + echo "$FW_MD5 $fwimage" | md5sum -c --status \ + || die "fwimage MD5 mismatch (expected $FW_MD5) - firmware corrupt or wrong version" + ok "fwimage extracted and verified (md5 $FW_MD5)" + fi + + # Install the *raw* GSP package for the host nvidia.ko physical-GSP boot + log "Installing GSP firmware to /lib/firmware/nvidia/{tu102,tu104}/gsp ..." + local SUDO; SUDO=$(need_sudo) + local f + for f in gsp-${GSP_VERSION}.bin bootloader-${GSP_VERSION}.bin \ + booter_load-${GSP_VERSION}.bin booter_unload-${GSP_VERSION}.bin; do + [[ -f "$FW_DIR/$f" ]] || curl -fSL --retry 3 -o "$FW_DIR/$f" "$FW_RAW_BASE/$f" \ + || warn "could not fetch $f (host GSP boot may fail)" + done + for arch in tu102 tu104; do + $SUDO mkdir -p "/lib/firmware/nvidia/$arch/gsp" + for f in gsp-${GSP_VERSION}.bin bootloader-${GSP_VERSION}.bin \ + booter_load-${GSP_VERSION}.bin booter_unload-${GSP_VERSION}.bin; do + [[ -f "$FW_DIR/$f" ]] && $SUDO cp -f "$FW_DIR/$f" "/lib/firmware/nvidia/$arch/gsp/$f" + done + done + # fwimage location used by scripts/stack_manager.sh install-firmware + $SUDO cp -f "$fwimage" "/lib/firmware/gsp-${GSP_VERSION}.fwimage.bin" + ok "firmware installed" +} + +# ---------------------------------------------------------------------------- +stage_build() { + log "Building open-iov-vfio.ko (generates Module.symvers)..." + make -C "$OPEN_IOV_ROOT" kernel + + log "Building NVIDIA OpenRM (3-step, KBUILD_EXTRA_SYMBOLS)..." + make -C "$NVIDIA_DIR/src/nvidia" -j"$(nproc)" + make -C "$NVIDIA_DIR/src/nvidia-modeset" -j"$(nproc)" + make -C "$NVIDIA_DIR/kernel-open" -j"$(nproc)" \ + KBUILD_EXTRA_SYMBOLS="$OPEN_IOV_ROOT/kernel/Module.symvers" modules + + log "Verifying vGPU VFIO symbols resolved (modinfo depends)..." + modinfo "$NVIDIA_DIR/kernel-open/nvidia.ko" | grep -q "^depends:.*open-iov-vfio" \ + || die "nvidia.ko missing 'depends: open-iov-vfio' - KBUILD_EXTRA_SYMBOLS resolution failed" + + log "Building userspace (libiov.so + examples)..." + make -C "$OPEN_IOV_ROOT" userspace examples + ok "build complete" +} + +# ---------------------------------------------------------------------------- +stage_system() { + log "Configuring host system..." + local SUDO; SUDO=$(need_sudo) + + # VFIO/mdev modules at boot + if [[ ! -f /etc/modules-load.d/open-iov-vfio.conf ]]; then + printf 'vfio\nvfio_iommu_type1\nvfio_pci\nmdev\nvfio_mdev\n' \ + | $SUDO tee /etc/modules-load.d/open-iov-vfio.conf >/dev/null + ok "wrote /etc/modules-load.d/open-iov-vfio.conf" + else + ok "modules-load.d already configured" + fi + $SUDO modprobe vfio vfio_iommu_type1 vfio_pci mdev vfio_mdev 2>/dev/null || true + + # Hugepages (used by the reference QEMU launcher; harmless otherwise) + if [[ "$(cat /proc/sys/vm/nr_hugepages)" -lt "$NR_HUGEPAGES" ]]; then + echo "$NR_HUGEPAGES" | $SUDO tee /proc/sys/vm/nr_hugepages >/dev/null + ok "nr_hugepages=$NR_HUGEPAGES" + fi + if ! mountpoint -q /dev/hugepages 2>/dev/null; then + $SUDO mkdir -p /dev/hugepages + $SUDO mount -t hugetlbfs hugetlbfs /dev/hugepages 2>/dev/null || true + fi + + # IOMMU + blacklist sanity checks (informational, never fatal) + local cmdline; cmdline=$(cat /proc/cmdline) + if ! grep -qE 'intel_iommu=on|amd_iommu=on' <<<"$cmdline"; then + warn "IOMMU does not appear enabled in kernel cmdline." + warn "Add 'intel_iommu=on' (Intel) or 'amd_iommu=on' (AMD) to" + warn "GRUB_CMDLINE_LINUX_DEFAULT in /etc/default/grub and run update-grub." + fi + if grep -q 'module_blacklist=.*nvidia' <<<"$cmdline"; then + warn "kernel cmdline blacklists nvidia modules - remove from GRUB and reboot." + fi + ok "system stage complete" +} + +# ---------------------------------------------------------------------------- +stage_load() { + log "Loading driver stack..." + "$OPEN_IOV_ROOT/scripts/stack_manager.sh" load-stack +} + +stage_mdev() { + log "Creating mdev device..." + "$OPEN_IOV_ROOT/scripts/stack_manager.sh" mdev-create +} + +# ---------------------------------------------------------------------------- +main() { + local stages=() + local do_load=1 + for a in "$@"; do + case "$a" in + --no-load) do_load=0 ;; + -h|--help) + sed -n '2,33p' "$0"; exit 0 ;; + *) stages+=("$a") ;; + esac + done + if [[ ${#stages[@]} -eq 0 ]]; then + stages=(deps nvidia firmware build system) + [[ $do_load -eq 1 ]] && stages+=(load mdev) + fi + + log "Open-IOV install - stages: ${stages[*]}" + log "repo root: $OPEN_IOV_ROOT" + local s + for s in "${stages[@]}"; do + case "$s" in + deps|nvidia|firmware|build|system|load|mdev) "stage_$s" ;; + *) die "unknown stage: $s (valid: deps nvidia firmware build system load mdev)" ;; + esac + done + + echo + ok "=== Open-IOV install complete ===" + echo "Next: attach the vGPU to a VM with the wrapper of your choice:" + echo " raw QEMU : scripts/launch_qemu_vgpu.sh" + echo " libvirt : <hostdev mode='subsystem' type='mdev' ...> with the printed UUID" + echo " status : scripts/stack_manager.sh status" +} + +main "$@" diff --git a/kernel/Makefile b/kernel/Makefile index c927390..aef7129 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -22,8 +22,8 @@ ccflags-y := -I$(src)/include -I$(src)/architecture -I$(src)/version -Wall -Wext # mdev dependencies KBUILD_MODPOST_WARN := 1 -# Use symbols from host nvidia.ko -KBUILD_EXTRA_SYMBOLS := $OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open/Module.symvers +# Use symbols from host nvidia.ko (path resolved relative to this repo) +KBUILD_EXTRA_SYMBOLS := $(abspath $(src)/../nvidia-open-gpu-kernel-modules/kernel-open/Module.symvers) all: $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules diff --git a/patches/nvidia/0001-Build-fixes-and-vGPU-VFIO-interface-integration.patch b/patches/nvidia/0001-Build-fixes-and-vGPU-VFIO-interface-integration.patch index eb5d282..1c7c286 100644 --- a/patches/nvidia/0001-Build-fixes-and-vGPU-VFIO-interface-integration.patch +++ b/patches/nvidia/0001-Build-fixes-and-vGPU-VFIO-interface-integration.patch @@ -1,4 +1,4 @@ -From d189201451e4abc29b57a1ec710d894e778790bf Mon Sep 17 00:00:00 2001 +From 1fe277fb1c48a7ba73fea588102062de281b610a Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson <arthur@vgpu.io> Date: Mon, 21 Sep 2026 04:42:04 -0400 Subject: [PATCH] Build fixes and vGPU VFIO interface integration @@ -22,7 +22,7 @@ Subject: [PATCH] Build fixes and vGPU VFIO interface integration create mode 100644 kernel-open/common/inc/nv-vgpu-vfio-interface.h diff --git a/kernel-open/Kbuild b/kernel-open/Kbuild -index d1811c6..978ef1d 100644 +index d1811c6..b56f1c0 100644 --- a/kernel-open/Kbuild +++ b/kernel-open/Kbuild @@ -79,7 +79,7 @@ ccflags-y += -I$(src)/common/inc @@ -42,7 +42,7 @@ index d1811c6..978ef1d 100644 endif +# Always include extra symbols from open-iov-vfio.ko for vGPU VFIO interface -+KBUILD_EXTRA_SYMBOLS := $OPEN_IOV_ROOT/kernel/Module.symvers ++KBUILD_EXTRA_SYMBOLS := $(abspath $(src)/../../kernel/Module.symvers) + # Some Android kernels prohibit driver use of filesystem functions like # filp_open() and kernel_read(). Disable the NV_FILESYSTEM_ACCESS_AVAILABLE @@ -57,7 +57,7 @@ index d1811c6..978ef1d 100644 NV_CONFTEST_HEADER := $(obj)/conftest/headers.h diff --git a/kernel-open/Makefile b/kernel-open/Makefile -index 341d55b..6558e19 100644 +index 341d55b..ac78628 100644 --- a/kernel-open/Makefile +++ b/kernel-open/Makefile @@ -94,8 +94,12 @@ else @@ -67,7 +67,7 @@ index 341d55b..6558e19 100644 - NV_VERBOSE ?= - SPECTRE_V2_RETPOLINE ?= 0 +# KBUILD_EXTRA_SYMBOLS for vGPU VFIO interface symbols from open-iov-vfio.ko -+KBUILD_EXTRA_SYMBOLS := $OPEN_IOV_ROOT/kernel/Module.symvers ++KBUILD_EXTRA_SYMBOLS := $(abspath $(CURDIR)/../../kernel/Module.symvers) + +NV_VERBOSE ?= +SPECTRE_V2_RETPOLINE ?= 0 @@ -79,108 +79,108 @@ index 341d55b..6558e19 100644 KBUILD_PARAMS += NV_KERNEL_MODULES="$(NV_KERNEL_MODULES)" KBUILD_PARAMS += INSTALL_MOD_DIR="$(INSTALL_MOD_DIR)" KBUILD_PARAMS += NV_SPECTRE_V2=$(SPECTRE_V2_RETPOLINE) -+ KBUILD_PARAMS += KBUILD_EXTRA_SYMBOLS=$OPEN_IOV_ROOT/kernel/Module.symvers ++ KBUILD_PARAMS += KBUILD_EXTRA_SYMBOLS=$(KBUILD_EXTRA_SYMBOLS) export ORIG_PAHOLE := $(PAHOLE) diff --git a/kernel-open/Module.symvers.open-iov b/kernel-open/Module.symvers.open-iov new file mode 100644 -index 0000000..24fd736 +index 0000000..553549f --- /dev/null +++ b/kernel-open/Module.symvers.open-iov @@ -0,0 +1,92 @@ -+0x805e4667 nv_vgpu_register_type_info $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x2510f9e4 nv_vgpu_find_type_info $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xb1cd7273 nv_vgpu_create_mdev_types $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xa3720b3e nv_vgpu_free_type_info $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x3ba3d107 nvidia_vgpu_vfio_get_ops $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xc61962f1 nvidia_vgpu_vfio_set_ops $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x536fe9e2 nv_vgpu_mdev_driver_register $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xe2db996e nv_vgpu_mdev_driver_unregister $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x71713730 nv_vgpu_mdev_register_parent $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xb0486899 nv_vgpu_mdev_unregister_parent $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xdb194166 nvidia_vgpu_vfio_probe $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xc34501b8 nvidia_vgpu_vfio_remove $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x3972fd88 gsp_client_init $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xaf9978fc gsp_client_fini $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x0994eed6 gsp_rpc_alloc $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x01ed1ec0 gsp_rpc_free $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x014ef707 gsp_rpc_map $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x577cba23 gsp_rpc_unmap $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xf55eb4ab gsp_rpc_pin $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x7f0922ac gsp_rpc_unpin $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xc7abb90f gsp_rpc_schedule $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xf64c68e2 gsp_rpc_yield $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x2111f684 gsp_rpc_bar_trap $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xd0b1131e gsp_rpc_migrate_save $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xbd8c881d gsp_rpc_migrate_restore $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x0f989cd0 gsp_wpr_write_mutex_acquire $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x03323922 gsp_wpr_write_mutex_release $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xae7f48d2 gsp_wpr_scratch_mutex_acquire $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xa2d5ed20 gsp_wpr_scratch_mutex_release $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x04a15dd5 gsp_rpc_emulation_set $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x980c5e08 gsp_rpc_emulation_is_enabled $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xa6751cae gsp_rpc_notify_register $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x46ab86c5 gsp_rpc_notify_unregister $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x42fe196a nv_vgpu_gsp_rpc_integration_init $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x3af2d161 nv_vgpu_gsp_rpc_integration_cleanup $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xd748c59b nv_vgpu_gsp_rpc_alloc_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x153fa014 nv_vgpu_gsp_rpc_free_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x637c076f nv_vgpu_gsp_rpc_map_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x34f998ca nv_vgpu_gsp_rpc_unmap_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xeb1110bd nv_vgpu_gsp_rpc_pin_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xf3037685 nv_vgpu_gsp_rpc_unpin_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x21d884f8 nv_vgpu_gsp_rpc_schedule_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xab6ad746 nv_vgpu_gsp_rpc_yield_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x1c4d1f08 nv_vgpu_gsp_rpc_bar_trap_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x708f30d4 nv_vgpu_gsp_rpc_migrate_save_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x8e7872e6 nv_vgpu_gsp_rpc_migrate_restore_fw $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x2ade3dd6 nv_vgpu_gsp_wpr_write_mutex_acquire $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x8dff96a4 nv_vgpu_gsp_wpr_write_mutex_release $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xec05ceea nv_vgpu_gsp_wpr_scratch_mutex_acquire $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x4b246598 nv_vgpu_gsp_wpr_scratch_mutex_release $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xfe33a432 nv_vgpu_gsp_get_primary_dispatcher $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xc1979caa nv_vgpu_gsp_get_secondary_dispatcher $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x89b21e78 nv_vgpu_gsp_get_alloc_main $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x972e5e45 nv_vgpu_gsp_get_schedule_main $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xa6809adb nv_vgpu_gsp_get_bar_trap_handler_jt $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0xcf2a6ed8 nv_vgpu_gsp_get_bar_trap_handler_primary $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x1ba6baef nv_vgpu_gsp_get_entry_point $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL -+0x4fa3b818 fw_analyzer_analyze $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x424272d4 fw_analyzer_load_and_analyze $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x7f7c5e6c fw_analyzer_print_result $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x8c7dcd3c dynamic_dispatcher_init $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x01dca6c2 dynamic_dispatcher_fini $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xcd45da5f dynamic_dispatcher_use_emulation $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x5c69b70b dynamic_dispatcher_alloc $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x436485b4 dynamic_dispatcher_free $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xd4547701 dynamic_dispatcher_map $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xa69f56a4 dynamic_dispatcher_unmap $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x444e6538 dynamic_dispatcher_pin $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x8eeace2b dynamic_dispatcher_unpin $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x39142c5c dynamic_dispatcher_schedule $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xb5bb4890 dynamic_dispatcher_yield $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x5bd014a6 dynamic_dispatcher_bar_trap $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xc6bc85d2 dynamic_dispatcher_migrate_save $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x2f480b10 dynamic_dispatcher_migrate_restore $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xd1994a67 dynamic_dispatcher_notify_register $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xeb6d1fbc dynamic_dispatcher_notify_unregister $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x4c369e34 dynamic_dispatcher_get_version $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xc5d8eb77 dynamic_dispatcher_get_fw_addresses $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x48cbd40e arch_get_by_name $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x90c7d953 arch_get_by_pci_id $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x750f8f00 arch_get_default $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xa505fad7 arch_get_bar_config $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x6707466a arch_get_doorbell_offset $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xc9ed25fd arch_get_shared_mem_offset $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x1d6af5c9 arch_get_msix_vectors $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x835587f5 arch_supports_mig $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x9d3f1b2f arch_supports_sriov $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xd2409aef arch_get_gsp_version $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xc75537a2 version_get_interface $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xe7818f08 version_get_by_kernel_version $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0x12d7fadb version_has_feature $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL -+0xfab46c68 version_get_string $OPEN_IOV_ROOT/kernel/open-iov-vfio EXPORT_SYMBOL_GPL ++0x805e4667 nv_vgpu_register_type_info open-iov-vfio EXPORT_SYMBOL ++0x2510f9e4 nv_vgpu_find_type_info open-iov-vfio EXPORT_SYMBOL ++0xb1cd7273 nv_vgpu_create_mdev_types open-iov-vfio EXPORT_SYMBOL ++0xa3720b3e nv_vgpu_free_type_info open-iov-vfio EXPORT_SYMBOL ++0x3ba3d107 nvidia_vgpu_vfio_get_ops open-iov-vfio EXPORT_SYMBOL ++0xc61962f1 nvidia_vgpu_vfio_set_ops open-iov-vfio EXPORT_SYMBOL ++0x536fe9e2 nv_vgpu_mdev_driver_register open-iov-vfio EXPORT_SYMBOL ++0xe2db996e nv_vgpu_mdev_driver_unregister open-iov-vfio EXPORT_SYMBOL ++0x71713730 nv_vgpu_mdev_register_parent open-iov-vfio EXPORT_SYMBOL ++0xb0486899 nv_vgpu_mdev_unregister_parent open-iov-vfio EXPORT_SYMBOL ++0xdb194166 nvidia_vgpu_vfio_probe open-iov-vfio EXPORT_SYMBOL ++0xc34501b8 nvidia_vgpu_vfio_remove open-iov-vfio EXPORT_SYMBOL ++0x3972fd88 gsp_client_init open-iov-vfio EXPORT_SYMBOL ++0xaf9978fc gsp_client_fini open-iov-vfio EXPORT_SYMBOL ++0x0994eed6 gsp_rpc_alloc open-iov-vfio EXPORT_SYMBOL ++0x01ed1ec0 gsp_rpc_free open-iov-vfio EXPORT_SYMBOL ++0x014ef707 gsp_rpc_map open-iov-vfio EXPORT_SYMBOL ++0x577cba23 gsp_rpc_unmap open-iov-vfio EXPORT_SYMBOL ++0xf55eb4ab gsp_rpc_pin open-iov-vfio EXPORT_SYMBOL ++0x7f0922ac gsp_rpc_unpin open-iov-vfio EXPORT_SYMBOL ++0xc7abb90f gsp_rpc_schedule open-iov-vfio EXPORT_SYMBOL ++0xf64c68e2 gsp_rpc_yield open-iov-vfio EXPORT_SYMBOL ++0x2111f684 gsp_rpc_bar_trap open-iov-vfio EXPORT_SYMBOL ++0xd0b1131e gsp_rpc_migrate_save open-iov-vfio EXPORT_SYMBOL ++0xbd8c881d gsp_rpc_migrate_restore open-iov-vfio EXPORT_SYMBOL ++0x0f989cd0 gsp_wpr_write_mutex_acquire open-iov-vfio EXPORT_SYMBOL ++0x03323922 gsp_wpr_write_mutex_release open-iov-vfio EXPORT_SYMBOL ++0xae7f48d2 gsp_wpr_scratch_mutex_acquire open-iov-vfio EXPORT_SYMBOL ++0xa2d5ed20 gsp_wpr_scratch_mutex_release open-iov-vfio EXPORT_SYMBOL ++0x04a15dd5 gsp_rpc_emulation_set open-iov-vfio EXPORT_SYMBOL ++0x980c5e08 gsp_rpc_emulation_is_enabled open-iov-vfio EXPORT_SYMBOL ++0xa6751cae gsp_rpc_notify_register open-iov-vfio EXPORT_SYMBOL ++0x46ab86c5 gsp_rpc_notify_unregister open-iov-vfio EXPORT_SYMBOL ++0x42fe196a nv_vgpu_gsp_rpc_integration_init open-iov-vfio EXPORT_SYMBOL ++0x3af2d161 nv_vgpu_gsp_rpc_integration_cleanup open-iov-vfio EXPORT_SYMBOL ++0xd748c59b nv_vgpu_gsp_rpc_alloc_fw open-iov-vfio EXPORT_SYMBOL ++0x153fa014 nv_vgpu_gsp_rpc_free_fw open-iov-vfio EXPORT_SYMBOL ++0x637c076f nv_vgpu_gsp_rpc_map_fw open-iov-vfio EXPORT_SYMBOL ++0x34f998ca nv_vgpu_gsp_rpc_unmap_fw open-iov-vfio EXPORT_SYMBOL ++0xeb1110bd nv_vgpu_gsp_rpc_pin_fw open-iov-vfio EXPORT_SYMBOL ++0xf3037685 nv_vgpu_gsp_rpc_unpin_fw open-iov-vfio EXPORT_SYMBOL ++0x21d884f8 nv_vgpu_gsp_rpc_schedule_fw open-iov-vfio EXPORT_SYMBOL ++0xab6ad746 nv_vgpu_gsp_rpc_yield_fw open-iov-vfio EXPORT_SYMBOL ++0x1c4d1f08 nv_vgpu_gsp_rpc_bar_trap_fw open-iov-vfio EXPORT_SYMBOL ++0x708f30d4 nv_vgpu_gsp_rpc_migrate_save_fw open-iov-vfio EXPORT_SYMBOL ++0x8e7872e6 nv_vgpu_gsp_rpc_migrate_restore_fw open-iov-vfio EXPORT_SYMBOL ++0x2ade3dd6 nv_vgpu_gsp_wpr_write_mutex_acquire open-iov-vfio EXPORT_SYMBOL ++0x8dff96a4 nv_vgpu_gsp_wpr_write_mutex_release open-iov-vfio EXPORT_SYMBOL ++0xec05ceea nv_vgpu_gsp_wpr_scratch_mutex_acquire open-iov-vfio EXPORT_SYMBOL ++0x4b246598 nv_vgpu_gsp_wpr_scratch_mutex_release open-iov-vfio EXPORT_SYMBOL ++0xfe33a432 nv_vgpu_gsp_get_primary_dispatcher open-iov-vfio EXPORT_SYMBOL ++0xc1979caa nv_vgpu_gsp_get_secondary_dispatcher open-iov-vfio EXPORT_SYMBOL ++0x89b21e78 nv_vgpu_gsp_get_alloc_main open-iov-vfio EXPORT_SYMBOL ++0x972e5e45 nv_vgpu_gsp_get_schedule_main open-iov-vfio EXPORT_SYMBOL ++0xa6809adb nv_vgpu_gsp_get_bar_trap_handler_jt open-iov-vfio EXPORT_SYMBOL ++0xcf2a6ed8 nv_vgpu_gsp_get_bar_trap_handler_primary open-iov-vfio EXPORT_SYMBOL ++0x1ba6baef nv_vgpu_gsp_get_entry_point open-iov-vfio EXPORT_SYMBOL ++0x4fa3b818 fw_analyzer_analyze open-iov-vfio EXPORT_SYMBOL_GPL ++0x424272d4 fw_analyzer_load_and_analyze open-iov-vfio EXPORT_SYMBOL_GPL ++0x7f7c5e6c fw_analyzer_print_result open-iov-vfio EXPORT_SYMBOL_GPL ++0x8c7dcd3c dynamic_dispatcher_init open-iov-vfio EXPORT_SYMBOL_GPL ++0x01dca6c2 dynamic_dispatcher_fini open-iov-vfio EXPORT_SYMBOL_GPL ++0xcd45da5f dynamic_dispatcher_use_emulation open-iov-vfio EXPORT_SYMBOL_GPL ++0x5c69b70b dynamic_dispatcher_alloc open-iov-vfio EXPORT_SYMBOL_GPL ++0x436485b4 dynamic_dispatcher_free open-iov-vfio EXPORT_SYMBOL_GPL ++0xd4547701 dynamic_dispatcher_map open-iov-vfio EXPORT_SYMBOL_GPL ++0xa69f56a4 dynamic_dispatcher_unmap open-iov-vfio EXPORT_SYMBOL_GPL ++0x444e6538 dynamic_dispatcher_pin open-iov-vfio EXPORT_SYMBOL_GPL ++0x8eeace2b dynamic_dispatcher_unpin open-iov-vfio EXPORT_SYMBOL_GPL ++0x39142c5c dynamic_dispatcher_schedule open-iov-vfio EXPORT_SYMBOL_GPL ++0xb5bb4890 dynamic_dispatcher_yield open-iov-vfio EXPORT_SYMBOL_GPL ++0x5bd014a6 dynamic_dispatcher_bar_trap open-iov-vfio EXPORT_SYMBOL_GPL ++0xc6bc85d2 dynamic_dispatcher_migrate_save open-iov-vfio EXPORT_SYMBOL_GPL ++0x2f480b10 dynamic_dispatcher_migrate_restore open-iov-vfio EXPORT_SYMBOL_GPL ++0xd1994a67 dynamic_dispatcher_notify_register open-iov-vfio EXPORT_SYMBOL_GPL ++0xeb6d1fbc dynamic_dispatcher_notify_unregister open-iov-vfio EXPORT_SYMBOL_GPL ++0x4c369e34 dynamic_dispatcher_get_version open-iov-vfio EXPORT_SYMBOL_GPL ++0xc5d8eb77 dynamic_dispatcher_get_fw_addresses open-iov-vfio EXPORT_SYMBOL_GPL ++0x48cbd40e arch_get_by_name open-iov-vfio EXPORT_SYMBOL_GPL ++0x90c7d953 arch_get_by_pci_id open-iov-vfio EXPORT_SYMBOL_GPL ++0x750f8f00 arch_get_default open-iov-vfio EXPORT_SYMBOL_GPL ++0xa505fad7 arch_get_bar_config open-iov-vfio EXPORT_SYMBOL_GPL ++0x6707466a arch_get_doorbell_offset open-iov-vfio EXPORT_SYMBOL_GPL ++0xc9ed25fd arch_get_shared_mem_offset open-iov-vfio EXPORT_SYMBOL_GPL ++0x1d6af5c9 arch_get_msix_vectors open-iov-vfio EXPORT_SYMBOL_GPL ++0x835587f5 arch_supports_mig open-iov-vfio EXPORT_SYMBOL_GPL ++0x9d3f1b2f arch_supports_sriov open-iov-vfio EXPORT_SYMBOL_GPL ++0xd2409aef arch_get_gsp_version open-iov-vfio EXPORT_SYMBOL_GPL ++0xc75537a2 version_get_interface open-iov-vfio EXPORT_SYMBOL_GPL ++0xe7818f08 version_get_by_kernel_version open-iov-vfio EXPORT_SYMBOL_GPL ++0x12d7fadb version_has_feature open-iov-vfio EXPORT_SYMBOL_GPL ++0xfab46c68 version_get_string open-iov-vfio EXPORT_SYMBOL_GPL diff --git a/kernel-open/common/inc/nv-linux.h b/kernel-open/common/inc/nv-linux.h index 7733e80..8331487 100644 --- a/kernel-open/common/inc/nv-linux.h diff --git a/patches/nvidia/README.md b/patches/nvidia/README.md new file mode 100644 index 0000000..9c74f75 --- /dev/null +++ b/patches/nvidia/README.md @@ -0,0 +1,45 @@ +# NVIDIA open-gpu-kernel-modules Patches + +Open-IOV does **not** vendor the NVIDIA OpenRM source tree. Instead, the +installer clones upstream at a pinned tag and applies this patch series. + +## Pinned upstream + +- Repository: <https://github.com/NVIDIA/open-gpu-kernel-modules.git> +- Tag: `610.57.04` +- Pristine-tree SHA256 of the delta base: tag `610.57.04` + +## Applying + +```bash +git clone --depth 1 --branch 610.57.04 \ + https://github.com/NVIDIA/open-gpu-kernel-modules.git +cd open-gpu-kernel-modules +git apply /path/to/open-iov/patches/nvidia/*.patch +``` + +(`install.sh` does this automatically.) + +## Regenerating the series + +If you modify the NVIDIA tree, regenerate the patches with: + +```bash +cd nvidia-open-gpu-kernel-modules +git format-patch 610.57.04..HEAD -o ../patches/nvidia/ +``` + +## What the series does + +`0001-Build-fixes-and-vGPU-VFIO-interface-integration.patch`: + +- Adds the vGPU VFIO interface headers (`nv-vgpu-vfio-interface.h`, + `nv-vgpu-vfio-interface-nvidia.h`) that `open-iov-vfio.ko` and + `nvidia.ko` share. +- Exports the vGPU VFIO symbols from `nvidia.ko` + (`nvidia_vgpu_vfio_probe/remove`, `nvidia_vgpu_vfio_get_ops/set_ops`, + `nv_vgpu_mdev_register_parent/unregister_parent`, ...). +- Reports version `570.144` to match the GSP firmware the guest driver + stack is validated against. +- Build fixes for kernel 6.8 out-of-tree builds and the + `Module.symvers.open-iov` reference list. diff --git a/scripts/auto-opencode-setup.sh b/scripts/auto-opencode-setup.sh index 3f98eac..4126d3d 100755 --- a/scripts/auto-opencode-setup.sh +++ b/scripts/auto-opencode-setup.sh @@ -71,7 +71,7 @@ command -v opencode &>/dev/null || exit 0 [[ -n "$TMUX" ]] && exit 0 # Target directory -TARGET_DIR="$OPEN_IOV_ROOT/.." +TARGET_DIR="__OPEN_IOV_PARENT__" # Session name SESSION_NAME="auto-opencode" @@ -103,6 +103,10 @@ sleep 1 exec tmux attach-session -t "$SESSION_NAME" SCRIPTEOF + # Bake in the repo parent dir (resolved at setup time) + local repo_parent + repo_parent="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + sed -i "s|__OPEN_IOV_PARENT__|$repo_parent|g" "$AUTO_OPENCODE_SCRIPT" chmod +x "$AUTO_OPENCODE_SCRIPT" log_ok "Created auto-opencode script: $AUTO_OPENCODE_SCRIPT" } @@ -115,7 +119,7 @@ setup_graphical_autologin() { sudo tee /etc/gdm3/custom.conf > /dev/null <<EOF [daemon] AutomaticLoginEnable=true -AutomaticLogin=rmcore +AutomaticLogin=${USER} WaylandEnable=false [security] diff --git a/scripts/build_nvidia_openrm.sh b/scripts/build_nvidia_openrm.sh index dfcbe59..89e058b 100755 --- a/scripts/build_nvidia_openrm.sh +++ b/scripts/build_nvidia_openrm.sh @@ -4,7 +4,7 @@ set -e -OPEN_IOV_ROOT="$OPEN_IOV_ROOT" +OPEN_IOV_ROOT="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" NVIDIA_OPENRM_DIR="$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules" KERNEL_DIR="$OPEN_IOV_ROOT/kernel" SYMVERS_FILE="$KERNEL_DIR/Module.symvers" diff --git a/scripts/load_vgpu_stack.sh b/scripts/load_vgpu_stack.sh index 5e7d6fd..fcabb06 100755 --- a/scripts/load_vgpu_stack.sh +++ b/scripts/load_vgpu_stack.sh @@ -6,7 +6,7 @@ set -euo pipefail FIRMWARE_NAME="${1:-gsp-570.144.fwimage.bin}" -BASE_DIR="$OPEN_IOV_ROOT" +BASE_DIR="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" NVIDIA_KO="$BASE_DIR/nvidia-open-gpu-kernel-modules/kernel-open/nvidia.ko" VFIO_KO="$BASE_DIR/kernel/open-iov-vfio.ko" FIRMWARE_PATH="$BASE_DIR/firmware/57X-GSP/570.144/$FIRMWARE_NAME" diff --git a/scripts/stack_manager.sh b/scripts/stack_manager.sh index fe0a73a..c4685da 100755 --- a/scripts/stack_manager.sh +++ b/scripts/stack_manager.sh @@ -4,7 +4,7 @@ set -euo pipefail -BASE_DIR="$OPEN_IOV_ROOT" +BASE_DIR="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" NVIDIA_KO="$BASE_DIR/nvidia-open-gpu-kernel-modules/kernel-open/nvidia.ko" VFIO_KO="$BASE_DIR/kernel/open-iov-vfio.ko" FIRMWARE_PATH="$BASE_DIR/firmware/57X-GSP/570.144/gsp-570.144.fwimage.bin" diff --git a/scripts/unload_vgpu_stack.sh b/scripts/unload_vgpu_stack.sh index 4f57b86..e02474a 100755 --- a/scripts/unload_vgpu_stack.sh +++ b/scripts/unload_vgpu_stack.sh @@ -3,7 +3,7 @@ set -euo pipefail -BASE_DIR="$OPEN_IOV_ROOT" +BASE_DIR="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" echo "==========================================" echo "Unloading vGPU Driver Stack" diff --git a/tools/module_ops/dry_run.sh b/tools/module_ops/dry_run.sh index 8bda820..e6bb6f2 100755 --- a/tools/module_ops/dry_run.sh +++ b/tools/module_ops/dry_run.sh @@ -4,7 +4,7 @@ set -euo pipefail -VGPU_KERNEL="$OPEN_IOV_ROOT/kernel" +VGPU_KERNEL="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}/kernel" echo "=== Open-IOV vGPU Dry-Run Validation ===" echo "" diff --git a/tools/module_ops/load_converged.sh b/tools/module_ops/load_converged.sh index 77cde09..dbd090e 100755 --- a/tools/module_ops/load_converged.sh +++ b/tools/module_ops/load_converged.sh @@ -4,7 +4,7 @@ set -euo pipefail -OPEN_IOV_ROOT="$OPEN_IOV_ROOT" +OPEN_IOV_ROOT="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" NVIDIA_OPEN_KERNEL="$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open" VGPU_KERNEL="$OPEN_IOV_ROOT/kernel" diff --git a/validate.sh b/validate.sh index fc80c21..642401e 100755 --- a/validate.sh +++ b/validate.sh @@ -13,7 +13,7 @@ log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } -PROJECT_ROOT="$OPEN_IOV_ROOT" +PROJECT_ROOT="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" cd "$PROJECT_ROOT" |