From 68cdd7728a7cc6715bbbff7decfe2b11c3bdf558 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- .../scripts/DecompileFunctions.java | 143 ++++++++++++++ .../scripts/analyze_gsp_fwimage.py | 207 +++++++++++++++++++++ .../gsp-tu104-570.144/scripts/basic_analysis.py | 108 +++++++++++ .../gsp-tu104-570.144/scripts/extract_fwimage.py | 56 ++++++ .../scripts/extract_gsp_version.py | 30 +++ 5 files changed, 544 insertions(+) create mode 100644 tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java create mode 100644 tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py create mode 100644 tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py create mode 100644 tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py create mode 100644 tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py (limited to 'tools/ghidra/gsp-tu104-570.144') diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java b/tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java new file mode 100644 index 0000000..e1d78c3 --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java @@ -0,0 +1,143 @@ +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionManager; +import ghidra.program.model.listing.Instruction; +import ghidra.program.model.listing.Program; +import ghidra.program.model.mem.Memory; +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileResults; +import ghidra.util.task.TaskMonitor; +import java.io.*; + +public class DecompileFunctions extends GhidraScript { + + private DecompInterface decompiler; + + @Override + protected void run() throws Exception { + decompiler = new DecompInterface(); + decompiler.openProgram(currentProgram); + + String[] args = getScriptArgs(); + if (args.length == 0) { + println("Usage: DecompileFunctions.java "); + return; + } + + String[] addresses = args[0].split(","); + + println("=== Targeted Decompilation ==="); + for (String addrStr : addresses) { + addrStr = addrStr.trim(); + if (addrStr.isEmpty()) continue; + + long addrVal; + try { + addrVal = Long.parseUnsignedLong(addrStr.startsWith("0x") ? addrStr.substring(2) : addrStr, 16); + } catch (NumberFormatException e) { + println("Invalid address: " + addrStr); + continue; + } + + Address addr = currentProgram.getAddressFactory().getAddress("0x" + Long.toHexString(addrVal)); + if (addr == null) { + println("Invalid address: " + addrStr); + continue; + } + + decompileFunction(addr); + } + + println("\n=== Decompilation Complete ==="); + } + + private void decompileFunction(Address entryAddr) throws Exception { + FunctionManager fm = currentProgram.getFunctionManager(); + Memory mem = currentProgram.getMemory(); + + // Manually disassemble from entry point by creating instructions + Address addr = entryAddr; + int maxBytes = 2000; + int bytes = 0; + + while (bytes < maxBytes && !monitor.isCancelled()) { + Instruction instr = currentProgram.getListing().getInstructionAt(addr); + if (instr != null) { + addr = instr.getMaxAddress().next(); + bytes += instr.getLength(); + continue; + } + + // Try to create instruction from bytes + try { + byte[] buf = new byte[4]; + int read = mem.getBytes(addr, buf); + if (read <= 0) break; + + // Create instruction prototype and try to disassemble + instr = currentProgram.getListing().createInstruction(addr, + currentProgram.getLanguage().getInstructionPrototype(buf), + new ghidra.program.model.mem.ByteMemBuffer(buf, 0, read), + currentProgram.getProgramContext(), 0); + + if (instr != null) { + addr = instr.getMaxAddress().next(); + bytes += instr.getLength(); + } else { + addr = addr.next().next(); + bytes += 2; + } + } catch (Exception e) { + addr = addr.next().next(); + bytes += 2; + } + } + println(" Disassembled " + bytes + " bytes from " + entryAddr); + + // Get or create function + Function func = fm.getFunctionAt(entryAddr); + if (func == null) { + func = fm.createFunction("FUNC_" + Long.toHexString(entryAddr.getOffset()).toUpperCase(), + entryAddr, null, ghidra.program.model.symbol.SourceType.ANALYSIS); + if (func == null) { + println("Failed to create function at " + entryAddr); + return; + } + println("Created function: " + func.getName() + " @ " + func.getEntryPoint()); + } else { + println("Existing function: " + func.getName() + " @ " + func.getEntryPoint()); + } + + // Decompile + println("\n--- Decompiling " + func.getName() + " @ " + func.getEntryPoint() + " ---"); + DecompileResults results = decompiler.decompileFunction(func, 120, monitor); + + if (results.decompileCompleted()) { + String cCode = results.getDecompiledFunction().getC(); + + // Save to file + String filename = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP/decompiled_" + + func.getName() + ".c"; + try (PrintWriter pw = new PrintWriter(new FileWriter(filename))) { + pw.println("// Function: " + func.getName()); + pw.println("// Address: " + func.getEntryPoint()); + pw.println("// Stack frame: " + func.getStackFrame().getFrameSize() + " bytes"); + pw.println(); + pw.print(cCode); + } + println(" Saved to: " + filename); + + // Print first 100 lines + String[] lines = cCode.split("\n"); + for (int i = 0; i < Math.min(100, lines.length); i++) { + println(" " + lines[i]); + } + if (lines.length > 100) { + println(" ... (" + (lines.length - 100) + " more lines)"); + } + } else { + println(" Decompilation failed: " + results.getErrorMessage()); + } + } +} diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py b/tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py new file mode 100644 index 0000000..4a6fb56 --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py @@ -0,0 +1,207 @@ +# Ghidra script to analyze GSP .fwimage binary +# Usage: analyzeHeadless ... -postScript analyze_gsp_fwimage.py + +import sys +import os +import json + +# Get the current program +program = getCurrentProgram() +listing = program.getListing() +memory = program.getMemory() +function_manager = program.getFunctionManager() + +print("=== GSP Firmware .fwimage Analysis ===") +print(f"Program: {program.getName()}") +print(f"Architecture: {program.getLanguage().toString()}") + +# Create output directory +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) + +# 1. Find all functions +print("\n--- Finding Functions ---") +functions = [] +func_iter = function_manager.getFunctions(True) +while func_iter.hasNext(): + func = func_iter.next() + functions.append({ + "name": func.getName(), + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses(), + "signature": func.getSignature().toString() if func.getSignature() else "" + }) + +print(f"Found {len(functions)} functions") + +# Save function list +with open(os.path.join(output_dir, "functions.json"), "w") as f: + json.dump(functions, f, indent=2) + +# 2. Find RPC-related functions (vRPC/pRPC handlers) +print("\n--- Finding RPC Handlers ---") +rpc_keywords = [ + "rpc", "Rpc", "RPC", + "vgpu", "Vgpu", "VGPU", + "vmiop", "Vmiop", "VMIOP", + "hal", "Hal", "HAL", + "bar", "Bar", "BAR", + "migrate", "Migrate", "MIGRATE", + "init", "Init", "INIT", + "boot", "Boot", "BOOT", + "kgsp", "Kgsp", "KGSP" +] + +rpc_functions = [] +for func in functions: + name_lower = func["name"].lower() + for kw in rpc_keywords: + if kw.lower() in name_lower: + rpc_functions.append(func) + break + +print(f"Found {len(rpc_functions)} potential RPC-related functions") +with open(os.path.join(output_dir, "rpc_functions.json"), "w") as f: + json.dump(rpc_functions, f, indent=2) + +# 3. Find strings related to vGPU initialization +print("\n--- Finding vGPU-related Strings ---") +vgpu_strings = [] +string_iter = listing.getDefinedData(True) +while string_iter.hasNext(): + data = string_iter.next() + if data.getDataType().getName() == "string": + value = data.getValue() + if value: + val_str = str(value).lower() + if any(kw in val_str for kw in ["vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version"]): + vgpu_strings.append({ + "address": str(data.getAddress()), + "value": str(value) + }) + +print(f"Found {len(vgpu_strings)} vGPU-related strings") +with open(os.path.join(output_dir, "vgpu_strings.json"), "w") as f: + json.dump(vgpu_strings, f, indent=2) + +# 4. Find entry points and initialization functions +print("\n--- Finding Entry Points ---") +entry_points = [] +# Look for common entry point names +entry_names = [ + "kgspInitRm_IMPL", "kgspInitRm", "kgspBootGspRm", "kgspBootGspRm_IMPL", + "gspBoot", "gspInit", "gspMain", + "nv_vgpu_vfio_get_ops", "nv_vgpu_vfio_set_ops", + "vgpu_init", "vgpu_init_IMPL", + "vmiop_init", "vmiop_init_IMPL", + "hal_init", "hal_init_IMPL" +] + +for name in entry_names: + func = function_manager.getFunction(name) + if func: + entry_points.append({ + "name": name, + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses() + }) + print(f" Found: {name} at {func.getEntryPoint()}") + +# Also check for functions called from reset vector / entry +# Look at the first few instructions +entry_addr = program.getMinAddress() +print(f"\nProgram entry address: {entry_addr}") + +# 5. Find WPR (Write Protected Region) related code +print("\n--- Finding WPR-related Code ---") +wpr_functions = [] +for func in functions: + name_lower = func["name"].lower() + if "wpr" in name_lower or "write_protect" in name_lower or "protect" in name_lower: + wpr_functions.append(func) + +print(f"Found {len(wpr_functions)} WPR-related functions") +with open(os.path.join(output_dir, "wpr_functions.json"), "w") as f: + json.dump(wpr_functions, f, indent=2) + +# 6. Find memory management functions +print("\n--- Finding Memory Management Functions ---") +mem_keywords = ["alloc", "free", "map", "unmap", "pin", "unpin", "dma", "iommu", "mmio", "bar"] +mem_functions = [] +for func in functions: + name_lower = func["name"].lower() + for kw in mem_keywords: + if kw in name_lower: + mem_functions.append(func) + break + +print(f"Found {len(mem_functions)} memory management functions") +with open(os.path.join(output_dir, "mem_functions.json"), "w") as f: + json.dump(mem_functions, f, indent=2) + +# 7. Find interrupt/MSI-X related functions +print("\n--- Finding Interrupt/MSI-X Functions ---") +irq_keywords = ["irq", "interrupt", "msi", "msix", "eventfd", "doorbell"] +irq_functions = [] +for func in functions: + name_lower = func["name"].lower() + for kw in irq_keywords: + if kw in name_lower: + irq_functions.append(func) + break + +print(f"Found {len(irq_functions)} interrupt-related functions") +with open(os.path.join(output_dir, "irq_functions.json"), "w") as f: + json.dump(irq_functions, f, indent=2) + +# 8. Generate summary report +print("\n--- Generating Summary Report ---") +report = { + "firmware": "gsp-570.144", + "architecture": program.getLanguage().toString(), + "total_functions": len(functions), + "rpc_functions": len(rpc_functions), + "vgpu_strings": len(vgpu_strings), + "entry_points": entry_points, + "wpr_functions": len(wpr_functions), + "mem_functions": len(mem_functions), + "irq_functions": len(irq_functions) +} + +with open(os.path.join(output_dir, "analysis_summary.json"), "w") as f: + json.dump(report, f, indent=2) + +# Print summary +print(json.dumps(report, indent=2)) + +# 9. Decompile key functions and save +print("\n--- Decompiling Key Functions ---") +from ghidra.app.decompiler import DecompInterface +from ghidra.util.task import ConsoleTaskMonitor + +decompiler = DecompInterface() +decompiler.openProgram(program) +monitor = ConsoleTaskMonitor() + +key_functions = entry_points + rpc_functions[:20] + wpr_functions + mem_functions[:10] + irq_functions[:10] +decompiled = {} + +for func_info in key_functions: + func = function_manager.getFunctionAt(func_info["entry"]) + if func: + result = decompiler.decompileFunction(func, 60, monitor) + if result.decompileCompleted(): + decompiled[func_info["name"]] = result.getDecompiledFunction().getC() + print(f" Decompiled: {func_info['name']}") + else: + print(f" Failed to decompile: {func_info['name']}") + +# Save decompiled code +with open(os.path.join(output_dir, "decompiled_key_functions.c"), "w") as f: + for name, code in decompiled.items(): + f.write(f"\n// ===== {name} =====\n") + f.write(code) + f.write("\n") + +print(f"\nDecompiled {len(decompiled)} key functions") +print("Analysis complete!") \ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py b/tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py new file mode 100644 index 0000000..4992106 --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py @@ -0,0 +1,108 @@ +# Ghidra script to do basic disassembly analysis of GSP .fwimage binary +# Usage: analyzeHeadless ... -postScript basic_analysis.py + +import sys +import os +import json + +# Get the current program +program = getCurrentProgram() +listing = program.getListing() +memory = program.getMemory() +function_manager = program.getFunctionManager() + +print("=== GSP Firmware .fwimage Basic Analysis ===") +print(f"Program: {program.getName()}") +print(f"Architecture: {program.getLanguage().toString()}") + +# Create output directory +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) + +# 1. Check memory blocks +print("\n--- Memory Blocks ---") +mem_blocks = [] +for block in memory.getBlocks(): + mem_blocks.append({ + "name": block.getName(), + "start": str(block.getStart()), + "end": str(block.getEnd()), + "size": block.getSize(), + "execute": block.isExecute(), + "read": block.isRead(), + "write": block.isWrite() + }) + print(f" {block.getName()}: {block.getStart()} - {block.getEnd()} ({block.getSize()} bytes) r={block.isRead()} w={block.isWrite()} x={block.isExecute()}") + +with open(os.path.join(output_dir, "memory_blocks.json"), "w") as f: + json.dump(mem_blocks, f, indent=2) + +# 2. Find all functions (but don't decompile yet) +print("\n--- Finding Functions ---") +functions = [] +func_iter = function_manager.getFunctions(True) +count = 0 +while func_iter.hasNext(): + func = func_iter.next() + functions.append({ + "name": func.getName(), + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses() + }) + count += 1 + if count % 1000 == 0: + print(f" Found {count} functions...") + +print(f"Total functions: {len(functions)}") + +with open(os.path.join(output_dir, "functions.json"), "w") as f: + json.dump(functions, f, indent=2) + +# 3. Find strings +print("\n--- Finding Strings ---") +strings = [] +string_iter = listing.getDefinedData(True) +while string_iter.hasNext(): + data = string_iter.next() + if data.getDataType().getName() == "string": + value = data.getValue() + if value and len(str(value)) > 4: + strings.append({ + "address": str(data.getAddress()), + "value": str(value) + }) + +print(f"Found {len(strings)} strings") + +# Filter for interesting strings +interesting_keywords = ["vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version", "error", "fail", "assert", "panic", "nvidia", "rmapi", "gsp", "falcon", "mailbox", "interrupt", "dma", "iommu", "page", "mmu", "pde", "pte"] +interesting_strings = [s for s in strings if any(kw in s['value'].lower() for kw in interesting_keywords)] +print(f"Interesting strings: {len(interesting_strings)}") + +with open(os.path.join(output_dir, "interesting_strings.json"), "w") as f: + json.dump(interesting_strings, f, indent=2) + +# 4. Find entry point (offset 0) +print("\n--- Entry Point Analysis ---") +entry_addr = program.getMinAddress() +print(f"Program entry (min address): {entry_addr}") + +# Check what's at address 0 +func_at_zero = function_manager.getFunctionAt(entry_addr) +if func_at_zero: + print(f"Function at entry: {func_at_zero.getName()}") +else: + print("No function at entry point - creating one...") + # Create function at entry point + func = function_manager.createFunction("entry_point", entry_addr, None, None) + if func: + print(f"Created function: {func.getName()}") + +# 5. Look for RISC-V standard function prologues +print("\n--- Looking for Function Prologues ---") +# RISC-V typical prologue: addi sp, sp, -X; sd ra, X-8(sp); sd s0, X-16(sp); addi s0, sp, X +# Let's search for common patterns + +# 6. Find cross-references to interesting addresses +print("\n--- Analysis Complete ---") +print(f"Results saved to {output_dir}") \ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py b/tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py new file mode 100644 index 0000000..d8b0bfd --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py @@ -0,0 +1,56 @@ +# Ghidra script to extract .fwimage section from GSP firmware ELF +# Usage: analyzeHeadless ... -postScript extract_fwimage.py + +import sys +import os + +# Get the current program +program = getCurrentProgram() + +# Get the .fwimage section +fwimage_section = program.getSection(".fwimage") +if not fwimage_section: + print("No .fwimage section found") + sys.exit(1) + +print(f"Found .fwimage section: {fwimage_section.getStart()} - {fwimage_section.getEnd()}") +print(f"Size: {fwimage_section.getLength()} bytes") + +# Get the data from the section +data = getBytes(fwimage_section.getStart(), fwimage_section.getLength()) + +# Save to file +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) +fwimage_path = os.path.join(output_dir, "gsp-570.144.fwimage.bin") + +with open(fwimage_path, "wb") as f: + f.write(data) + +print(f"Saved .fwimage to {fwimage_path}") + +# Also extract .fwversion +fwversion_section = program.getSection(".fwversion") +if fwversion_section: + version_data = getBytes(fwversion_section.getStart(), fwversion_section.getLength()) + version_str = version_data.decode('ascii').strip('\x00') + print(f"GSP Firmware Version: {version_str}") + + with open(os.path.join(output_dir, "gsp_version.txt"), "w") as f: + f.write(version_str) + +# Extract signatures +for i in range(3): + sig_section = program.getSection(f".fwsignature{i}") + if sig_section: + sig_data = getBytes(sig_section.getStart(), sig_section.getLength()) + sig_path = os.path.join(output_dir, f"gsp-570.144.fwsignature{i}.bin") + with open(sig_path, "wb") as f: + f.write(sig_data) + print(f"Saved signature {i} to {sig_path} ({len(sig_data)} bytes)") + +# Create a new Ghidra project for the .fwimage binary +print("Creating Ghidra project for .fwimage binary...") + +# We'll do this via a separate script call +print("Done extracting sections.") \ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py b/tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py new file mode 100644 index 0000000..ef44d7d --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py @@ -0,0 +1,30 @@ +# Ghidra script to extract version from .fwversion section of GSP binary +# Usage: analyzeHeadless ... -postScript extract_gsp_version.py + +import sys + +# Get the current program +program = getCurrentProgram() + +# Get the .fwversion section +fwversion_section = program.getSection(".fwversion") +if fwversion_section: + # Get the data from the section + data = getBytes(fwversion_section.getStart(), fwversion_section.getLength()) + version_str = data.decode('ascii').strip('\x00') + print(f"GSP Firmware Version: {version_str}") + + # Also save to a file + with open("/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/ghidra/gsp-tu104-570.144/gsp_version.txt", "w") as f: + f.write(version_str) +else: + print("No .fwversion section found") + + # Try to find version in memory + listing = program.getListing() + for i in range(listing.getNumAddresses()): + addr = listing.getAddress(i) + data = getBytes(addr, 32) + if data and b"570.144" in data: + print(f"Found version at {addr}: {data}") + break -- cgit v1.2.3