Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 187 insertions, 0 deletions

diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h
new file mode 100644
index 0000000..a82656f
--- /dev/null
+++ b/kernel/gsp_rm_emulation.h
@@ -0,0 +1,187 @@
+/*
+ * GSP-RM Emulation (physical-path) for Host-Side vGPU
+ *
+ * Emulates the GSP-RM firmware's msgq-based RPC channel for a guest driver
+ * that believes it is driving a real physical GPU. The guest driver:
+ *   - boots the "GSP" (falcon stages emulated in gsp_emulation.c)
+ *   - programs libos init args address into GSP MAILBOX0/1
+ *   - expects GSP-RM to msgqInit the status queue in *guest system memory*
+ *     (shared command/status queue buffer described by GSP_ARGUMENTS_CACHED)
+ *   - polls the status queue for NV_VGPU_MSG_EVENT_GSP_INIT_DONE and then
+ *     issues the RM RPC sequence on the command queue.
+ *
+ * Since there is no real GSP engine, this module performs the "virtual DMA"
+ * on behalf of the device using vfio_dma_rw() (CPU copies through the VFIO
+ * container's IOMMU mappings, IOVA == guest physical address).
+ */
+
+#ifndef _GSP_RM_EMULATION_H_
+#define _GSP_RM_EMULATION_H_
+
+#include <linux/types.h>
+
+#include "nvidia-vgpu-vfio.h"
+
+struct gsp_emu_context;
+
+/* Wire constants (from NVIDIA OpenRM 570.144 msgq/rpc headers - the guest
+ * driver is the 570.144 stock runfile, whose GSP_MSG_QUEUE_ELEMENT has NO
+ * MCTP/NVDM transport headers: authTag[16]+aad[16]+checkSum+seqNum+elemCount
+ * +rpc@0x30). */
+#define GSPRM_MSGQ_VERSION		0
+#define GSPRM_MSGQ_FLAGS_SWAP_RX	1
+#define GSPRM_RPC_SIGNATURE		0x43505256  /* NV_VGPU_MSG_SIGNATURE_VALID */
+#define GSPRM_RPC_HEADER_VERSION	0x03000000  /* MAJOR_TOT(3)<<24|MINOR_TOT(0)<<16 */
+
+/* RPC function/event numbers (rpc_global_enums.h) */
+#define GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO	1
+#define GSPRM_RPC_FUNC_ALLOC_ROOT		2
+#define GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO	65
+#define GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO	72
+#define GSPRM_RPC_FUNC_SET_REGISTRY		73
+#define GSPRM_RPC_FUNC_CONTINUATION_RECORD	71
+#define GSPRM_RPC_FUNC_GSP_RM_CONTROL		76
+#define GSPRM_RPC_FUNC_GSP_RM_ALLOC		103
+#define GSPRM_RPC_FUNC_FREE			10
+#define GSPRM_RPC_EVENT_GSP_INIT_DONE		0x1001
+
+/* rpc_set_guest_system_info_v03_00: version fields the driver reads back
+ * from the reply to negotiate the RPC interface version (ipVersion). */
+#define GSPRM_SET_GUEST_INFO_SIZE	0x324
+#define GSPRM_VGX_VERSION_MAJOR		0x29
+#define GSPRM_VGX_VERSION_MINOR		0x0C
+
+/* "RMARGS" packed big-endian into u64 by _kgspGenerateInitArgId() */
+#define GSPRM_LIBOS_ID_RMARGS		0x524D41524753ULL
+
+#define GSPRM_MAX_LIBOS_REGIONS		32
+#define GSPRM_QUEUE_ELEMENT_SIZE	0x1000
+#define GSPRM_STATUS_QUEUE_SIZE		0x40000
+
+/* msgqTxHeader (msgq_priv.h) */
+struct gsprm_msgq_tx_header {
+	u32 version;
+	u32 size;
+	u32 msgSize;
+	u32 msgCount;
+	u32 writePtr;
+	u32 flags;
+	u32 rxHdrOff;
+	u32 entryOff;
+};
+
+/* rpc_message_header_v03_00 (g_rpc-message-header.h) */
+struct gsprm_rpc_header {
+	u32 header_version;
+	u32 signature;
+	u32 length;
+	u32 function;
+	u32 rpc_result;
+	u32 rpc_result_private;
+	u32 sequence;
+	u32 spare;		/* rpc_message_rpc_union_field_v (spare/cpuRmGfid) */
+};
+
+/* LibosMemoryRegionInitArgument (libos_init_args.h) - natural alignment, 32B */
+struct gsprm_libos_region {
+	u64 id8;
+	u64 pa;
+	u64 size;
+	u8  kind;
+	u8  loc;
+	u8  pad[6];
+};
+
+/* MESSAGE_QUEUE_INIT_ARGUMENTS (gsp_init_args.h), x86_64 NvLength=u64 */
+struct gsprm_mq_init_args {
+	u64 sharedMemPhysAddr;
+	u32 pageTableEntryCount;
+	u32 pad0;
+	u64 cmdQueueOffset;
+	u64 statQueueOffset;
+	u64 queueElementHdrSize;
+	u64 queueElementSizeMin;
+	u64 queueElementSizeMax;
+	u32 queueHeaderAlign;
+	u32 queueElementAlign;
+};
+
+struct gsp_rm_emu {
+	struct nv_vgpu_device *vgpu;
+
+	bool		parsed;		/* queue chain parsed + status queue linked */
+	bool		active;
+
+	unsigned long	last_parse_retry;	/* jiffies of last on_booter_done retry */
+
+	u64		libos_args_gpa;	/* from GSP MAILBOX0/1 */
+	u64		gsp_args_gpa;	/* RMARGS region pa */
+
+	/* queue buffer description (from GSP_ARGUMENTS_CACHED) */
+	struct gsprm_mq_init_args mq;
+	u64		*ptes;		/* pageTableEntryCount guest-phys addrs */
+
+	/* computed queue geometry. cmd_off/stat_off are offsets into the shared
+	 * buffer (linear); GPAs must be computed per-access via the page table
+	 * (buffer is NV_MEMORY_NONCONTIGUOUS). */
+	u64		cmd_off;
+	u64		stat_off;
+	u64		cmd_gpa;	/* GPA of command queue base (diagnostic) */
+	u64		stat_gpa;	/* GPA of status queue base (diagnostic) */
+	u32		cmd_size;	/* bytes */
+	u32		stat_size;	/* bytes */
+	u32		cmd_msgcount;
+	u32		stat_msgcount;
+	u32		entry_off;	/* 0x1000 (msgq entryOff, both queues) */
+
+	/* GSP-side cursors */
+	u32		cmd_read;	/* elements consumed from command queue */
+	u32		stat_write;	/* elements written to status queue */
+	u32		stat_seq;	/* status record sequence numbers */
+	bool		init_done_sent;
+
+	/*
+	 * Client page-table geometry, learned from
+	 * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106):
+	 * the client hands us the PHYSICAL addresses of its page-table
+	 * levels (root PDE ... leaf PTE) plus the VA range/page shifts.
+	 * Used to translate client GPU VAs (e.g. semaphore/pushbuffer VAs)
+	 * to guest physical addresses.
+	 */
+	struct {
+		u64	va_lo;
+		u64	va_hi;
+		u64	page_size;	/* coverage of the root level */
+		u32	num_levels;
+		struct {
+			u64 phys;
+			u64 size;
+			u32 aperture;	/* 1=sysmem(PA), 2=vidmem(offset) */
+			u8  page_shift;
+		} levels[6];
+	} pt;
+
+	/* stats */
+	u64		rpcs_seen;
+	u64		rpcs_replied;
+	u64		unknown_rpcs;
+	u32		alloc_dumps;	/* raw GSP_RM_ALLOC dumps emitted */
+	u32		alloc_param_dumps;	/* alloc param dumps emitted */
+	u32		ctrl_param_dumps;	/* control param dumps emitted */
+};
+
+int  gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx);
+void gsp_rm_emu_fini(struct gsp_emu_context *ctx);
+int  gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle,
+			   u64 start, u64 end, u32 max_hits);
+int  gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat,
+			       u32 npat, u64 start, u64 end, u32 max_hits);
+int  gsp_rm_scan_guest_ram_pt(struct nv_vgpu_device *vgpu, u64 pa,
+			      u64 start, u64 end, u32 max_hits);
+int  gsp_rm_scan_guest_ram_masked(struct nv_vgpu_device *vgpu, u64 value,
+				  u64 mask, u64 start, u64 end, u32 max_hits);
+void gsp_rm_emu_reset(struct gsp_emu_context *ctx);
+void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx);
+void gsp_rm_emu_poll(struct gsp_emu_context *ctx);
+
+#endif /* _GSP_RM_EMULATION_H_ */