Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 531 insertions, 0 deletions

diff --git a/tools/debug/nv2080_ctrl.c b/tools/debug/nv2080_ctrl.c
new file mode 100644
index 0000000..46ade02
--- /dev/null
+++ b/tools/debug/nv2080_ctrl.c
@@ -0,0 +1,531 @@
+/*
+ * NV2080 Control Command Sender
+ * C program to send NV2080 RM control commands via ioctl
+ * Compile: gcc -o nv2080_ctrl nv2080_ctrl.c
+ * Usage: sudo ./nv2080_ctrl /dev/nvidia0 <command> [args...]
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <errno.h>
+
+// NV_IOCTL definitions from nv-ioctl.h
+#define NV_IOCTL_MAGIC        0x46
+#define NV_ESC_RM_CONTROL     0x54
+#define NV_ESC_CHECK_VERSION_STR  0x32
+#define NV_ESC_RM_ALLOC       0x21
+#define NV_ESC_RM_FREE        0x00
+#define NV_ESC_ALLOC_OS_EVENT 0x24
+
+// IOCTL macros
+#define _IOC_NRBITS   8
+#define _IOC_TYPEBITS 8
+#define _IOC_SIZEBITS 14
+#define _IOC_DIRBITS  2
+
+#define _IOC_NRSHIFT   0
+#define _IOC_TYPESHIFT (_IOC_NRSHIFT + _IOC_NRBITS)
+#define _IOC_SIZESHIFT (_IOC_TYPESHIFT + _IOC_TYPEBITS)
+#define _IOC_DIRSHIFT  (_IOC_SIZESHIFT + _IOC_SIZEBITS)
+
+#define _IOC_NONE  0U
+#define _IOC_WRITE 1U
+#define _IOC_READ  2U
+
+#define _IOC(dir, type, nr, size) \
+    (((dir)  << _IOC_DIRSHIFT) | \
+     ((type) << _IOC_TYPESHIFT) | \
+     ((nr)   << _IOC_NRSHIFT) | \
+     ((size) << _IOC_SIZESHIFT))
+
+#define _IOWR(type, nr, size) _IOC(_IOC_READ | _IOC_WRITE, type, nr, size)
+
+// NV_CTRL_RES = _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS)
+#define NV_CTRL_RES _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, 128)
+
+// NV2080 Control Commands (from ctrl2080unix.h and ctrl2080vgpumgrinternal.h)
+#define NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT          0x20803d01
+#define NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF        0x20803d02
+#define NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER         0x20803d03
+#define NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS      0x20803d04
+#define NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER       0x20803d05
+#define NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER               0x20803d06
+#define NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS   0x20803d07
+#define NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS           0x20803d08
+
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK  0x20804001
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK  0x20804002
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE             0x20804003
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU        0x20804004
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO            0x20804005
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE              0x20804006
+#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY      0x20804007
+#define NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP                     0x20804008
+
+// Action values
+#define NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC   1
+#define NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC   2
+#define NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW    1
+#define NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW 2
+
+// NVOS54_PARAMETERS structure (from nv-ioctl.h)
+typedef struct {
+    uint32_t hClient;
+    uint32_t hObject;
+    uint32_t cmd;
+    uint32_t flags;
+    void *params;
+    uint32_t paramsSize;
+    uint32_t status;
+} NVOS54_PARAMETERS;
+
+// NVOS21_PARAMETERS for RM_ALLOC
+typedef struct {
+    uint32_t hRoot;
+    uint32_t hObjectParent;
+    uint32_t hObjectNew;
+    uint32_t hClass;
+    void *pAllocParms;
+    uint32_t status;
+} NVOS21_PARAMETERS;
+
+// NVOS00_PARAMETERS for RM_FREE
+typedef struct {
+    uint32_t hRoot;
+    uint32_t hObjectParent;
+    uint32_t hObjectOld;
+    uint32_t status;
+} NVOS00_PARAMETERS;
+
+// Version check structure
+typedef struct {
+    uint32_t cmd;
+    char versionString[256];
+    uint32_t reply;
+} nv_ioctl_rm_api_version_t;
+
+// OS Event allocation
+typedef struct {
+    uint32_t hClient;
+    uint32_t hDevice;
+    int fd;
+    uint32_t Status;
+} nv_ioctl_alloc_os_event_t;
+
+// GC6 Blocker Refcnt Params
+typedef struct {
+    uint32_t action;
+} NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS;
+
+// Allow/Disallow GCOFF Params
+typedef struct {
+    uint32_t action;
+} NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS;
+
+// Audio Dynamic Power Params
+typedef struct {
+    uint32_t bEnter;
+} NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS;
+
+// Vidmem Persistence Status Params
+typedef struct {
+    uint32_t bVidmemPersistent;
+} NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS;
+
+// TGP Status Params
+typedef struct {
+    uint32_t bUpdateTGP;
+} NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS;
+
+// VGPU Bootload Params (simplified)
+typedef struct {
+    uint32_t dbdf;
+    uint32_t gfid;
+    uint32_t vgpuType;
+    uint32_t vmPid;
+    uint32_t swizzId;
+    uint32_t numChannels;
+    uint32_t numPluginChannels;
+    uint32_t bDisableDefaultSmcExecPartRestore;
+    uint32_t numGuestFbSegments;
+    uint64_t guestFbPhysAddrList[384];
+    uint64_t guestFbLengthList[384];
+    uint64_t pluginHeapMemoryPhysAddr;
+    uint64_t pluginHeapMemoryLength;
+    uint64_t ctrlBuffOffset;
+} NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS;
+
+// VGPU Shutdown Params
+typedef struct {
+    uint32_t gfid;
+} NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS;
+
+// VGPU Add Type Params
+typedef struct {
+    uint32_t discardVgpuTypes;
+    uint64_t vgpuInfo; // NVA081_CTRL_VGPU_INFO
+} NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS;
+
+// VGPU Enumerate Params
+typedef struct {
+    uint32_t numVgpu;
+    uint64_t vgpuGuest[32]; // NV2080_VGPU_GUEST array
+} NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS;
+
+// VGPU Clear Guest Params
+typedef struct {
+    uint32_t gfid;
+} NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS;
+
+// VGPU FB Usage Params
+typedef struct {
+    uint32_t vgpuCount;
+    struct {
+        uint32_t gfid;
+        uint64_t fbUsed;
+    } vgpuFbUsage[32];
+} NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS;
+
+// VGPU Encoder Capacity Params
+typedef struct {
+    uint32_t gfid;
+    uint32_t encoderCapacity;
+} NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS;
+
+// VGPU Plugin Cleanup Params
+typedef struct {
+    uint32_t gfid;
+} NV2080_CTRL_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS;
+
+// Send RM Control command
+int send_rm_control(int fd, uint32_t client, uint32_t object, uint32_t cmd, void *params, uint32_t params_size) {
+    NVOS54_PARAMETERS ctrl = {
+        .hClient = client,
+        .hObject = object,
+        .cmd = cmd,
+        .flags = 0,
+        .params = params,
+        .paramsSize = params_size,
+        .status = 0
+    };
+    
+    int ret = ioctl(fd, NV_CTRL_RES, &ctrl);
+    if (ret == -1) {
+        perror("ioctl NV_CTRL_RES failed");
+        return -1;
+    }
+    
+    if (ctrl.status != 0) {
+        fprintf(stderr, "RM Control failed: cmd=0x%x, status=0x%x\n", cmd, ctrl.status);
+        return ctrl.status;
+    }
+    
+    return 0;
+}
+
+// Version check
+int version_check(int fd, const char *version) {
+    nv_ioctl_rm_api_version_t vc = {0};
+    vc.cmd = 0x32; // NV_ESC_CHECK_VERSION_STR
+    if (version) {
+        strncpy(vc.versionString, version, sizeof(vc.versionString) - 1);
+    }
+    
+    int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_CHECK_VERSION_STR, sizeof(vc)), &vc);
+    if (ret == -1) {
+        perror("ioctl version check failed");
+        return -1;
+    }
+    
+    printf("Version check: reply=%u, version=%s\n", vc.reply, vc.versionString);
+    return vc.reply == 1 ? 0 : -1;
+}
+
+// Allocate RM resource
+int rm_alloc(int fd, uint32_t root, uint32_t parent, uint32_t object, uint32_t class, void *data) {
+    NVOS21_PARAMETERS alloc = {
+        .hRoot = root,
+        .hObjectParent = parent,
+        .hObjectNew = object,
+        .hClass = class,
+        .pAllocParms = data,
+        .status = 0
+    };
+    
+    int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, sizeof(alloc)), &alloc);
+    if (ret == -1) {
+        perror("ioctl RM_ALLOC failed");
+        return -1;
+    }
+    
+    if (alloc.status != 0) {
+        fprintf(stderr, "RM_ALLOC failed: status=0x%x\n", alloc.status);
+        return alloc.status;
+    }
+    
+    printf("Allocated object 0x%x (class 0x%x)\n", alloc.hObjectNew, class);
+    return alloc.hObjectNew;
+}
+
+// Free RM resource
+int rm_free(int fd, uint32_t root, uint32_t parent, uint32_t object) {
+    NVOS00_PARAMETERS free_params = {
+        .hRoot = root,
+        .hObjectParent = parent,
+        .hObjectOld = object,
+        .status = 0
+    };
+    
+    int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_FREE, sizeof(free_params)), &free_params);
+    if (ret == -1) {
+        perror("ioctl RM_FREE failed");
+        return -1;
+    }
+    
+    return free_params.status == 0 ? 0 : free_params.status;
+}
+
+// Print usage
+void print_usage(const char *prog) {
+    fprintf(stderr, "Usage: %s <device> <command> [args...]\n", prog);
+    fprintf(stderr, "\nDevice: /dev/nvidia0, /dev/nvidiactl, etc.\n");
+    fprintf(stderr, "\nCommands:\n");
+    fprintf(stderr, "  version-check [version]           - Check RM API version\n");
+    fprintf(stderr, "  gc6-inc <client> <object>         - Increment GC6 blocker refcount\n");
+    fprintf(stderr, "  gc6-dec <client> <object>         - Decrement GC6 blocker refcount\n");
+    fprintf(stderr, "  gcoff-allow <client> <object>     - Allow GCOFF\n");
+    fprintf(stderr, "  gcoff-disallow <client> <object>  - Disallow GCOFF\n");
+    fprintf(stderr, "  audio-power <client> <object> <enter> - Audio dynamic power (1=enter, 0=exit)\n");
+    fprintf(stderr, "  vidmem-persistence <client> <object> - Query vidmem persistence\n");
+    fprintf(stderr, "  tgp-status <client> <object> <update> - Update TGP status (1=update, 0=don't)\n");
+    fprintf(stderr, "  vgpu-bootload <client> <object> <dbdf> <gfid> <vgpuType> <vmPid> ... - Bootload GSP VGPU plugin\n");
+    fprintf(stderr, "  vgpu-shutdown <client> <object> <gfid> - Shutdown GSP VGPU plugin\n");
+    fprintf(stderr, "  vgpu-add-type <client> <object> <discard> <vgpuInfo> - Add vGPU type\n");
+    fprintf(stderr, "  vgpu-enumerate <client> <object>  - Enumerate vGPUs per pGPU\n");
+    fprintf(stderr, "  vgpu-clear-guest <client> <object> <gfid> - Clear guest VM info\n");
+    fprintf(stderr, "  vgpu-fb-usage <client> <object>   - Get vGPU FB usage\n");
+    fprintf(stderr, "  vgpu-encoder-cap <client> <object> <gfid> <capacity> - Set encoder capacity (0-100)\n");
+    fprintf(stderr, "  vgpu-plugin-cleanup <client> <object> <gfid> - Cleanup VGPU plugin\n");
+    fprintf(stderr, "  alloc <root> <parent> <object> <class> - Allocate RM resource\n");
+    fprintf(stderr, "  free <root> <parent> <object>     - Free RM resource\n");
+}
+
+int main(int argc, char *argv[]) {
+    if (argc < 3) {
+        print_usage(argv[0]);
+        return 1;
+    }
+    
+    const char *device = argv[1];
+    const char *command = argv[2];
+    
+    int fd = open(device, O_RDWR);
+    if (fd < 0) {
+        perror("Failed to open device");
+        return 1;
+    }
+    
+    int ret = 0;
+    
+    if (strcmp(command, "version-check") == 0) {
+        const char *version = argc > 3 ? argv[3] : NULL;
+        ret = version_check(fd, version);
+    }
+    else if (strcmp(command, "gc6-inc") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "gc6-dec") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "gcoff-allow") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "gcoff-disallow") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "audio-power") == 0) {
+        if (argc < 6) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t enter = strtoul(argv[5], NULL, 0);
+            NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS params = { .bEnter = enter };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vidmem-persistence") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS params = {0};
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS, &params, sizeof(params));
+            if (ret == 0) {
+                printf("Vidmem persistent: %s\n", params.bVidmemPersistent ? "yes" : "no");
+            }
+        }
+    }
+    else if (strcmp(command, "tgp-status") == 0) {
+        if (argc < 6) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t update = strtoul(argv[5], NULL, 0);
+            NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS params = { .bUpdateTGP = update };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vgpu-bootload") == 0) {
+        if (argc < 12) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS params = {0};
+            params.dbdf = strtoul(argv[5], NULL, 0);
+            params.gfid = strtoul(argv[6], NULL, 0);
+            params.vgpuType = strtoul(argv[7], NULL, 0);
+            params.vmPid = strtoul(argv[8], NULL, 0);
+            params.swizzId = strtoul(argv[9], NULL, 0);
+            params.numChannels = strtoul(argv[10], NULL, 0);
+            params.numPluginChannels = strtoul(argv[11], NULL, 0);
+            // Additional params would need more args
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vgpu-shutdown") == 0) {
+        if (argc < 6) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t gfid = strtoul(argv[5], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS params = { .gfid = gfid };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vgpu-add-type") == 0) {
+        if (argc < 7) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t discard = strtoul(argv[5], NULL, 0);
+            uint64_t vgpuInfo = strtoull(argv[6], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS params = { .discardVgpuTypes = discard, .vgpuInfo = vgpuInfo };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vgpu-enumerate") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS params = {0};
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU, &params, sizeof(params));
+            if (ret == 0) {
+                printf("Enumerated %u vGPUs\n", params.numVgpu);
+            }
+        }
+    }
+    else if (strcmp(command, "vgpu-clear-guest") == 0) {
+        if (argc < 6) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t gfid = strtoul(argv[5], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS params = { .gfid = gfid };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vgpu-fb-usage") == 0) {
+        if (argc < 5) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS params = {0};
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE, &params, sizeof(params));
+            if (ret == 0) {
+                printf("vGPU count: %u\n", params.vgpuCount);
+                for (uint32_t i = 0; i < params.vgpuCount; i++) {
+                    printf("  GFID 0x%x: FB used %lu bytes\n", params.vgpuFbUsage[i].gfid, params.vgpuFbUsage[i].fbUsed);
+                }
+            }
+        }
+    }
+    else if (strcmp(command, "vgpu-encoder-cap") == 0) {
+        if (argc < 7) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t gfid = strtoul(argv[5], NULL, 0);
+            uint32_t capacity = strtoul(argv[6], NULL, 0);
+            NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS params = { .gfid = gfid, .encoderCapacity = capacity };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "vgpu-plugin-cleanup") == 0) {
+        if (argc < 6) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t client = strtoul(argv[3], NULL, 0);
+            uint32_t object = strtoul(argv[4], NULL, 0);
+            uint32_t gfid = strtoul(argv[5], NULL, 0);
+            NV2080_CTRL_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS params = { .gfid = gfid };
+            ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP, &params, sizeof(params));
+        }
+    }
+    else if (strcmp(command, "alloc") == 0) {
+        if (argc < 7) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t root = strtoul(argv[3], NULL, 0);
+            uint32_t parent = strtoul(argv[4], NULL, 0);
+            uint32_t object = strtoul(argv[5], NULL, 0);
+            uint32_t class = strtoul(argv[6], NULL, 0);
+            ret = rm_alloc(fd, root, parent, object, class, NULL);
+        }
+    }
+    else if (strcmp(command, "free") == 0) {
+        if (argc < 6) { print_usage(argv[0]); ret = 1; }
+        else {
+            uint32_t root = strtoul(argv[3], NULL, 0);
+            uint32_t parent = strtoul(argv[4], NULL, 0);
+            uint32_t object = strtoul(argv[5], NULL, 0);
+            ret = rm_free(fd, root, parent, object);
+        }
+    }
+    else {
+        fprintf(stderr, "Unknown command: %s\n", command);
+        print_usage(argv[0]);
+        ret = 1;
+    }
+    
+    close(fd);
+    return ret;
+}
\ No newline at end of file