From 2302fc09080bad51c73f293294e19fd4db464eb0 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- tools/debug/nv2080_ctrl.c | 531 +++++++++++++++++++++++++++++++++++++++++++++ tools/debug/nv2080_ctrl.sh | 168 ++++++++++++++ 2 files changed, 699 insertions(+) create mode 100644 tools/debug/nv2080_ctrl.c create mode 100755 tools/debug/nv2080_ctrl.sh (limited to 'tools/debug') diff --git a/tools/debug/nv2080_ctrl.c b/tools/debug/nv2080_ctrl.c new file mode 100644 index 0000000..46ade02 --- /dev/null +++ b/tools/debug/nv2080_ctrl.c @@ -0,0 +1,531 @@ +/* + * NV2080 Control Command Sender + * C program to send NV2080 RM control commands via ioctl + * Compile: gcc -o nv2080_ctrl nv2080_ctrl.c + * Usage: sudo ./nv2080_ctrl /dev/nvidia0 [args...] + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +// NV_IOCTL definitions from nv-ioctl.h +#define NV_IOCTL_MAGIC 0x46 +#define NV_ESC_RM_CONTROL 0x54 +#define NV_ESC_CHECK_VERSION_STR 0x32 +#define NV_ESC_RM_ALLOC 0x21 +#define NV_ESC_RM_FREE 0x00 +#define NV_ESC_ALLOC_OS_EVENT 0x24 + +// IOCTL macros +#define _IOC_NRBITS 8 +#define _IOC_TYPEBITS 8 +#define _IOC_SIZEBITS 14 +#define _IOC_DIRBITS 2 + +#define _IOC_NRSHIFT 0 +#define _IOC_TYPESHIFT (_IOC_NRSHIFT + _IOC_NRBITS) +#define _IOC_SIZESHIFT (_IOC_TYPESHIFT + _IOC_TYPEBITS) +#define _IOC_DIRSHIFT (_IOC_SIZESHIFT + _IOC_SIZEBITS) + +#define _IOC_NONE 0U +#define _IOC_WRITE 1U +#define _IOC_READ 2U + +#define _IOC(dir, type, nr, size) \ + (((dir) << _IOC_DIRSHIFT) | \ + ((type) << _IOC_TYPESHIFT) | \ + ((nr) << _IOC_NRSHIFT) | \ + ((size) << _IOC_SIZESHIFT)) + +#define _IOWR(type, nr, size) _IOC(_IOC_READ | _IOC_WRITE, type, nr, size) + +// NV_CTRL_RES = _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) +#define NV_CTRL_RES _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, 128) + +// NV2080 Control Commands (from ctrl2080unix.h and ctrl2080vgpumgrinternal.h) +#define NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT 0x20803d01 +#define NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF 0x20803d02 +#define NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER 0x20803d03 +#define NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS 0x20803d04 +#define NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER 0x20803d05 +#define NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER 0x20803d06 +#define NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS 0x20803d07 +#define NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS 0x20803d08 + +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK 0x20804001 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK 0x20804002 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE 0x20804003 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU 0x20804004 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO 0x20804005 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE 0x20804006 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY 0x20804007 +#define NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP 0x20804008 + +// Action values +#define NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC 1 +#define NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC 2 +#define NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW 1 +#define NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW 2 + +// NVOS54_PARAMETERS structure (from nv-ioctl.h) +typedef struct { + uint32_t hClient; + uint32_t hObject; + uint32_t cmd; + uint32_t flags; + void *params; + uint32_t paramsSize; + uint32_t status; +} NVOS54_PARAMETERS; + +// NVOS21_PARAMETERS for RM_ALLOC +typedef struct { + uint32_t hRoot; + uint32_t hObjectParent; + uint32_t hObjectNew; + uint32_t hClass; + void *pAllocParms; + uint32_t status; +} NVOS21_PARAMETERS; + +// NVOS00_PARAMETERS for RM_FREE +typedef struct { + uint32_t hRoot; + uint32_t hObjectParent; + uint32_t hObjectOld; + uint32_t status; +} NVOS00_PARAMETERS; + +// Version check structure +typedef struct { + uint32_t cmd; + char versionString[256]; + uint32_t reply; +} nv_ioctl_rm_api_version_t; + +// OS Event allocation +typedef struct { + uint32_t hClient; + uint32_t hDevice; + int fd; + uint32_t Status; +} nv_ioctl_alloc_os_event_t; + +// GC6 Blocker Refcnt Params +typedef struct { + uint32_t action; +} NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS; + +// Allow/Disallow GCOFF Params +typedef struct { + uint32_t action; +} NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS; + +// Audio Dynamic Power Params +typedef struct { + uint32_t bEnter; +} NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS; + +// Vidmem Persistence Status Params +typedef struct { + uint32_t bVidmemPersistent; +} NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS; + +// TGP Status Params +typedef struct { + uint32_t bUpdateTGP; +} NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS; + +// VGPU Bootload Params (simplified) +typedef struct { + uint32_t dbdf; + uint32_t gfid; + uint32_t vgpuType; + uint32_t vmPid; + uint32_t swizzId; + uint32_t numChannels; + uint32_t numPluginChannels; + uint32_t bDisableDefaultSmcExecPartRestore; + uint32_t numGuestFbSegments; + uint64_t guestFbPhysAddrList[384]; + uint64_t guestFbLengthList[384]; + uint64_t pluginHeapMemoryPhysAddr; + uint64_t pluginHeapMemoryLength; + uint64_t ctrlBuffOffset; +} NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS; + +// VGPU Shutdown Params +typedef struct { + uint32_t gfid; +} NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS; + +// VGPU Add Type Params +typedef struct { + uint32_t discardVgpuTypes; + uint64_t vgpuInfo; // NVA081_CTRL_VGPU_INFO +} NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS; + +// VGPU Enumerate Params +typedef struct { + uint32_t numVgpu; + uint64_t vgpuGuest[32]; // NV2080_VGPU_GUEST array +} NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS; + +// VGPU Clear Guest Params +typedef struct { + uint32_t gfid; +} NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS; + +// VGPU FB Usage Params +typedef struct { + uint32_t vgpuCount; + struct { + uint32_t gfid; + uint64_t fbUsed; + } vgpuFbUsage[32]; +} NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS; + +// VGPU Encoder Capacity Params +typedef struct { + uint32_t gfid; + uint32_t encoderCapacity; +} NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS; + +// VGPU Plugin Cleanup Params +typedef struct { + uint32_t gfid; +} NV2080_CTRL_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS; + +// Send RM Control command +int send_rm_control(int fd, uint32_t client, uint32_t object, uint32_t cmd, void *params, uint32_t params_size) { + NVOS54_PARAMETERS ctrl = { + .hClient = client, + .hObject = object, + .cmd = cmd, + .flags = 0, + .params = params, + .paramsSize = params_size, + .status = 0 + }; + + int ret = ioctl(fd, NV_CTRL_RES, &ctrl); + if (ret == -1) { + perror("ioctl NV_CTRL_RES failed"); + return -1; + } + + if (ctrl.status != 0) { + fprintf(stderr, "RM Control failed: cmd=0x%x, status=0x%x\n", cmd, ctrl.status); + return ctrl.status; + } + + return 0; +} + +// Version check +int version_check(int fd, const char *version) { + nv_ioctl_rm_api_version_t vc = {0}; + vc.cmd = 0x32; // NV_ESC_CHECK_VERSION_STR + if (version) { + strncpy(vc.versionString, version, sizeof(vc.versionString) - 1); + } + + int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_CHECK_VERSION_STR, sizeof(vc)), &vc); + if (ret == -1) { + perror("ioctl version check failed"); + return -1; + } + + printf("Version check: reply=%u, version=%s\n", vc.reply, vc.versionString); + return vc.reply == 1 ? 0 : -1; +} + +// Allocate RM resource +int rm_alloc(int fd, uint32_t root, uint32_t parent, uint32_t object, uint32_t class, void *data) { + NVOS21_PARAMETERS alloc = { + .hRoot = root, + .hObjectParent = parent, + .hObjectNew = object, + .hClass = class, + .pAllocParms = data, + .status = 0 + }; + + int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, sizeof(alloc)), &alloc); + if (ret == -1) { + perror("ioctl RM_ALLOC failed"); + return -1; + } + + if (alloc.status != 0) { + fprintf(stderr, "RM_ALLOC failed: status=0x%x\n", alloc.status); + return alloc.status; + } + + printf("Allocated object 0x%x (class 0x%x)\n", alloc.hObjectNew, class); + return alloc.hObjectNew; +} + +// Free RM resource +int rm_free(int fd, uint32_t root, uint32_t parent, uint32_t object) { + NVOS00_PARAMETERS free_params = { + .hRoot = root, + .hObjectParent = parent, + .hObjectOld = object, + .status = 0 + }; + + int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_FREE, sizeof(free_params)), &free_params); + if (ret == -1) { + perror("ioctl RM_FREE failed"); + return -1; + } + + return free_params.status == 0 ? 0 : free_params.status; +} + +// Print usage +void print_usage(const char *prog) { + fprintf(stderr, "Usage: %s [args...]\n", prog); + fprintf(stderr, "\nDevice: /dev/nvidia0, /dev/nvidiactl, etc.\n"); + fprintf(stderr, "\nCommands:\n"); + fprintf(stderr, " version-check [version] - Check RM API version\n"); + fprintf(stderr, " gc6-inc - Increment GC6 blocker refcount\n"); + fprintf(stderr, " gc6-dec - Decrement GC6 blocker refcount\n"); + fprintf(stderr, " gcoff-allow - Allow GCOFF\n"); + fprintf(stderr, " gcoff-disallow - Disallow GCOFF\n"); + fprintf(stderr, " audio-power - Audio dynamic power (1=enter, 0=exit)\n"); + fprintf(stderr, " vidmem-persistence - Query vidmem persistence\n"); + fprintf(stderr, " tgp-status - Update TGP status (1=update, 0=don't)\n"); + fprintf(stderr, " vgpu-bootload ... - Bootload GSP VGPU plugin\n"); + fprintf(stderr, " vgpu-shutdown - Shutdown GSP VGPU plugin\n"); + fprintf(stderr, " vgpu-add-type - Add vGPU type\n"); + fprintf(stderr, " vgpu-enumerate - Enumerate vGPUs per pGPU\n"); + fprintf(stderr, " vgpu-clear-guest - Clear guest VM info\n"); + fprintf(stderr, " vgpu-fb-usage - Get vGPU FB usage\n"); + fprintf(stderr, " vgpu-encoder-cap - Set encoder capacity (0-100)\n"); + fprintf(stderr, " vgpu-plugin-cleanup - Cleanup VGPU plugin\n"); + fprintf(stderr, " alloc - Allocate RM resource\n"); + fprintf(stderr, " free - Free RM resource\n"); +} + +int main(int argc, char *argv[]) { + if (argc < 3) { + print_usage(argv[0]); + return 1; + } + + const char *device = argv[1]; + const char *command = argv[2]; + + int fd = open(device, O_RDWR); + if (fd < 0) { + perror("Failed to open device"); + return 1; + } + + int ret = 0; + + if (strcmp(command, "version-check") == 0) { + const char *version = argc > 3 ? argv[3] : NULL; + ret = version_check(fd, version); + } + else if (strcmp(command, "gc6-inc") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "gc6-dec") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "gcoff-allow") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "gcoff-disallow") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "audio-power") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t enter = strtoul(argv[5], NULL, 0); + NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS params = { .bEnter = enter }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vidmem-persistence") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS params = {0}; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS, ¶ms, sizeof(params)); + if (ret == 0) { + printf("Vidmem persistent: %s\n", params.bVidmemPersistent ? "yes" : "no"); + } + } + } + else if (strcmp(command, "tgp-status") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t update = strtoul(argv[5], NULL, 0); + NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS params = { .bUpdateTGP = update }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-bootload") == 0) { + if (argc < 12) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS params = {0}; + params.dbdf = strtoul(argv[5], NULL, 0); + params.gfid = strtoul(argv[6], NULL, 0); + params.vgpuType = strtoul(argv[7], NULL, 0); + params.vmPid = strtoul(argv[8], NULL, 0); + params.swizzId = strtoul(argv[9], NULL, 0); + params.numChannels = strtoul(argv[10], NULL, 0); + params.numPluginChannels = strtoul(argv[11], NULL, 0); + // Additional params would need more args + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-shutdown") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS params = { .gfid = gfid }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-add-type") == 0) { + if (argc < 7) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t discard = strtoul(argv[5], NULL, 0); + uint64_t vgpuInfo = strtoull(argv[6], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS params = { .discardVgpuTypes = discard, .vgpuInfo = vgpuInfo }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-enumerate") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS params = {0}; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU, ¶ms, sizeof(params)); + if (ret == 0) { + printf("Enumerated %u vGPUs\n", params.numVgpu); + } + } + } + else if (strcmp(command, "vgpu-clear-guest") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS params = { .gfid = gfid }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-fb-usage") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS params = {0}; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE, ¶ms, sizeof(params)); + if (ret == 0) { + printf("vGPU count: %u\n", params.vgpuCount); + for (uint32_t i = 0; i < params.vgpuCount; i++) { + printf(" GFID 0x%x: FB used %lu bytes\n", params.vgpuFbUsage[i].gfid, params.vgpuFbUsage[i].fbUsed); + } + } + } + } + else if (strcmp(command, "vgpu-encoder-cap") == 0) { + if (argc < 7) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + uint32_t capacity = strtoul(argv[6], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS params = { .gfid = gfid, .encoderCapacity = capacity }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-plugin-cleanup") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + NV2080_CTRL_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS params = { .gfid = gfid }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "alloc") == 0) { + if (argc < 7) { print_usage(argv[0]); ret = 1; } + else { + uint32_t root = strtoul(argv[3], NULL, 0); + uint32_t parent = strtoul(argv[4], NULL, 0); + uint32_t object = strtoul(argv[5], NULL, 0); + uint32_t class = strtoul(argv[6], NULL, 0); + ret = rm_alloc(fd, root, parent, object, class, NULL); + } + } + else if (strcmp(command, "free") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t root = strtoul(argv[3], NULL, 0); + uint32_t parent = strtoul(argv[4], NULL, 0); + uint32_t object = strtoul(argv[5], NULL, 0); + ret = rm_free(fd, root, parent, object); + } + } + else { + fprintf(stderr, "Unknown command: %s\n", command); + print_usage(argv[0]); + ret = 1; + } + + close(fd); + return ret; +} \ No newline at end of file diff --git a/tools/debug/nv2080_ctrl.sh b/tools/debug/nv2080_ctrl.sh new file mode 100755 index 0000000..22c7117 --- /dev/null +++ b/tools/debug/nv2080_ctrl.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# NV2080 Control Command Tool +# Sends NV2080 RM control commands to the NVIDIA kernel module via ioctl +# Usage: ./nv2080_ctrl.sh [params...] + +set -euo pipefail + +# NV2080 Control Command Definitions (from ctrl2080unix.h and ctrl2080vgpumgrinternal.h) +# These are the same commands used by GVM userspace via NV_CTRL_RES ioctl + +# NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT (0x20803d01) +# Params: action (INC=1, DEC=2) +NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT=0x20803d01 +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC=1 +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC=2 + +# NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF (0x20803d02) +# Params: action (ALLOW=1, DISALLOW=2) +NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF=0x20803d02 +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW=1 +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW=2 + +# NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER (0x20803d03) +# Params: bEnter (true/false) +NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER=0x20803d03 + +# NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS (0x20803d04) +NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS=0x20803d04 + +# NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER (0x20803d05) +NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER=0x20803d05 + +# NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER (0x20803d06) +NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER=0x20803d06 + +# NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS (0x20803d07) +NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS=0x20803d07 + +# NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS (0x20803d08) +NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS=0x20803d08 + +# VGPU Manager Internal Commands +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK=0x20804001 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK=0x20804002 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE=0x20804003 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU=0x20804004 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO=0x20804005 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE=0x20804006 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY=0x20804007 +NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP=0x20804008 + +# NV_IOCTL_MAGIC and NV_ESC_RM_CONTROL +NV_IOCTL_MAGIC=0x46 +NV_ESC_RM_CONTROL=0x54 + +# NV_CTRL_RES = _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) +NV_CTRL_RES=$(printf "0x%x" $(( (2 << 30) | (NV_IOCTL_MAGIC << 8) | (NV_ESC_RM_CONTROL) | (128 << 16) ))) + +# Helper to send ioctl +send_ioctl() { + local fd=$1 + local cmd=$2 + local data=$3 + local size=$4 + + # This would need a C program to actually send the ioctl + # For now, we document the commands + echo "Command: 0x$(printf "%x" $cmd)" + echo "Data size: $size" +} + +# Main command dispatcher +case "${1:-help}" in + gc6-inc) + echo "Increment GC6 blocker refcount" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, action: $NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC})" + ;; + gc6-dec) + echo "Decrement GC6 blocker refcount" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, action: $NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC})" + ;; + gcoff-allow) + echo "Allow GCOFF" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, action: $NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW})" + ;; + gcoff-disallow) + echo "Disallow GCOFF" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, action: $NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW})" + ;; + audio-power) + echo "Audio dynamic power: $2" + ;; + profiler-install) + echo "Install profiler hooks" + ;; + profiler-flush) + echo "Flush snapshot buffer" + ;; + profiler-stop) + echo "Stop profiler" + ;; + vidmem-persistence) + echo "Query vidmem persistence status" + ;; + tgp-status) + echo "Update TGP status: $2" + ;; + vgpu-bootload) + echo "Bootload GSP VGPU plugin task" + echo "Params: dbdf, gfid, vgpuType, vmPid, swizzId, numChannels, numPluginChannels, bDisableSmcPartitionRestore, guestFbPhysAddrList, guestFbLengthList, pluginHeapMemoryPhysAddr, pluginHeapMemoryLength, ctrlBuffOffset" + ;; + vgpu-shutdown) + echo "Shutdown GSP VGPU plugin task" + echo "Params: gfid" + ;; + vgpu-add-type) + echo "Add vGPU type to pGPU" + echo "Params: discardVgpuTypes, vgpuInfo" + ;; + vgpu-enumerate) + echo "Enumerate vGPUs per pGPU" + ;; + vgpu-clear-guest) + echo "Clear guest VM info" + echo "Params: gfid" + ;; + vgpu-fb-usage) + echo "Get vGPU FB usage" + ;; + vgpu-encoder-cap) + echo "Set vGPU encoder capacity" + echo "Params: gfid, encoderCapacity (0-100)" + ;; + vgpu-plugin-cleanup) + echo "Cleanup VGPU plugin" + echo "Params: gfid" + ;; + list) + echo "Available NV2080 Control Commands:" + echo " OS Unix Commands:" + echo " gc6-inc - Increment GC6 blocker refcount" + echo " gc6-dec - Decrement GC6 blocker refcount" + echo " gcoff-allow - Allow GCOFF" + echo " gcoff-disallow - Disallow GCOFF" + echo " audio-power - Audio dynamic power (enter/exit)" + echo " profiler-install - Install profiler hooks (HWPM)" + echo " profiler-flush - Flush snapshot buffer" + echo " profiler-stop - Stop profiler" + echo " vidmem-persistence- Query vidmem persistence status" + echo " tgp-status - Update TGP status" + echo "" + echo " VGPU Manager Internal Commands:" + echo " vgpu-bootload - Bootload GSP VGPU plugin task" + echo " vgpu-shutdown - Shutdown GSP VGPU plugin task" + echo " vgpu-add-type - Add vGPU type to pGPU" + echo " vgpu-enumerate - Enumerate vGPUs per pGPU" + echo " vgpu-clear-guest - Clear guest VM info" + echo " vgpu-fb-usage - Get vGPU FB usage" + echo " vgpu-encoder-cap - Set vGPU encoder capacity" + echo " vgpu-plugin-cleanup - Cleanup VGPU plugin" + ;; + help|*) + echo "NV2080 Control Command Tool" + echo "Usage: $0 [args...]" + echo "" + echo "Run '$0 list' to see all available commands" + ;; +esac \ No newline at end of file -- cgit v1.2.3