/* * GSP-RM Emulation (physical-path) for Host-Side vGPU * * Emulates the GSP-RM msgq RPC channel in guest system memory. * See gsp_rm_emulation.h for the design overview. */ #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_emulation.h" #include "gsp_rm_emulation.h" #include "gsp_rm_tu104.h" /* ============================================================ * Guest-memory access ("virtual DMA" on behalf of the device) * ============================================================ */ static int gsp_rm_read(struct gsp_emu_context *ctx, u64 gpa, void *buf, size_t len) { struct nv_vgpu_device *vgpu = ctx->vgpu; if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) return -ENODEV; return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int gsp_rm_write(struct gsp_emu_context *ctx, u64 gpa, void *buf, size_t len) { struct nv_vgpu_device *vgpu = ctx->vgpu; if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) return -ENODEV; return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* Translate a shared-buffer offset to a GPA via the buffer's page table */ static u64 gsp_rm_off_to_gpa(struct gsp_rm_emu *rm, u64 off) { u64 idx = off >> 12; if (idx >= rm->mq.pageTableEntryCount) return 0; return rm->ptes[idx] + (off & 0xfff); } /* ============================================================ * Status queue element construction * ============================================================ */ /* XOR-fold checksum over u64s (message_queue_priv.h _checkSum32) */ static u32 gsp_rm_checksum32(const void *data, u32 len) { const u64 *p = data; const u64 *end = (const u64 *)((const u8 *)data + len); u64 sum = 0; while (p < end) sum ^= *p++; return (u32)(sum >> 32) ^ (u32)sum; } /* * 570.144 GSP_MSG_QUEUE_ELEMENT wire layout (guest driver is 570.144 stock): * 0x00 authTagBuffer[16] (zero - no CC) * 0x10 aadBuffer[16] (zero - no CC) * 0x20 checkSum (XOR-fold over msgLen, makes total 0) * 0x24 seqNum (per-record sequence) * 0x28 elemCount (elements this record spans) * 0x30 rpc_message_header_v (32 bytes) + payload * GSP_MSG_QUEUE_ELEMENT_HDR_SIZE = offsetof(rpc) = 0x30. */ #define GSPRM_ELEM_HDR_SIZE 0x30 /* * Max RPC data that fits in one queue element: * 4096 (element) - 0x30 (element hdr) - 32 (rpc hdr) = 4016 bytes. */ #define GSPRM_MAX_SINGLE_ELEM_DATA (GSPRM_QUEUE_ELEMENT_SIZE - \ GSPRM_ELEM_HDR_SIZE - \ sizeof(struct gsprm_rpc_header)) /* * Build a multi-element record into a contiguous buffer. * * The record spans elem_count consecutive queue elements: * [0]: element header (0x30) + rpc header (32) + data[0..4015] * [1..N-1]: continuation data only (no headers) * * Returns the total record size in bytes (elem_count * 4096). * Sets *out_elem_count to the number of elements used. */ static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size, u32 function, u32 sequence, u32 result, const void *data, u32 data_len, u32 *out_elem_count) { struct gsprm_rpc_header *hdr = (struct gsprm_rpc_header *)(buf + GSPRM_ELEM_HDR_SIZE); u32 rpc_len = sizeof(*hdr) + data_len; u32 msg_len = GSPRM_ELEM_HDR_SIZE + rpc_len; u32 aligned_len; u32 elem_count; u32 first_data; u32 remaining; u32 i; /* How many elements does this record span? */ elem_count = (msg_len + GSPRM_QUEUE_ELEMENT_SIZE - 1) / GSPRM_QUEUE_ELEMENT_SIZE; if (elem_count == 0) elem_count = 1; if (elem_count * GSPRM_QUEUE_ELEMENT_SIZE > buf_size) { pr_err("GSPRM: build_record overflow: need %u have %u\n", elem_count * GSPRM_QUEUE_ELEMENT_SIZE, buf_size); return 0; } memset(buf, 0, elem_count * GSPRM_QUEUE_ELEMENT_SIZE); /* First element: header + rpc header + start of data */ memcpy(buf + 0x24, &rm->stat_seq, 4); /* seqNum */ memcpy(buf + 0x28, &elem_count, 4); /* elemCount */ hdr->header_version = GSPRM_RPC_HEADER_VERSION; hdr->signature = GSPRM_RPC_SIGNATURE; hdr->length = rpc_len; hdr->function = function; hdr->rpc_result = result; hdr->rpc_result_private = result; hdr->sequence = sequence; hdr->spare = 0; /* Copy data across element boundaries */ first_data = min(data_len, (u32)GSPRM_MAX_SINGLE_ELEM_DATA); if (data && first_data) memcpy((u8 *)(hdr + 1), data, first_data); remaining = data_len - first_data; if (remaining > 0 && data) { const u8 *src = (const u8 *)data + first_data; u8 *dst = buf + GSPRM_QUEUE_ELEMENT_SIZE; for (i = 1; i < elem_count && remaining > 0; i++) { u32 chunk = min(remaining, (u32)GSPRM_QUEUE_ELEMENT_SIZE); memcpy(dst, src, chunk); src += chunk; dst += GSPRM_QUEUE_ELEMENT_SIZE; remaining -= chunk; } } /* checksum covers msg_len zero-padded to 8 (buffer already zeroed) */ aligned_len = (msg_len + 7) & ~7U; *(u32 *)(buf + 0x20) = gsp_rm_checksum32(buf, aligned_len); *out_elem_count = elem_count; return elem_count * GSPRM_QUEUE_ELEMENT_SIZE; } /* * Write a status queue record (1+ elements) and advance the writePtr. */ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, u32 sequence, u32 result, const void *data, u32 data_len) { struct gsp_rm_emu *rm = ctx->rm_emu; u8 *buf; u64 elem_gpa; u32 slot; u32 new_wp; u32 elem_count; u32 record_size; u32 buf_size; int ret; /* Allocate buffer for worst-case multi-element record */ buf_size = GSPRM_QUEUE_ELEMENT_SIZE * ((data_len + GSPRM_MAX_SINGLE_ELEM_DATA + GSPRM_QUEUE_ELEMENT_SIZE - 1) / GSPRM_QUEUE_ELEMENT_SIZE + 1); if (buf_size < GSPRM_QUEUE_ELEMENT_SIZE) buf_size = GSPRM_QUEUE_ELEMENT_SIZE; buf = kzalloc(buf_size, GFP_KERNEL); if (!buf) return -ENOMEM; record_size = gsp_rm_build_record(rm, buf, buf_size, function, sequence, result, data, data_len, &elem_count); if (!record_size) { kfree(buf); return -EOVERFLOW; } /* Check we don't overflow the status queue */ if (elem_count > rm->stat_msgcount) { nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u", elem_count, rm->stat_msgcount); kfree(buf); return -EOVERFLOW; } /* Write all elements to consecutive queue slots */ slot = rm->stat_write % rm->stat_msgcount; elem_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off + rm->entry_off + (u64)slot * GSPRM_QUEUE_ELEMENT_SIZE); /* Handle wrap-around: write may span the end of the circular queue */ { u32 elems_to_end = rm->stat_msgcount - slot; u32 first_chunk = min(elem_count, elems_to_end); u32 second_chunk = elem_count - first_chunk; ret = gsp_rm_write(ctx, elem_gpa, buf, first_chunk * GSPRM_QUEUE_ELEMENT_SIZE); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed gpa=0x%llx ret=%d", elem_gpa, ret); kfree(buf); return ret; } if (second_chunk > 0) { u64 wrap_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off + rm->entry_off); ret = gsp_rm_write(ctx, wrap_gpa, buf + first_chunk * GSPRM_QUEUE_ELEMENT_SIZE, second_chunk * GSPRM_QUEUE_ELEMENT_SIZE); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d", ret); kfree(buf); return ret; } } } kfree(buf); rm->stat_write += elem_count; rm->stat_seq++; /* bump status TX header writePtr (offset 0x10) */ new_wp = rm->stat_write; ret = gsp_rm_write(ctx, gsp_rm_off_to_gpa(rm, rm->stat_off) + 0x10, &new_wp, sizeof(new_wp)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret); return ret; } /* Notify the guest (MSI-X vector 0). Boot path polls anyway; this * covers the driver's interrupt-driven drain later on. */ gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq); return 0; } /* ============================================================ * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED) * ============================================================ */ static int gsp_rm_parse_queue_chain(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; struct gsprm_libos_region *regions; int ret, i; u64 gsp_args_gpa = 0; u32 total_size; regions = kmalloc_array(GSPRM_MAX_LIBOS_REGIONS, sizeof(*regions), GFP_KERNEL); if (!regions) return -ENOMEM; /* 1. Read the libos init args array */ ret = gsp_rm_read(ctx, rm->libos_args_gpa, regions, GSPRM_MAX_LIBOS_REGIONS * sizeof(*regions)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: libos args read failed gpa=0x%llx ret=%d", rm->libos_args_gpa, ret); goto out; } /* 2. Find the RMARGS region -> GSP_ARGUMENTS_CACHED */ for (i = 0; i < GSPRM_MAX_LIBOS_REGIONS; i++) { if (regions[i].id8 == 0) break; nv_vgpu_info(ctx->vgpu, "GSPRM: libos region[%d] id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u", i, regions[i].id8, regions[i].pa, regions[i].size, regions[i].kind, regions[i].loc); if (regions[i].id8 == GSPRM_LIBOS_ID_RMARGS) gsp_args_gpa = regions[i].pa; } if (!gsp_args_gpa) { nv_vgpu_err(ctx->vgpu, "GSPRM: RMARGS region not found"); ret = -ENOENT; goto out; } rm->gsp_args_gpa = gsp_args_gpa; /* 3. Read GSP_ARGUMENTS_CACHED (MESSAGE_QUEUE_INIT_ARGUMENTS first) */ ret = gsp_rm_read(ctx, gsp_args_gpa, &rm->mq, sizeof(rm->mq)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_ARGUMENTS_CACHED read failed ret=%d", ret); goto out; } nv_vgpu_info(ctx->vgpu, "GSPRM: queue args sharedPA=0x%llx ptes=%u cmdOff=0x%llx statOff=0x%llx hdrSize=0x%llx elemMin=0x%llx elemMax=0x%llx hdrAlign=%u elemAlign=%u", rm->mq.sharedMemPhysAddr, rm->mq.pageTableEntryCount, rm->mq.cmdQueueOffset, rm->mq.statQueueOffset, rm->mq.queueElementHdrSize, rm->mq.queueElementSizeMin, rm->mq.queueElementSizeMax, rm->mq.queueHeaderAlign, rm->mq.queueElementAlign); print_hex_dump(KERN_INFO, "GSPRM args raw: ", DUMP_PREFIX_OFFSET, 16, 8, &rm->mq, sizeof(rm->mq), false); if (!rm->mq.sharedMemPhysAddr || !rm->mq.pageTableEntryCount || rm->mq.pageTableEntryCount > 1024) { nv_vgpu_err(ctx->vgpu, "GSPRM: invalid queue args"); ret = -EINVAL; goto out; } /* 4. Read the page table (RmPhysAddr per 4K page of the shared buffer) */ rm->ptes = kmalloc_array(rm->mq.pageTableEntryCount, sizeof(u64), GFP_KERNEL); if (!rm->ptes) { ret = -ENOMEM; goto out; } ret = gsp_rm_read(ctx, rm->mq.sharedMemPhysAddr, rm->ptes, rm->mq.pageTableEntryCount * sizeof(u64)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: page table read failed ret=%d", ret); goto out; } /* 5. Compute queue geometry. NOTE: the shared buffer is * NV_MEMORY_NONCONTIGUOUS - only queue-offsets are linear, GPAs are * not. Every access must translate through rm->ptes (gsp_rm_off_to_gpa). */ rm->cmd_off = rm->mq.cmdQueueOffset; rm->stat_off = rm->mq.statQueueOffset; rm->cmd_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off); rm->stat_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off); rm->cmd_size = (u32)(rm->mq.statQueueOffset - rm->mq.cmdQueueOffset); total_size = rm->mq.pageTableEntryCount << 12; rm->stat_size = total_size - (u32)rm->mq.statQueueOffset; rm->entry_off = 0x1000; /* ALIGN_UP(0x20 + 4, 1 << elemAlign(12)) */ rm->cmd_msgcount = (rm->cmd_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE; rm->stat_msgcount = (rm->stat_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE; if (!rm->cmd_gpa || !rm->stat_gpa || !rm->cmd_msgcount || !rm->stat_msgcount) { nv_vgpu_err(ctx->vgpu, "GSPRM: bad queue geometry cmd_gpa=0x%llx stat_gpa=0x%llx cmd_mc=%u stat_mc=%u", rm->cmd_gpa, rm->stat_gpa, rm->cmd_msgcount, rm->stat_msgcount); ret = -EINVAL; goto out; } nv_vgpu_info(ctx->vgpu, "GSPRM: cmd queue gpa=0x%llx size=0x%x mc=%u; status queue gpa=0x%llx size=0x%x mc=%u", rm->cmd_gpa, rm->cmd_size, rm->cmd_msgcount, rm->stat_gpa, rm->stat_size, rm->stat_msgcount); ret = 0; out: kfree(regions); return ret; } /* * Write the status queue TX header (what GSP-RM's msgqTxCreate produces). */ static int gsp_rm_link_status_queue(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; struct gsprm_msgq_tx_header tx; u32 zero = 0; int ret; memset(&tx, 0, sizeof(tx)); tx.version = GSPRM_MSGQ_VERSION; tx.size = rm->stat_size; tx.msgSize = GSPRM_QUEUE_ELEMENT_SIZE; tx.msgCount = rm->stat_msgcount; tx.writePtr = 0; tx.flags = GSPRM_MSGQ_FLAGS_SWAP_RX; tx.rxHdrOff = 0x20; /* ALIGN_UP(sizeof(msgqTxHeader)=32, 1<entry_off; ret = gsp_rm_write(ctx, gsp_rm_off_to_gpa(rm, rm->stat_off), &tx, sizeof(tx)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: status TX header write failed ret=%d", ret); return ret; } /* status queue RX header (our command-queue read position, swap mode) */ ret = gsp_rm_write(ctx, gsp_rm_off_to_gpa(rm, rm->stat_off) + tx.rxHdrOff, &zero, sizeof(zero)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: status RX header write failed ret=%d", ret); return ret; } nv_vgpu_info(ctx->vgpu, "GSPRM: status queue linked (size=0x%x msgCount=%u)", tx.size, tx.msgCount); return 0; } /* ============================================================ * Command processing * ============================================================ */ static const char *gsp_rm_rpc_name(u32 function) { switch (function) { case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: return "SET_GUEST_SYSTEM_INFO"; case GSPRM_RPC_FUNC_ALLOC_ROOT: return "ALLOC_ROOT"; case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO: return "GET_GSP_STATIC_INFO"; case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO: return "GSP_SET_SYSTEM_INFO"; case GSPRM_RPC_FUNC_SET_REGISTRY: return "SET_REGISTRY"; case GSPRM_RPC_FUNC_CONTINUATION_RECORD: return "CONTINUATION_RECORD"; case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL"; default: return "UNKNOWN"; } } /* * SET_GUEST_SYSTEM_INFO: the driver negotiates the RPC interface version * from the reply's vgxVersionMajorNum/MinorNum fields. Without them the * version-gated HALs stay stubs and every subsequent RPC fails with * NV_VGPU_MSG_RESULT_RPC_UNKNOWN_FUNCTION (0xff100001). Echo the driver's * own version back (no host-guest version skew). */ static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { struct gsp_rm_emu *rm = ctx->rm_emu; const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE + sizeof(struct gsprm_rpc_header); u8 *reply; u32 major, minor; memcpy(&major, req_data + 0, 4); memcpy(&minor, req_data + 4, 4); if (!major) major = GSPRM_VGX_VERSION_MAJOR; if (!minor && major != GSPRM_VGX_VERSION_MAJOR) minor = GSPRM_VGX_VERSION_MINOR; reply = kzalloc(GSPRM_SET_GUEST_INFO_SIZE, GFP_KERNEL); if (!reply) { nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply alloc failed"); return; } memcpy(reply + 0, &major, 4); memcpy(reply + 4, &minor, 4); nv_vgpu_info(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO seq=%u -> VGX %u.%u", sequence, major, minor); if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO, sequence, 0, reply, GSPRM_SET_GUEST_INFO_SIZE)) nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply send failed"); else rm->rpcs_replied++; kfree(reply); } /* ============================================================ * GET_GSP_STATIC_INFO (function 65) * * Builds the 1656-byte GspStaticConfigInfo for TU104. * ============================================================ */ static void gsp_rm_build_static_info(u8 *buf) { u64 val64; u32 val32; u8 *p; memset(buf, 0, TU104_GSCI_SIZE); /* grCapsBits (offset 0, 23 bytes) - basic GR caps */ buf[0] = 0x01; /* GR present */ /* gidInfo (offset 24, 268 bytes): index(4) + flags(4) + length(4) + data(256) */ p = buf + TU104_GSCI_OFF_gidInfo; val32 = 0; memcpy(p + 0, &val32, 4); /* index */ val32 = 0x80000000; memcpy(p + 4, &val32, 4); /* flags: valid */ val32 = 16; memcpy(p + 8, &val32, 4); /* length = 16 (UUID) */ /* GPU UUID - deterministic fake for TU104 */ memcpy(p + 12, "GPU-9da5e221-18", 15); /* SKUInfo (offset 292, 48 bytes) */ p = buf + TU104_GSCI_OFF_SKUInfo; memcpy(p + 4, "TU104", 5); /* chipSKU */ memcpy(p + 13, "400", 3); /* chipSKUMod */ memcpy(p + 20, "N18E", 4); /* project */ memcpy(p + 24, "G3", 2); /* projectSKU */ /* fbRegionInfoParams (offset 344, 776 bytes) */ p = buf + TU104_GSCI_OFF_fbRegionInfoParams; val32 = 1; memcpy(p, &val32, 4); /* numFBRegions = 1 */ /* Region 0: base=0, limit=256MB-1, reserved=0 */ { struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */ val64 = 0; memcpy(®ion->base, &val64, 8); val64 = 0x0FFFFFFF; memcpy(®ion->limit, &val64, 8); val64 = 0; memcpy(®ion->reserved, &val64, 8); val32 = 0; memcpy(®ion->performance, &val32, 4); region->supportCompressed = 0; /* memmgrCalcReservedFbSpace asserts that SOME region supports * ISO "even if we have no display" - without it the reserved * space split fails and kdisp can't allocate its channel * table. A single ISO-capable region satisfies fast/slow/iso * assignment. */ region->supportISO = 1; region->bProtected = 0; memset(region->blackList, 0, sizeof(region->blackList)); } /* sriovCaps (offset 1120, 80 bytes) - all zeros (not SR-IOV) */ /* sriovMaxGfid (offset 1200, 4 bytes) */ val32 = 0; memcpy(buf + TU104_GSCI_OFF_sriovMaxGfid, &val32, 4); /* engineCaps (offset 1204, 12 bytes) */ val32 = 0x60003E02; /* GR0|CE0-4|NVDEC0|NVDEC1 */ memcpy(buf + TU104_GSCI_OFF_engineCaps, &val32, 4); val32 = 0x00362020; /* NVENC0|SW|SEC2|HOST|PMU|FBFLCN */ memcpy(buf + TU104_GSCI_OFF_engineCaps + 4, &val32, 4); val32 = 0; memcpy(buf + TU104_GSCI_OFF_engineCaps + 8, &val32, 4); /* poisonFuseEnabled (offset 1216, 1 byte) */ buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0; /* fb_length (offset 1224, 8 bytes) - 256MB */ val64 = 0x10000000ULL; memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8); /* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */ val64 = 0xFF; memcpy(buf + TU104_GSCI_OFF_fbio_mask, &val64, 8); /* fb_bus_width (offset 1240, 4 bytes) - 256-bit */ val32 = 256; memcpy(buf + TU104_GSCI_OFF_fb_bus_width, &val32, 4); /* fb_ram_type (offset 1244, 4 bytes) - GDDR6 = 6 */ val32 = 6; memcpy(buf + TU104_GSCI_OFF_fb_ram_type, &val32, 4); /* fbp_mask (offset 1248, 8 bytes) - 6 FBPs for 6 GPCs */ val64 = 0x3F; memcpy(buf + TU104_GSCI_OFF_fbp_mask, &val64, 8); /* l2_cache_size (offset 1256, 4 bytes) - 4MB */ val32 = 0x400000; memcpy(buf + TU104_GSCI_OFF_l2_cache_size, &val32, 4); /* gpuNameString (offset 1260, 64 bytes) */ strncpy((char *)(buf + TU104_GSCI_OFF_gpuNameString), "NVIDIA GeForce RTX 2080 SUPER", 64); /* gpuShortNameString (offset 1324, 64 bytes) */ strncpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString), "TU104", 64); /* gpuNameString_Unicode (offset 1388, 128 bytes) - skip for now */ /* Branding booleans */ buf[TU104_GSCI_OFF_bIsMobile] = 1; /* Mobile/Max-Q */ /* bar1PdeBase (offset 1536, 8 bytes) */ val64 = 0; memcpy(buf + TU104_GSCI_OFF_bar1PdeBase, &val64, 8); /* bar2PdeBase (offset 1544, 8 bytes) */ val64 = 0; memcpy(buf + TU104_GSCI_OFF_bar2PdeBase, &val64, 8); /* bVbiosValid (offset 1552, 1 byte) */ buf[TU104_GSCI_OFF_bVbiosValid] = 1; /* vbiosSubVendor (offset 1556, 4 bytes) - ASUS */ val32 = 0x1043; memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val32, 4); /* vbiosSubDevice (offset 1560, 4 bytes) */ val32 = 0x1F01; memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val32, 4); /* hInternalClient/Device/Subdevice */ val32 = TU104_H_INTERNAL_CLIENT; memcpy(buf + TU104_GSCI_OFF_hInternalClient, &val32, 4); val32 = TU104_H_INTERNAL_DEVICE; memcpy(buf + TU104_GSCI_OFF_hInternalDevice, &val32, 4); val32 = TU104_H_INTERNAL_SUBDEVICE; memcpy(buf + TU104_GSCI_OFF_hInternalSubdevice, &val32, 4); /* ecidInfo (offset 1616, 24 bytes) - leave zeroed */ /* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */ val64 = 0; memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8); val64 = 0xFE00000; /* frtsOffset = fbSize - 2MB */ memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8); } static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx, u32 sequence) { struct gsp_rm_emu *rm = ctx->rm_emu; u8 *reply; reply = kzalloc(TU104_GSCI_SIZE, GFP_KERNEL); if (!reply) { nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply alloc failed"); return; } gsp_rm_build_static_info(reply); nv_vgpu_info(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)", sequence, TU104_GSCI_SIZE, 0x10000000ULL >> 20, 0x60003E02, 0x362020); if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0, reply, TU104_GSCI_SIZE)) nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply send failed"); else rm->rpcs_replied++; kfree(reply); } /* ============================================================ * GSP_RM_CONTROL (function 76) - sub-command dispatch * * rpc_gsp_rm_control_v03_00: hClient(4) + hObject(4) + cmd(4) + status(4) * + paramsSize(4) + flags(4) + params[] * ============================================================ */ struct gsp_rm_control_req { u32 hClient; u32 hObject; u32 cmd; u32 status; u32 paramsSize; u32 flags; /* params[] follows */ }; #define GSP_RM_CTRL_HDR_SIZE 24 static void gsp_rm_handle_fifo_device_table(struct gsp_emu_context *ctx, u32 sequence, struct gsp_rm_control_req *req, const u8 *req_params, u32 *out_status, u8 *out_params, u32 *out_params_size) { struct tu104_fifo_get_device_info_params *fifo_params; u32 base, count, i, copy_count; fifo_params = (struct tu104_fifo_get_device_info_params *)out_params; base = 0; /* baseIndex is the first field of the request params, which live in * the queue element buffer (NOT after the local header struct!). Only * readable when the whole request fits in the first element. */ if (req->paramsSize >= 4 && req->paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA) memcpy(&base, req_params, 4); if (base >= TU104_FIFO_TABLE_MAX_DEVICES) base = 0; /* How many entries to return in this page */ count = TU104_ENGINE_TABLE_SIZE - base; if (count > TU104_FIFO_TABLE_MAX_ENTRIES) count = TU104_FIFO_TABLE_MAX_ENTRIES; copy_count = count; if (base >= TU104_ENGINE_TABLE_SIZE) copy_count = 0; fifo_params->baseIndex = base; fifo_params->numEntries = copy_count; fifo_params->bMore = (base + copy_count) < TU104_ENGINE_TABLE_SIZE; for (i = 0; i < copy_count; i++) memcpy(&fifo_params->entries[i], &tu104_engine_table[base + i], sizeof(struct tu104_fifo_device_entry)); *out_params_size = offsetof(struct tu104_fifo_get_device_info_params, entries) + copy_count * sizeof(struct tu104_fifo_device_entry); *out_status = 0; /* NV_OK */ nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_DEVICE_INFO_TABLE base=%u count=%u more=%u", base, copy_count, fifo_params->bMore); } static void gsp_rm_handle_intr_kernel_table(struct gsp_emu_context *ctx, u32 sequence, u32 *out_status, u8 *out_params, u32 *out_params_size) { struct tu104_intr_get_kernel_table_params *intr_params; u32 copy_size; intr_params = (struct tu104_intr_get_kernel_table_params *)out_params; intr_params->tableLen = TU104_INTR_KERNEL_TABLE_SIZE; copy_size = TU104_INTR_KERNEL_TABLE_SIZE * sizeof(struct tu104_intr_table_entry); memcpy(intr_params->table, tu104_intr_kernel_table, copy_size); /* Subtree map: all entries in DEFAULT category (0) */ memset(intr_params->subtreeMap, 0, sizeof(intr_params->subtreeMap)); *out_params_size = offsetof(struct tu104_intr_get_kernel_table_params, table) + copy_size + sizeof(intr_params->subtreeMap); *out_status = 0; nv_vgpu_info(ctx->vgpu, "GSPRM: INTR_GET_KERNEL_TABLE tableLen=%u", TU104_INTR_KERNEL_TABLE_SIZE); } static void gsp_rm_handle_constructed_falcon_info(struct gsp_emu_context *ctx, u32 sequence, u32 *out_status, u8 *out_params, u32 *out_params_size) { struct tu104_get_constructed_falcon_info_params *falcon_params; falcon_params = (struct tu104_get_constructed_falcon_info_params *)out_params; falcon_params->numConstructedFalcons = TU104_FALCON_TABLE_SIZE; memcpy(falcon_params->constructedFalconsTable, tu104_falcon_table, TU104_FALCON_TABLE_SIZE * sizeof(struct tu104_constructed_falcon_info)); *out_params_size = offsetof(struct tu104_get_constructed_falcon_info_params, constructedFalconsTable) + TU104_FALCON_TABLE_SIZE * sizeof(struct tu104_constructed_falcon_info); *out_status = 0; nv_vgpu_info(ctx->vgpu, "GSPRM: GET_CONSTRUCTED_FALCON_INFO numFalcons=%u", TU104_FALCON_TABLE_SIZE); } static void gsp_rm_handle_internal_device_table(struct gsp_emu_context *ctx, u32 sequence, u32 *out_status, u8 *out_params, u32 *out_params_size) { struct tu104_internal_get_device_info_params *dev_params; u32 copy_size; dev_params = (struct tu104_internal_get_device_info_params *)out_params; dev_params->numEntries = TU104_INTERNAL_DEVICE_TABLE_SIZE; copy_size = TU104_INTERNAL_DEVICE_TABLE_SIZE * sizeof(struct tu104_internal_device_info); memcpy(dev_params->deviceInfoTable, tu104_internal_device_table, copy_size); *out_params_size = offsetof(struct tu104_internal_get_device_info_params, deviceInfoTable) + copy_size; *out_status = 0; nv_vgpu_info(ctx->vgpu, "GSPRM: INTERNAL_GET_DEVICE_INFO_TABLE numEntries=%u", TU104_INTERNAL_DEVICE_TABLE_SIZE); } /* * INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP (0x20800a41) * * Returns a compressed register access map telling the guest which BAR0 * registers are userspace-accessible. For our emulated GPU, return an * empty map (no compressed data, no profiling ranges). */ static void gsp_rm_handle_user_register_access_map(struct gsp_emu_context *ctx, u32 sequence, u32 *out_status, u8 *out_params, u32 *out_params_size) { /* NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS: * userRegisterAccessMapSize(4) + compressedSize(4) + * compressedData[4096] + profilingRangesSize(4) + profilingRanges[4096] * Total: 8204 bytes */ u32 val32 = 0; memset(out_params, 0, 8204); memcpy(out_params + 0, &val32, 4); /* userRegisterAccessMapSize */ memcpy(out_params + 4, &val32, 4); /* compressedSize */ /* compressedData[4096] = zeros */ memcpy(out_params + 4104, &val32, 4); /* profilingRangesSize */ /* profilingRanges[4096] = zeros */ *out_params_size = 8204; *out_status = 0; nv_vgpu_info(ctx->vgpu, "GSPRM: USER_REGISTER_ACCESS_MAP -> empty map"); } static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { struct gsp_rm_emu *rm = ctx->rm_emu; const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE + sizeof(struct gsprm_rpc_header); struct gsp_rm_control_req req; u32 reply_data_len; u8 *reply; u32 out_status = 0; u32 out_params_size = 0; memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE); nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x", req.cmd, req.hClient, req.hObject, req.paramsSize, req.flags); /* Allocate reply: control header + max possible params */ reply_data_len = GSP_RM_CTRL_HDR_SIZE + max(req.paramsSize, (u32)4096); reply = kzalloc(reply_data_len, GFP_KERNEL); if (!reply) { nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply alloc failed"); return; } switch (req.cmd) { case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE: gsp_rm_handle_fifo_device_table(ctx, sequence, &req, req_data + GSP_RM_CTRL_HDR_SIZE, &out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; case TU104_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE: gsp_rm_handle_intr_kernel_table(ctx, sequence, &out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; case TU104_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO: gsp_rm_handle_constructed_falcon_info(ctx, sequence, &out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; case TU104_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE: gsp_rm_handle_internal_device_table(ctx, sequence, &out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; case 0x20800a41: /* INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP */ gsp_rm_handle_user_register_access_map(ctx, sequence, &out_status, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; default: /* Unknown control command - return success with zeroed params */ nv_vgpu_info(ctx->vgpu, "GSPRM: RM_CONTROL unknown cmd=0x%x -> generic SUCCESS", req.cmd); out_status = 0; out_params_size = req.paramsSize; break; } /* Build the reply header */ memcpy(reply + 0, &req.hClient, 4); memcpy(reply + 4, &req.hObject, 4); memcpy(reply + 8, &req.cmd, 4); memcpy(reply + 12, &out_status, 4); memcpy(reply + 16, &out_params_size, 4); memcpy(reply + 20, &req.flags, 4); if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply send failed cmd=0x%x", req.cmd); else rm->rpcs_replied++; kfree(reply); } /* * Handle one command-queue record (first element). For async init RPCs we * just consume. Synchronous RPCs get a reply; per-function handlers are * implemented iteratively (unknown ones get a generic success reply). */ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) { struct gsp_rm_emu *rm = ctx->rm_emu; const struct gsprm_rpc_header *hdr = (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); u32 function = hdr->function; u32 sequence = hdr->sequence; rm->rpcs_seen++; switch (function) { case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO: case GSPRM_RPC_FUNC_SET_REGISTRY: case GSPRM_RPC_FUNC_CONTINUATION_RECORD: /* Async init RPCs - fire and forget, just consume. */ nv_vgpu_info(ctx->vgpu, "GSPRM: async RPC %s(%u) seq=%u len=%u consumed", gsp_rm_rpc_name(function), function, sequence, hdr->length); return; case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: gsp_rm_handle_set_guest_system_info(ctx, elem, sequence); return; case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO: gsp_rm_handle_get_static_info(ctx, sequence); return; case GSPRM_RPC_FUNC_GSP_RM_CONTROL: gsp_rm_handle_rm_control(ctx, elem, sequence); return; default: /* Synchronous RPC: echo function+sequence with success so the * driver's rpcRecvPoll completes. */ rm->unknown_rpcs++; nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)", gsp_rm_rpc_name(function), function, sequence, hdr->length, rm->unknown_rpcs); if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "GSPRM: reply send failed fn=%u", function); else rm->rpcs_replied++; return; } } /* * Poll the command queue TX writePtr; consume and handle new records. * A record spans elemCount consecutive queue elements. */ static void gsp_rm_poll_commands(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; struct gsprm_msgq_tx_header tx; int ret; ret = gsp_rm_read(ctx, gsp_rm_off_to_gpa(rm, rm->cmd_off), &tx, sizeof(tx)); if (ret) { /* VM may be gone or device closed; stay quiet, retry later */ return; } if (tx.writePtr == rm->cmd_read) return; if (tx.writePtr < rm->cmd_read || (tx.writePtr - rm->cmd_read) > rm->cmd_msgcount) { /* The writePtr is monotonic for the life of a boot. A backward * or insane jump means the queue buffer was torn down - the * guest driver unloaded (or crashed) without an observable * GSP shutdown. Declare the emulated GSP dead: clear WPR2 / * RISCV / boot stages so the next probe's WPR2-up check * passes and it re-boots from scratch. */ nv_vgpu_warn(ctx->vgpu, "GSPRM: cmd writePtr jumped %u -> %u, GSP boot declared dead", rm->cmd_read, tx.writePtr); gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue"); return; } while (rm->cmd_read < tx.writePtr) { u8 *elem; u64 elem_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off + rm->entry_off + (u64)(rm->cmd_read % rm->cmd_msgcount) * GSPRM_QUEUE_ELEMENT_SIZE); u32 signature, elem_count; const struct gsprm_rpc_header *hdr; elem = kmalloc(GSPRM_QUEUE_ELEMENT_SIZE, GFP_KERNEL); if (!elem) return; ret = gsp_rm_read(ctx, elem_gpa, elem, GSPRM_QUEUE_ELEMENT_SIZE); if (ret) { kfree(elem); return; } hdr = (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); signature = hdr->signature; memcpy(&elem_count, elem + 0x28, 4); if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 || elem_count > rm->cmd_msgcount) { /* The driver always writes a valid element BEFORE * bumping writePtr. An advanced writePtr with garbage * content means the queue buffer is stale (freed and * reused by the guest after unload/crash) - declare * the emulated GSP dead so the next probe re-boots. */ nv_vgpu_warn(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, GSP boot declared dead", rm->cmd_read, signature, elem_count, hdr->function); kfree(elem); gsp_emu_clear_boot_state(ctx, "garbage GSP-RM queue"); return; } gsp_rm_handle_element(ctx, elem); kfree(elem); /* consume the whole record (all its elements) */ rm->cmd_read += elem_count; } /* Publish our command-queue read position into the status queue RX * header (MSGQ_FLAGS_SWAP_RX: read pointers live in the peer's queue). */ gsp_rm_write(ctx, gsp_rm_off_to_gpa(rm, rm->stat_off) + 0x20, &rm->cmd_read, sizeof(rm->cmd_read)); } /* ============================================================ * Public interface * ============================================================ */ int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm; rm = kzalloc(sizeof(*rm), GFP_KERNEL); if (!rm) return -ENOMEM; rm->vgpu = vgpu; ctx->rm_emu = rm; return 0; } void gsp_rm_emu_fini(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; if (!rm) return; kfree(rm->ptes); kfree(rm); ctx->rm_emu = NULL; } /* * Reset the GSP-RM emulation state on GSP ENGINE reset (guest driver reboots * the GSP on every probe/unload). The queue buffer GPAs from the previous * boot are stale (freed by the guest) - drop all parsed state so the next * Booter completion re-parses the chain. */ void gsp_rm_emu_reset(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; if (!rm || !rm->parsed) return; nv_vgpu_info(ctx->vgpu, "GSPRM: reset (rpcs=%llu replied=%llu unknown=%llu)", rm->rpcs_seen, rm->rpcs_replied, rm->unknown_rpcs); kfree(rm->ptes); rm->ptes = NULL; rm->parsed = false; rm->active = false; rm->init_done_sent = false; rm->libos_args_gpa = 0; rm->gsp_args_gpa = 0; rm->cmd_read = 0; rm->stat_write = 0; rm->stat_seq = 0; memset(&rm->mq, 0, sizeof(rm->mq)); rm->cmd_gpa = 0; rm->stat_gpa = 0; rm->cmd_size = 0; rm->stat_size = 0; rm->cmd_msgcount = 0; rm->stat_msgcount = 0; } /* * Called when the guest's Booter Load completes. The GSP MAILBOX0/1 now hold * the libos init args GPA; parse the queue chain, link the status queue and * emit GSP_INIT_DONE so the driver's kgspWaitForRmInitDone completes. */ void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; u32 mb_lo, mb_hi; u32 b_is_d3hot = 0; int ret; if (!rm || rm->parsed) return; mb_lo = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0); mb_hi = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0 + 4); rm->libos_args_gpa = ((u64)mb_hi << 32) | mb_lo; nv_vgpu_info(ctx->vgpu, "GSPRM: libos args at gpa=0x%llx (mailbox 0x%x:0x%x)", rm->libos_args_gpa, mb_hi, mb_lo); if (!rm->libos_args_gpa) { nv_vgpu_warn(ctx->vgpu, "GSPRM: libos args GPA is 0, will retry"); return; } ret = gsp_rm_parse_queue_chain(ctx); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: queue chain parse failed: %d", ret); return; } ret = gsp_rm_link_status_queue(ctx); if (ret) return; rm->parsed = true; rm->active = true; /* Emit GSP_INIT_DONE (function 0x1001, sequence 0, init_done_v17_00) */ ret = gsp_rm_send(ctx, GSPRM_RPC_EVENT_GSP_INIT_DONE, 0, 0, &b_is_d3hot, sizeof(b_is_d3hot)); if (ret) { nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_INIT_DONE send failed: %d", ret); return; } rm->init_done_sent = true; nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_INIT_DONE emitted, RPC loop active"); } void gsp_rm_emu_poll(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; if (!rm) return; /* Retry chain parsing if the first attempt at BOOTER_DONE failed * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE. */ if (!rm->parsed) { if (ctx->boot_stage == 2) gsp_rm_emu_on_booter_done(ctx); return; } if (!rm->active) return; gsp_rm_poll_commands(ctx); }