#include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_comm.h" #include "gsp_rpc_integration.h" #define NV_VGPU_GSP_BOOTLOADER_FILE "nvidia/tu102/gsp/bootloader-535.113.01.bin" #define NV_VGPU_GSP_BOOTER_LOAD_FILE "nvidia/tu102/gsp/booter_load-535.113.01.bin" #define NV_VGPU_GSP_GSP_FILE "nvidia/tu102/gsp/gsp-535.113.01.bin" static void nv_vgpu_gsp_rpc_work(struct work_struct *work) { struct nv_vgpu_gsp_comm_ctx *ctx = container_of(work, struct nv_vgpu_gsp_comm_ctx, rpc_work); struct nv_vgpu_gsp_rpc_entry *entry, *tmp; LIST_HEAD(completed); mutex_lock(&ctx->queue_lock); list_splice_tail_init(&ctx->complete_queue, &completed); mutex_unlock(&ctx->queue_lock); list_for_each_entry_safe(entry, tmp, &completed, list) { list_del(&entry->list); complete(&entry->done); } } static void nv_vgpu_gsp_poll_work(struct work_struct *work) { struct nv_vgpu_gsp_comm_ctx *ctx = container_of(work, struct nv_vgpu_gsp_comm_ctx, poll_work.work); u32 status; if (!ctx || !ctx->irq_enabled || !ctx->gsp_doorbell) goto reschedule; status = readl(ctx->gsp_doorbell + 0x100); if (status & 0x1) { writel(0x1, ctx->gsp_doorbell + 0x100); schedule_work(&ctx->rpc_work); } reschedule: if (ctx->poll_active) schedule_delayed_work(&ctx->poll_work, msecs_to_jiffies(1)); } static irqreturn_t nv_vgpu_gsp_irq_handler(int irq, void *data) { struct nv_vgpu_device *vgpu = data; struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; u32 status; if (!ctx || !ctx->irq_enabled) return IRQ_NONE; status = readl(ctx->gsp_doorbell + 0x100); if (status & 0x1) { writel(0x1, ctx->gsp_doorbell + 0x100); schedule_work(&ctx->rpc_work); return IRQ_HANDLED; } return IRQ_NONE; } static int nv_vgpu_gsp_load_firmware_file(struct nv_vgpu_device *vgpu, const char *filename, void **data, size_t *size) { const struct firmware *fw; int ret; ret = request_firmware(&fw, filename, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "Failed to load firmware %s: %d", filename, ret); return ret; } /* Use vmalloc for large firmware files (GSP is ~23MB) */ *data = vzalloc(fw->size); if (!*data) { release_firmware(fw); return -ENOMEM; } memcpy(*data, fw->data, fw->size); *size = fw->size; release_firmware(fw); nv_vgpu_info(vgpu, "Loaded firmware %s (%zu bytes)", filename, *size); return 0; } int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu) { struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; int ret; if (!ctx) return -ENODEV; if (ctx->firmware_loaded) return 0; if (vgpu->dry_run) { nv_vgpu_info(vgpu, "DRY-RUN: validating GSP firmware files"); /* Just verify firmware files exist without loading */ ret = request_firmware(NULL, NV_VGPU_GSP_BOOTLOADER_FILE, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTLOADER_FILE, ret); return ret; } release_firmware(NULL); ret = request_firmware(NULL, NV_VGPU_GSP_BOOTER_LOAD_FILE, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTER_LOAD_FILE, ret); return ret; } release_firmware(NULL); ret = request_firmware(NULL, NV_VGPU_GSP_GSP_FILE, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_GSP_FILE, ret); return ret; } release_firmware(NULL); nv_vgpu_info(vgpu, "DRY-RUN: All GSP firmware files present"); return 0; } ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_BOOTLOADER_FILE, &ctx->firmware.bootloader, &ctx->firmware.bootloader_size); if (ret) return ret; ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_BOOTER_LOAD_FILE, &ctx->firmware.booter_load, &ctx->firmware.booter_load_size); if (ret) goto err_bootloader; ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_GSP_FILE, &ctx->firmware.gsp, &ctx->firmware.gsp_size); if (ret) goto err_booter_load; ctx->firmware_loaded = true; nv_vgpu_info(vgpu, "GSP firmware loaded successfully"); return 0; err_booter_load: vfree(ctx->firmware.booter_load); ctx->firmware.booter_load = NULL; ctx->firmware.booter_load_size = 0; err_bootloader: vfree(ctx->firmware.bootloader); ctx->firmware.bootloader = NULL; ctx->firmware.bootloader_size = 0; return ret; } void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu) { struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; if (!ctx || !ctx->firmware_loaded) return; vfree(ctx->firmware.bootloader); ctx->firmware.bootloader = NULL; ctx->firmware.bootloader_size = 0; vfree(ctx->firmware.booter_load); ctx->firmware.booter_load = NULL; ctx->firmware.booter_load_size = 0; vfree(ctx->firmware.gsp); ctx->firmware.gsp = NULL; ctx->firmware.gsp_size = 0; ctx->firmware_loaded = false; nv_vgpu_info(vgpu, "GSP firmware released"); } static int nv_vgpu_gsp_write_firmware(struct nv_vgpu_device *vgpu, void *data, size_t size, u64 offset) { struct nv_vgpu_bar *bar = &vgpu->bars[0]; void __iomem *dst; if (!bar->vaddr) return -ENODEV; dst = bar->vaddr + offset; memcpy_toio(dst, data, size); return 0; } int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu) { struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; int ret; if (!ctx || !ctx->firmware_loaded) return -ENODEV; if (vgpu->dry_run) { nv_vgpu_info(vgpu, "DRY-RUN: GSP boot sequence validated (would write firmware to BAR0)"); nv_vgpu_info(vgpu, "DRY-RUN: Would write bootloader (%zu bytes) to offset 0x0", ctx->firmware.bootloader_size); nv_vgpu_info(vgpu, "DRY-RUN: Would write booter_load (%zu bytes) to offset 0x10000", ctx->firmware.booter_load_size); nv_vgpu_info(vgpu, "DRY-RUN: Would write GSP binary (%zu bytes) to offset 0x200000", ctx->firmware.gsp_size); nv_vgpu_info(vgpu, "DRY-RUN: Would kick GSP doorbell at offset 0x%llx", (u64)NV_VGPU_GSP_DOORBELL_OFFSET); return 0; } nv_vgpu_info(vgpu, "Starting GSP boot sequence"); ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.bootloader, ctx->firmware.bootloader_size, 0x0); if (ret) { nv_vgpu_err(vgpu, "Failed to write bootloader: %d", ret); return ret; } ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.booter_load, ctx->firmware.booter_load_size, 0x10000); if (ret) { nv_vgpu_err(vgpu, "Failed to write booter_load: %d", ret); return ret; } ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.gsp, ctx->firmware.gsp_size, 0x200000); if (ret) { nv_vgpu_err(vgpu, "Failed to write GSP binary: %d", ret); return ret; } writel(0x1, ctx->gsp_doorbell); nv_vgpu_info(vgpu, "GSP boot sequence initiated"); return 0; } int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu) { struct nv_vgpu_gsp_comm_ctx *ctx; struct nv_vgpu_bar *bar = &vgpu->bars[0]; int ret; if (vgpu->dry_run) { nv_vgpu_info(vgpu, "DRY-RUN: GSP init validation"); if (!bar->vaddr) { nv_vgpu_err(vgpu, "DRY-RUN: BAR0 not mapped, cannot validate GSP offsets"); return -ENODEV; } nv_vgpu_info(vgpu, "DRY-RUN: BAR0 mapped at %p, GSP doorbell offset 0x%llx, shared mem offset 0x%llx", bar->vaddr, (u64)NV_VGPU_GSP_DOORBELL_OFFSET, (u64)NV_VGPU_GSP_SHARED_MEM_OFFSET); /* Validate firmware files exist */ ret = request_firmware(NULL, NV_VGPU_GSP_BOOTLOADER_FILE, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTLOADER_FILE, ret); return ret; } release_firmware(NULL); ret = request_firmware(NULL, NV_VGPU_GSP_BOOTER_LOAD_FILE, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTER_LOAD_FILE, ret); return ret; } release_firmware(NULL); ret = request_firmware(NULL, NV_VGPU_GSP_GSP_FILE, &vgpu->pdev->dev); if (ret) { nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_GSP_FILE, ret); return ret; } release_firmware(NULL); nv_vgpu_info(vgpu, "DRY-RUN: GSP init validation passed"); return 0; } ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; ctx->vgpu = vgpu; mutex_init(&ctx->queue_lock); INIT_LIST_HEAD(&ctx->pending_queue); INIT_LIST_HEAD(&ctx->complete_queue); init_waitqueue_head(&ctx->wait_queue); atomic_set(&ctx->sequence_counter, 0); spin_lock_init(&ctx->irq_lock); if (bar->vaddr) { ctx->gsp_doorbell = bar->vaddr + NV_VGPU_GSP_DOORBELL_OFFSET; ctx->gsp_shared_mem = bar->vaddr + NV_VGPU_GSP_SHARED_MEM_OFFSET; ctx->shared_mem_size = NV_VGPU_GSP_SHARED_MEM_SIZE; } ctx->rpc_wq = alloc_workqueue("nv-vgpu-gsp-rpc", WQ_UNBOUND | WQ_HIGHPRI, 1); if (!ctx->rpc_wq) { ret = -ENOMEM; goto err_free; } INIT_WORK(&ctx->rpc_work, nv_vgpu_gsp_rpc_work); INIT_DELAYED_WORK(&ctx->poll_work, nv_vgpu_gsp_poll_work); /* Use polling instead of IRQ since nvidia.ko owns the MSI interrupt */ ctx->irq_enabled = true; ctx->poll_active = true; schedule_delayed_work(&ctx->poll_work, msecs_to_jiffies(1)); vgpu->gsp_comm_ctx = ctx; /* Initialize new firmware-decompiled RPC client */ ret = nv_vgpu_gsp_rpc_integration_init(vgpu, ctx); if (ret) { nv_vgpu_warn(vgpu, "Failed to initialize GSP RPC integration: %d (non-fatal)", ret); /* Continue anyway - legacy RPC still works */ } ret = nv_vgpu_gsp_load_firmware(vgpu); if (ret) { nv_vgpu_warn(vgpu, "Failed to load GSP firmware: %d", ret); } ret = nv_vgpu_gsp_boot(vgpu); if (ret) { nv_vgpu_warn(vgpu, "Failed to boot GSP: %d", ret); } return 0; err_free: kfree(ctx); return ret; } void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu) { struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; struct nv_vgpu_gsp_rpc_entry *entry, *tmp; if (!ctx) return; if (vgpu->dry_run) { nv_vgpu_info(vgpu, "DRY-RUN: GSP cleanup (no hardware resources to release)"); return; } /* Cleanup new firmware-decompiled RPC client */ nv_vgpu_gsp_rpc_integration_cleanup(vgpu); /* Stop polling */ ctx->poll_active = false; cancel_delayed_work_sync(&ctx->poll_work); /* Cancel RPC work */ cancel_work_sync(&ctx->rpc_work); if (ctx->rpc_wq) { destroy_workqueue(ctx->rpc_wq); ctx->rpc_wq = NULL; } mutex_lock(&ctx->queue_lock); list_for_each_entry_safe(entry, tmp, &ctx->pending_queue, list) { list_del(&entry->list); entry->result = -ENODEV; complete(&entry->done); } list_for_each_entry_safe(entry, tmp, &ctx->complete_queue, list) { list_del(&entry->list); entry->result = -ENODEV; complete(&entry->done); } mutex_unlock(&ctx->queue_lock); nv_vgpu_gsp_release_firmware(vgpu); kfree(ctx); vgpu->gsp_comm_ctx = NULL; } int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size) { struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; struct nv_vgpu_gsp_rpc_entry *entry; struct nv_vgpu_gsp_rpc_msg *rpc_msg; int ret; if (!ctx || !ctx->gsp_doorbell || !ctx->gsp_shared_mem) return -ENODEV; if (size > NV_VGPU_GSP_RPC_MAX_SIZE - sizeof(struct nv_vgpu_gsp_rpc_header)) return -EINVAL; entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return -ENOMEM; init_completion(&entry->done); entry->msg = kmalloc(size + sizeof(struct nv_vgpu_gsp_rpc_header), GFP_KERNEL); if (!entry->msg) { kfree(entry); return -ENOMEM; } rpc_msg = entry->msg; rpc_msg->hdr.type = NV_GSP_RPC_TYPE_ALLOC; rpc_msg->hdr.sequence = nv_vgpu_gsp_next_sequence(ctx); rpc_msg->hdr.size = size; rpc_msg->hdr.flags = 0; rpc_msg->hdr.timestamp = ktime_get_ns(); memcpy(rpc_msg->data, msg, size); mutex_lock(&ctx->queue_lock); list_add_tail(&entry->list, &ctx->pending_queue); mutex_unlock(&ctx->queue_lock); memcpy(ctx->gsp_shared_mem, rpc_msg, size + sizeof(struct nv_vgpu_gsp_rpc_header)); writel(rpc_msg->hdr.sequence, ctx->gsp_doorbell); ret = wait_for_completion_timeout(&entry->done, msecs_to_jiffies(NV_VGPU_GSP_RPC_TIMEOUT_MS)); if (ret == 0) { ret = -ETIMEDOUT; } else { ret = entry->result; } mutex_lock(&ctx->queue_lock); list_del(&entry->list); mutex_unlock(&ctx->queue_lock); kfree(entry->msg); kfree(entry); return ret; } int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms) { struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; struct nv_vgpu_gsp_rpc_msg *rpc_msg; int ret; if (!ctx || !ctx->gsp_shared_mem) return -ENODEV; ret = wait_event_timeout(ctx->wait_queue, !list_empty(&ctx->complete_queue), msecs_to_jiffies(timeout_ms)); if (ret == 0) return -ETIMEDOUT; mutex_lock(&ctx->queue_lock); if (list_empty(&ctx->complete_queue)) { mutex_unlock(&ctx->queue_lock); return -ENOENT; } rpc_msg = ctx->gsp_shared_mem; if (rpc_msg->hdr.size > size) { mutex_unlock(&ctx->queue_lock); return -EOVERFLOW; } memcpy(msg, rpc_msg->data, rpc_msg->hdr.size); ret = rpc_msg->hdr.size; mutex_unlock(&ctx->queue_lock); return ret; } int nv_vgpu_gsp_alloc(struct nv_vgpu_device *vgpu, u64 size, u32 align, u64 *handle) { struct { u64 size; u32 align; u64 handle; } __packed msg = { .size = size, .align = align }; int ret; ret = nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); if (ret >= 0) *handle = msg.handle; return ret; } int nv_vgpu_gsp_free(struct nv_vgpu_device *vgpu, u64 handle) { struct { u64 handle; } __packed msg = { .handle = handle }; return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); } int nv_vgpu_gsp_map(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size, u32 flags) { struct { u64 handle; u64 iova; u64 size; u32 flags; } __packed msg = { .handle = handle, .iova = iova, .size = size, .flags = flags }; return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); } int nv_vgpu_gsp_unmap(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size) { struct { u64 handle; u64 iova; u64 size; } __packed msg = { .handle = handle, .iova = iova, .size = size }; return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); } int nv_vgpu_gsp_schedule(struct nv_vgpu_device *vgpu, u32 runlist_id, u32 channel_id) { struct { u32 runlist_id; u32 channel_id; } __packed msg = { .runlist_id = runlist_id, .channel_id = channel_id }; return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); } int nv_vgpu_gsp_notify_bar_trap(struct nv_vgpu_device *vgpu, u64 offset, u32 size, void *val) { struct { u64 offset; u32 size; u64 val; } __packed msg = { .offset = offset, .size = size, .val = *(u64 *)val }; return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); } /* ============================================================ * New Firmware-Decompiled RPC Operations (enhanced APIs) * ============================================================ */ int nv_vgpu_gsp_rpc_alloc(struct nv_vgpu_device *vgpu, u64 size, u64 alignment, u32 mem_type, u32 flags, u64 addr_hint, u32 pid, u64 *gpu_va, u64 *handle, u64 *actual_size) { return nv_vgpu_gsp_rpc_alloc_fw(vgpu, size, alignment, mem_type, flags, addr_hint, pid, gpu_va, handle, actual_size); } int nv_vgpu_gsp_rpc_free(struct nv_vgpu_device *vgpu, u64 handle) { return nv_vgpu_gsp_rpc_free_fw(vgpu, handle); } int nv_vgpu_gsp_rpc_map(struct nv_vgpu_device *vgpu, u64 handle, u64 gpu_va, u64 size, u32 prot, u32 flags, u64 offset, u64 *mapped_va) { return nv_vgpu_gsp_rpc_map_fw(vgpu, handle, gpu_va, size, prot, flags, offset, mapped_va); } int nv_vgpu_gsp_rpc_unmap(struct nv_vgpu_device *vgpu, u64 handle, u64 gpu_va, u64 size) { return nv_vgpu_gsp_rpc_unmap_fw(vgpu, handle, gpu_va, size); } int nv_vgpu_gsp_rpc_pin(struct nv_vgpu_device *vgpu, u64 handle, u64 offset, u64 size, u32 flags, u64 *dma_addr, u32 *num_pages) { return nv_vgpu_gsp_rpc_pin_fw(vgpu, handle, offset, size, flags, dma_addr, num_pages); } int nv_vgpu_gsp_rpc_unpin(struct nv_vgpu_device *vgpu, u64 handle, u64 offset, u64 size) { return nv_vgpu_gsp_rpc_unpin_fw(vgpu, handle, offset, size); } int nv_vgpu_gsp_rpc_schedule(struct nv_vgpu_device *vgpu, u64 work_handle, u64 callback, u64 arg, u32 priority, u32 flags) { return nv_vgpu_gsp_rpc_schedule_fw(vgpu, work_handle, callback, arg, priority, flags); } int nv_vgpu_gsp_rpc_yield(struct nv_vgpu_device *vgpu) { return nv_vgpu_gsp_rpc_yield_fw(vgpu); } int nv_vgpu_gsp_rpc_bar_trap(struct nv_vgpu_device *vgpu, u64 bar_addr, u32 access_type, u32 size, u64 value, u32 bar_num, u64 *ret_value) { return nv_vgpu_gsp_rpc_bar_trap_fw(vgpu, bar_addr, access_type, size, value, bar_num, ret_value); } int nv_vgpu_gsp_rpc_migrate_save(struct nv_vgpu_device *vgpu, u64 src_gpu_id, u64 dst_gpu_id, u64 memory_bitmap, u64 page_table_ptr, u64 *checkpoint) { return nv_vgpu_gsp_rpc_migrate_save_fw(vgpu, src_gpu_id, dst_gpu_id, memory_bitmap, page_table_ptr, checkpoint); } int nv_vgpu_gsp_rpc_migrate_restore(struct nv_vgpu_device *vgpu, u64 src_gpu_id, u64 dst_gpu_id, u64 memory_bitmap, u64 page_table_ptr, u64 checkpoint) { return nv_vgpu_gsp_rpc_migrate_restore_fw(vgpu, src_gpu_id, dst_gpu_id, memory_bitmap, page_table_ptr, checkpoint); }