From c9bf1cf098c8ea4daeb3e6c9b274a9025c03045d Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 15:13:06 -0400 Subject: kernel: GR context-promote bisection modes 4-17 and server-managed-ctxbuf registry variant; complete vgsp rename; docs/tooling refresh - gsp_phys_booter: GR promote bisection modes 4-17 (two-phase zero-field VA, batched entries, hVirtMemory, bare VA, KMD SKIP_CTXBUFFER_ALLOC channel, old-interface GPU_INITIALIZE_CTX, FBMEM context buffer, UVM-faithful PA-only, PATCH/PM auxiliary promotes, server-managed no-promote path); PATCH/PM geometry queries and buffers; pre-boot SET_REGISTRY profile selection - gsp_phys_gpu: gsp_phys_rmcall_gr_promote clamp 0..17 and gsp_phys_srv_ctxbuf parameter; registry image variant with RMSetClientRMAllocatedCtxBuffer=0 - vgsp rename across kernel, scripts, tools and docs (module vgsp.ko, /sys/kernel/debug/vgsp/, vgsp-: log prefixes) - risc-v-gsp-rm-debugger user-facing rename and docs/CI refresh; guest-diag tool updates; README/SBOM/install/Makefile sync Live status: CE ladder and P6 guest-copy forwarding green; GR engine context commit still open (all promote shapes accepted or cleanly refused; the channel stays CTXNOTVALID) - see Handoff-vGPU-IO.md Addendum 21 (local). --- kernel/gsp_phys_booter.c | 1938 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 1505 insertions(+), 433 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9232961..c77e2e2 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -56,6 +56,7 @@ #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" #include "gsp_phys_registry_img.h" +#include "gsp_phys_registry_img_srvctx.h" static bool gsp_phys_boot_param; module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644); @@ -223,6 +224,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CLASS_SUBDEVICE 0x2080u #define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM 0x0081u #define GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER 0x0040u +#define GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL 0x0070u #define GSP_PHYS_RMCALL_CLASS_TSG 0xa06cu #define GSP_PHYS_RMCALL_CLASS_CHANNEL 0xc46fu #define GSP_PHYS_RMCALL_CLASS_CE 0xc5b5u @@ -307,11 +309,41 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707u #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO 0x20801109u #define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX 0x2080012bu +/* + * NV2080_CTRL_CMD_GPU_INITIALIZE_CTX (0x2080012d, ctrl2080gpu.h:1047-1063): + * the old-interface physical-context init. The server routes it to + * grctxInitializeAPI, which stores the PA/size/physAttr memdesc as the + * GraphicsContext's context buffer; a following PROMOTE_CTX with a + * hVirtMemory/bare-VA virtual memdesc then finds that memdesc and binds the + * context. This is the documented client-managed flow the KMD promote path + * expects. + */ +#define GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX 0x2080012du #define GSP_PHYS_RMCALL_CTRL_GR_GET_CTX_BUFFER_INFO 0x20801219u #define GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID 0u #define GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT 3u #define GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR 0x6u +/* + * VIDMEM + GPU cacheable no (NV2080_CTRL_GPU_INITIALIZE_CTX_APERTURE_VIDMEM + * 0, _GPU_CACHEABLE_NO 1), the attribute the reference client derives for an + * FBMEM context memdesc (kernel_graphics_context.c:1814-1843). + */ +#define GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB 0x4u #define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN 0u +#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PM 1u +#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH 2u +/* + * GR_PATCH engine id in NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES + * (ctrl/ctrl0080/ctrl0080fifo.h:137 -> 0x10), the size source for the + * PATCH promote entry (kernel_graphics_context.c:1227). + */ +#define GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID 0x10u +/* + * GR_PM engine id (ctrl0080fifo.h:132 -> 0x09), the PM ctxsw buffer size + * source the client uses for the externally-owned-VAS pre-allocation + * (kernel_graphics_context.c:1296). + */ +#define GSP_PHYS_RMCALL_GR_PM_ENGINE_ID 0x09u #define GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE (32u * 1024u * 1024u) /* One 4 KB PT page covers one 2 MB PD0 slot; 32 MB max -> 16 pages max. */ #define GSP_PHYS_RMCALL_GR_CTX_PT_MAX \ @@ -323,8 +355,58 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE 1u #define GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE 2u #define GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW 3u +#define GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO 4u +#define GSP_PHYS_RMCALL_GR_PROMOTE_BATCHED 5u +#define GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT 6u +#define GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA 7u +#define GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE 8u +#define GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA 9u +#define GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA 10u +#define GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB 11u +/* + * Modes 12/13 (2026-10-02): the UVM/vGPU-faithful PA-only flow. The real + * guest client never promotes the context VAs on an externally-owned VAS; it + * sends bInitialize=1 bNonmapped=1 entries and lets the server commit the + * engine context, while UVM owns the page tables. Mode 2 step 1 proved that + * exact MAIN entry is ACCEPTED (status=0x0) on the V5 channel; only the + * follow-up VA-only step wedged. Mode 12 sends the accepted entry alone and + * continues to BIND/SCHEDULE/submit. Mode 13 additionally promotes the PATCH + * buffer first (the real client's promote set always contains PATCH), sized + * from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10, in case the commit path + * requires it. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY 12u +#define GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH 13u +/* + * Modes 14/15 (2026-10-02): the KMD client-context-buffer path (V6 + * SKIP_CTXBUFFER_ALLOC channel) is the only flow that books the context VA in + * the server's engine-context vaList and commits ENG_CTX_VALID (the CTXNOTVALID + * wall the V5 PA-only modes cannot clear). Modes 8-11 reached it but returned + * 0x40 before the commit; the real client always promotes the PATCH buffer, so + * modes 14/15 promote PATCH PA-only first, then the single-entry MAIN + * (bInitialize=1 bNonmapped=0, PA+VA). Mode 15 backs MAIN with a server FBMEM + * allocation and the VIDMEM physAttr (0x4), the buffer type a vGPU host + * context uses. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD 14u +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB 15u +/* + * Mode 16: mode 14 plus the PM ctxsw buffer PA-only promote. The client + * pre-allocates PM for externally-owned VAS and the promote-time + * preemption/commit HALs consume it; its absence is the next candidate for + * the shared 0x40 site after PATCH failed to clear it. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM 16u +/* + * Mode 17 (2026-10-02): no client PROMOTE_CTX at all, paired with + * gsp_phys_srv_ctxbuf=1. On the server-managed path the GSP-RM allocates and + * commits the engine context itself; the ladder just proceeds to + * BIND/SCHEDULE/submit. Used to test whether the server-side commit makes the + * GR channel valid where every client-owned-buffer promote returned 0x40. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_SERVER 17u #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW) + (GSP_PHYS_RMCALL_GR_PROMOTE_SERVER) /* * Work-submit token notifier slot. The default index is @@ -498,9 +580,12 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 0xcaf00014u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 0xcaf00024u +#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V6 0xcaf00034u #define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM 0xcaf00028u #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u +#define GSP_PHYS_RMCALL_HANDLE_MEM_VIRT 0xcaf00030u +#define GSP_PHYS_RMCALL_HANDLE_CTX_FB 0xcaf00038u /* * NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225); @@ -510,6 +595,20 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER (1u << 23) +/* + * NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC bit 31 (alloc_channel.h:285-287): the + * client owns the engine context buffer and supplies it through PROMOTE_CTX + * (KMD virtual context). The server records it as bSkipCtxBufferAlloc + * (kernel_channel.c:211) and a single MAIN promote entry then takes the + * client-context-buffer path, which creates the virtual context memdesc + * itself instead of resolving the VA against a server-side booking (the + * mode-1/2/4 assert) - live mode 6 showed the hVirtMemory no-entry branch + * returns NV_ERR_INVALID_ARGUMENT without this bit on the channel. Mode 8 + * selects the V5 shape plus this bit; no other mode and no COPY0 step sets + * it. + */ +#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC (1u << 31) + /* * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc @@ -1138,11 +1237,30 @@ static struct gsp_phys_boot_stage { */ bool rmcall_ctx_prepared; bool rmcall_ctx_valid; + /* mode 11: the context buffer is a server-owned FBMEM allocation */ + bool rmcall_ctx_fbmem; u64 rmcall_ctx_pa; /* context buffer DMA base */ u64 rmcall_ctx_va; /* promoted GPU VA window base */ u32 rmcall_ctx_size; /* promoted total size */ u32 rmcall_ctx_pages; /* 4 KB pages mapped */ u32 rmcall_ctx_pt_pages; /* small-PT pages used */ + /* + * Mode 13: the client PATCH context buffer (PA-only promote). Prepared + * lazily in the promote step from the engine-context geometry query + * (engineId 0x10); freed by gsp_phys_rmcall_fini(). + */ + bool rmcall_patch_prepared; + u64 rmcall_patch_pa; /* PATCH buffer DMA base */ + u32 rmcall_patch_size; /* queried patch save size */ + struct gsp_phys_boot_chunk rmcall_patch; + /* + * Mode 16: the client PM ctxsw buffer (PA-only promote), pre-allocated + * for externally-owned VAS. + */ + bool rmcall_pm_prepared; + u64 rmcall_pm_pa; /* PM buffer DMA base */ + u32 rmcall_pm_size; /* queried PM save size */ + struct gsp_phys_boot_chunk rmcall_pm; /* * GR context-promote bisection snapshot (stage 3, GR0 only): the 0..3 * gsp_phys_rmcall_gr_promote value, the number of accepted promote @@ -1157,6 +1275,7 @@ static struct gsp_phys_boot_stage { bool rmcall_gr_promote_warned; /* promote-site wedge warning logged */ u32 rmcall_gr_promote_step; u32 rmcall_gr_promote_size; + u32 rmcall_gr_hvirt; /* mode-6 NV01_MEMORY_VIRTUAL handle */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ @@ -1310,7 +1429,7 @@ static int gsp_phys_chunk_alloc(struct pci_dev *pdev, goto err; } - pr_warn("gsp-phys-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n", + pr_warn("vgsp-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n", pages, pages); return 0; @@ -1404,12 +1523,12 @@ static int gsp_phys_radix3_build(struct pci_dev *pdev, memset(r3, 0, sizeof(*r3)); if (!size) { - pr_err("gsp-phys-boot: radix3 image size is zero; refusing\n"); + pr_err("vgsp-boot: radix3 image size is zero; refusing\n"); return -EINVAL; } if (data_pages > GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE) { - pr_err("gsp-phys-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n", + pr_err("vgsp-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n", data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE); @@ -1427,7 +1546,7 @@ static int gsp_phys_radix3_build(struct pci_dev *pdev, u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES); if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) { - pr_err("gsp-phys-boot: radix3 needs more than %u chunks\n", + pr_err("vgsp-boot: radix3 needs more than %u chunks\n", GSP_PHYS_RADIX3_MAX_CHUNKS); ret = -ENOSPC; goto err; @@ -1544,16 +1663,16 @@ static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot) args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; - pr_info("gsp-phys-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", + pr_info("vgsp-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_LOGINIT, login_pa, GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc); - pr_info("gsp-phys-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", + pr_info("vgsp-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa, GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc); - pr_info("gsp-phys-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n", + pr_info("vgsp-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma, (u64)boot->gsp_args_size, args[2].kind, args[2].loc); - pr_info("gsp-phys-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n"); + pr_info("vgsp-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n"); } static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot, @@ -1576,7 +1695,7 @@ static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot, args->profiler.pa = cpu_to_le64(0); args->profiler.size = cpu_to_le64(0); - pr_info("gsp-phys-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n", + pr_info("vgsp-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n", (u64)gsp_phys_chunk_page_dma(queue, 0), GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); @@ -1608,7 +1727,7 @@ static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) 2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL; if (r3->data_pages != expected_pages || r3->nr_chunks == 0) { - pr_err("gsp-phys-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n", + pr_err("vgsp-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n", r3->data_pages, expected_pages, r3->nr_chunks); return -EINVAL; } @@ -1621,7 +1740,7 @@ static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) gsp_phys_radix3_page_dma(r3, last_used) || (l2_unused && l2_unused[first_unused % GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) { - pr_err("gsp-phys-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n"); + pr_err("vgsp-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n"); return -EINVAL; } tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); @@ -1636,7 +1755,7 @@ static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) || le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) != gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) { - pr_err("gsp-phys-boot: self-check queue PTEs/header FAIL\n"); + pr_err("vgsp-boot: self-check queue PTEs/header FAIL\n"); return -EINVAL; } if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) != @@ -1658,13 +1777,13 @@ static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma || le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size || le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) { - pr_err("gsp-phys-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n"); + pr_err("vgsp-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n"); return -EINVAL; } for (i = 0; i < 3; i++) { if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS || libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) { - pr_err("gsp-phys-boot: self-check libos args[%u] kind/loc FAIL\n", + pr_err("vgsp-boot: self-check libos args[%u] kind/loc FAIL\n", i); return -EINVAL; } @@ -1679,7 +1798,7 @@ static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != gsp_phys_chunk_page_dma(&boot_stage.logrm, GSP_PHYS_LOG_BUFFER_PAGES - 1)) { - pr_err("gsp-phys-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n"); + pr_err("vgsp-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n"); return -EINVAL; } if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) || @@ -1688,11 +1807,11 @@ static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) !gsp_phys_chunk_page_dma(q, 0) || !gsp_phys_chunk_page_dma(&boot_stage.login, 0) || !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) { - pr_err("gsp-phys-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n"); + pr_err("vgsp-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n"); return -EINVAL; } - pr_info("gsp-phys-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n", + pr_info("vgsp-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n", r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE, @@ -1711,7 +1830,7 @@ static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where) struct gsp_phys_scratch14 s; gsp_phys_fwsec_scratch14_read(bar0, &s); - pr_info("gsp-phys-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n", + pr_info("vgsp-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n", s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload, s.booter_reload_acr_unload, s.acr_unload_booter_unload, s.booter_binary_version, where); @@ -1748,17 +1867,25 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_object = 0; boot_stage.rmcall_ctx_prepared = false; boot_stage.rmcall_ctx_valid = false; + boot_stage.rmcall_ctx_fbmem = false; boot_stage.rmcall_ctx_pa = 0; boot_stage.rmcall_ctx_va = 0; boot_stage.rmcall_ctx_size = 0; boot_stage.rmcall_ctx_pages = 0; boot_stage.rmcall_ctx_pt_pages = 0; + boot_stage.rmcall_patch_prepared = false; + boot_stage.rmcall_patch_pa = 0; + boot_stage.rmcall_patch_size = 0; + boot_stage.rmcall_pm_prepared = false; + boot_stage.rmcall_pm_pa = 0; + boot_stage.rmcall_pm_size = 0; boot_stage.rmcall_gr_promote = 0; boot_stage.rmcall_gr_promote_valid = false; boot_stage.rmcall_gr_promote_change_logged = false; boot_stage.rmcall_gr_promote_warned = false; boot_stage.rmcall_gr_promote_step = 0; boot_stage.rmcall_gr_promote_size = 0; + boot_stage.rmcall_gr_hvirt = 0; boot_stage.rmcall_token = 0; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; @@ -1807,16 +1934,16 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, !src->fw_signature || !src->fw_image_size) return -EINVAL; if (!boot->initialized || !boot->wpr_meta_va) { - pr_err("gsp-phys-boot: boot layout not initialised; refusing stage 3d staging\n"); + pr_err("vgsp-boot: boot layout not initialised; refusing stage 3d staging\n"); return -EINVAL; } if (boot_stage.primed) { - pr_err("gsp-phys-boot: staging already primed; refusing a second stage 3d staging\n"); + pr_err("vgsp-boot: staging already primed; refusing a second stage 3d staging\n"); return -EBUSY; } if (src->fw_image_size != boot->size_of_radix3_elf || src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) { - pr_err("gsp-phys-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n", + pr_err("vgsp-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n", src->fw_image_size, boot->size_of_radix3_elf, src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE); return -EINVAL; @@ -1834,7 +1961,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, */ boot_stage.pdev = pdev; gsp_phys_rmcall_reset(); - pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n", + pr_info("vgsp-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n", dryrun ? "prime" : "staging", pci_name(pdev), src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE, GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE, @@ -1845,7 +1972,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow"); /* The staging reads are the only device access here. */ - pr_info("gsp-phys-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n", + pr_info("vgsp-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n", readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), readl(bar0 + GSP_PHYS_GFW_BOOT), @@ -1859,7 +1986,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0), gsp_phys_gsp_rm_boot_image, sizeof(gsp_phys_gsp_rm_boot_image)); - pr_info("gsp-phys-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n", + pr_info("vgsp-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n", sizeof(gsp_phys_gsp_rm_boot_image), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), GSP_PHYS_FMC_APP_VERSION); @@ -1868,7 +1995,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, src->fw_image_size); if (ret) goto err; - pr_info("gsp-phys-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n", + pr_info("vgsp-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n", src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), @@ -1879,7 +2006,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, goto err; memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0), src->fw_signature, GSP_PHYS_SIGNATURE_SIZE); - pr_info("gsp-phys-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n", + pr_info("vgsp-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n", GSP_PHYS_SIGNATURE_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0)); @@ -1896,7 +2023,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, goto err; gsp_phys_chunk_zero(&boot_stage.logrm); gsp_phys_log_chunk_fill(&boot_stage.logrm); - pr_info("gsp-phys-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n", + pr_info("vgsp-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n", GSP_PHYS_LOG_BUFFER_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), @@ -1905,7 +2032,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, ret = gsp_phys_queue_init(pdev, &boot_stage.queue); if (ret) goto err; - pr_info("gsp-phys-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n", + pr_info("vgsp-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n", GSP_PHYS_QUEUE_TOTAL_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); @@ -1931,7 +2058,7 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, goto err; boot_stage.primed = true; - pr_info("gsp-phys-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n", + pr_info("vgsp-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n", (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0), @@ -1955,7 +2082,7 @@ err: static void gsp_phys_boot_telemetry(void __iomem *bar0) { - pr_info("gsp-phys-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n", + pr_info("vgsp-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), @@ -1981,19 +2108,19 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, if (!boot || !pdev || !bar0) return -EINVAL; if (!boot_stage.primed || !boot->initialized) { - pr_err("gsp-phys-boot: stage 3d FAILED: host staging is not primed\n"); + pr_err("vgsp-boot: stage 3d FAILED: host staging is not primed\n"); return -EINVAL; } if (!gsp_phys_fwsec_wpr2_up(bar0)) { - pr_err("gsp-phys-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n"); + pr_err("vgsp-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n"); return -EBUSY; } if (boot_stage.dryrun) { - pr_err("gsp-phys-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n"); + pr_err("vgsp-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n"); return -EINVAL; } if (gsp_phys_fwsec_board_debug(bar0)) { - pr_err("gsp-phys-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n"); + pr_err("vgsp-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n"); return -EPERM; } @@ -2009,7 +2136,7 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, if (!boot_stage.preinit_submitted) { ret = gsp_phys_preinit_rpcs(bar0, pdev); if (ret) { - pr_err("gsp-phys-boot: stage 3d FAILED: pre-boot async init RPCs (fn %u/%u) were not queued (%d); refusing to start GSP-RM without system info\n", + pr_err("vgsp-boot: stage 3d FAILED: pre-boot async init RPCs (fn %u/%u) were not queued (%d); refusing to start GSP-RM without system info\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret); return ret; @@ -2017,12 +2144,12 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, boot_stage.preinit_submitted = true; } - pr_info("gsp-phys-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n", + pr_info("vgsp-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n", (u64)boot->libos_args_dma); gsp_phys_scratch14_log(bar0, "pre-gsp-reset"); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) { - pr_err("gsp-phys-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n", + pr_err("vgsp-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n", ret); return ret; } @@ -2030,12 +2157,12 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); - pr_info("gsp-phys-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n", + pr_info("vgsp-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n", (u64)boot->libos_args_dma, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1)); - pr_info("gsp-phys-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n", + pr_info("vgsp-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n", (u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, @@ -2045,7 +2172,7 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, GSP_PHYS_BOOTER_LOAD_DATA_OFF); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE); if (ret) { - pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 falcon reset error %d\n", + pr_err("vgsp-boot: stage 3d FAILED: SEC2 falcon reset error %d\n", ret); return ret; } @@ -2070,7 +2197,7 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, writel(upper_32_bits((u64)boot->wpr_meta_dma), bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL); - pr_info("gsp-phys-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n", + pr_info("vgsp-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n", (u64)boot->wpr_meta_dma, readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), @@ -2083,19 +2210,19 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, GSP_PHYS_BOOTER_HALT_TIMEOUT_MS); m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); - pr_info("gsp-phys-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n", + pr_info("vgsp-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n", m0, m1, readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); gsp_phys_scratch14_log(bar0, "post-booter-halt"); if (ret) { gsp_phys_boot_telemetry(bar0); - pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n", + pr_err("vgsp-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n", GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1); return ret; } if (m0 != 0) { gsp_phys_boot_telemetry(bar0); - pr_err("gsp-phys-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n", + pr_err("vgsp-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n", m0, m1); return -EIO; } @@ -2105,7 +2232,7 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { gsp_phys_boot_telemetry(bar0); - pr_err("gsp-phys-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n", + pr_err("vgsp-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n", riscv); return -ENODEV; } @@ -2114,7 +2241,7 @@ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, boot_stage.ran = true; boot_stage.riscv_active = true; gsp_phys_boot_telemetry(bar0); - pr_info("gsp-phys-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n", + pr_info("vgsp-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n", riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), GSP_PHYS_FMC_APP_VERSION, readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), @@ -2191,10 +2318,10 @@ static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off, u32 dump_dwords = min_t(u32, cmd_index, GSP_PHYS_SEQ_DUMP_DWORDS); u32 index = 0, ops = 0; - pr_info("gsp-phys-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n", + pr_info("vgsp-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n", payload_off, buffer_size, cmd_index, dump_dwords); if (dump_dwords) - print_hex_dump(KERN_INFO, "gsp-phys-boot: seq cmd: ", + print_hex_dump(KERN_INFO, "vgsp-boot: seq cmd: ", DUMP_PREFIX_OFFSET, 16, 4, cmd, dump_dwords * 4u, true); @@ -2207,7 +2334,7 @@ static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off, index++; if (args == U32_MAX || (u64)index + (u64)args > (u64)cmd_index) { - pr_info("gsp-phys-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n", + pr_info("vgsp-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n", at, opcode, args, cmd_index - at); break; } @@ -2221,34 +2348,34 @@ static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off, switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: - pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n", + pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n", ops - 1, at, a0, a1); break; case GSP_PHYS_SEQ_OP_REG_MODIFY: - pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n", + pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n", ops - 1, at, a0, a1, a2); break; case GSP_PHYS_SEQ_OP_REG_POLL: - pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n", + pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n", ops - 1, at, a0, a1, a2, a3, a4); break; case GSP_PHYS_SEQ_OP_DELAY_US: - pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n", + pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n", ops - 1, at, a0); break; case GSP_PHYS_SEQ_OP_REG_STORE: - pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n", + pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n", ops - 1, at, a0, a1); break; default: - pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n", + pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n", ops - 1, at, gsp_phys_seq_op_name(opcode), a0, a1, a2, a3, a4); break; } } if (ops == GSP_PHYS_SEQ_DUMP_OPS && index < cmd_index) - pr_info("gsp-phys-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n", + pr_info("vgsp-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n", ops, index, cmd_index); } @@ -2260,16 +2387,16 @@ static bool gsp_phys_seq_addr_valid(void __iomem *bar0, resource_size_t bar0_len, u32 addr) { if (!bar0 || !bar0_len) { - pr_err("gsp-phys-boot: sequencer register access without a BAR0 window\n"); + pr_err("vgsp-boot: sequencer register access without a BAR0 window\n"); return false; } if (addr & 0x3u) { - pr_err("gsp-phys-boot: sequencer register address 0x%08x is not 4-byte aligned\n", + pr_err("vgsp-boot: sequencer register address 0x%08x is not 4-byte aligned\n", addr); return false; } if ((u64)addr + 4u > (u64)bar0_len) { - pr_err("gsp-phys-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n", + pr_err("vgsp-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n", addr, (u64)bar0_len); return false; } @@ -2300,7 +2427,7 @@ static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask, observed = readl(bar0 + addr); while ((observed & mask) != val) { if (time_after_eq(jiffies, deadline)) { - pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", + pr_err("vgsp-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", timeout_us, addr, mask, val, observed, error); return -ETIMEDOUT; } @@ -2325,11 +2452,11 @@ static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot, u32 m0, riscv; int ret; - pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n", + pr_info("vgsp-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n", (u64)boot->libos_args_dma); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) { - pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n", + pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n", ret); return ret; } @@ -2348,7 +2475,7 @@ static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot, break; if (time_after_eq(jiffies, deadline)) { gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout"); - pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n", + pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n", GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); return -ETIMEDOUT; } @@ -2357,18 +2484,18 @@ static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot, m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); if (m0 != 0) { - pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n", + pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n", m0); return -EIO; } riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { - pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n", + pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n", riscv); return -ENODEV; } gsp_phys_scratch14_log(bar0, "sequencer-resume-done"); - pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n", + pr_info("vgsp-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n", readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), riscv); return 0; @@ -2386,29 +2513,29 @@ static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len, u32 bs, ci; if (elem_count != 1) { - pr_err("gsp-phys-boot: sequencer message spans %u elements; only single-element buffers are supported\n", + pr_err("vgsp-boot: sequencer message spans %u elements; only single-element buffers are supported\n", elem_count); return -EOPNOTSUPP; } if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) { - pr_err("gsp-phys-boot: sequencer payload too small (%u bytes, need %u)\n", + pr_err("vgsp-boot: sequencer payload too small (%u bytes, need %u)\n", payload_len, GSP_PHYS_SEQ_REQUEST_SIZE); return -EINVAL; } bs = gsp_phys_seq_dw(payload, 0); ci = gsp_phys_seq_dw(payload, 1); if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) { - pr_err("gsp-phys-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n", + pr_err("vgsp-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n", bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS); return -EINVAL; } if (ci >= bs) { - pr_err("gsp-phys-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n", + pr_err("vgsp-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n", ci, bs); return -EINVAL; } if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) { - pr_err("gsp-phys-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n", + pr_err("vgsp-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n", ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u, payload_len); return -EINVAL; @@ -2441,12 +2568,12 @@ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, index++; if (args == U32_MAX) { - pr_err("gsp-phys-boot: sequencer unknown opcode %u at dword %u\n", + pr_err("vgsp-boot: sequencer unknown opcode %u at dword %u\n", opcode, index - 1); return -EINVAL; } if ((u64)index + (u64)args > (u64)cmd_index) { - pr_err("gsp-phys-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n", + pr_err("vgsp-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n", opcode, index, cmd_index); return -EINVAL; } @@ -2485,7 +2612,7 @@ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, case GSP_PHYS_SEQ_OP_DELAY_US: if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) { - pr_info("gsp-phys-boot: sequencer DELAY_US %u clamped to %u us\n", + pr_info("vgsp-boot: sequencer DELAY_US %u clamped to %u us\n", a0, GSP_PHYS_SEQ_DELAY_MAX_US); a0 = GSP_PHYS_SEQ_DELAY_MAX_US; } @@ -2499,7 +2626,7 @@ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) { - pr_err("gsp-phys-boot: sequencer REG_STORE index %u out of range (max %u)\n", + pr_err("vgsp-boot: sequencer REG_STORE index %u out of range (max %u)\n", a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1); return -EINVAL; } @@ -2571,7 +2698,7 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, int ret; if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) { - pr_err("gsp-phys-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n", + pr_err("vgsp-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n", rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE); return -EINVAL; } @@ -2602,20 +2729,20 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, elem_count, &buffer_size, &cmd_index); if (ret) { - pr_err("gsp-phys-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n", + pr_err("vgsp-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, ret, ref_buffer_size); return ret; } legacy_layout = true; - pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n", + pr_warn("vgsp-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ref_buffer_size, GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len); } else { - pr_err("gsp-phys-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n", + pr_err("vgsp-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ret, ref_buffer_size); return ret; @@ -2632,7 +2759,7 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, gsp_phys_seq_log_buffer(payload, offset, buffer_size, cmd_index); } - pr_info("gsp-phys-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n", + pr_info("vgsp-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n", legacy_layout ? "legacy" : "reference", offset, buffer_size, cmd_index); @@ -2933,7 +3060,7 @@ static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec) source, engine, rec->seq_num); if (gsp_phys_nocat_param) - pr_info("gsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n", + pr_info("vgsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n", rec->seq_num, rec_type, error_code, subsystem, bugcheck, len, source, engine); } @@ -2942,14 +3069,14 @@ static void gsp_phys_nocat_summary_dump(void) { u32 i; - pr_info("gsp-nocat: summary: %u unique, %u total, %u overflow\n", + pr_info("vgsp-nocat: summary: %u unique, %u total, %u overflow\n", gsp_phys_nocat_slot_count, gsp_phys_nocat_record_count, gsp_phys_nocat_overflow); for (i = 0; i < gsp_phys_nocat_slot_count; i++) { const struct gsp_phys_nocat_slot *slot = &gsp_phys_nocat_slots[i]; - pr_info("gsp-nocat: entry[%u]: count=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x firstSeq=%u lastSeq=%u source=\"%s\" faultingEngine=\"%s\"\n", + pr_info("vgsp-nocat: entry[%u]: count=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x firstSeq=%u lastSeq=%u source=\"%s\" faultingEngine=\"%s\"\n", i, slot->count, slot->rec_type, slot->error_code, slot->subsystem, slot->bugcheck, slot->first_seq, slot->last_seq, slot->source, slot->faulting_engine); @@ -3008,7 +3135,7 @@ static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq, * consumed records (GSP-RM derives its free space from it). */ if (boot_stage.status_read_ptr >= msg_count) { - pr_warn("gsp-phys-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n", + pr_warn("vgsp-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n", boot_stage.status_read_ptr, msg_count); boot_stage.status_read_ptr = 0; } @@ -3018,7 +3145,7 @@ static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq, sq->read_ptr = boot_stage.status_read_ptr; sq->last_wp = wp; sq->last_wp_logged = wp; - pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n", + pr_info("vgsp-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n", size, msg_size, msg_count, le32_to_cpu(READ_ONCE(tx->flags)), rx_hdr_off, entry_off, wp, sq->read_ptr == 0 ? "initial, scanning" : "resuming", @@ -3026,7 +3153,7 @@ static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq, return 0; } if (wp != sq->last_wp_logged) { - pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", + pr_info("vgsp-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", wp, version, size, msg_size, msg_count); sq->last_wp_logged = wp; } @@ -3061,12 +3188,12 @@ static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq, (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))); dma_rmb(); if (wp >= sq->msg_count) { - pr_err("gsp-phys-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n", + pr_err("vgsp-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n", wp, sq->msg_count); return -EIO; } if (wp != sq->last_wp) { - pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n", + pr_info("vgsp-boot: status queue writePtr (ring slot) %u -> %u\n", sq->last_wp, wp); sq->last_wp = wp; } @@ -3089,19 +3216,19 @@ static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq, dma_rmb(); rpc_length = le32_to_cpu(READ_ONCE(e->rpc_length)); if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) { - pr_err("gsp-phys-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n", + pr_err("vgsp-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n", slot, elem_count, GSP_PHYS_MSG_ELEMENTS_MAX); return -EIO; } if (elem_count > available) { - pr_err("gsp-phys-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n", + pr_err("vgsp-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n", slot, elem_count, available); return -EIO; } if (rpc_length < GSP_PHYS_MSG_RPC_HEADER_SIZE || (u64)GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length > (u64)elem_count * GSP_PHYS_MSG_SIZE) { - pr_err("gsp-phys-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n", + pr_err("vgsp-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n", slot, rpc_length, elem_count); return -EIO; } @@ -3220,7 +3347,7 @@ static int gsp_phys_status_wait_locked(struct gsp_phys_status_queue *sq, gsp_phys_fault_observe(&one, &fi); gsp_phys_fault_wait_unmatched++; } else { - pr_debug("gsp-phys-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", + pr_debug("vgsp-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", one.first_slot, one.seq_num, one.rpc_length, one.checksum, one.function, one.rpc_result); } @@ -3270,13 +3397,13 @@ static void gsp_phys_rx_readptr_publish(u32 read_ptr) if (!(flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) || rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 || rx_hdr_off + 4 > entry_off) { - pr_warn("gsp-phys-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n", + pr_warn("vgsp-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n", read_ptr, flags, rx_hdr_off, entry_off); return; } rx_ptr = (__le32 *)((u8 *)cmd_hdr + rx_hdr_off); - pr_debug("gsp-phys-boot: status-queue readPtr -> %u (cmd+0x%x)\n", + pr_debug("vgsp-boot: status-queue readPtr -> %u (cmd+0x%x)\n", read_ptr, rx_hdr_off); WRITE_ONCE(*rx_ptr, cpu_to_le32(read_ptr)); dma_wmb(); @@ -3311,7 +3438,7 @@ static void gsp_phys_rx_readptr_publish(u32 read_ptr) #define GSP_PHYS_OSERR_LEN_ERRSTRING 256 #define GSP_PHYS_OSERR_OFF_PREVXID 268 -/* One decoded line: "gsp-phys-status:" prefix in dmesg and in the seq_file. */ +/* One decoded line: "vgsp-status:" prefix in dmesg and in the seq_file. */ static void gsp_phys_status_emit(struct seq_file *m, const char *fmt, ...) { char line[GSP_PHYS_STATUS_LINE_MAX]; @@ -3323,9 +3450,9 @@ static void gsp_phys_status_emit(struct seq_file *m, const char *fmt, ...) va_end(args); if (n <= 0) return; - pr_info("gsp-phys-status: %s\n", line); + pr_info("vgsp-status: %s\n", line); if (m) - seq_printf(m, "gsp-phys-status: %s\n", line); + seq_printf(m, "vgsp-status: %s\n", line); } /* @@ -3465,7 +3592,7 @@ static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, gsp_phys_fault_seen++; gsp_phys_fault_epoch++; if (gsp_phys_fault_relay_enabled()) - pr_info_ratelimited("gsp-phys-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", + pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", rec->seq_num, out->except_type, out->runlist_id, out->chid, out->prev_xid, out->err); } @@ -3774,7 +3901,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, * so a failed boot always leaves the NOCAT record summary in dmesg. */ if (!boot || !boot_stage.ran) { - pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n"); + pr_warn("vgsp-boot: stage 3e skipped: stage 3d did not complete\n"); gsp_phys_nocat_reset(); goto out; } @@ -3783,7 +3910,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, gsp_phys_status_queue_init(&sq); start_jiffies = jiffies; deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS); - pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n", + pr_info("vgsp-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n", GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF, GSP_PHYS_RM_INIT_TIMEOUT_MS); @@ -3799,12 +3926,12 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, if (rret == -EAGAIN) break; if (rret) { - pr_err("gsp-phys-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n", + pr_err("vgsp-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n", rret); goto out; } - pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", + pr_info("vgsp-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", rec.first_slot, rec.seq_num, rec.elem_count, rec.function, rec.rpc_result); @@ -3828,7 +3955,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, int sret; if (!bar0 || !bar0_len) { - pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", + pr_err("vgsp-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", rec.function); gsp_phys_rx_record_free(&rec); goto out; @@ -3836,7 +3963,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, if (rec.rpc_length > GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_RPC_HEADER_OFF) { - pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n", + pr_err("vgsp-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n", rec.function, rec.rpc_length, GSP_PHYS_MSG_SIZE, @@ -3852,7 +3979,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, &seq_buffer, &seq_index, &seq_cmds, &seq_off); if (sret) { - pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n", + pr_err("vgsp-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n", rec.function, sret, rec.first_slot, rec.seq_num); @@ -3861,7 +3988,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, } seq_payload = rec.buf + seq_off; gsp_phys_fwsec_scratch14_read(bar0, &s14); - pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n", + pr_info("vgsp-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n", seq_index, seq_cmds, seq_buffer, rec.rpc_length, seq_off, gsp_phys_seq_dw(seq_payload, 2), @@ -3879,7 +4006,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) { if (rec.rpc_result == 0) { boot_stage.rm_init_done = true; - pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n", + pr_info("vgsp-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n", rec.function, rec.seq_num, jiffies_to_msecs(jiffies - @@ -3888,7 +4015,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, gsp_phys_rx_record_free(&rec); goto out; } - pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", + pr_info("vgsp-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", rec.function, rec.rpc_result, rec.seq_num); gsp_phys_rx_record_free(&rec); @@ -3899,7 +4026,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, } if (time_after_eq(jiffies, deadline)) { - pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n", + pr_info("vgsp-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, GSP_PHYS_RM_INIT_TIMEOUT_MS, sq.linked ? sq.last_wp : 0); @@ -3959,11 +4086,11 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, bool stat_linked, stat_rx_swapped, peer_from_status; if (!bar0 || !boot_stage.primed || !q->pages) { - pr_err("gsp-phys-boot: TX refused: command-queue staging is not primed\n"); + pr_err("vgsp-boot: TX refused: command-queue staging is not primed\n"); return -EINVAL; } if (payload_len > GSP_PHYS_MSG_PAYLOAD_MAX) { - pr_err("gsp-phys-boot: TX refused: payload %u exceeds %u bytes\n", + pr_err("vgsp-boot: TX refused: payload %u exceeds %u bytes\n", payload_len, GSP_PHYS_MSG_PAYLOAD_MAX); return -EINVAL; } @@ -3972,7 +4099,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, aligned_len = (msg_len + 7u) & ~7u; elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE); if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) { - pr_err("gsp-phys-boot: TX refused: message spans %u elements (max %u)\n", + pr_err("vgsp-boot: TX refused: message spans %u elements (max %u)\n", elem_count, GSP_PHYS_MSG_ELEMENTS_MAX); return -EINVAL; } @@ -3996,7 +4123,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, entry_off != GSP_PHYS_PAGE_SIZE || rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 || rx_hdr_off + 4 > entry_off || wp >= msg_count) { - pr_err("gsp-phys-boot: TX refused: command-queue TX header mismatch (version=%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr=%u); GSP-RM did not link the staged header\n", + pr_err("vgsp-boot: TX refused: command-queue TX header mismatch (version=%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr=%u); GSP-RM did not link the staged header\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp); return -EIO; @@ -4040,7 +4167,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, */ if (stat_linked) { if (cmd_read >= msg_count) { - pr_err("gsp-phys-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n", + pr_err("vgsp-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n", cmd_read, msg_count); return -EIO; } @@ -4048,7 +4175,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, if (tx_free >= msg_count) tx_free -= msg_count; if (tx_free < elem_count) { - pr_err("gsp-phys-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n", + pr_err("vgsp-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n", tx_free, elem_count, cmd_read, wp, msg_count); return -EBUSY; } @@ -4056,7 +4183,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, tx_free = msg_count - 1; } - pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n", + pr_info("vgsp-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp, tx_free, rx_hdr_off, cmd_read_cmd, stat_rx_hdr_off, cmd_read_status, stat_flags, stat_rx_swapped, @@ -4133,7 +4260,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, */ writel(0, bar0 + GSP_PHYS_QUEUE_HEAD_REG); - pr_info("gsp-phys-boot: TX submitted: function=%u (0x%x) slot=%u length=%u (rpc.length=%u elemCount=%u seqNum=%u checksum=0x%08x) writePtr slot %u -> %u; doorbell QUEUE_HEAD(%u)@0x%x<-0\n", + pr_info("vgsp-boot: TX submitted: function=%u (0x%x) slot=%u length=%u (rpc.length=%u elemCount=%u seqNum=%u checksum=0x%08x) writePtr slot %u -> %u; doorbell QUEUE_HEAD(%u)@0x%x<-0\n", function, function, slot, msg_len, GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len, elem_count, boot_stage.tx_seq_num, checksum, wp, new_wp, @@ -4460,7 +4587,7 @@ static void gsp_phys_preinit_fill_system_info( mirror_id = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE); mirror_rev = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x08); mirror_sub = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x2c); - pr_info("gsp-phys-boot: NV_PCFG mirror @0x%x: id=0x%08x rev=0x%08x sub=0x%08x (config @0x00/0x08/0x2c: 0x%08x/0x%08x/0x%08x)\n", + pr_info("vgsp-boot: NV_PCFG mirror @0x%x: id=0x%08x rev=0x%08x sub=0x%08x (config @0x00/0x08/0x2c: 0x%08x/0x%08x/0x%08x)\n", GSP_PHYS_PCI_CONFIG_MIRROR_BASE, mirror_id, mirror_rev, mirror_sub, cfg_id, cfg_rev, cfg_sub); info->oor_arch = 0; /* OOR_ARCH_X86_64 */ @@ -4560,7 +4687,7 @@ static u64 gsp_phys_timer_set_current_time(void __iomem *bar0) plm = readl(bar0 + GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK); if (!(plm & GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0)) { - pr_warn("gsp-phys-boot: tmrSetCurrentTime skipped: PTIMER level-0 write protection is not enabled (PLM=0x%08x)\n", + pr_warn("vgsp-boot: tmrSetCurrentTime skipped: PTIMER level-0 write protection is not enabled (PLM=0x%08x)\n", plm); return 0; } @@ -4573,7 +4700,7 @@ static u64 gsp_phys_timer_set_current_time(void __iomem *bar0) writel(upper_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_1); writel(lower_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_0); - pr_info("gsp-phys-boot: tmrSetCurrentTime: PTIMER_TIME_1/0 = 0x%08x%08x (%llu ns wall clock, %u s + %u us); sysTimerOffsetNs=0 (GV100/TU10x HAL programs PTIMER directly)\n", + pr_info("vgsp-boot: tmrSetCurrentTime: PTIMER_TIME_1/0 = 0x%08x%08x (%llu ns wall clock, %u s + %u us); sysTimerOffsetNs=0 (GV100/TU10x HAL programs PTIMER directly)\n", upper_32_bits(ns), lower_32_bits(ns), ns, seconds, useconds); return 0; } @@ -4593,6 +4720,8 @@ static u64 gsp_phys_timer_set_current_time(void __iomem *bar0) static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev) { struct gsp_phys_gsp_system_info *info; + const u8 *reg_img; + u32 reg_len; u64 sys_timer_offset_ns; int ret; @@ -4602,7 +4731,7 @@ static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev) if (!info) return -ENOMEM; gsp_phys_preinit_fill_system_info(info, pdev, bar0, sys_timer_offset_ns); - pr_info("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) prepared: gpuPhys=0x%llx fb=0x%llx inst=0x%llx io=0x%llx dbf=0x%llx mirror=0x%x/0x%x ids=%08x:%08x:%08x maxUserVa=0x%llx pageSize=0x%llx flr=%u behindBridge=%u l1ss=0x%x lnkCap=0x%x primary=%u sysTimerOffsetNs=0x%llx\n", + pr_info("vgsp-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) prepared: gpuPhys=0x%llx fb=0x%llx inst=0x%llx io=0x%llx dbf=0x%llx mirror=0x%x/0x%x ids=%08x:%08x:%08x maxUserVa=0x%llx pageSize=0x%llx flr=%u behindBridge=%u l1ss=0x%x lnkCap=0x%x primary=%u sysTimerOffsetNs=0x%llx\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, le64_to_cpu(info->gpu_phys_addr), le64_to_cpu(info->gpu_phys_fb_addr), @@ -4621,7 +4750,7 @@ static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev) le32_to_cpu(info->pcie_config_reg_link_cap), info->b_is_primary, le64_to_cpu(info->sys_timer_offset_ns)); - pr_info("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) chipset block: clPdbProperties=0x%llx chipset=%u (CS_UNKNOWN) FHBBusInfo=%04x:%04x sub=%04x:%04x rev=%02x chipsetIDInfo=%04x:%04x sub=%04x:%04x rev=%02x acpiMethodData bValid=0 (676-byte body zeroed; host ACPI not read)\n", + pr_info("vgsp-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) chipset block: clPdbProperties=0x%llx chipset=%u (CS_UNKNOWN) FHBBusInfo=%04x:%04x sub=%04x:%04x rev=%02x chipsetIDInfo=%04x:%04x sub=%04x:%04x rev=%02x acpiMethodData bValid=0 (676-byte body zeroed; host ACPI not read)\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, le64_to_cpu(info->cl_pdb_properties), le32_to_cpu(info->chipset), @@ -4640,7 +4769,7 @@ static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev) info, sizeof(*info)); kfree(info); if (ret) { - pr_err("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) submit failed: %d\n", + pr_err("vgsp-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) submit failed: %d\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, ret); return ret; } @@ -4655,22 +4784,36 @@ static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev) * the record is 8-byte aligned (1392) and the TX producer zero-pads * any tail and folds the checksum over the padded length, matching the * reference checksum. + * + * With gsp_phys_srv_ctxbuf=1 the variant table is sent instead: + * RMSetClientRMAllocatedCtxBuffer=0, which makes the GSP-RM manage the + * GR context buffers on the server (the physical-RM path) instead of + * requiring a client-owned-buffer GPU_PROMOTE_CTX. The rest of the + * profile is byte-identical. */ - pr_info("gsp-phys-boot: pre-boot SET_REGISTRY (fn%u) prepared: size=%u numEntries=%u (host default profile + RMDevidCheckIgnore=1, display enabled)\n", - GSP_PHYS_RPC_FUNC_SET_REGISTRY, - (unsigned int)GSP_PHYS_REGISTRY_IMAGE_SIZE, - get_unaligned_le32(gsp_phys_registry_image + 4)); + if (gsp_phys_srv_ctxbuf_value()) { + reg_img = gsp_phys_registry_image_srvctx; + reg_len = GSP_PHYS_REGISTRY_IMAGE_SRVCTX_SIZE; + } else { + reg_img = gsp_phys_registry_image; + reg_len = GSP_PHYS_REGISTRY_IMAGE_SIZE; + } + pr_info("vgsp-boot: pre-boot SET_REGISTRY (fn%u) prepared: size=%u numEntries=%u (%s + RMDevidCheckIgnore=1, display enabled)\n", + GSP_PHYS_RPC_FUNC_SET_REGISTRY, (unsigned int)reg_len, + get_unaligned_le32(reg_img + 4), + gsp_phys_srv_ctxbuf_value() ? + "server-managed ctx buffers (RMSetClientRMAllocatedCtxBuffer=0)" : + "host default profile"); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_SET_REGISTRY, - gsp_phys_registry_image, - GSP_PHYS_REGISTRY_IMAGE_SIZE); + reg_img, reg_len); if (ret) { - pr_err("gsp-phys-boot: pre-boot SET_REGISTRY (fn%u) submit failed: %d\n", + pr_err("vgsp-boot: pre-boot SET_REGISTRY (fn%u) submit failed: %d\n", GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret); return ret; } - pr_info("gsp-phys-boot: pre-boot async init RPCs queued (fn%u then fn%u); GSP-RM will consume them at early init before OBJGPU creation\n", + pr_info("vgsp-boot: pre-boot async init RPCs queued (fn%u then fn%u); GSP-RM will consume them at early init before OBJGPU creation\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, GSP_PHYS_RPC_FUNC_SET_REGISTRY); return 0; @@ -4696,7 +4839,7 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) * calls this unconditionally on a healthy board. */ if (!boot_stage.primed || !boot_stage.ran) { - pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: stage 3d did not run (primed=%u ran=%u)\n", + pr_debug("vgsp-boot: UNLOADING_GUEST_DRIVER skipped: stage 3d did not run (primed=%u ran=%u)\n", boot_stage.primed, boot_stage.ran); ret = -EINVAL; goto out; @@ -4709,14 +4852,14 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) * falls through to quiesce + the existing fail-safe. */ if (!boot_stage.rm_init_done) { - pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n", + pr_debug("vgsp-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); ret = -EINVAL; goto out; } if (boot_stage.rm_unload_submitted) { - pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n"); + pr_info("vgsp-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n"); ret = -EALREADY; goto out; } @@ -4724,13 +4867,13 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) /* Normal unload: bInPMTransition=0, bGc6Entering=0, newLevel=0. */ memset(¶ms, 0, sizeof(params)); gsp_phys_scratch14_log(bar0, "pre-unload-rpc"); - pr_info("gsp-phys-boot: sending UNLOADING_GUEST_DRIVER (function %u): bInPMTransition=0 bGc6Entering=0 newLevel=0\n", + pr_info("vgsp-boot: sending UNLOADING_GUEST_DRIVER (function %u): bInPMTransition=0 bGc6Entering=0 newLevel=0\n", GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER, ¶ms, sizeof(params)); if (ret) { - pr_err("gsp-phys-boot: UNLOADING_GUEST_DRIVER submit failed: %d; GSP-RM stays up, WPR2 teardown keeps its existing fail-safe\n", + pr_err("vgsp-boot: UNLOADING_GUEST_DRIVER submit failed: %d; GSP-RM stays up, WPR2 teardown keeps its existing fail-safe\n", ret); goto out; } @@ -4748,7 +4891,7 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) if (time_after_eq(jiffies, deadline)) { gsp_phys_scratch14_log(bar0, "unload-rpc-suspend-timeout"); - pr_err("gsp-phys-boot: GSP-RM processor suspend not observed within %u ms (GSP MAILBOX0=0x%08x, want 0x%08x); the unload RPC may have been dropped (checksum/seqNum/link) or GSP-RM is stuck\n", + pr_err("vgsp-boot: GSP-RM processor suspend not observed within %u ms (GSP MAILBOX0=0x%08x, want 0x%08x); the unload RPC may have been dropped (checksum/seqNum/link) or GSP-RM is stuck\n", GSP_PHYS_RM_UNLOAD_TIMEOUT_MS, mailbox, GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED); ret = -ETIMEDOUT; @@ -4763,10 +4906,10 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) all_done = s14.booter_load && s14.ahesasc_booter_reload && s14.booter_reload_acr_unload && s14.acr_unload_booter_unload; gsp_phys_scratch14_log(bar0, "post-unload-rpc"); - pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER done: GSP-RM processor suspended (GSP MAILBOX0=0x%08x) after %u ms; scratch14 handoffs 24|25|26|27 all DONE=%u\n", + pr_info("vgsp-boot: UNLOADING_GUEST_DRIVER done: GSP-RM processor suspended (GSP MAILBOX0=0x%08x) after %u ms; scratch14 handoffs 24|25|26|27 all DONE=%u\n", mailbox, jiffies_to_msecs(jiffies - start), all_done); if (!all_done) { - pr_err("gsp-phys-boot: GSP-RM suspended but scratch14 handoff bits are not all DONE; the booter-unload gate will refuse (WPR2 stays up, fail-safe keeps the hand-back)\n"); + pr_err("vgsp-boot: GSP-RM suspended but scratch14 handoff bits are not all DONE; the booter-unload gate will refuse (WPR2 stays up, fail-safe keeps the hand-back)\n"); ret = -EIO; goto out; } @@ -4844,18 +4987,18 @@ static void gsp_phys_sci_decode(const u8 *p, u32 len) gsp_phys_sci_string(short_name, sizeof(short_name), p, len, GSP_PHYS_SCI_GPU_SHORT_NAME_OFF); - pr_info("gsp-phys-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n", + pr_info("vgsp-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n", len, GSP_PHYS_SCI_MIN_SIZE, len < GSP_PHYS_SCI_MIN_SIZE ? " SHORT: only present fields are decoded" : ""); - pr_info("gsp-phys-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n", + pr_info("vgsp-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n", gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF)); - pr_info("gsp-phys-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n", + pr_info("vgsp-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n", gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF), @@ -4864,7 +5007,7 @@ static void gsp_phys_sci_decode(const u8 *p, u32 len) gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF)); - pr_info("gsp-phys-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n", + pr_info("vgsp-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n", name, short_name, gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF), @@ -4872,7 +5015,7 @@ static void gsp_phys_sci_decode(const u8 *p, u32 len) gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF)); - pr_info("gsp-phys-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n", + pr_info("vgsp-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n", gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8), @@ -4979,7 +5122,7 @@ static void gsp_phys_sci_decode(const u8 *p, u32 len) gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF); gsp_phys_backend_static.valid = boot_stage.rm_probe_internal_valid; - pr_info("gsp-phys-boot: rm-probe: static snapshot cached: generation=%u fw_crc32=0x%08x payload=%u name='%s' handles=0x%08x/0x%08x/0x%08x valid=%u\n", + pr_info("vgsp-boot: rm-probe: static snapshot cached: generation=%u fw_crc32=0x%08x payload=%u name='%s' handles=0x%08x/0x%08x/0x%08x valid=%u\n", gsp_phys_backend_static.generation, gsp_phys_backend_static.fw_crc32, gsp_phys_backend_static.payload_len, @@ -5027,11 +5170,11 @@ static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1) req, GSP_PHYS_SCI_MIN_SIZE); kvfree(req); if (ret) { - pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n", + pr_err("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n", ret); return ret; } - pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %u B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n", + pr_info("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %u B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n", GSP_PHYS_SCI_MIN_SIZE, tx_seq, after_fn1 ? ", after SET_GUEST_SYSTEM_INFO" : "", GSP_PHYS_RM_PROBE_TIMEOUT_MS); @@ -5042,12 +5185,12 @@ static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1) false, 0, GSP_PHYS_RM_PROBE_TIMEOUT_MS, &rec); if (ret) { - pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n", + pr_err("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n", GSP_PHYS_RM_PROBE_TIMEOUT_MS, ret); return ret; } - pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n", + pr_info("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n", rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num, rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, rec.checksum, @@ -5057,7 +5200,7 @@ static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1) boot_stage.rm_probe_last_payload_len = rec.payload_len; if (rec.rpc_result != 0) { - pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n", + pr_err("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n", rec.rpc_result); gsp_phys_rm_probe_internal_clear(); gsp_phys_rx_record_free(&rec); @@ -5105,11 +5248,11 @@ static int gsp_phys_rm_probe_fn1(void __iomem *bar0) req, sizeof(*req)); kvfree(req); if (ret) { - pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n", + pr_err("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n", ret); return ret; } - pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n", + pr_info("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n", sizeof(*req), 35817632, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS); gsp_phys_status_queue_init(&sq); @@ -5118,11 +5261,11 @@ static int gsp_phys_rm_probe_fn1(void __iomem *bar0) false, 0, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, &rec); if (ret) { - pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n", + pr_err("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n", GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, ret); return ret; } - pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n", + pr_info("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n", rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num, rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, rec.checksum); @@ -5157,7 +5300,7 @@ static int gsp_phys_rm_probe_cycle(void __iomem *bar0) * init-done, so sending here could be dropped. */ if (!boot_stage.rm_init_done) { - pr_info("gsp-phys-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n", + pr_info("vgsp-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE); return -EINVAL; } @@ -5170,7 +5313,7 @@ static int gsp_phys_rm_probe_cycle(void __iomem *bar0) boot_stage.rm_probe_attempts++; fn1_ret = gsp_phys_rm_probe_fn1(bar0); if (fn1_ret) - pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; issuing GET_GSP_STATIC_INFO (65) anyway\n", + pr_err("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; issuing GET_GSP_STATIC_INFO (65) anyway\n", fn1_ret); ret = gsp_phys_rm_probe_fn65(bar0, true); boot_stage.rm_probe_done = true; @@ -5217,7 +5360,7 @@ int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY || !gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) { - pr_info("gsp-phys-boot: backend rpc fn%u refused: not ready (state=%d claimed=%u rm_init_done=%u unload_submitted=%u)\n", + pr_info("vgsp-boot: backend rpc fn%u refused: not ready (state=%d claimed=%u rm_init_done=%u unload_submitted=%u)\n", function, gsp_phys_backend_state(), gsp_phys_gpu_claimed(), boot_stage.rm_init_done, boot_stage.rm_unload_submitted); @@ -5227,7 +5370,7 @@ int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, generation = gsp_phys_backend_generation(); bar0 = gsp_phys_gpu_bar0(); if (!bar0) { - pr_info("gsp-phys-boot: backend rpc fn%u refused: no BAR0\n", + pr_info("vgsp-boot: backend rpc fn%u refused: no BAR0\n", function); return -ENODEV; } @@ -5239,7 +5382,7 @@ int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, u32 gen_now = gsp_phys_backend_generation(); mutex_unlock(&gsp_phys_rm_lock); - pr_info("gsp-phys-boot: backend rpc fn%u refused: session changed while waiting for the GSP lock (generation %u -> %u)\n", + pr_info("vgsp-boot: backend rpc fn%u refused: session changed while waiting for the GSP lock (generation %u -> %u)\n", function, generation, gen_now); return -EAGAIN; } @@ -5247,7 +5390,7 @@ int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len); if (ret) { mutex_unlock(&gsp_phys_rm_lock); - pr_err("gsp-phys-boot: backend rpc fn%u submit refused: %d\n", + pr_err("vgsp-boot: backend rpc fn%u submit refused: %d\n", function, ret); return ret; } @@ -5257,17 +5400,17 @@ int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, ret = gsp_phys_status_wait(&sq, function, false, 0, timeout_ms, &rec); if (ret) { mutex_unlock(&gsp_phys_rm_lock); - pr_err("gsp-phys-boot: backend rpc fn%u no reply within %u ms (%d)\n", + pr_err("vgsp-boot: backend rpc fn%u no reply within %u ms (%d)\n", function, timeout_ms, ret); return ret; } - pr_info("gsp-phys-boot: backend rpc fn%u reply: payload=%u B seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", + pr_info("vgsp-boot: backend rpc fn%u reply: payload=%u B seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", function, rec.payload_len, rec.seq_num, rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, rec.checksum); if (rec.rpc_result != 0) { - pr_err("gsp-phys-boot: backend rpc fn%u server rpc_result=0x%x (not success)\n", + pr_err("vgsp-boot: backend rpc fn%u server rpc_result=0x%x (not success)\n", function, rec.rpc_result); ret = -EIO; goto out; @@ -5382,7 +5525,7 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) if (!gsp_phys_backend_static.valid || gsp_phys_backend_static.generation != generation) { mutex_unlock(&gsp_phys_rm_lock); - pr_debug("gsp-phys-boot: backend static snapshot: no valid cache for generation %u\n", + pr_debug("vgsp-boot: backend static snapshot: no valid cache for generation %u\n", generation); return -ENODATA; } @@ -5663,6 +5806,41 @@ struct gsp_phys_rmcall_promote_params { struct gsp_phys_rmcall_promote_entry entry[16]; }; +/* + * NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS (56 bytes, ctrl2080gpu.h:1051-1063): + * engineType/hClient/ChID/hChanClient/hObject/hVirtMemory, physAddress, + * physAttr, hDmaHandle, index, size. Mode 10 sends it before the promote to + * install the physical context memdesc on the GraphicsContext. + */ +struct gsp_phys_rmcall_init_ctx_params { + __le32 engine_type; + __le32 h_client; + __le32 ch_id; + __le32 h_chan_client; + __le32 h_object; + __le32 h_virt_memory; + __le64 phys_address; + __le32 phys_attr; + __le32 h_dma_handle; + __le32 index; + __le32 pad; + __le64 size; +}; + +/* + * NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS (cl0070.h:66-70), 24 bytes: a VA-range + * reservation object (class NV01_MEMORY_VIRTUAL 0x70). mode 6 allocates one + * on the ladder VAS at the context window and passes its handle as + * PROMOTE_CTX hVirtMemory, so the server creates the memdesc/vaList entry for + * the range itself (the documented no-entry branch; ctrl2080gpu.h:934-947). + */ +struct gsp_phys_rmcall_mem_virtual_params { + __le64 offset; + __le64 limit; + __le32 h_va_space; + __le32 pad; +}; + /* NV2080_CTRL_GR_CTX_BUFFER_INFO (88 bytes), * ctrl/ctrl2080/ctrl2080gr.h:1066-1081. */ struct gsp_phys_rmcall_ctx_buffer_info { @@ -5734,7 +5912,7 @@ static void gsp_phys_rmcall_dump_reply(const struct gsp_phys_rx_record *rec) if (!rec->buf || !rec->payload_len) return; n = min_t(u32, 48u, rec->payload_len); - print_hex_dump_debug("gsp-phys-boot: rm-call: reply: ", + print_hex_dump_debug("vgsp-boot: rm-call: reply: ", DUMP_PREFIX_OFFSET, 16, 4, rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, n, false); @@ -5766,12 +5944,12 @@ gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec) words = shown / 4; if (rec->function == GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC) - pr_info("gsp-phys-boot: rm-call: failed reply (fn103) payload=%u: [0](request hClient)=0x%08x [16](status)=0x%08x [32](params[0])=0x%08x%s\n", + pr_info("vgsp-boot: rm-call: failed reply (fn103) payload=%u: [0](request hClient)=0x%08x [16](status)=0x%08x [32](params[0])=0x%08x%s\n", rec->payload_len, gsp_phys_rmcall_reply_u32(rec, 0), gsp_phys_rmcall_reply_u32(rec, 16), gsp_phys_rmcall_reply_u32(rec, 32), tail); else - pr_info("gsp-phys-boot: rm-call: failed reply (fn%u) payload=%u: raw words [0]=0x%08x [16]=0x%08x [32]=0x%08x%s\n", + pr_info("vgsp-boot: rm-call: failed reply (fn%u) payload=%u: raw words [0]=0x%08x [16]=0x%08x [32]=0x%08x%s\n", rec->function, rec->payload_len, gsp_phys_rmcall_reply_u32(rec, 0), gsp_phys_rmcall_reply_u32(rec, 16), @@ -5782,7 +5960,7 @@ gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec) pos += scnprintf(hex + pos, sizeof(hex) - pos, "%08x", gsp_phys_rmcall_reply_u32(rec, i * 4)); if (pos) - pr_info("gsp-phys-boot: rm-call: hex=%s\n", hex); + pr_info("vgsp-boot: rm-call: hex=%s\n", hex); } /* @@ -5805,7 +5983,7 @@ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, memset(rec, 0, sizeof(*rec)); ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len); if (ret) { - pr_err("gsp-phys-boot: rm-call: fn%u submit refused: %d\n", + pr_err("vgsp-boot: rm-call: fn%u submit refused: %d\n", function, ret); return ret; } @@ -5815,12 +5993,12 @@ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, ret = gsp_phys_status_wait(&sq, function, false, 0, GSP_PHYS_RMCALL_TIMEOUT_MS, rec); if (ret) { - pr_err("gsp-phys-boot: rm-call: fn%u no reply within %u ms (%d)\n", + pr_err("vgsp-boot: rm-call: fn%u no reply within %u ms (%d)\n", function, GSP_PHYS_RMCALL_TIMEOUT_MS, ret); return ret; } - pr_info("gsp-phys-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", + pr_info("vgsp-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", function, tx_seq, rec->rpc_length, rec->payload_len, rec->seq_num, rec->rpc_sequence, rec->rpc_result, rec->rpc_result_private, rec->checksum); @@ -5829,7 +6007,7 @@ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, gsp_phys_rmcall_dump_reply(rec); if (rec->rpc_result != 0) { - pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n", + pr_err("vgsp-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n", function, rec->rpc_result, gsp_phys_rmcall_status_str(rec->rpc_result)); if (dump_on_fail) @@ -5837,7 +6015,7 @@ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, return -EIO; } if (rec->payload_len < reply_min) { - pr_err("gsp-phys-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n", + pr_err("vgsp-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n", function, rec->payload_len, reply_min); return -EIO; } @@ -5879,7 +6057,7 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, out_rpc_result); kvfree(req); if (ret) { - pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n", + pr_err("vgsp-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n", what, h_class, ret); gsp_phys_rx_record_free(&rec); return ret; @@ -5891,7 +6069,7 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, * read; the rpc helper already guaranteed >= 20 B for status. */ if (object_is_out && rec.payload_len < 36) { - pr_err("gsp-phys-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n", + pr_err("vgsp-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n", what, rec.payload_len); gsp_phys_rx_record_free(&rec); return -EIO; @@ -5901,16 +6079,16 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, *out_status = status; if (object_is_out) { *out_object = gsp_phys_rmcall_reply_u32(&rec, 32); - pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n", + pr_info("vgsp-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n", what, h_class, status, *out_object); } else { - pr_info("gsp-phys-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n", + pr_info("vgsp-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n", what, h_parent, h_object, h_class, status); } gsp_phys_rx_record_free(&rec); if (status != 0) { - pr_err("gsp-phys-boot: rm-call: %s alloc server status=0x%x (%s)\n", + pr_err("vgsp-boot: rm-call: %s alloc server status=0x%x (%s)\n", what, status, gsp_phys_rmcall_status_str(status)); return -EIO; } @@ -5952,7 +6130,7 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index) req, (u32)req_len, 0, &rec, true, NULL); kvfree(req); if (ret) { - pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n", + pr_err("vgsp-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n", index, handle, pages, (u64)gsp_phys_chunk_page_dma(c, 0), ret); gsp_phys_rx_record_free(&rec); @@ -5962,7 +6140,7 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index) boot_stage.rmcall_mem_handles[index] = handle; boot_stage.rmcall_mem_registered[index] = true; - pr_info("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n", + pr_info("vgsp-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n", index, handle, GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM, pages, (u64)gsp_phys_chunk_page_dma(c, 0), c->page_dma ? " (per-page PTEs, noncontiguous flag)" : @@ -6028,7 +6206,7 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) req, req_len, 20, &rec, true, NULL); kvfree(req); if (ret) { - pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n", + pr_err("vgsp-boot: rm-call: FBMEM error-notifier alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n", GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM, GSP_PHYS_RMCALL_NOTIFIER_SIZE, ret); gsp_phys_rx_record_free(&rec); @@ -6039,7 +6217,7 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) if (status != 0 || rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) + 8) { - pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", + pr_err("vgsp-boot: rm-call: FBMEM error-notifier alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", status, rec.payload_len, offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) + 8); @@ -6054,12 +6232,81 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) boot_stage.rmcall_notifier_mem = GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM; boot_stage.rmcall_notifier_base = base; boot_stage.rmcall_notifier_valid = true; - pr_info("gsp-phys-boot: rm-call: FBMEM error-notifier allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx (notifyIndex=%u)\n", + pr_info("vgsp-boot: rm-call: FBMEM error-notifier allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx (notifyIndex=%u)\n", boot_stage.rmcall_notifier_mem, GSP_PHYS_RMCALL_NOTIFIER_SIZE, boot_stage.rmcall_notifier_base, GSP_PHYS_RMCALL_NOTIF_INDEX); return 0; } +/* + * Allocate a server-owned FBMEM buffer through GSP_RM_ALLOC class + * NV01_MEMORY_LOCAL_USER (0x40), same shape as the error-notifier + * allocation. Mode 11 uses it for the GR context buffer so the KMD promote + * path sees a VIDMEM context memdesc; returns the frame-buffer base offset + * the server echoed in NV_MEMORY_ALLOCATION_PARAMS.offset. + */ +static int gsp_phys_rmcall_alloc_fb_mem(void __iomem *bar0, u32 handle, + u32 size, u64 *out_base, + const char *what) +{ + struct gsp_phys_rmcall_memory_alloc_params mem; + struct gsp_phys_rmcall_rm_alloc_req *req; + struct gsp_phys_rx_record rec; + u32 req_len = sizeof(*req) + sizeof(mem); + u32 status; + u64 base; + int ret; + + memset(&mem, 0, sizeof(mem)); + mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC); + mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE); + mem.size = cpu_to_le64(size); + mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED); + mem.internalflags = + cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB); + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(boot_stage.rmcall_client); + req->h_parent = cpu_to_le32(boot_stage.rmcall_device); + req->h_object = cpu_to_le32(handle); + req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER); + req->params_size = cpu_to_le32(sizeof(mem)); + memcpy(req->params, &mem, sizeof(mem)); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, + req, req_len, 20, &rec, true, NULL); + kvfree(req); + if (ret) { + pr_err("vgsp-boot: rm-call: %s FBMEM alloc (hObject=0x%x class=0x40 size=0x%x) RPC failed: %d\n", + what, handle, size, ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + + status = gsp_phys_rmcall_reply_u32(&rec, 16); + if (status != 0 || + rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8) { + pr_err("vgsp-boot: rm-call: %s FBMEM alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", + what, status, rec.payload_len, + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 + + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset)); + gsp_phys_rx_record_free(&rec); + + *out_base = base; + pr_info("vgsp-boot: rm-call: %s FBMEM allocated: hObject=0x%x class=0x40 size=0x%x FB base=0x%llx\n", + what, handle, size, base); + return 0; +} + /* * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional) @@ -6098,7 +6345,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, out_rpc_result); kvfree(req); if (ret) { - pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n", + pr_err("vgsp-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n", what, cmd, h_object, ret); gsp_phys_rx_record_free(&rec); return ret; @@ -6106,7 +6353,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, /* Bound the status read by the received payload (reply_min >= 16). */ if (rec.payload_len < 16) { - pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n", + pr_err("vgsp-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n", what, cmd, rec.payload_len); gsp_phys_rx_record_free(&rec); return -EIO; @@ -6121,10 +6368,10 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, } gsp_phys_rx_record_free(&rec); - pr_info("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n", + pr_info("vgsp-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n", what, cmd, h_object, params_len, status); if (status != 0) { - pr_err("gsp-phys-boot: rm-call: %s server status=0x%x (%s)\n", + pr_err("vgsp-boot: rm-call: %s server status=0x%x (%s)\n", what, status, gsp_phys_rmcall_status_str(status)); return -EIO; } @@ -6178,7 +6425,7 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0) return ret; boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE; - pr_info("gsp-phys-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", + pr_info("vgsp-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice); return 0; @@ -6200,12 +6447,12 @@ static int gsp_phys_rmcall_alloc_chunks(void) if (c->pages) continue; if (!boot_stage.pdev) { - pr_err("gsp-phys-boot: rm-call: no claimed pci_dev for chunk %u\n", + pr_err("vgsp-boot: rm-call: no claimed pci_dev for chunk %u\n", i); return -EINVAL; } if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) { - pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n", + pr_err("vgsp-boot: rm-call: buffer %u allocation failed\n", i); return -ENOMEM; } @@ -6239,13 +6486,13 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0) continue; ret = gsp_phys_rmcall_register_mem(bar0, i); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 2 aborted at buffer %u\n", + pr_err("vgsp-boot: rm-call: stage 2 aborted at buffer %u\n", i); return ret; } } - pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n", + pr_info("vgsp-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n", GSP_PHYS_RMCALL_BUF_COUNT); return 0; } @@ -6306,6 +6553,25 @@ gsp_phys_rmcall_channel_variants[] = { GSP_PHYS_RMCALL_HANDLE_CHANNEL }, }; +/* + * Mode-8 channel variant (GR ladder only): the V5 shape (TSG parent, + * page-directory-backed VAS, client memdescs, SKIP_SCRUBBER, privilege + * kernel) plus NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC, which marks the channel as + * a KMD-managed virtual context whose engine context buffer the client owns + * and promotes. It is kept out of the shared variant walk so the COPY0 + * ladder and promote modes 0-7 keep their exact historical variant sequence; + * only gsp_phys_rmcall_alloc_channel selects it, and only for promote mode + * 8. + */ +static const struct gsp_phys_rmcall_channel_variant +gsp_phys_rmcall_channel_variant_skipctx = { + "tsg-vas-pdb-skip-ctxbuf-alloc", 6, true, + GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER | + GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC, + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false, + GSP_PHYS_RMCALL_HANDLE_CHANNEL_V6, +}; + /* * H1 engine snapshot. gsp_phys_rmcall_engine is a load-time module * parameter (0=COPY0, 1=GR0); the ladder snapshots it when stage 3 starts @@ -6325,13 +6591,13 @@ static void gsp_phys_rmcall_engine_snapshot(void) boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_GR0; } else { if (v != 0) - pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_engine=%d is neither 0 (COPY0) nor 1 (GR0); using COPY0\n", + pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_engine=%d is neither 0 (COPY0) nor 1 (GR0); using COPY0\n", v); boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_COPY0; } boot_stage.rmcall_engine_valid = true; if (boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0) - pr_info("gsp-phys-boot: rm-call: engine snapshot: gsp_phys_rmcall_engine=1 -> GR0 (TSG/channel engineType=0x1, TURING_COMPUTE_A 0xc5c0, BIND 0x1, GR semaphore release)\n"); + pr_info("vgsp-boot: rm-call: engine snapshot: gsp_phys_rmcall_engine=1 -> GR0 (TSG/channel engineType=0x1, TURING_COMPUTE_A 0xc5c0, BIND 0x1, GR semaphore release)\n"); } static bool gsp_phys_rmcall_engine_is_gr(void) @@ -6356,7 +6622,7 @@ static void gsp_phys_rmcall_engine_check_change(void) boot_stage.rmcall_engine_change_logged) return; boot_stage.rmcall_engine_change_logged = true; - pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_engine changed to %d after the stage-3 snapshot; the ladder keeps engine=%s for the rest of this staging (mid-session changes are ignored)\n", + pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_engine changed to %d after the stage-3 snapshot; the ladder keeps engine=%s for the rest of this staging (mid-session changes are ignored)\n", v, gsp_phys_rmcall_engine_name()); } @@ -6369,6 +6635,34 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "two-phase PA-only nonmapped then VA-only"; case GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW: return "single-entry bInitialize=1 bNonmapped=0 size=raw"; + case GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO: + return "two-phase PA-only nonmapped then zero-field VA-only"; + case GSP_PHYS_RMCALL_GR_PROMOTE_BATCHED: + return "batched entryCount=2 PA-only init + zero-field VA-only"; + case GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT: + return "no-entry hVirtMemory (NV01_MEMORY_VIRTUAL on the VAS)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA: + return "no-entry bare virtAddress+size"; + case GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + single-entry MAIN init+VA"; + case GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + zero-field VA-only MAIN entry"; + case GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then bare VA+size promote"; + case GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB: + return "as mode 10 with a server FBMEM context buffer (VIDMEM physAttr 0x4, local-aperture PTEs)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY: + return "V5 UVM-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1), no VA, continue"; + case GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH: + return "as mode 12 plus a PATCH PA-only promote first"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD: + return "KMD channel (V6) + PATCH PA-only then single-entry MAIN init+VA"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB: + return "as mode 14 with a server FBMEM MAIN context buffer (VIDMEM physAttr 0x4)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM: + return "as mode 14 plus a PM ctxsw PA-only promote first"; + case GSP_PHYS_RMCALL_GR_PROMOTE_SERVER: + return "no client promote; server-managed ctx buffers (gsp_phys_srv_ctxbuf=1)"; default: return "off (PROMOTE_CTX never sent)"; } @@ -6376,7 +6670,7 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) /* * H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a - * load-time module parameter (0..3, see gsp_phys_gpu.c); stage 3 snapshots it + * load-time module parameter (0..13, see gsp_phys_gpu.c); stage 3 snapshots it * next to the engine so every later step of this staging uses one shape. The * module parameter's set callback already clamps out-of-range values with a * warning; the check here is a second line of defense against a future @@ -6391,15 +6685,15 @@ static void gsp_phys_rmcall_gr_promote_snapshot(void) return; v = gsp_phys_rmcall_gr_promote_value(); if (v < 0 || v > (int)GSP_PHYS_RMCALL_GR_PROMOTE_MAX) { - pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_gr_promote=%d is outside 0..3; clamping to 0 (no PROMOTE_CTX)\n", - v); + pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_gr_promote=%d is outside 0..%u; clamping to 0 (no PROMOTE_CTX)\n", + v, GSP_PHYS_RMCALL_GR_PROMOTE_MAX); v = 0; } boot_stage.rmcall_gr_promote = (u32)v; boot_stage.rmcall_gr_promote_valid = true; if (!gsp_phys_rmcall_engine_is_gr()) return; - pr_info("gsp-phys-boot: rm-call: stage-3 gr_promote_param=%d (%s); PROMOTE_CTX is opt-in, every attempt warns (a failed promote can wedge GSP-RM)\n", + pr_info("vgsp-boot: rm-call: stage-3 gr_promote_param=%d (%s); PROMOTE_CTX is opt-in, every attempt warns (a failed promote can wedge GSP-RM)\n", v, gsp_phys_rmcall_gr_promote_name()); } @@ -6414,7 +6708,7 @@ static void gsp_phys_rmcall_gr_promote_check_change(void) boot_stage.rmcall_gr_promote_change_logged) return; boot_stage.rmcall_gr_promote_change_logged = true; - pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_gr_promote changed to %d after the stage-3 snapshot; the ladder keeps gr_promote_param=%u for the rest of this staging (mid-session changes are ignored; reload the module for another variant)\n", + pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_gr_promote changed to %d after the stage-3 snapshot; the ladder keeps gr_promote_param=%u for the rest of this staging (mid-session changes are ignored; reload the module for another variant)\n", v, boot_stage.rmcall_gr_promote); } @@ -6430,7 +6724,7 @@ static bool gsp_phys_rmcall_gr_ce_refuse(const char *what) return false; if (!boot_stage.rmcall_gr_ce_refused_logged) { boot_stage.rmcall_gr_ce_refused_logged = true; - pr_info("gsp-phys-work: %s refused: the ladder channel is a %s engine channel (gsp_phys_rmcall_engine snapshot); CE methods are never submitted on a GR channel (logged once per staging)\n", + pr_info("vgsp-work: %s refused: the ladder channel is a %s engine channel (gsp_phys_rmcall_engine snapshot); CE methods are never submitted on a GR channel (logged once per staging)\n", what, gsp_phys_rmcall_engine_name()); } return true; @@ -6502,6 +6796,8 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) { struct gsp_phys_rmcall_channel_params ch; + const struct gsp_phys_rmcall_channel_variant *vars = + gsp_phys_rmcall_channel_variants; u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; u32 attempts = gsp_phys_rmcall_engine_is_gr() ? 1u : @@ -6509,10 +6805,52 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) u32 i; int ret; + /* + * Modes 8/9/10/11 are the client-context-buffer shapes: only the V6 + * variant (V5 plus SKIP_CTXBUFFER_ALLOC) makes the server treat the + * channel as KMD-managed and route the single MAIN promote entry to + * its client-context-buffer path (or let the old-interface + * INITIALIZE_CTX+PROMOTE flow bind a client-owned buffer). It is + * tried alone, with no fallback: a V5 win would put the promote back + * on the VA/bookkeeping resolve that asserts GSP-RM in modes 1/2/4. + */ + if (gsp_phys_rmcall_engine_is_gr() && + (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM)) { + vars = &gsp_phys_rmcall_channel_variant_skipctx; + attempts = 1; + } + /* + * Modes 12/13 are the non-KMD UVM/vGPU-faithful PA-only flow + * (bInitialize=1 bNonmapped=1): the mode-2 step-1 entry was ACCEPTED + * on the V5 channel, so pin the walk to that exact winning variant + * with no fallback. The V6 skip-ctx channel would route the MAIN + * entry to the KMD client-context path instead. + */ + if (gsp_phys_rmcall_engine_is_gr() && + (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH)) { + vars = &gsp_phys_rmcall_channel_variants[0]; + attempts = 1; + } + ret = -EIO; for (i = 0; i < attempts; i++) { - const struct gsp_phys_rmcall_channel_variant *v = - &gsp_phys_rmcall_channel_variants[i]; + const struct gsp_phys_rmcall_channel_variant *v = &vars[i]; u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg : boot_stage.rmcall_device; @@ -6525,7 +6863,7 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), false, NULL, v->name, false, &v_rpc[i], &v_status[i]); - pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n", + pr_info("vgsp-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n", v->id, i + 1, ARRAY_SIZE(gsp_phys_rmcall_channel_variants), v->name, parent, v->handle, v->flags, @@ -6534,16 +6872,15 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) boot_stage.rmcall_channel = v->handle; boot_stage.rmcall_channel_variant = v->id; boot_stage.rmcall_channel_server_mem = v->server_mem; - pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n", + pr_info("vgsp-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n", v->id, v->name, v->handle); return 0; } } if (attempts == 1) { - pr_err("gsp-phys-boot: rm-call: stage 3: the single GR0 channel attempt (V%u %s) failed (ret=%d rpc_result=0x%x status=0x%x); the GR ladder aborts with no fallback variants and no retry\n", - gsp_phys_rmcall_channel_variants[0].id, - gsp_phys_rmcall_channel_variants[0].name, ret, + pr_err("vgsp-boot: rm-call: stage 3: the single GR0 channel attempt (V%u %s) failed (ret=%d rpc_result=0x%x status=0x%x); the GR ladder aborts with no fallback variants and no retry\n", + vars[0].id, vars[0].name, ret, v_rpc[0], v_status[0]); return ret; } @@ -6559,7 +6896,7 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) gsp_phys_rmcall_channel_variants[i].id, gsp_phys_rmcall_channel_variants[i].name, v_rpc[i], v_status[i]); - pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n", + pr_err("vgsp-boot: rm-call: stage 3: all %zu channel variants failed: %s\n", ARRAY_SIZE(gsp_phys_rmcall_channel_variants), summary); } return ret; @@ -6594,7 +6931,7 @@ static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0) GSP_PHYS_RMCALL_CLASS_COMPUTE, &gr, sizeof(gr), false, NULL, "TURING_COMPUTE_A", true, NULL, &status); - pr_info("gsp-phys-boot: rm-call: TURING_COMPUTE_A alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n", + pr_info("vgsp-boot: rm-call: TURING_COMPUTE_A alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n", GSP_PHYS_RMCALL_CLASS_COMPUTE, status, GSP_PHYS_RMCALL_HANDLE_GR_OBJ, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_GR_PARAM_VERSION, 0u, @@ -6627,6 +6964,18 @@ static u64 gsp_phys_rmcall_pte(u64 pa) return 0xfull | ((pa >> 12) << 8); } +/* + * GP10X leaf PTE for a frame-buffer (VIDEO_MEMORY aperture 0) address: + * VALID=1 (bit 0), APERTURE=0 (bits 2:1), VOL=1 (bit 3), frame-buffer byte + * offset >> 12 in the VIDEO address field (bits 32:8; dev_mmu.h:592-595, + * uvm_pascal_mmu.c:235-303). Used for the mode-11 FBMEM context buffer; the + * page-table pages that hold these PTEs stay sysmem. + */ +static u64 gsp_phys_rmcall_pte_vid(u64 fb_offset) +{ + return 0x9ull | ((fb_offset >> 12) << 8); +} + /* * Build the channel's page-directory hierarchy around the 2 MB VA window * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO, pushbuffer and the CE copy's @@ -6746,10 +7095,10 @@ static void gsp_phys_rmcall_pdb_build(void) put_unaligned_le64(guest_pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); - pr_info("gsp-phys-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n", + pr_info("vgsp-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n", boot_stage.rmcall_guest_gpa, guest_hpa); } else { - pr_warn("gsp-phys-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n", + pr_warn("vgsp-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n", boot_stage.rmcall_guest_gpa, gpa_ret); } } @@ -6804,22 +7153,31 @@ static void gsp_phys_rmcall_pdb_build(void) for (i = 0; i < boot_stage.rmcall_ctx_pages; i++) { u32 pt_page = i / (GSP_PHYS_PAGE_SIZE / 8u); u32 pt_slot = i % (GSP_PHYS_PAGE_SIZE / 8u); - u64 ctx_pte = gsp_phys_rmcall_pte( - (u64)gsp_phys_chunk_page_dma( - &boot_stage.rmcall_ctx, i)); + u64 ctx_pte; + + if (boot_stage.rmcall_ctx_fbmem) + ctx_pte = gsp_phys_rmcall_pte_vid( + boot_stage.rmcall_ctx_pa + + (u64)i * GSP_PHYS_PAGE_SIZE); + else + ctx_pte = gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_ctx, i)); put_unaligned_le64(ctx_pte, (u8 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_ctx_pt[pt_page], 0) + (size_t)pt_slot * 8); } - pr_info("gsp-phys-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (fresh slots above the scratch pool; no fetched line rewritten)\n", + pr_info("vgsp-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (%s; fresh slots above the scratch pool; no fetched line rewritten)\n", GSP_PHYS_RMCALL_VA_GR_CTX, boot_stage.rmcall_ctx_size, boot_stage.rmcall_ctx_pages, boot_stage.rmcall_ctx_pt_pages, (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xffu), (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xffu) + - boot_stage.rmcall_ctx_pt_pages - 1); + boot_stage.rmcall_ctx_pt_pages - 1, + boot_stage.rmcall_ctx_fbmem ? + "VIDMEM aperture PTEs" : "sysmem PTEs"); } dma_wmb(); @@ -6837,7 +7195,7 @@ static void gsp_phys_rmcall_pdb_build(void) boot_stage.rmcall_pt_gpfifo = pt_gpfifo; boot_stage.rmcall_pt_pb = pt_pb; - pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n", + pr_info("vgsp-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n", GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, @@ -6847,9 +7205,9 @@ static void gsp_phys_rmcall_pdb_build(void) (u64)gsp_phys_chunk_page_dma(pd1, 0), (u64)gsp_phys_chunk_page_dma(pd0, 0), (u64)gsp_phys_chunk_page_dma(pt, 0)); - pr_info("gsp-phys-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n", + pr_info("vgsp-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n", root_ent, pd2_ent, pd1_ent, pd0_small); - pr_info("gsp-phys-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n", + pr_info("vgsp-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n", gpfifo_idx, pt_gpfifo, (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], @@ -6857,11 +7215,11 @@ static void gsp_phys_rmcall_pdb_build(void) pb_idx, pt_pb, (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); - pr_info("gsp-phys-boot: rm-call: channel PDB CE PT: pt[%u]=0x%016llx (src dma=0x%llx) pt[%u]=0x%016llx (dst dma=0x%llx) pt[%u]=0x%016llx (sem dma=0x%llx)\n", + pr_info("vgsp-boot: rm-call: channel PDB CE PT: pt[%u]=0x%016llx (src dma=0x%llx) pt[%u]=0x%016llx (dst dma=0x%llx) pt[%u]=0x%016llx (sem dma=0x%llx)\n", src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); - pr_info("gsp-phys-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", + pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", GSP_PHYS_RMCALL_GUEST_PT_PAGES, GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, GSP_PHYS_RMCALL_GUEST_PT_PAGES * @@ -6903,7 +7261,7 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) * not allocate over the live chunks: reuse the prepared geometry. */ if (boot_stage.rmcall_ctx_prepared) { - pr_info("gsp-phys-boot: rm-call: GR ctx already prepared (pa=0x%llx va=0x%llx size=0x%x); reusing\n", + pr_info("vgsp-boot: rm-call: GR ctx already prepared (pa=0x%llx va=0x%llx size=0x%x); reusing\n", boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size); return 0; @@ -6918,13 +7276,13 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) "GR_GET_ENGINE_CONTEXT_PROPERTIES"); raw = le32_to_cpu(props.size); alignment = le32_to_cpu(props.alignment); - pr_info("gsp-phys-boot: rm-call: GR ctx geometry: engineId=%u rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x (hDevice=0x%x)\n", + pr_info("vgsp-boot: rm-call: GR ctx geometry: engineId=%u rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x (hDevice=0x%x)\n", GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID, raw, alignment, status, rpc_result, boot_stage.rmcall_device); if (ret) return ret; if (!raw || alignment < GSP_PHYS_PAGE_SIZE) { - pr_err("gsp-phys-boot: rm-call: GR ctx geometry invalid (rawSize=0x%x alignment=0x%x); aborting the ladder with no retry\n", + pr_err("vgsp-boot: rm-call: GR ctx geometry invalid (rawSize=0x%x alignment=0x%x); aborting the ladder with no retry\n", raw, alignment); return -EIO; } @@ -6944,7 +7302,7 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) fifo, sizeof(*fifo), &rpc_result, &status, "GR_FIFO_GET_INFO_MAX_SUBCONTEXT"); max_subctx = le32_to_cpu(fifo->fifo_info_tbl[0].data); - pr_info("gsp-phys-boot: rm-call: GR ctx maxSubctx=%u (index=%u engineType=0x%x status=0x%x rpc_result=0x%x hSubdevice=0x%x)\n", + pr_info("vgsp-boot: rm-call: GR ctx maxSubctx=%u (index=%u engineType=0x%x status=0x%x rpc_result=0x%x hSubdevice=0x%x)\n", max_subctx, GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT, GSP_PHYS_RMCALL_ENGINE_GR0, status, rpc_result, boot_stage.rmcall_subdevice); @@ -6955,14 +7313,14 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) total = ALIGN((u64)raw, GSP_PHYS_PAGE_SIZE) + (u64)max_subctx * GSP_PHYS_PAGE_SIZE; if (!total || total > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) { - pr_err("gsp-phys-boot: rm-call: GR ctx MAIN size 0x%llx (raw=0x%x maxSubctx=%u) is outside the 1..0x%x sanity bound; aborting the ladder with no retry\n", + pr_err("vgsp-boot: rm-call: GR ctx MAIN size 0x%llx (raw=0x%x maxSubctx=%u) is outside the 1..0x%x sanity bound; aborting the ladder with no retry\n", total, raw, max_subctx, GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE); return -E2BIG; } pages = (u32)(total / GSP_PHYS_PAGE_SIZE); pt_pages = DIV_ROUND_UP(pages, GSP_PHYS_PAGE_SIZE / 8u); if (pt_pages > GSP_PHYS_RMCALL_GR_CTX_PT_MAX) { - pr_err("gsp-phys-boot: rm-call: GR ctx MAIN needs %u PT pages, above the %u-page window (size=0x%x); aborting the ladder with no retry\n", + pr_err("vgsp-boot: rm-call: GR ctx MAIN needs %u PT pages, above the %u-page window (size=0x%x); aborting the ladder with no retry\n", pt_pages, GSP_PHYS_RMCALL_GR_CTX_PT_MAX, (u32)total); return -E2BIG; } @@ -6975,44 +7333,71 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW) ? raw : (u32)total; boot_stage.rmcall_gr_promote_size = entry_size; - pr_info("gsp-phys-boot: rm-call: GR ctx MAIN: raw=0x%x align=0x%x maxSubctx=%u total=0x%x pages=%u ptPages=%u gr_promote_param=%u entrySize=0x%x\n", + pr_info("vgsp-boot: rm-call: GR ctx MAIN: raw=0x%x align=0x%x maxSubctx=%u total=0x%x pages=%u ptPages=%u gr_promote_param=%u entrySize=0x%x\n", raw, alignment, max_subctx, (u32)total, pages, pt_pages, boot_stage.rmcall_gr_promote, entry_size); - ret = gsp_phys_chunk_alloc(boot_stage.pdev, ctx, pages); - if (ret) { - pr_err("gsp-phys-boot: rm-call: GR ctx buffer allocation (%u pages, 0x%x bytes) failed: %d; aborting the ladder with no retry\n", - pages, (u32)total, ret); - return ret; - } - if (ctx->page_dma) { - pr_err("gsp-phys-boot: rm-call: GR ctx buffer is not physically contiguous (per-page fallback); PROMOTE_CTX takes one PA+size, so the GR context cannot be promoted; aborting the ladder with no retry\n"); - gsp_phys_chunk_free(boot_stage.pdev, ctx); - return -ENOMEM; + /* + * Modes 11/15 back the context buffer with a server-owned FBMEM + * allocation (VIDMEM aperture, the shape the KMD promote path + * expects); every other mode keeps the host sysmem chunk. The PT + * pages are always host sysmem pages behind the sysmem-aperture PD0. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB) { + u64 fb_base = 0; + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_CTX_FB, (u32)total, + &fb_base, "GR ctx"); + if (ret) { + pr_err("vgsp-boot: rm-call: GR ctx FBMEM allocation (0x%x bytes) failed: %d; aborting the ladder with no retry\n", + (u32)total, ret); + return ret; + } + boot_stage.rmcall_ctx_fbmem = true; + boot_stage.rmcall_ctx_pa = fb_base; + } else { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, ctx, pages); + if (ret) { + pr_err("vgsp-boot: rm-call: GR ctx buffer allocation (%u pages, 0x%x bytes) failed: %d; aborting the ladder with no retry\n", + pages, (u32)total, ret); + return ret; + } + if (ctx->page_dma) { + pr_err("vgsp-boot: rm-call: GR ctx buffer is not physically contiguous (per-page fallback); PROMOTE_CTX takes one PA+size, so the GR context cannot be promoted; aborting the ladder with no retry\n"); + gsp_phys_chunk_free(boot_stage.pdev, ctx); + return -ENOMEM; + } + gsp_phys_chunk_zero(ctx); + boot_stage.rmcall_ctx_pa = (u64)ctx->dma; } - gsp_phys_chunk_zero(ctx); for (i = 0; i < pt_pages; i++) { ret = gsp_phys_chunk_alloc(boot_stage.pdev, &boot_stage.rmcall_ctx_pt[i], 1u); if (ret) { - pr_err("gsp-phys-boot: rm-call: GR ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n", + pr_err("vgsp-boot: rm-call: GR ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n", i, pt_pages, ret); goto err_free; } gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]); } - boot_stage.rmcall_ctx_pa = (u64)ctx->dma; boot_stage.rmcall_ctx_va = GSP_PHYS_RMCALL_VA_GR_CTX; boot_stage.rmcall_ctx_size = (u32)total; boot_stage.rmcall_ctx_pages = pages; boot_stage.rmcall_ctx_pt_pages = pt_pages; boot_stage.rmcall_ctx_prepared = true; - pr_info("gsp-phys-boot: rm-call: GR ctx buffer prepared: total=0x%x pages=%u ptPages=%u pa=0x%llx va=0x%llx (contiguous, zeroed; PDB mapping in gsp_phys_rmcall_pdb_build)\n", + pr_info("vgsp-boot: rm-call: GR ctx buffer prepared: total=0x%x pages=%u ptPages=%u pa=0x%llx va=0x%llx (%s; PDB mapping in gsp_phys_rmcall_pdb_build)\n", boot_stage.rmcall_ctx_size, boot_stage.rmcall_ctx_pages, boot_stage.rmcall_ctx_pt_pages, boot_stage.rmcall_ctx_pa, - boot_stage.rmcall_ctx_va); + boot_stage.rmcall_ctx_va, + boot_stage.rmcall_ctx_fbmem ? + "server FBMEM, VIDMEM aperture PTEs" : + "host sysmem, contiguous, zeroed"); return 0; err_free: @@ -7024,23 +7409,178 @@ err_free: } /* - * Fill the single MAIN promote entry with the prepared PA/VA and the - * snapshot's entry size. initialize/nonmapped carry the bisection shape: - * mode 1/3 use {1,0} (PA+VA in one control), mode 2 step 1 uses {1,1} (PA - * only, va=0) and step 2 uses {0,0} (VA only, same PA/size). + * Auxiliary GR context-buffer prepare (PATCH/PM): query the save size for + * `engine_id` (GET_ENGINE_CONTEXT_PROPERTIES), allocate a zeroed physically + * contiguous sysmem buffer and record its PA/size. The real client allocates + * and PA-promotes the PATCH buffer for every GR object and pre-allocates the + * PM ctxsw buffer when the VAS is externally owned + * (kernel_graphics_context.c:1407-1419,2583-2607); the promote-time + * preemption/commit HALs consume them. Lazy: only runs when the mode reaches + * the promote step; a failure aborts the ladder with no retry. + */ +static int gsp_phys_rmcall_gr_aux_prepare(void __iomem *bar0, + struct gsp_phys_boot_chunk *chunk, + u32 engine_id, const char *name, + u64 *pa_out, u32 *size_out, + bool *prepared) +{ + struct gsp_phys_rmcall_engine_ctx_props props; + u32 pages, raw, rpc_result = 0, status = 0; + int ret; + + if (!boot_stage.pdev) + return -EINVAL; + if (*prepared) { + pr_info("vgsp-boot: rm-call: GR %s already prepared (pa=0x%llx size=0x%x); reusing\n", + name, *pa_out, *size_out); + return 0; + } + + memset(&props, 0, sizeof(props)); + props.engine_id = cpu_to_le32(engine_id); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES, + &props, sizeof(props), 24 + sizeof(props), + &props, sizeof(props), &rpc_result, &status, + "GR_GET_ENGINE_CONTEXT_PROPERTIES (aux)"); + raw = le32_to_cpu(props.size); + pr_info("vgsp-boot: rm-call: GR %s geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n", + name, engine_id, raw, le32_to_cpu(props.alignment), + status, rpc_result); + if (ret) + return ret; + if (!raw || raw > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) { + pr_err("vgsp-boot: rm-call: GR %s size 0x%x is outside the 1..0x%x sanity bound; aborting the ladder with no retry\n", + name, raw, GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE); + return -E2BIG; + } + + pages = DIV_ROUND_UP(raw, GSP_PHYS_PAGE_SIZE); + ret = gsp_phys_chunk_alloc(boot_stage.pdev, chunk, pages); + if (ret) { + pr_err("vgsp-boot: rm-call: GR %s buffer allocation (%u pages, 0x%x bytes) failed: %d; aborting the ladder with no retry\n", + name, pages, raw, ret); + return ret; + } + if (chunk->page_dma) { + pr_err("vgsp-boot: rm-call: GR %s buffer is not physically contiguous (per-page fallback); the PA-only promote takes one PA+size; aborting the ladder with no retry\n", + name); + gsp_phys_chunk_free(boot_stage.pdev, chunk); + return -ENOMEM; + } + gsp_phys_chunk_zero(chunk); + *pa_out = (u64)chunk->dma; + *size_out = raw; + *prepared = true; + pr_info("vgsp-boot: rm-call: GR %s buffer prepared: size=0x%x pages=%u pa=0x%llx va=0 (PA-only promote, no PDB mapping needed)\n", + name, raw, pages, *pa_out); + return 0; +} + +static int gsp_phys_rmcall_gr_patch_prepare(void __iomem *bar0) +{ + return gsp_phys_rmcall_gr_aux_prepare(bar0, &boot_stage.rmcall_patch, + GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID, "PATCH", + &boot_stage.rmcall_patch_pa, + &boot_stage.rmcall_patch_size, + &boot_stage.rmcall_patch_prepared); +} + +static int gsp_phys_rmcall_gr_pm_prepare(void __iomem *bar0) +{ + return gsp_phys_rmcall_gr_aux_prepare(bar0, &boot_stage.rmcall_pm, + GSP_PHYS_RMCALL_GR_PM_ENGINE_ID, "PM", + &boot_stage.rmcall_pm_pa, + &boot_stage.rmcall_pm_size, + &boot_stage.rmcall_pm_prepared); +} + +/* + * Fill one MAIN promote entry with the prepared PA/VA and the snapshot's + * entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5 + * entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the + * full PA/VA/size; modes 4 and 5 entry 1 use gsp_phys_rmcall_gr_promote_ + * entry_zero_idx() instead, which matches the reference client's zeroed + * second entry (portMemSet params, only bufferId/gpuVirtAddr/bNonmapped set). */ +static void gsp_phys_rmcall_gr_promote_entry_idx( + struct gsp_phys_rmcall_promote_params *p, unsigned int idx, u64 va, + u8 initialize, u8 nonmapped) +{ + p->entry[idx].gpu_phys_addr = cpu_to_le64(boot_stage.rmcall_ctx_pa); + p->entry[idx].gpu_virt_addr = cpu_to_le64(va); + p->entry[idx].size = cpu_to_le64(boot_stage.rmcall_gr_promote_size); + p->entry[idx].phys_attr = cpu_to_le32(GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR); + p->entry[idx].buffer_id = cpu_to_le16( + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN); + p->entry[idx].b_initialize = initialize; + p->entry[idx].b_nonmapped = nonmapped; +} + static void gsp_phys_rmcall_gr_promote_entry( struct gsp_phys_rmcall_promote_params *p, u64 va, u8 initialize, u8 nonmapped) { - p->entry[0].gpu_phys_addr = cpu_to_le64(boot_stage.rmcall_ctx_pa); - p->entry[0].gpu_virt_addr = cpu_to_le64(va); - p->entry[0].size = cpu_to_le64(boot_stage.rmcall_gr_promote_size); - p->entry[0].phys_attr = cpu_to_le32(GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR); - p->entry[0].buffer_id = cpu_to_le16( - GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN); - p->entry[0].b_initialize = initialize; - p->entry[0].b_nonmapped = nonmapped; + gsp_phys_rmcall_gr_promote_entry_idx(p, 0, va, initialize, nonmapped); +} + +/* + * Reference-client zero-field entry: only bufferId, gpuVirtAddr and + * bNonmapped are set; PA/size/physAttr stay zero (the client zeroes the + * whole params struct first and the VA-only prepare path writes only those + * three fields). + */ +static void gsp_phys_rmcall_gr_promote_entry_zero_idx( + struct gsp_phys_rmcall_promote_params *p, unsigned int idx, u64 va, + u8 initialize, u8 nonmapped) +{ + struct gsp_phys_rmcall_promote_entry *e = &p->entry[idx]; + + memset(e, 0, sizeof(*e)); + e->gpu_virt_addr = cpu_to_le64(va); + e->buffer_id = cpu_to_le16(GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN); + e->b_initialize = initialize; + e->b_nonmapped = nonmapped; +} + +/* + * Mode 6 setup: reserve the context VA window on the ladder VAS with a + * class-0x70 NV01_MEMORY_VIRTUAL object so the no-entry PROMOTE_CTX + * hVirtMemory branch can ask the server to create the memdesc/vaList entry + * for the range itself (ctrl2080gpu.h:934-947). One allocation per staging; + * a failure aborts mode 6 before any promote, with no retry. + */ +static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0) +{ + struct gsp_phys_rmcall_mem_virtual_params v; + u32 rpc_result = 0, status = 0; + u64 limit; + int ret; + + if (boot_stage.rmcall_gr_hvirt) + return 0; + + limit = boot_stage.rmcall_ctx_va + (u64)boot_stage.rmcall_ctx_size; + memset(&v, 0, sizeof(v)); + v.offset = cpu_to_le64(boot_stage.rmcall_ctx_va); + v.limit = cpu_to_le64(limit); + v.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_HANDLE_MEM_VIRT, + GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL, + &v, sizeof(v), false, NULL, + "NV01_MEMORY_VIRTUAL", true, + &rpc_result, &status); + if (ret) { + pr_err("vgsp-boot: rm-call: mode 6 aborted: NV01_MEMORY_VIRTUAL alloc on hVas=0x%x offset=0x%llx limit=0x%llx failed (rpc_result=0x%x status=0x%x); no PROMOTE_CTX is sent and the ladder aborts with no retry\n", + boot_stage.rmcall_vas, boot_stage.rmcall_ctx_va, limit, + rpc_result, status); + return ret; + } + boot_stage.rmcall_gr_hvirt = GSP_PHYS_RMCALL_HANDLE_MEM_VIRT; + return 0; } /* @@ -7051,22 +7591,55 @@ static void gsp_phys_rmcall_gr_promote_entry( * - mode 2: reference external-VAS shape, a PA-only nonmapped control * (bInitialize=1/bNonmapped=1, no VA) followed - only on success - by a * VA-only control (bInitialize=0/bNonmapped=0, same PA/VA/size); - * - mode 3: like 1 but the entry size is the exact raw save size. + * - mode 3: like 1 but the entry size is the exact raw save size; + * - mode 4: like 2 but step 2 is the reference client's zero-field VA-only + * entry (only bufferId/gpuVirtAddr/bNonmapped set; PA/size/physAttr 0); + * - mode 5: one control, entryCount=2 - entry 0 the PA-only nonmapped + * initialize entry, entry 1 the zero-field VA-only entry; + * - mode 6: no promoteEntry at all; first allocate an NV01_MEMORY_VIRTUAL + * (class 0x70) VA-range object on the ladder VAS at the context window, + * then one PROMOTE_CTX with hVirtMemory = that handle, virtAddress/size + * set and entryCount=0, so the server creates the memdesc/vaList entry + * itself; + * - mode 7: no promoteEntry and no hVirtMemory; one PROMOTE_CTX with the + * bare (virtAddress, size) pair and entryCount=0 (the documented other + * no-entry branch); + * - mode 8: mode 1's single MAIN entry (PA + external-VAS VA + size, + * bInitialize=1, bNonmapped=0) on the V6 KMD-managed channel + * (SKIP_CTXBUFFER_ALLOC): the server routes the single MAIN entry to + * its client-context-buffer path, initializes the context into the + * physical buffer, creates the virtual memdesc itself and commits it, + * with no VA/bookkeeping resolve; + * - mode 9: the stock 570 client's KMD-managed shape - the same V6 + * channel, but the single MAIN entry is the zero-field VA-only entry + * (bInitialize=0, bNonmapped=0, PA/size/physAttr zero) the client + * sends when `kchannelIsCtxBufferAllocSkipped` makes it skip the + * initialize entry; + * - mode 10: the old-interface client-managed flow on the same V6 + * channel - GPU_INITIALIZE_CTX (0x2080012d) with the physical context + * buffer first (installs the PA/size/physAttr memdesc the promote's + * client-memdesc path requires), then PROMOTE_CTX with entryCount=0 + * and the bare (virtAddress, size) pair; + * - mode 11: mode 10 with the context buffer backing swapped to a + * server-owned FBMEM allocation (VIDMEM physAttr 0x4, local-aperture + * leaf PTEs) instead of host sysmem. * Mode 0 never reaches this function (stage 3 stops before it); the early * return is a belt-and-braces no-op. Success means the server installed the * context memdesc, added the VA to the engine-context vaList, committed the * context (ENG_CTX_VALID=1) and ran the GR golden-image bind; only then may * stage 4 BIND/submit run. * - * OBSERVED WEDGE: mode 1 asserted GSP-RM on submission (NOCAT recType=5 - * errorCode=0x52a8b04 source="ASSERT", then no fn76 reply within the 2000 ms - * bound; no 0x1006, no engine fault) and WPR2 could not be cleared afterwards - * (fuse-locked PLM -> reboot with human approval). The first promote control - * of any staging is therefore warned once below; on failure the function - * aborts with no retry, reports exactly which step was last accepted and - * leaves the context buffer allocated until staging free/rmmod. The control - * targets the subdevice, like the reference client promote - * (kernel_graphics_object.c:134-139; kernel_falcon.c:271-272). + * OBSERVED WEDGE: modes 1, 2 and 4 asserted GSP-RM on submission (NOCAT + * recType=5 errorCode=0x52a8b04 source="ASSERT", then no fn76 reply within + * the 2000 ms bound; no 0x1006, no engine fault) and WPR2 could not be + * cleared afterwards (fuse-locked PLM -> reboot with human approval); the + * PA-only nonmapped step of modes 2/4 is accepted every time, and the + * LOGINIT ring shows NV_ERR_OBJECT_NOT_FOUND (0x57) on the VA resolve. The + * first promote control of any staging is therefore warned once below; on + * failure the function aborts with no retry, reports exactly which step was + * last accepted and leaves the context buffer allocated until staging + * free/rmmod. The control targets the subdevice, like the reference client + * promote (kernel_graphics_object.c:134-139; kernel_falcon.c:271-272). */ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) { @@ -7081,9 +7654,48 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) return -EINVAL; if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF) return -EOPNOTSUPP; + /* + * Modes 8/9/10 must run on the V6 channel: without the + * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping + * route that asserted GSP-RM in modes 1/2/4 (or rejects the + * hVirtMemory/bare-VA memdesc with INVALID_ARGUMENT). The variant is + * selected at channel alloc and recorded here; a mismatch aborts + * before any PROMOTE_CTX instead of risking a wedge. + */ + if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) && + boot_stage.rmcall_channel_variant != + gsp_phys_rmcall_channel_variant_skipctx.id) { + pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", + mode, gsp_phys_rmcall_channel_variant_skipctx.id, + gsp_phys_rmcall_channel_variant_skipctx.name, + boot_stage.rmcall_channel_variant); + return -EINVAL; + } + /* + * Modes 12/13 must run on the V5 channel: the mode-2 step-1 PA-only + * entry was accepted there, and the V6 skip-ctx channel would take the + * KMD client-context-buffer route instead (which stayed at 0x40 in + * modes 8-11). + */ + if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) && + boot_stage.rmcall_channel_variant != + gsp_phys_rmcall_channel_variants[0].id) { + pr_err("vgsp-boot: rm-call: mode %u requires the V5 channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", + mode, gsp_phys_rmcall_channel_variants[0].id, + gsp_phys_rmcall_channel_variants[0].name, + boot_stage.rmcall_channel_variant); + return -EINVAL; + } if (!boot_stage.rmcall_gr_promote_warned) { boot_stage.rmcall_gr_promote_warned = true; - pr_warn("gsp-phys-boot: rm-call: WARNING: gr_promote_param=%u sends GPU_PROMOTE_CTX: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n", + pr_warn("vgsp-boot: rm-call: WARNING: gr_promote_param=%u sends GPU_PROMOTE_CTX: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n", mode); } @@ -7101,7 +7713,40 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) p->size = cpu_to_le64(0); p->entry_count = cpu_to_le32(1); - if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE) { + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_BATCHED) { + /* + * Mode 5: one control, entryCount=2 - entry 0 the PA-only + * nonmapped initialize entry, entry 1 the reference client's + * zero-field VA-only promote entry. This mirrors the client + * batching both entries into one params struct when a MAIN + * buffer is both initialized and promoted in the same call. + */ + p->entry_count = cpu_to_le32(2); + gsp_phys_rmcall_gr_promote_entry_idx(p, 0, 0, 1, 1); + gsp_phys_rmcall_gr_promote_entry_zero_idx(p, 1, + boot_stage.rmcall_ctx_va, 0, 0); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (batched init+VA)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX batched: entryCount=2 entry0{pa=0x%llx va=0 size=0x%x physAttr=0x%x bInitialize=1 bNonmapped=1} entry1{pa=0 va=0x%llx size=0 physAttr=0 bInitialize=0 bNonmapped=0} status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x bufferId=MAIN gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_pa, size, + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, + boot_stage.rmcall_ctx_va, status, rpc_result, + boot_stage.rmcall_channel, boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX batched failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 2; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO) { /* Step 1/2: PA-only nonmapped entry (no VA). */ gsp_phys_rmcall_gr_promote_entry(p, 0, 1, 1); ret = gsp_phys_rmcall_control(bar0, @@ -7110,12 +7755,12 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, "GPU_PROMOTE_CTX (PA-only)"); - pr_info("gsp-phys-boot: rm-call: PROMOTE_CTX PA-only (step 1/2): pa=0x%llx va=0 size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n", + pr_info("vgsp-boot: rm-call: PROMOTE_CTX PA-only (step 1/2): pa=0x%llx va=0 size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n", boot_stage.rmcall_ctx_pa, size, status, rpc_result, boot_stage.rmcall_channel, boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { - pr_err("gsp-phys-boot: rm-call: PROMOTE_CTX PA-only (step 1/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/2); no further controls are sent and the ladder aborts before BIND with no retry\n", + pr_err("vgsp-boot: rm-call: PROMOTE_CTX PA-only (step 1/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/2); no further controls are sent and the ladder aborts before BIND with no retry\n", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); kvfree(p); @@ -7123,9 +7768,18 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } boot_stage.rmcall_gr_promote_step = 1; - /* Step 2/2: VA-only entry (same PA/VA/size, initialize off). */ - gsp_phys_rmcall_gr_promote_entry(p, boot_stage.rmcall_ctx_va, - 0, 0); + /* + * Step 2/2: VA-only entry. Mode 2 repeats the full PA/VA/ + * size; mode 4 sends the reference client's zero-field shape + * (only bufferId/gpuVirtAddr/bNonmapped set) to isolate + * whether the VA path objects to the repeated PA/size. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO) + gsp_phys_rmcall_gr_promote_entry_zero_idx(p, 0, + boot_stage.rmcall_ctx_va, 0, 0); + else + gsp_phys_rmcall_gr_promote_entry(p, + boot_stage.rmcall_ctx_va, 0, 0); rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, @@ -7134,24 +7788,201 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, "GPU_PROMOTE_CTX (VA-only)"); - pr_info("gsp-phys-boot: rm-call: PROMOTE_CTX VA-only (step 2/2): pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=0 bNonmapped=0 gr_promote_param=%u)\n", - boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, - size, status, rpc_result, boot_stage.rmcall_channel, + pr_info("vgsp-boot: rm-call: PROMOTE_CTX VA-only (step 2/2): pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=0 bNonmapped=0 gr_promote_param=%u)\n", + mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO ? 0 : + boot_stage.rmcall_ctx_pa, + boot_stage.rmcall_ctx_va, + mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO ? 0 : size, + status, rpc_result, boot_stage.rmcall_channel, boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { - pr_err("gsp-phys-boot: rm-call: PROMOTE_CTX VA-only (step 2/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: PA-only nonmapped accepted (1/2); no further controls are sent and the ladder aborts before BIND with no retry\n", + pr_err("vgsp-boot: rm-call: PROMOTE_CTX VA-only (step 2/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: PA-only nonmapped accepted (1/2); no further controls are sent and the ladder aborts before BIND with no retry\n", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); kvfree(p); return ret; } boot_stage.rmcall_gr_promote_step = 2; - } else { + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA) { /* - * Modes 1 and 3: one MAIN entry carrying both the PA + * Modes 6/7: the documented no-entry branches. Mode 6 first + * reserves the context window with an NV01_MEMORY_VIRTUAL + * object on the ladder VAS and passes it as hVirtMemory + * (hClient is its owner); mode 7 sends the bare + * (virtAddress, size) pair with no hVirtMemory. entryCount=0 + * in both, so the server creates the memdesc/vaList entry + * instead of resolving one of our raw-PTE entries. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT) { + ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0); + if (ret) { + kvfree(p); + return ret; + } + p->h_client = cpu_to_le32(boot_stage.rmcall_client); + p->h_virt_memory = + cpu_to_le32(boot_stage.rmcall_gr_hvirt); + } + p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va); + p->size = cpu_to_le64(boot_stage.rmcall_ctx_size); + p->entry_count = cpu_to_le32(0); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + "GPU_PROMOTE_CTX (hVirtMemory)" : + "GPU_PROMOTE_CTX (bare VA+size)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hClient=0x%x hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", + mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + "hVirtMemory" : "bare VA+size", + le32_to_cpu(p->h_virt_memory), + boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size, + status, rpc_result, le32_to_cpu(p->h_client), + boot_stage.rmcall_channel, boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the GR context is not committed and the ladder aborts before BIND with no retry\n", + mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + "hVirtMemory" : "bare VA+size", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 2; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA) { + /* + * Mode 9: the stock 570 client's KMD-managed channel shape. + * `kgrctxPrepareInitializeCtxBuffer` returns without filling + * the MAIN entry when `kchannelIsCtxBufferAllocSkipped`, then + * `kgrctxPreparePromoteCtxBuffer` fills only the VA fields, so + * the client sends entryCount=1, bufferId=MAIN, bInitialize=0, + * bNonmapped=0, gpuVirtAddr set and PA/size/physAttr zero. On + * the V6 channel the server routes this to its + * client-context-buffer path and binds without the PA + * initialization or the VA/bookkeeping resolve. + */ + gsp_phys_rmcall_gr_promote_entry_zero_idx(p, 0, + boot_stage.rmcall_ctx_va, 0, 0); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (KMD VA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX KMD VA-only: pa=0 va=0x%llx size=0 status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=0 bNonmapped=0 gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_va, status, rpc_result, + boot_stage.rmcall_channel, boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX KMD VA-only failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB) { + /* + * Modes 10/11: the old-interface client-managed flow. Step 1 + * sends GPU_INITIALIZE_CTX (0x2080012d) with the physical + * context buffer, which installs the PA/size/physAttr + * memdesc on the GraphicsContext (grctxInitializeAPI) that + * the following promote's client-memdesc path requires. + * Mode 11's buffer is the server FBMEM allocation with the + * VIDMEM physAttr (0x4); mode 10's is the host sysmem chunk + * with NCOH_SYS (0x6). Step 2 sends PROMOTE_CTX with + * entryCount=0 and the bare (virtAddress, size) pair; the + * server builds the virtual memdesc itself and, on the V6 + * KMD-managed channel, binds it without the VA/bookkeeping + * resolve. + */ + struct gsp_phys_rmcall_init_ctx_params *ic; + u32 ic_phys_attr = (mode == + GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB) ? + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR; + u32 ic_rpc = 0, ic_status = 0; + + ic = kvzalloc(sizeof(*ic), GFP_KERNEL); + if (!ic) { + kvfree(p); + return -ENOMEM; + } + ic->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0); + ic->h_client = cpu_to_le32(0); + ic->ch_id = cpu_to_le32(0); + ic->h_chan_client = cpu_to_le32(boot_stage.rmcall_client); + ic->h_object = cpu_to_le32(boot_stage.rmcall_channel); + ic->h_virt_memory = cpu_to_le32(0); + ic->phys_address = cpu_to_le64(boot_stage.rmcall_ctx_pa); + ic->phys_attr = cpu_to_le32(ic_phys_attr); + ic->h_dma_handle = cpu_to_le32(0); + ic->index = cpu_to_le32(0); + ic->size = cpu_to_le64(boot_stage.rmcall_ctx_size); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX, + ic, sizeof(*ic), 24, NULL, 0, + &ic_rpc, &ic_status, + "GPU_INITIALIZE_CTX (old interface)"); + kvfree(ic); + pr_info("vgsp-boot: rm-call: INITIALIZE_CTX: pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x hVirtMemory=0 gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size, + ic_phys_attr, ic_status, ic_rpc, + boot_stage.rmcall_channel, boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: INITIALIZE_CTX failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no PROMOTE_CTX is sent and the ladder aborts with no retry\n", + ic_status, ic_rpc, + gsp_phys_rmcall_status_str(ic_status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + + /* Step 2: PROMOTE_CTX entryCount=0, bare (virtAddress, size). */ + p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va); + p->size = cpu_to_le64(boot_stage.rmcall_ctx_size); + p->entry_count = cpu_to_le32(0); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (KMD bare VA+size after init)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size after init: entryCount=0 virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size, + status, rpc_result, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 2; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE) { + /* + * Modes 1, 3 and 8: one MAIN entry carrying both the PA * (bInitialize=1) and the external-VAS VA (bNonmapped=0); - * mode 3 carries the exact raw size. + * mode 3 carries the exact raw size. Mode 8 runs on the V6 + * KMD-managed channel, so the server takes its + * client-context-buffer path: it initializes the context into + * our physical buffer, creates the virtual context memdesc + * itself and commits it, without resolving the VA against a + * server-side booking (the mode-1/2/4 assert). */ gsp_phys_rmcall_gr_promote_entry(p, boot_stage.rmcall_ctx_va, 1, 0); @@ -7160,19 +7991,257 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, "GPU_PROMOTE_CTX MAIN"); - pr_info("gsp-phys-boot: rm-call: PROMOTE_CTX MAIN: pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=0 gr_promote_param=%u)\n", + pr_info("vgsp-boot: rm-call: PROMOTE_CTX MAIN: pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=0 gr_promote_param=%u)\n", boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, size, status, rpc_result, boot_stage.rmcall_channel, boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { - pr_err("gsp-phys-boot: rm-call: PROMOTE_CTX MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry (read the failed-reply dump above: 0x23 invalid client, 0x36 invalid parent, 0x56 not supported)\n", + pr_err("vgsp-boot: rm-call: PROMOTE_CTX MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry (read the failed-reply dump above: 0x23 invalid client, 0x36 invalid parent, 0x56 not supported)\n", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); kvfree(p); return ret; } boot_stage.rmcall_gr_promote_step = 1; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) { + /* + * Modes 12/13: the UVM/vGPU-faithful PA-only flow. On an + * externally-owned VAS the real client never promotes context + * VAs; it sends bInitialize=1 bNonmapped=1 entries so the + * server installs the memdescs and commits the engine context, + * while the page tables stay externally owned. The mode-2 + * step-1 MAIN entry in exactly this shape was ACCEPTED on the + * V5 channel (status=0x0); only the follow-up VA step wedged. + * Mode 13 first promotes the PATCH buffer the real client + * always allocates (PA-only, same shape), because its absence + * is the leading candidate for the 0x40 the KMD modes return. + * No VA promote is ever sent; on acceptance the ladder + * continues to BIND/SCHEDULE/submit. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) { + struct gsp_phys_rmcall_promote_entry *e = &p->entry[0]; + u32 patch_rpc = 0, patch_status = 0; + + ret = gsp_phys_rmcall_gr_patch_prepare(bar0); + if (ret) { + kvfree(p); + return ret; + } + memset(e, 0, sizeof(*e)); + e->gpu_phys_addr = + cpu_to_le64(boot_stage.rmcall_patch_pa); + e->size = cpu_to_le64(boot_stage.rmcall_patch_size); + e->phys_attr = cpu_to_le32( + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR); + e->buffer_id = cpu_to_le16( + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH); + e->b_initialize = 1; + e->b_nonmapped = 1; + p->entry_count = cpu_to_le32(1); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &patch_rpc, &patch_status, + "GPU_PROMOTE_CTX (PATCH PA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only: pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=PATCH bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n", + boot_stage.rmcall_patch_pa, + boot_stage.rmcall_patch_size, + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, patch_status, + patch_rpc, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no MAIN promote is sent and the ladder aborts with no retry\n", + patch_status, patch_rpc, + gsp_phys_rmcall_status_str(patch_status), + ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + } + + /* MAIN PA-only nonmapped entry: exactly mode 2 step 1. */ + gsp_phys_rmcall_gr_promote_entry(p, 0, 1, 1); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (MAIN PA-only)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX MAIN PA-only: pa=0x%llx va=0 size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=1 gr_promote_param=%u); no VA promote is sent, continuing to BIND/SCHEDULE/submit\n", + boot_stage.rmcall_ctx_pa, size, status, rpc_result, + boot_stage.rmcall_channel, boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX MAIN PA-only failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: %s; the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret, + mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH ? + "PATCH PA-only accepted (1/2)" : "none (0/1)"); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) ? + 2 : 1; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) { + /* + * Modes 14/15/16: the KMD client-context-buffer path (V6) with + * the auxiliary buffers promoted first. The single MAIN entry + * carries both PA (bInitialize=1) and the external-VAS VA + * (bNonmapped=0): the server builds its own virtual memdesc + * via kchannelSetEngineContextMemDesc, which books the VA in + * the engine-context vaList the commit needs (the V5 PA-only + * modes stayed CTXNOTVALID because no VA was ever booked). + * PATCH is the real client's always-present buffer; mode 16 + * also promotes the PM ctxsw buffer, which the client + * pre-allocates for externally-owned VAS and the promote-time + * preemption/commit HALs consume. Mode 15 uses the server + * FBMEM context buffer with the VIDMEM physAttr 0x4 (a vGPU + * host context buffer type). + */ + struct gsp_phys_rmcall_promote_entry *e = &p->entry[0]; + u32 aux_rpc = 0, aux_status = 0; + + ret = gsp_phys_rmcall_gr_patch_prepare(bar0); + if (ret) { + kvfree(p); + return ret; + } + memset(e, 0, sizeof(*e)); + e->gpu_phys_addr = cpu_to_le64(boot_stage.rmcall_patch_pa); + e->size = cpu_to_le64(boot_stage.rmcall_patch_size); + e->phys_attr = cpu_to_le32( + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR); + e->buffer_id = cpu_to_le16( + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH); + e->b_initialize = 1; + e->b_nonmapped = 1; + p->entry_count = cpu_to_le32(1); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &aux_rpc, &aux_status, + "GPU_PROMOTE_CTX (PATCH PA-only, KMD)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only (KMD): pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=PATCH bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n", + boot_stage.rmcall_patch_pa, + boot_stage.rmcall_patch_size, + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, aux_status, + aux_rpc, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only (KMD) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no MAIN promote is sent and the ladder aborts with no retry\n", + aux_status, aux_rpc, + gsp_phys_rmcall_status_str(aux_status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) { + ret = gsp_phys_rmcall_gr_pm_prepare(bar0); + if (ret) { + kvfree(p); + return ret; + } + memset(e, 0, sizeof(*e)); + e->gpu_phys_addr = + cpu_to_le64(boot_stage.rmcall_pm_pa); + e->size = cpu_to_le64(boot_stage.rmcall_pm_size); + e->phys_attr = cpu_to_le32( + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR); + e->buffer_id = cpu_to_le16( + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PM); + e->b_initialize = 1; + e->b_nonmapped = 1; + p->entry_count = cpu_to_le32(1); + aux_rpc = 0; + aux_status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &aux_rpc, &aux_status, + "GPU_PROMOTE_CTX (PM PA-only, KMD)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX PM PA-only (KMD): pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=PM bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n", + boot_stage.rmcall_pm_pa, + boot_stage.rmcall_pm_size, + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, aux_status, + aux_rpc, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX PM PA-only (KMD) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no MAIN promote is sent and the ladder aborts with no retry\n", + aux_status, aux_rpc, + gsp_phys_rmcall_status_str(aux_status), + ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 2; + } + + /* MAIN single entry: the mode-8 shape (PA + VA + init). */ + gsp_phys_rmcall_gr_promote_entry(p, + boot_stage.rmcall_ctx_va, 1, 0); + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB) + p->entry[0].phys_attr = cpu_to_le32( + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (KMD MAIN init+VA)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX KMD MAIN: pa=0x%llx va=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=0 gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + size, + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ? + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, + status, rpc_result, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX KMD MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: %s; the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret, + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM ? + "PATCH+PM PA-only accepted (2/3)" : + "PATCH PA-only accepted (1/2)"); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) ? 3 : 2; + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_SERVER) { + /* + * Mode 17: server-managed ctx buffers. No client PROMOTE_CTX is + * sent; the GSP-RM allocated and committed the engine context + * itself (RMSetClientRMAllocatedCtxBuffer=0 in the pre-boot + * registry profile, gsp_phys_srv_ctxbuf=1). The ladder proceeds + * straight to BIND/SCHEDULE/submit. If the committed context is + * valid this is where REAL GR SEM OK should appear. + */ + pr_info("vgsp-boot: rm-call: gr_promote_param=17: no client PROMOTE_CTX is sent; the context buffers are server-managed (gsp_phys_srv_ctxbuf registry profile) and the server commits the engine context itself; continuing to BIND/SCHEDULE/submit\n"); + boot_stage.rmcall_gr_promote_step = 0; + } else { + pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", + mode); + kvfree(p); + return -EINVAL; } kvfree(p); @@ -7196,7 +8265,7 @@ static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0) p = kvzalloc(sizeof(*p), GFP_KERNEL); if (!p) { - pr_warn("gsp-phys-boot: rm-call: GR ctx info skipped: no memory for the %zu-byte best-effort query\n", + pr_warn("vgsp-boot: rm-call: GR ctx info skipped: no memory for the %zu-byte best-effort query\n", sizeof(*p)); return; } @@ -7208,7 +8277,7 @@ static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0) p, sizeof(*p), &rpc_result, &status, "GR_GET_CTX_BUFFER_INFO"); if (ret) { - pr_warn("gsp-phys-boot: rm-call: GR ctx info (best effort) failed: status=0x%x rpc_result=0x%x ret=%d; the promote result stands\n", + pr_warn("vgsp-boot: rm-call: GR ctx info (best effort) failed: status=0x%x rpc_result=0x%x ret=%d; the promote result stands\n", status, rpc_result, ret); kvfree(p); return; @@ -7216,13 +8285,13 @@ static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0) count = le32_to_cpu(p->buffer_count); shown = min_t(u32, count, 4u); - pr_info("gsp-phys-boot: rm-call: GR_GET_CTX_BUFFER_INFO: bufferCount=%u (showing %u)\n", + pr_info("vgsp-boot: rm-call: GR_GET_CTX_BUFFER_INFO: bufferCount=%u (showing %u)\n", count, shown); for (i = 0; i < shown; i++) { const struct gsp_phys_rmcall_ctx_buffer_info *ci = &p->ctx_buffer_info[i]; - pr_info("gsp-phys-boot: rm-call: GR ctx[%u]: bufferType=%u physAddr=0x%llx aperture=%u size=0x%llx pageCount=0x%llx alignment=0x%llx contiguous=%u\n", + pr_info("vgsp-boot: rm-call: GR ctx[%u]: bufferType=%u physAddr=0x%llx aperture=%u size=0x%llx pageCount=0x%llx alignment=0x%llx contiguous=%u\n", i, le32_to_cpu(ci->buffer_type), le64_to_cpu(ci->phys_addr), le32_to_cpu(ci->aperture), le64_to_cpu(ci->size), @@ -7295,7 +8364,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) "FERMI_VASPACE_A", true, &vas_rpc, &vas_status); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 3 aborted: FERMI_VASPACE_A alloc failed (rpc_result=0x%x status=0x%x); no VAS for the channel\n", + pr_err("vgsp-boot: rm-call: stage 3 aborted: FERMI_VASPACE_A alloc failed (rpc_result=0x%x status=0x%x); no VAS for the channel\n", vas_rpc, vas_status); return ret; } @@ -7316,7 +8385,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) &pdb, sizeof(pdb), 16, NULL, 0, &pdb_rpc, NULL, "DMA_SET_PAGE_DIRECTORY"); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n", + pr_err("vgsp-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n", boot_stage.rmcall_vas, le32_to_cpu(pdb.num_entries), pdb_rpc); return ret; @@ -7332,7 +8401,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) if (gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_gr_ctx_prepare(bar0); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 3 aborted: GR context buffer prepare failed (%d); the GR context is not committed and the ladder aborts with no retry\n", + pr_err("vgsp-boot: rm-call: stage 3 aborted: GR context buffer prepare failed (%d); the GR context is not committed and the ladder aborts with no retry\n", ret); return ret; } @@ -7377,7 +8446,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) */ ret = gsp_phys_rmcall_alloc_notifier_mem(bar0); if (ret) - pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n", + pr_warn("vgsp-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n", ret); ret = gsp_phys_rmcall_alloc_channel(bar0); @@ -7396,7 +8465,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) if (gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_alloc_gr_object(bar0); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 3 aborted: TURING_COMPUTE_A (0xc5c0) engine object alloc failed (%d); the GR ladder has no engine object to bind and is not retried\n", + pr_err("vgsp-boot: rm-call: stage 3 aborted: TURING_COMPUTE_A (0xc5c0) engine object alloc failed (%d); the GR ladder has no engine object to bind and is not retried\n", ret); return ret; } @@ -7410,7 +8479,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) */ if (boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_OFF) { - pr_info("gsp-phys-boot: rm-call: gr_promote_param=0: GPU_PROMOTE_CTX not sent; GR ctx prepared (pa=0x%llx va=0x%llx size=0x%x) and PDB-mapped, but not committed; the ladder stops before BIND/SCHEDULE/submit (safe half complete; set gsp_phys_rmcall_gr_promote=1/2/3 and reload to attempt the promote)\n", + pr_info("vgsp-boot: rm-call: gr_promote_param=0: GPU_PROMOTE_CTX not sent; GR ctx prepared (pa=0x%llx va=0x%llx size=0x%x) and PDB-mapped, but not committed; the ladder stops before BIND/SCHEDULE/submit (safe half complete; set gsp_phys_rmcall_gr_promote=1/2/3 and reload to attempt the promote)\n", boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size); @@ -7418,7 +8487,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) } ret = gsp_phys_rmcall_gr_ctx_promote(bar0); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 3 aborted: GPU_PROMOTE_CTX failed (%d); the GR context is not committed and BIND/SCHEDULE/submit are not attempted (no retry; UNLOAD + rmmod remains the fail-safe)\n", + pr_err("vgsp-boot: rm-call: stage 3 aborted: GPU_PROMOTE_CTX failed (%d); the GR context is not committed and BIND/SCHEDULE/submit are not attempted (no retry; UNLOAD + rmmod remains the fail-safe)\n", ret); return ret; } @@ -7433,7 +8502,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_promote_size); else promote_suffix[0] = '\0'; - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n", + pr_info("vgsp-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", @@ -7641,7 +8710,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { - pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } @@ -7662,7 +8731,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, */ e0 = (u32)pb_gpu_va & ~3u; if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) - pr_err("gsp-phys-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", + pr_err("vgsp-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", e0, (u64)(e0 & ~3u), pb_gpu_va); e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); @@ -7686,7 +8755,7 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, ch_id = boot_stage.rmcall_token & 0xfffu; doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); if (doorbell != boot_stage.rmcall_token) - pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + pr_warn("vgsp-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", boot_stage.rmcall_token, doorbell); /* * C3: snapshot the fault epoch immediately before the doorbell, so a @@ -7746,13 +8815,13 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) GSP_PHYS_RMCALL_SEM_PAYLOAD, true, &sub); if (ret) { - pr_err("gsp-phys-boot: rm-call: GR submission refused before the doorbell (%d)\n", + pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", ret); return ret; } userd = gsp_phys_chunk_page_va(userd_c, 0); - pr_info("gsp-phys-boot: rm-call: GR submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSem=0x%llx vaSrc=0x%llx vaDst=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + pr_info("vgsp-boot: rm-call: GR submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSem=0x%llx vaSrc=0x%llx vaDst=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, sub.words, sub.slot, sub.gpget, @@ -7771,7 +8840,7 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) GSP_PHYS_RMCALL_USERD_GPGET_OFF); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); - pr_err("gsp-phys-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n", + pr_err("vgsp-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput, gsp_phys_rmcall_engine_name()); @@ -7782,7 +8851,7 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) } gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - pr_info("gsp-phys-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n", + pr_info("vgsp-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); @@ -7957,7 +9026,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, mutex_unlock(&gsp_phys_rm_lock); gsp_phys_work_refused++; if ((gsp_phys_work_refused & 63) == 1) - pr_info("gsp-phys-work: shadow submit refused before the doorbell (backend changed) refused=%llu\n", + pr_info("vgsp-work: shadow submit refused before the doorbell (backend changed) refused=%llu\n", gsp_phys_work_refused); return -EAGAIN; } @@ -7969,7 +9038,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, if (ret) { gsp_phys_work_refused++; if ((gsp_phys_work_refused & 63) == 1) - pr_info("gsp-phys-work: shadow submit refused (ret %d) refused=%llu\n", + pr_info("vgsp-work: shadow submit refused (ret %d) refused=%llu\n", ret, gsp_phys_work_refused); return ret; } @@ -7987,7 +9056,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, if (time_after_eq(jiffies, deadline)) { gsp_phys_work_timed_out++; if ((gsp_phys_work_timed_out & 63) == 1) - pr_info("gsp-phys-work: shadow copy timeout after %u ms (sem=0x%08x want=0x%08x) timeouts=%llu\n", + pr_info("vgsp-work: shadow copy timeout after %u ms (sem=0x%08x want=0x%08x) timeouts=%llu\n", poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gsp_phys_work_timed_out); @@ -8007,7 +9076,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, memcmp(gsp_phys_chunk_page_va(src, 0), gsp_phys_chunk_page_va(dst, 0), bytes) != 0) { gsp_phys_work_refused++; - pr_info("gsp-phys-work: semaphore observed but the destination does not match the source after %u bytes (refused=%llu)\n", + pr_info("vgsp-work: semaphore observed but the destination does not match the source after %u bytes (refused=%llu)\n", bytes, gsp_phys_work_refused); return -EIO; } @@ -8333,7 +9402,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, (u64)src_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE || (u64)dst_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE) { gsp_phys_guest_refused++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n", + pr_err("vgsp-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n", src_hpa, src_off, dst_hpa, dst_off, bytes, GSP_PHYS_RMCALL_COPY_SIZE); ret = -EINVAL; @@ -8343,7 +9412,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, (!IS_ALIGNED(sem_hpa, GSP_PHYS_PAGE_SIZE) || (u64)sem_off + 4 > bytes || !sem_payload)) { gsp_phys_relay_refused++; - pr_err("gsp-phys-work: GUEST CE RELAY FAILED: invalid semaphore relay sem_hpa=0x%llx sem_off=%u sem_payload=0x%08x bytes=%u (want a page-aligned HPA, sem_off+4 <= copy size, nonzero payload)\n", + pr_err("vgsp-work: GUEST CE RELAY FAILED: invalid semaphore relay sem_hpa=0x%llx sem_off=%u sem_payload=0x%08x bytes=%u (want a page-aligned HPA, sem_off+4 <= copy size, nonzero payload)\n", sem_hpa, sem_off, sem_payload, bytes); ret = -EINVAL; goto out; @@ -8353,7 +9422,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if (!pfn_valid(pfn)) { gsp_phys_relay_refused++; - pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore HPA 0x%llx is not a valid RAM page\n", + pr_err("vgsp-work: GUEST CE RELAY FAILED: semaphore HPA 0x%llx is not a valid RAM page\n", sem_hpa); ret = -EINVAL; goto out; @@ -8413,14 +9482,14 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if (relay) { gsp_phys_relay_pool_exhausted++; if ((gsp_phys_relay_pool_exhausted & 63) == 1) - pr_err("gsp-phys-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", + pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", gsp_phys_relay_pool_exhausted, GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } else { gsp_phys_guest_pool_exhausted++; if ((gsp_phys_guest_pool_exhausted & 63) == 1) - pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", + pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", gsp_phys_guest_pool_exhausted, GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); @@ -8430,7 +9499,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, gsp_phys_relay_refused++; else gsp_phys_guest_refused++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n", + pr_err("vgsp-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n", ret, src_hpa, dst_hpa); } mutex_unlock(&gsp_phys_rm_lock); @@ -8458,7 +9527,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, gsp_phys_relay_refused++; else gsp_phys_guest_refused++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + pr_err("vgsp-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", ret, src_hpa, dst_hpa, bytes); goto out; } @@ -8481,7 +9550,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if (time_after_eq(jiffies, deadline)) { if (relay) { gsp_phys_relay_timed_out++; - pr_err("gsp-phys-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + pr_err("vgsp-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", poll_ms, poll, sem_payload, sem_hpa, sem_off, src_hpa, dst_hpa, bytes); /* @@ -8510,7 +9579,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, } } else { gsp_phys_guest_refused++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", + pr_err("vgsp-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_hpa, dst_hpa, bytes); @@ -8532,7 +9601,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, (u8 *)__va(src_hpa) + src_off, bytes) != 0) { if (!relay) { gsp_phys_guest_verify_fail++; - pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", + pr_err("vgsp-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", poll, dst_hpa, dst_off, src_hpa, src_off, bytes); ret = -EIO; @@ -8578,7 +9647,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, snap_dst_diff = memcmp(dp, snap, bytes) != 0; } - pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes (retried after 2 ms) diff@%u dst=%s src=%s snap=%s src_mutated=%u snap_vs_dst=%u sem_hpa=0x%llx+%u payload=0x%08x poll=0x%08x elapsed_ms=%u verify_fail=%llu verify_transient=%llu\n", + pr_err("vgsp-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes (retried after 2 ms) diff@%u dst=%s src=%s snap=%s src_mutated=%u snap_vs_dst=%u sem_hpa=0x%llx+%u payload=0x%08x poll=0x%08x elapsed_ms=%u verify_fail=%llu verify_transient=%llu\n", poll, dst_hpa, dst_off, src_hpa, src_off, bytes, diff, dbuf, sbuf, snap_str, src_mutated, snap_dst_diff, @@ -8592,7 +9661,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, } gsp_phys_relay_verify_transient++; if ((gsp_phys_relay_verify_transient & 63) == 1) - pr_info("gsp-phys-work: GUEST CE RELAY transient data mismatch resolved on retry src_hpa=0x%llx dst_hpa=0x%llx bytes=%u transient=%llu\n", + pr_info("vgsp-work: GUEST CE RELAY transient data mismatch resolved on retry src_hpa=0x%llx dst_hpa=0x%llx bytes=%u transient=%llu\n", src_hpa, dst_hpa, bytes, gsp_phys_relay_verify_transient); } @@ -8602,7 +9671,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if (relay) { gsp_phys_relay_ok++; if (gsp_phys_relay_ok == 1 || (gsp_phys_relay_ok & 63) == 0) - pr_info("gsp-phys-work: GUEST CE RELAY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem_hpa=0x%llx+%u sem=0x%08x relays=%llu\n", + pr_info("vgsp-work: GUEST CE RELAY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem_hpa=0x%llx+%u sem=0x%08x relays=%llu\n", src_hpa, dst_hpa, bytes, sem_hpa, sem_off, poll, gsp_phys_relay_ok); } else { @@ -8610,7 +9679,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, gsp_phys_guest_fwd_ok++; if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0) - pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", + pr_info("vgsp-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", src_hpa, dst_hpa, bytes, poll, gsp_phys_guest_fwd_ok); } @@ -8722,18 +9791,18 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) * submitted. Distinct error, nothing is scheduled or rung. */ if (boot_stage.rmcall_channel_server_mem) { - pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (V5 tsg-vas-pdb-skip-scrubber, V1 ref-device-skip-scrubber or V4 current-control)\n", + pr_err("vgsp-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (V5 tsg-vas-pdb-skip-scrubber, V1 ref-device-skip-scrubber or V4 current-control)\n", boot_stage.rmcall_channel_variant); return -EOPNOTSUPP; } if (!boot_stage.pdev) { - pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n"); + pr_err("vgsp-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n"); return -EINVAL; } bar0_len = pci_resource_len(boot_stage.pdev, 0); if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) { - pr_err("gsp-phys-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n", + pr_err("vgsp-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n", (u64)bar0_len, GSP_PHYS_RMCALL_DOORBELL_OFF); return -EIO; } @@ -8747,7 +9816,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) NULL, 0, &rpc_result, &status, "BIND_CHANNEL"); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: BIND_CHANNEL (0xa06f0104 engineType=0x%x) failed (rpc_result=0x%x status=0x%x)\n", + pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: BIND_CHANNEL (0xa06f0104 engineType=0x%x) failed (rpc_result=0x%x status=0x%x)\n", bind_params, rpc_result, status); return ret; } @@ -8761,7 +9830,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) NULL, 0, &rpc_result, &status, "GPFIFO_SCHEDULE"); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GPFIFO_SCHEDULE (0xa06f0103 bEnable=1 bSkipSubmit=0) failed (rpc_result=0x%x status=0x%x)\n", + pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: GPFIFO_SCHEDULE (0xa06f0103 bEnable=1 bSkipSubmit=0) failed (rpc_result=0x%x status=0x%x)\n", rpc_result, status); return ret; } @@ -8784,10 +9853,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) NULL, 0, &rpc_result, &status, "TSG_GPFIFO_SCHEDULE"); if (ret) - pr_warn("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n", + pr_warn("vgsp-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n", boot_stage.rmcall_tsg, rpc_result, status); else - pr_info("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n", + pr_info("vgsp-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n", boot_stage.rmcall_tsg, rpc_result, status); /* @@ -8806,10 +9875,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) NULL, 0, &rpc_result, &status, "SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); if (ret) - pr_warn("gsp-phys-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (0xc36f010a index=%u) advisory failure: rpc_result=0x%x status=0x%x (no error-context object: hErrorContext=0); the work-submit token is generated independently of the notifier, continuing to GET_WORK_SUBMIT_TOKEN\n", + pr_warn("vgsp-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (0xc36f010a index=%u) advisory failure: rpc_result=0x%x status=0x%x (no error-context object: hErrorContext=0); the work-submit token is generated independently of the notifier, continuing to GET_WORK_SUBMIT_TOKEN\n", notif_index, rpc_result, status); else - pr_info("gsp-phys-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010a index=%u accepted (rpc_result=0x%x status=0x%x)\n", + pr_info("vgsp-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010a index=%u accepted (rpc_result=0x%x status=0x%x)\n", notif_index, rpc_result, status); /* @@ -8828,17 +9897,17 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) sizeof(token), &rpc_result, &status, "GET_WORK_SUBMIT_TOKEN"); if (ret && token != 0) { - pr_warn("gsp-phys-boot: rm-call: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; the server wrote it before the notifier update, proceeding to submission\n", + pr_warn("vgsp-boot: rm-call: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; the server wrote it before the notifier update, proceeding to submission\n", rpc_result, status, token); ret = 0; } if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN (0xc36f0108 after BIND/SCHEDULE/notif-index=%u) failed (rpc_result=0x%x status=0x%x token=0x%x); refusing to ring the doorbell\n", + pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN (0xc36f0108 after BIND/SCHEDULE/notif-index=%u) failed (rpc_result=0x%x status=0x%x token=0x%x); refusing to ring the doorbell\n", notif_index, rpc_result, status, token); return ret; } if (!token) { - pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN returned a zero token; refusing to ring the doorbell\n"); + pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN returned a zero token; refusing to ring the doorbell\n"); return -EIO; } boot_stage.rmcall_token = token; @@ -8853,7 +9922,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) if (gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); if (ret) - pr_err("gsp-phys-boot: rm-call: stage 4 GR path failed (%d); no retry\n", + pr_err("vgsp-boot: rm-call: stage 4 GR path failed (%d); no retry\n", ret); return ret; } @@ -8874,12 +9943,12 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) GSP_PHYS_RMCALL_SEM_PAYLOAD, false, &sub); if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", + pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", ret); return ret; } - pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + pr_info("vgsp-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, @@ -8898,7 +9967,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) GSP_PHYS_RMCALL_USERD_GPGET_OFF); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); - pr_err("gsp-phys-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n", + pr_err("vgsp-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput); /* @@ -8917,13 +9986,13 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) if (memcmp(gsp_phys_chunk_page_va(src, 0), gsp_phys_chunk_page_va(dst, 0), GSP_PHYS_RMCALL_COPY_SIZE) != 0) { - pr_err("gsp-phys-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n", + pr_err("vgsp-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n", GSP_PHYS_RMCALL_COPY_SIZE); return -EIO; } gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - pr_info("gsp-phys-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", + pr_info("vgsp-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); @@ -8951,7 +10020,7 @@ static int gsp_phys_rmcall_gr_sem_rerun(void __iomem *bar0) !boot_stage.rmcall_channel || !boot_stage.rmcall_gr_object || !boot_stage.rmcall_token || boot_stage.rmcall_stage < 3) { mutex_unlock(&gsp_phys_rm_lock); - pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger 9 no-op: requires the GR ladder through stage 3 (gr=%u claimed=%u init_done=%u unload=%u hChannel=0x%x hGr=0x%x token=0x%x stage=%u)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: trigger 9 no-op: requires the GR ladder through stage 3 (gr=%u claimed=%u init_done=%u unload=%u hChannel=0x%x hGr=0x%x token=0x%x stage=%u)\n", gsp_phys_rmcall_engine_is_gr(), gsp_phys_gpu_claimed(), boot_stage.rm_init_done, boot_stage.rm_unload_submitted, boot_stage.rmcall_channel, boot_stage.rmcall_gr_object, @@ -8979,12 +10048,12 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0) if (!boot_stage.rm_probe_done || !boot_stage.rm_probe_internal_valid) { u32 tx_before = boot_stage.tx_seq_num; - pr_info("gsp-phys-boot: rm-call: stage 5: no persisted static-config handles in this session; running the rm-probe cycle inline (fn1 then fn65)\n"); + pr_info("vgsp-boot: rm-call: stage 5: no persisted static-config handles in this session; running the rm-probe cycle inline (fn1 then fn65)\n"); ret = gsp_phys_rm_probe_cycle(bar0); /* Count the inline probe's submitted RPCs in the ladder total. */ boot_stage.rmcall_rpcs += boot_stage.tx_seq_num - tx_before; if (ret) { - pr_err("gsp-phys-boot: rm-call: stage 5: inline rm-probe failed (%d); cannot adopt internal handles\n", + pr_err("vgsp-boot: rm-call: stage 5: inline rm-probe failed (%d); cannot adopt internal handles\n", ret); return ret; } @@ -8994,7 +10063,7 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0) !boot_stage.rm_probe_h_internal_client || !boot_stage.rm_probe_h_internal_device || !boot_stage.rm_probe_h_internal_subdevice) { - pr_err("gsp-phys-boot: rm-call: stage 5: static config carries no usable internal handles (valid=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x)\n", + pr_err("vgsp-boot: rm-call: stage 5: static config carries no usable internal handles (valid=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x)\n", boot_stage.rm_probe_internal_valid, boot_stage.rm_probe_h_internal_client, boot_stage.rm_probe_h_internal_device, @@ -9005,7 +10074,7 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0) boot_stage.rmcall_client = boot_stage.rm_probe_h_internal_client; boot_stage.rmcall_device = boot_stage.rm_probe_h_internal_device; boot_stage.rmcall_subdevice = boot_stage.rm_probe_h_internal_subdevice; - pr_info("gsp-phys-boot: rm-call: stage 5: adopted server internal handles: hClient=0x%x hDevice=0x%x hSubdevice=0x%x (stage 1 satisfied by adoption; write 4 to run stages 2-4 with them)\n", + pr_info("vgsp-boot: rm-call: stage 5: adopted server internal handles: hClient=0x%x hDevice=0x%x hSubdevice=0x%x (stage 1 satisfied by adoption; write 4 to run stages 2-4 with them)\n", boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice); return 0; @@ -9048,7 +10117,7 @@ static void gsp_phys_rmcall_guest_hpa_note(u64 gpa, u64 hpa) boot_stage.rmcall_guest_hpa_note_logged) return; boot_stage.rmcall_guest_hpa_note_logged = true; - pr_info("gsp-phys-boot: rm-call: guest HPA changed: gpa=0x%llx was 0x%llx now 0x%llx (re-translated at trigger time; informational, not a failure)\n", + pr_info("vgsp-boot: rm-call: guest HPA changed: gpa=0x%llx was 0x%llx now 0x%llx (re-translated at trigger time; informational, not a failure)\n", gpa, boot_stage.rmcall_guest_hpa, hpa); } @@ -9065,14 +10134,14 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) dom = gsp_phys_dma_domain_identity(); if (dom != 1) { - pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", hpa, pte, dom); return -EIO; } ret = gsp_phys_gpa_to_hpa(gpa, &hpa); if (ret) { - pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); return -EIO; } @@ -9089,13 +10158,13 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) hpa, 0, (u64)gsp_phys_chunk_page_dma(dst, 0), 0, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_GUEST_POLL_MS, &sem); if (ret) { - pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); return -EIO; } if (memcmp(dst_page, __va(hpa), GSP_PHYS_RMCALL_COPY_SIZE) != 0) { - pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n", + pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n", hpa, pte, sem, hpa, gpa); return -EIO; } @@ -9111,7 +10180,7 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) { /* Dump both heads to tell a bad translation from a bad copy. */ - pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n", + pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n", hpa, pte, hpa, gpa, __va(hpa), dst_page); return -EIO; } @@ -9119,7 +10188,7 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) boot_stage.rmcall_guest_hpa = hpa; boot_stage.rmcall_guest_r_ok = true; - pr_info("gsp-phys-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + pr_info("vgsp-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); return 0; } @@ -9141,21 +10210,21 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) * guest memory. */ if (!boot_stage.rmcall_guest_r_ok) { - pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n", + pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n", hpa, pte); return -EIO; } dom = gsp_phys_dma_domain_identity(); if (dom != 1) { - pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", hpa, pte, dom); return -EIO; } ret = gsp_phys_gpa_to_hpa(gpa, &hpa); if (ret) { - pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); return -EIO; } @@ -9174,7 +10243,7 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) { /* Dump the head so the stale contents can be compared. */ - pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n", + pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n", hpa, pte, hpa, __va(hpa)); return -EIO; } @@ -9188,20 +10257,20 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) (u64)gsp_phys_chunk_page_dma(src, 0), 0, hpa, 0, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_GUEST_POLL_MS, &sem); if (ret) { - pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); return -EIO; } if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { - pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n", + pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n", hpa, pte, sem, hpa, gpa); return -EIO; } boot_stage.rmcall_guest_hpa = hpa; boot_stage.rmcall_guest_w2g_ok = true; - pr_info("gsp-phys-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + pr_info("vgsp-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); return 0; } @@ -9238,7 +10307,7 @@ static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0) src_page = gsp_phys_chunk_page_va(src, 0); dst_page = gsp_phys_chunk_page_va(dst, 0); if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) { - pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n", + pr_err("vgsp-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n", src_page, dst_page, gsp_phys_chunk_page_va(pt, 0), src_dma); return -EIO; @@ -9257,20 +10326,20 @@ static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0) GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, &sem); if (ret) { - pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n", + pr_err("vgsp-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n", ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx, pte); return -EIO; } if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { - pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n", + pr_err("vgsp-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n", sem, src_dma, pt_idx, pte); return -EIO; } boot_stage.rmcall_guest_selftest_ok = true; - pr_info("gsp-phys-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n", + pr_info("vgsp-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n", src_dma, sem); return 0; } @@ -9289,7 +10358,7 @@ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) */ if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) { - pr_info("gsp-phys-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n", + pr_info("vgsp-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n", gsp_phys_gpu_claimed(), boot_stage.rm_init_done, boot_stage.rm_unload_submitted); mutex_unlock(&gsp_phys_rm_lock); @@ -9304,18 +10373,18 @@ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) */ boot_stage.rmcall_last_stage = 5; if (boot_stage.rmcall_stage >= 1) { - pr_info("gsp-phys-boot: rm-call: stage 5 skipped: stage 1 is already satisfied (hClient=0x%x hDevice=0x%x hSubdevice=0x%x); write 2/3/4 for the remaining stages\n", + pr_info("vgsp-boot: rm-call: stage 5 skipped: stage 1 is already satisfied (hClient=0x%x hDevice=0x%x hSubdevice=0x%x); write 2/3/4 for the remaining stages\n", boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice); } else { ret = gsp_phys_rmcall_stage5(bar0); if (ret) { - pr_err("gsp-phys-boot: rm-call: ladder aborted in stage 5 (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", + pr_err("vgsp-boot: rm-call: ladder aborted in stage 5 (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", ret, boot_stage.rmcall_rpcs); } else { boot_stage.rmcall_stage = 1; /* stage 1 satisfied */ - pr_info("gsp-phys-boot: rm-call: stage 5 (adopt) complete (%u RPCs so far); stage 2/3/4 will use the adopted handles\n", + pr_info("vgsp-boot: rm-call: stage 5 (adopt) complete (%u RPCs so far); stage 2/3/4 will use the adopted handles\n", boot_stage.rmcall_rpcs); } } @@ -9340,16 +10409,16 @@ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) break; } if (ret) { - pr_err("gsp-phys-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", + pr_err("vgsp-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", stage, ret, boot_stage.rmcall_rpcs); break; } boot_stage.rmcall_stage = stage; - pr_info("gsp-phys-boot: rm-call: stage %u complete (%u RPCs so far)\n", + pr_info("vgsp-boot: rm-call: stage %u complete (%u RPCs so far)\n", stage, boot_stage.rmcall_rpcs); } if (!ret && target <= boot_stage.rmcall_stage) - pr_info("gsp-phys-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n", + pr_info("vgsp-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n", boot_stage.rmcall_stage, target, boot_stage.rmcall_rpcs); } @@ -9376,6 +10445,8 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx); for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm); gsp_phys_rmcall_reset(); } @@ -9413,7 +10484,7 @@ static bool gsp_phys_state_tx_header(struct seq_file *m, const char *tag, } /* - * Read-only snapshot for /sys/kernel/debug/open-iov-vfio/gsp-phys-state. + * Read-only snapshot for /sys/kernel/debug/vgsp/gsp-phys-state. * Touches the device only when the boot/claim is live AND the stage-3d * staging is primed; otherwise it returns a one-line status and no hardware * access. No register or queue state is written and nothing is consumed: the @@ -9432,12 +10503,12 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) if (!gsp_phys_gpu_claimed() || !bar0) { seq_puts(m, - "gsp-phys-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n"); + "vgsp-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n"); return 0; } if (!boot_stage.primed) { seq_printf(m, - "gsp-phys-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n", + "vgsp-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n", boot_stage.kicked, boot_stage.ran, boot_stage.rm_init_done, boot_stage.dryrun, boot_stage.rm_unload_submitted); @@ -9453,7 +10524,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) GSP_PHYS_FALCON_CPUCTL); gsp_phys_fwsec_scratch14_read(bar0, &s14); - seq_printf(m, "gsp-phys-state: claimed=1 boot_param=%u dryrun_param=%u\n", + seq_printf(m, "vgsp-state: claimed=1 boot_param=%u dryrun_param=%u\n", gsp_phys_boot_enabled(), gsp_phys_boot_dryrun_enabled()); seq_printf(m, "stage: primed=%u kicked=%u ran=%u riscv_active=%u rm_init_done=%u rm_unload_submitted=%u dryrun=%u tx_seq_num=%u\n", @@ -9493,7 +10564,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) /* COPY0 sessions print no line (byte-identical state output). */ if (boot_stage.rmcall_ctx_prepared) seq_printf(m, - "rm_call_ctx: prepared=%u valid=%u pa=0x%llx va=0x%llx size=0x%x pages=%u ptPages=%u gr_promote_param=%u step=%u entrySize=0x%x\n", + "rm_call_ctx: prepared=%u valid=%u pa=0x%llx va=0x%llx size=0x%x pages=%u ptPages=%u gr_promote_param=%u step=%u entrySize=0x%x hVirt=0x%x\n", boot_stage.rmcall_ctx_prepared, boot_stage.rmcall_ctx_valid, boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, @@ -9502,7 +10573,8 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_ctx_pt_pages, boot_stage.rmcall_gr_promote, boot_stage.rmcall_gr_promote_step, - boot_stage.rmcall_gr_promote_size); + boot_stage.rmcall_gr_promote_size, + boot_stage.rmcall_gr_hvirt); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, @@ -9911,14 +10983,14 @@ static int gsp_phys_log_dump_show(struct seq_file *m, void *v) if (!gsp_phys_gpu_claimed() || !boot_stage.primed || !boot_stage.login.pages || !boot_stage.logrm.pages) { seq_printf(m, - "gsp-phys-log-dump: inactive: claimed=%u primed=%u login_pages=%u logrm_pages=%u; no log buffers to dump\n", + "vgsp-log-dump: inactive: claimed=%u primed=%u login_pages=%u logrm_pages=%u; no log buffers to dump\n", gsp_phys_gpu_claimed(), boot_stage.primed, boot_stage.login.pages, boot_stage.logrm.pages); return 0; } seq_puts(m, - "gsp-phys-boot: libos task log buffers (read-only; raw hex is authoritative, decode offline with backups/evidence-*/decode_logs.py)\n"); + "vgsp-boot: libos task log buffers (read-only; raw hex is authoritative, decode offline with backups/evidence-*/decode_logs.py)\n"); gsp_phys_log_dump_raw(m, "LOGINIT", &boot_stage.login); gsp_phys_log_dump_raw(m, "LOGRM", &boot_stage.logrm); gsp_phys_log_dump_decoded(m, "LOGINIT", &boot_stage.login); @@ -9954,7 +11026,7 @@ static int gsp_phys_status_drain_show(struct seq_file *m, void *v) if (!gsp_phys_gpu_claimed() || !boot_stage.primed || !boot_stage.queue.pages) { seq_printf(m, - "gsp-phys-status: inactive: claimed=%u primed=%u queue_pages=%u; no status queue to drain\n", + "vgsp-status: inactive: claimed=%u primed=%u queue_pages=%u; no status queue to drain\n", gsp_phys_gpu_claimed(), boot_stage.primed, boot_stage.queue.pages); return 0; @@ -10001,37 +11073,37 @@ static ssize_t gsp_phys_unload_rpc_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value != 1) { - pr_info("gsp-phys-boot: gsp-phys-unload-rpc: write ignored (write \"1\" to trigger)\n"); + pr_info("vgsp-boot: gsp-phys-unload-rpc: write ignored (write \"1\" to trigger)\n"); return len; } if (!gsp_phys_boot_enabled()) { - pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: gsp_phys_boot is not enabled\n"); + pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { - pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: the physical GPU is not claimed\n"); + pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.ran) { - pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", + pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } if (!boot_stage.rm_init_done) { - pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n", + pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { - pr_warn("gsp-phys-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n"); + pr_warn("vgsp-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n"); return len; } ret = gsp_phys_boot_rm_unload(bar0); - pr_info("gsp-phys-boot: gsp-phys-unload-rpc: UNLOADING_GUEST_DRIVER attempt returned %d (WPR2 untouched; run release/rmmod to tear down WPR2)\n", + pr_info("vgsp-boot: gsp-phys-unload-rpc: UNLOADING_GUEST_DRIVER attempt returned %d (WPR2 untouched; run release/rmmod to tear down WPR2)\n", ret); return len; } @@ -10069,37 +11141,37 @@ static ssize_t gsp_phys_rm_probe_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value != 1) { - pr_info("gsp-phys-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n"); + pr_info("vgsp-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n"); return len; } if (!gsp_phys_boot_enabled()) { - pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n"); + pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { - pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n"); + pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.ran) { - pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", + pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } if (!boot_stage.rm_init_done) { - pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", + pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { - pr_warn("gsp-phys-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n"); + pr_warn("vgsp-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n"); return len; } ret = gsp_phys_rm_probe(bar0); - pr_info("gsp-phys-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n", + pr_info("vgsp-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n", ret, boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); return len; @@ -10149,44 +11221,44 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value < 1 || value > 9) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, 8 for the guest PTE self-test, or 9 to re-run one GR semaphore release on a GR ladder; 6-8 need gsp_phys_guest_mem=1)\n"); + pr_info("vgsp-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, 8 for the guest PTE self-test, or 9 to re-run one GR semaphore release on a GR ladder; 6-8 need gsp_phys_guest_mem=1)\n"); return len; } if (!gsp_phys_boot_enabled()) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n"); + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n"); + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.primed) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n"); + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n"); return len; } if (!boot_stage.ran) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } if (!boot_stage.rm_init_done) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); return len; } if (boot_stage.rm_unload_submitted) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n"); + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n"); return len; } if (!boot_stage.pdev) { - pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n"); + pr_warn("vgsp-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n"); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { - pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n"); + pr_warn("vgsp-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n"); return len; } @@ -10200,7 +11272,7 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, */ if (value == 9) { ret = gsp_phys_rmcall_gr_sem_rerun(bar0); - pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger 9 returned %d (GR semaphore re-run; WPR2 untouched)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: trigger 9 returned %d (GR semaphore re-run; WPR2 untouched)\n", ret); return len; } @@ -10214,22 +11286,22 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, * no-op; the engine is never touched from here. */ if (!gsp_phys_guest_mem_enabled()) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n", + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n", value); return len; } if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n", value, gsp_phys_backend_state()); return len; } if (boot_stage.rmcall_stage < 3) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n", value, boot_stage.rmcall_stage); return len; } if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n", value); return len; } @@ -10241,7 +11313,7 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, else ret = gsp_phys_rmcall_guest_selftest(bar0); - pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n", value, ret, boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, boot_stage.rmcall_guest_r_ok, @@ -10251,7 +11323,7 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, } ret = gsp_phys_rmcall_run(bar0, (u32)value); - pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", + pr_info("vgsp-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", value, ret, boot_stage.rmcall_stage, boot_stage.rmcall_last_stage, boot_stage.rmcall_rpcs); return len; @@ -10288,24 +11360,24 @@ static ssize_t gsp_phys_guest_gpa_write(struct file *file, buf[len] = '\0'; if (kstrtoull(strim(buf), 0, &gpa)) { - pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: expected \"0x\"\n"); + pr_info("vgsp-boot: gsp-phys-guest-gpa: write ignored: expected \"0x\"\n"); return -EINVAL; } if (!PAGE_ALIGNED(gpa)) { - pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n", + pr_info("vgsp-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n", gpa); return -EINVAL; } if (boot_stage.rmcall_guest_gpa_valid && boot_stage.rmcall_guest_gpa != gpa) { - pr_err("gsp-phys-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", + pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", boot_stage.rmcall_guest_gpa); return -EBUSY; } boot_stage.rmcall_guest_gpa = gpa; boot_stage.rmcall_guest_gpa_valid = true; - pr_info("gsp-phys-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n", + pr_info("vgsp-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n", gpa); return len; } @@ -10358,34 +11430,34 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, p = strim(buf); if (!strcmp(p, "-")) { gsp_phys_guest_ram_clear(); - pr_info("gsp-phys-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); + pr_info("vgsp-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); return len; } path = gsp_phys_guest_ram_token(&p); tok = gsp_phys_guest_ram_token(&p); if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { - pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected \" \" or \"-\"\n"); + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \" \" or \"-\"\n"); return -EINVAL; } tok = gsp_phys_guest_ram_token(&p); if (!tok || kstrtoull(tok, 0, &file_offset)) { - pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); return -EINVAL; } tok = gsp_phys_guest_ram_token(&p); if (!tok || kstrtoull(tok, 0, &size)) { - pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); return -EINVAL; } ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); if (ret) { - pr_err("gsp-phys-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", path, gpa_base, file_offset, size, ret); return ret; } - pr_info("gsp-phys-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + pr_info("vgsp-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", path, gpa_base, file_offset, size); return len; } @@ -10401,13 +11473,13 @@ int gsp_phys_boot_debugfs_init(void) { struct dentry *file; - gsp_phys_debugfs_root = debugfs_create_dir("open-iov-vfio", NULL); + gsp_phys_debugfs_root = debugfs_create_dir("vgsp", NULL); if (IS_ERR_OR_NULL(gsp_phys_debugfs_root)) { long err = IS_ERR(gsp_phys_debugfs_root) ? PTR_ERR(gsp_phys_debugfs_root) : -ENODEV; gsp_phys_debugfs_root = NULL; - pr_warn("gsp-phys-boot: debugfs dir open-iov-vfio creation failed: %ld\n", + pr_warn("vgsp-boot: debugfs dir vgsp creation failed: %ld\n", err); return (int)err; } @@ -10419,7 +11491,7 @@ int gsp_phys_boot_debugfs_init(void) debugfs_remove_recursive(gsp_phys_debugfs_root); gsp_phys_debugfs_root = NULL; - pr_warn("gsp-phys-boot: debugfs file gsp-phys-unload-rpc creation failed: %ld\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-unload-rpc creation failed: %ld\n", err); return (int)err; } @@ -10433,7 +11505,7 @@ int gsp_phys_boot_debugfs_init(void) * Keep the existing trigger registered: only the snapshot is * missing. The caller's warning covers both files. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n", err); return (int)err; } @@ -10447,7 +11519,7 @@ int gsp_phys_boot_debugfs_init(void) * The probe trigger is optional: the unload trigger and the * state snapshot stay registered. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n", err); return (int)err; } @@ -10461,7 +11533,7 @@ int gsp_phys_boot_debugfs_init(void) * The RM-client ladder is optional: the other triggers and * the state snapshot stay registered. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n", err); return (int)err; } @@ -10475,7 +11547,7 @@ int gsp_phys_boot_debugfs_init(void) * The guest-GPA pin is optional: the other triggers and the * state snapshot stay registered. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n", err); return (int)err; } @@ -10489,7 +11561,7 @@ int gsp_phys_boot_debugfs_init(void) * The guest-RAM backing file is optional: the other triggers * and the state snapshot stay registered. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n", err); return (int)err; } @@ -10503,7 +11575,7 @@ int gsp_phys_boot_debugfs_init(void) * The log dump is diagnostic; the triggers and the state * snapshot stay registered. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-log-dump creation failed: %ld (triggers and state snapshot stay available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-log-dump creation failed: %ld (triggers and state snapshot stay available)\n", err); return (int)err; } @@ -10517,11 +11589,11 @@ int gsp_phys_boot_debugfs_init(void) * The status-queue drain is diagnostic; the triggers, the * log dump and the state snapshot stay registered. */ - pr_warn("gsp-phys-boot: debugfs file gsp-phys-status-drain creation failed: %ld (triggers, log dump and state snapshot stay available)\n", + pr_warn("vgsp-boot: debugfs file gsp-phys-status-drain creation failed: %ld (triggers, log dump and state snapshot stay available)\n", err); return (int)err; } - pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); + pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } @@ -10539,21 +11611,21 @@ void gsp_phys_boot_quiesce(void __iomem *bar0) if (!boot_stage.kicked) return; - pr_info("gsp-phys-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n", + pr_info("vgsp-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n", boot_stage.riscv_active); gsp_phys_scratch14_log(bar0, "quiesce-pre-reset"); gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); gsp_phys_scratch14_log(bar0, "quiesce-post-reset"); - pr_info("gsp-phys-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n", + pr_info("vgsp-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_RISCV_STATUS), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI)); if (gsp_phys_fwsec_wpr2_up(bar0)) - pr_info("gsp-phys-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n"); + pr_info("vgsp-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n"); else - pr_info("gsp-phys-boot: release: WPR2 already down after the quiesce\n"); + pr_info("vgsp-boot: release: WPR2 already down after the quiesce\n"); } void gsp_phys_boot_staging_free(struct pci_dev *pdev) @@ -10566,7 +11638,7 @@ void gsp_phys_boot_staging_free(struct pci_dev *pdev) boot_stage.queue.pages || boot_stage.radix3.table.pages || boot_stage.login.pages || boot_stage.logrm.pages || boot_stage.radix3.nr_chunks || boot_stage.rmcall_chunks[0].pages) - pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n"); + pr_info("vgsp-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n"); gsp_phys_rmcall_fini(pdev); gsp_phys_radix3_free(pdev, &boot_stage.radix3); -- cgit v1.2.3