diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 864 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 86 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 12 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 721 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 11 | ||||
| -rw-r--r-- | kernel/main.c | 18 |
6 files changed, 1622 insertions, 90 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index d5585e4..b26f2bd 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -609,6 +609,21 @@ bool gsp_phys_boot_requested(void) ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) /* + * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources, + * allocated lazily under gsp_phys_rm_lock and freed with the staging. The + * Tier-1 window pool holds small-PT pages installed with DMA_UPDATE_PDE_2 at + * the guest's own 2 MiB VA windows (one PDE per window); the mapper's + * conservative no-cached-line rule only allows a new PTE in a fetch line the + * current submission installed, and never rewrites a stale one. The Tier-2 + * shadow chunk backs relocated parameter pages whose content was captured as + * inline writes. Both are deliberately outside rmcall_chunks so the COPY0 + * stage-2 registration walk stays byte-identical. + */ +#define GSP_PHYS_RMCALL_GR_PT_PAGES 8u +#define GSP_PHYS_RMCALL_GR_PT_ENTRIES 32u +#define GSP_PHYS_RMCALL_GR_SHADOW_PAGES 4u + +/* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: @@ -1418,6 +1433,29 @@ static struct gsp_phys_boot_stage { */ u64 rmcall_va_guest_pt; /* guest scratch-PT VA window base */ u32 rmcall_guest_pt_next_line; /* next never-used 128 B PTE line */ + /* + * GR forwarding (gsp_phys_gr_forward=1; inert otherwise). The exec + * gate latches only after the stage-4 real GR submission logs + * REAL GR SEM OK, so the forward path can never pretend to work while + * the ladder itself cannot execute GR. The window pool and shadow + * chunk are allocated lazily by the forwarding submit. + */ + bool rmcall_gr_exec_ok; /* a real GR submission proved out */ + u32 rmcall_gr_submit_epoch; /* per-submission mapper epoch */ + u32 rmcall_gr_pt_used; /* Tier-1 window pool slots used */ + struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES]; + struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */ + struct { + u32 pde_index; + u8 pt_slot; + u8 used; + u8 pte_count; + struct { + u32 idx; /* small-PT index in the 2 MiB window */ + u32 epoch; /* submission that installed it */ + u64 hpa; /* the PTE's mapped host PA */ + } pte[GSP_PHYS_RMCALL_GR_PT_ENTRIES]; + } rmcall_gr_win[GSP_PHYS_RMCALL_GR_PT_PAGES]; struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -2016,6 +2054,13 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_guest_selftest_ok = false; boot_stage.rmcall_va_guest_pt = 0; boot_stage.rmcall_guest_pt_next_line = 0; + boot_stage.rmcall_gr_exec_ok = false; + boot_stage.rmcall_gr_submit_epoch = 0; + boot_stage.rmcall_gr_pt_used = 0; + memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt)); + memset(&boot_stage.rmcall_gr_shadow, 0, + sizeof(boot_stage.rmcall_gr_shadow)); + memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win)); for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -9434,84 +9479,39 @@ struct gsp_phys_rmcall_ce_submit { }; /* - * One CE copy block on the ladder channel: clear the module semaphore, build - * the methods for bytes into the module pushbuffer, publish the GPFIFO entry - * and USERD GP_PUT and ring the doorbell. The caller must hold - * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical - * lock-across-poll; the P5 work API polls without the lock). Every guard of - * the original stage-4 submission body is preserved here: server-memory - * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the - * token check. The semaphore is cleared before the doorbell so a stale release - * from a previous run cannot be observed as this submission's completion. - * P6c: sem_payload is the release value the method block carries; legacy - * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is - * still cleared even when the caller polls a guest page: the clear only - * guards the module-sem poll path and is a harmless non-guest write. - * - * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release - * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and - * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call - * site passes false, so the default COPY0 ladder emits the same bytes as - * before. + * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET, + * encode and write the GPFIFO entry for the module pushbuffer VA, advance + * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted + * verbatim from gsp_phys_rmcall_ce_submit_locked so the CE path and the GR + * launch path publish byte-identically; the CE caller's semaphore clear and + * method-block build happen before this call. The caller must hold + * gsp_phys_rm_lock. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, + u32 words, + struct gsp_phys_rmcall_ce_submit *out) { - struct gsp_phys_boot_chunk *pb = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; - resource_size_t bar0_len; - u32 *pb_va, *userd, *ring_va; - u32 words, gpput, gpget, slot, e0, e1; + u32 *ring_va, *userd; + u32 gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; - u64 pb_gpu_va; - if (boot_stage.rmcall_channel_server_mem) - return -EOPNOTSUPP; - if (!boot_stage.pdev) - return -EINVAL; - bar0_len = pci_resource_len(boot_stage.pdev, 0); - if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) - return -EIO; - if (!bar0 || !boot_stage.rmcall_token || !bytes || - bytes > GSP_PHYS_RMCALL_COPY_SIZE) - return -EINVAL; - pb_gpu_va = boot_stage.rmcall_va_pb; - if (!pb_gpu_va || (pb_gpu_va & 3)) - return -EIO; - - pb_va = gsp_phys_chunk_page_va(pb, 0); ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); - if (!pb_va || !ring_va || !userd || - !gsp_phys_chunk_page_va(sem, 0) || - !gsp_phys_chunk_page_va(src, 0) || - !gsp_phys_chunk_page_va(dst, 0)) + if (!ring_va || !userd) return -EIO; - /* A stale release from a previous submission must not be observed. */ - WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); - wmb(); - - if (gr) - words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va, - sem_payload); - else - words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, - dst_va, bytes, - sem_payload); - gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { @@ -9590,6 +9590,87 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, } /* + * One CE copy block on the ladder channel: clear the module semaphore, build + * the methods for bytes into the module pushbuffer, publish the GPFIFO entry + * and USERD GP_PUT and ring the doorbell. The caller must hold + * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical + * lock-across-poll; the P5 work API polls without the lock). Every guard of + * the original stage-4 submission body is preserved here: server-memory + * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the + * token check. The semaphore is cleared before the doorbell so a stale release + * from a previous run cannot be observed as this submission's completion. + * P6c: sem_payload is the release value the method block carries; legacy + * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is + * still cleared even when the caller polls a guest page: the clear only + * guards the module-sem poll path and is a harmless non-guest write. + * + * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release + * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and + * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call + * site passes false, so the default COPY0 ladder emits the same bytes as + * before. + */ +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + struct gsp_phys_boot_chunk *ring = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + struct gsp_phys_boot_chunk *userd_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + resource_size_t bar0_len; + u32 *pb_va, *userd, *ring_va; + u32 words; + u64 pb_gpu_va; + + if (boot_stage.rmcall_channel_server_mem) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_gpu_va = boot_stage.rmcall_va_pb; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + + pb_va = gsp_phys_chunk_page_va(pb, 0); + ring_va = gsp_phys_chunk_page_va(ring, 0); + userd = gsp_phys_chunk_page_va(userd_c, 0); + if (!pb_va || !ring_va || !userd || + !gsp_phys_chunk_page_va(sem, 0) || + !gsp_phys_chunk_page_va(src, 0) || + !gsp_phys_chunk_page_va(dst, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + if (gr) + words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va, + sem_payload); + else + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, + dst_va, bytes, + sem_payload); + + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); +} + +/* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll; @@ -9660,6 +9741,12 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); + /* + * Latch the GR execution gate: this is the only place a real GR + * submission is proven on the ladder, and gsp_phys_backend_gr_ready() + * refuses every forwarded launch until this is observed. + */ + boot_stage.rmcall_gr_exec_ok = true; return 0; } @@ -9769,6 +9856,36 @@ bool gsp_phys_backend_work_ready(void) } EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready); +/* + * GR forwarding readiness: the parameter, a live client-memory GR ladder + * channel and the latched boot proof that a real GR submission executed + * (rmcall_gr_exec_ok, set only by a REAL GR SEM OK). This is deliberately + * separate from gsp_phys_backend_ladder_ready(), which refuses CE work on a + * GR0 ladder; a GR0 ladder is exactly the GR forwarding prerequisite. Every + * submit re-checks the same state under gsp_phys_rm_lock. + */ +bool gsp_phys_backend_gr_ready(void) +{ + if (!gsp_phys_gr_forward_enabled()) + return false; + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return false; + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) + return false; + if (!gsp_phys_rmcall_engine_is_gr() || !boot_stage.rmcall_gr_exec_ok) + return false; + if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || + !boot_stage.rmcall_token) + return false; + if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo || + !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_sem || + !boot_stage.rmcall_pdb_root) + return false; + return true; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_ready); + bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va, u32 *bytes) { @@ -10525,6 +10642,595 @@ int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); +/* ------------------------------------------------------------------ */ +/* GR (compute) forwarding backend (gsp_phys_gr_forward) */ +/* ------------------------------------------------------------------ */ + +/* + * C5C0 inline-QMD method window: SET_INLINE_QMD_ADDRESS_A/B carry the + * shifted QMD device address and LOAD_INLINE_QMD_DATA the 64 QMD words; the + * guest emits the window as one INC run at 0x318 with count 66 (A, B, then 64 + * words), and the launch is replayed in exactly that shape with the QMD words + * the sema service decoded (release0 and, under Tier 2, relocated pointers + * rewritten). The channel release is the same A..D block as the H1 GR sem + * verify, with a payload distinct from the CE/module sentinel. + */ +#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A 0x318u +#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_B 0x31cu +#define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA 0x320u +/* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */ +#define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS 66u +#define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD GSP_PHYS_RMCALL_SEM_PAYLOAD +#define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD 0xc0ffee02u +/* Method block bound: SET_OBJECT(2) + inline run(67) + REPORT_SEM(8). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 96u + +static u64 gsp_phys_gr_submitted; +static u64 gsp_phys_gr_ok; +static u64 gsp_phys_gr_refused; +static u64 gsp_phys_gr_timed_out; +static u64 gsp_phys_gr_pool_exhausted; + +void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, + u64 *timed_out, u64 *pool_exhausted) +{ + if (submitted) + *submitted = gsp_phys_gr_submitted; + if (ok) + *ok = gsp_phys_gr_ok; + if (refused) + *refused = gsp_phys_gr_refused; + if (timed_out) + *timed_out = gsp_phys_gr_timed_out; + if (pool_exhausted) + *pool_exhausted = gsp_phys_gr_pool_exhausted; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_counters); + +/* + * GR launch method block: SET_OBJECT on the compute subchannel, the inline + * QMD run at its original shape and the channel REPORT_SEMAPHORE release. + * Returns the block size in dwords. + */ +static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd, + u64 qmd_va, u64 sem_va, + u32 sem_payload) +{ + u32 *p = pb; + + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT, + GSP_PHYS_RMCALL_CLASS_COMPUTE); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) | + ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) | + (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2); + *p++ = (u32)(qmd_va >> 40); + *p++ = (u32)((qmd_va >> 8) & 0xffffffffu); + memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4); + p += GSP_PHYS_GR_QMD_WORDS; + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + return (u32)(p - pb); +} + +/* + * Tier-1 window pool: find (or install) the small-PT page for one guest + * 2 MiB VA window through DMA_UPDATE_PDE_2. Installation is once per window + * per staging, so a later launch in the same window writes PTEs into the same + * server-installed PT page. Called with gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_win_slot_locked(void __iomem *bar0, + u32 pde_index, u32 *slot_out) +{ + struct gsp_phys_rmcall_update_pde2_params p; + struct gsp_phys_boot_chunk *pt; + u32 rpc_result = 0, status = 0; + u32 i, slot; + int ret; + + for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) { + if (boot_stage.rmcall_gr_win[i].used && + boot_stage.rmcall_gr_win[i].pde_index == pde_index) { + *slot_out = i; + return 0; + } + } + if (boot_stage.rmcall_gr_pt_used >= GSP_PHYS_RMCALL_GR_PT_PAGES) + return -ENOSPC; + slot = boot_stage.rmcall_gr_pt_used; + pt = &boot_stage.rmcall_gr_pt[slot]; + if (!pt->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, pt, 1u); + if (ret) + return ret; + gsp_phys_chunk_zero(pt); + } + memset(&p, 0, sizeof(p)); + p.pde_index = cpu_to_le32(pde_index); + p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); + p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0)); + p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); + p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); + p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + p.pde_buffer = 0; + p.sub_device_id = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2, + &p, sizeof(p), sizeof(p), &p, sizeof(p), + &rpc_result, &status, "GR forward PDE_UPDATE"); + if (ret) { + pr_err("vgsp-gr: forward PDE install refused pdeIndex=%u (window VA 0x%llx) status=0x%x rpc_result=0x%x ret=%d\n", + pde_index, (u64)pde_index << 21, status, rpc_result, ret); + return ret; + } + boot_stage.rmcall_gr_win[slot].pde_index = pde_index; + boot_stage.rmcall_gr_win[slot].pt_slot = (u8)slot; + boot_stage.rmcall_gr_win[slot].used = true; + boot_stage.rmcall_gr_win[slot].pte_count = 0; + boot_stage.rmcall_gr_pt_used = slot + 1; + pr_info("vgsp-gr: forward PDE installed pdeIndex=%u slot=%u ptPa=0x%llx status=0x%x\n", + pde_index, slot, (u64)gsp_phys_chunk_page_dma(pt, 0), status); + *slot_out = slot; + return 0; +} + +/* + * Tier-1 map: write one guest page's HPA into the PT page of its own 2 MiB + * VA window at the guest VA itself, so the QMD and every pointer inside the + * mapped pages resolve without rewriting. The GMMU caches 128 B PTE fetch + * lines with no invalidate, so the mapper never rewrites an installed PTE + * (a different HPA for the same VA is refused rather than made stale) and + * only adds a new PTE to a fetch line this same submission already touched. + * Every refusal is fail-closed: the caller drops to Tier 2 or refuses the + * launch and the sema service falls back. Called with gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, + u64 hpa, u32 epoch) +{ + u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 pde_index = (u32)(page >> 21); + u32 pte_idx = (u32)((page >> 12) & 0x1ffu); + u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; + struct gsp_phys_boot_chunk *pt; + u8 *pt_va; + u32 slot, i; + int ret; + + ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); + if (ret) + return ret; + for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) { + if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) { + if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == hpa) + return 0; /* already mapped, no write */ + return -EALREADY; /* stale: never rewrite */ + } + if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch && + boot_stage.rmcall_gr_win[slot].pte[i].idx / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line) + return -EBUSY; /* fetch line may be cached */ + } + if (boot_stage.rmcall_gr_win[slot].pte_count >= + GSP_PHYS_RMCALL_GR_PT_ENTRIES) + return -ENOSPC; + pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot]; + pt_va = gsp_phys_chunk_page_va(pt, 0); + if (!pt_va) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + pt_va + (size_t)pte_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; + boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch; + boot_stage.rmcall_gr_win[slot].pte[i].hpa = hpa; + boot_stage.rmcall_gr_win[slot].pte_count++; + return 0; +} + +/* + * Tier-2 map: consume one never-reused 128 B fetch line from the guest + * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the + * line's base VA. The shadow chunk is allocated lazily; page slot `slot` + * supplies the backing page. Called with gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, + u64 *va_out) +{ + struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; + struct gsp_phys_boot_chunk *pt; + u32 line = boot_stage.rmcall_guest_pt_next_line; + u32 page, line_in_page; + u64 va; + u8 *pt_va; + int ret; + + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) + return ret; + gsp_phys_chunk_zero(sh); + } + if (slot >= sh->pages) + return -ENOSPC; + if (!boot_stage.rmcall_va_guest_pt) + return -EIO; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + return -ENOSPC; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; + if (!pt->pages) + return -EIO; + pt_va = gsp_phys_chunk_page_va(pt, 0); + if (!pt_va) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page) * 8); + dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1; + va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + ((u64)line_in_page << 16); + *va_out = va; + return 0; +} + +/* GR forward fault observation is armed by the GR parameters, not the CE + * ones (gsp_phys_fault_relay_active() requires gsp_phys_work_forward). */ +static bool gsp_phys_gr_fault_active(void) +{ + return gsp_phys_fault_relay_enabled() && + gsp_phys_gr_forward_enabled() && + gsp_phys_guest_mem_enabled(); +} + +/* + * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The + * caller has resolved the launch's pages; every mapping is fresh or proven + * unchanged, the QMD release0 and the channel release are authored either by + * the engine into ladder-mapped shadow pages or into the module semaphore, + * and the GPFIFO/USERD/doorbell tail is the shared publish helper. The + * release poll runs without gsp_phys_rm_lock; a timeout drains the status + * queue (epoch-gated) into out->fault. Every refusal moves the GR counters + * and returns an errno the sema service falls back on. + */ +int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, + struct gsp_phys_backend_copy_result *out) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + void __iomem *bar0; + struct gsp_phys_rmcall_ce_submit sub; + struct gsp_phys_fault_info fault = { }; + /* + * Static scratch: the only caller is the single-inflight sema work + * item (ordered gsp-phys-work queue), so no concurrent submission can + * touch these. Keeping the 96-dword method block and the 64-word QMD + * copy off the stack keeps the frame under the warning bound. + */ + static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; + static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + unsigned long start = jiffies; + unsigned long deadline; + u8 *sem_va; + u32 epoch, words, i, n_shadow = 0; + u32 want_qmd = 0, want_ch = 0; + bool qmd_rel = false, ch_rel = false; + u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; + u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; + u32 qmd_rel_off = 0, ch_rel_off = 0; + u32 poll = 0, w0 = 0, w1 = 0; + int ret = 0; + + if (out) + memset(out, 0, sizeof(*out)); + if (!req || !req->qmd || + req->qmd_words != GSP_PHYS_GR_QMD_WORDS || + req->n_pages > GSP_PHYS_GR_MAX_PAGES || + req->n_inl > GSP_PHYS_GR_MAX_INLINE) { + gsp_phys_gr_refused++; + return -EINVAL; + } + if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() || + gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { + gsp_phys_gr_refused++; + ret = -EAGAIN; + goto out; + } + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) { + gsp_phys_gr_refused++; + ret = -EAGAIN; + goto out; + } + sem_va = gsp_phys_chunk_page_va(sem, 0); + if (!sem_va || !gsp_phys_chunk_page_va(pb, 0)) { + gsp_phys_gr_refused++; + ret = -EIO; + goto out; + } + + gsp_phys_gr_submitted++; + mutex_lock(&gsp_phys_rm_lock); + if (!gsp_phys_backend_gr_ready()) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + ret = -EAGAIN; + goto out; + } + epoch = ++boot_stage.rmcall_gr_submit_epoch; + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + memcpy(qmd, req->qmd, sizeof(qmd)); + + /* + * QMD release0: when the launch has a one-word absolute release and + * the caller resolved the guest release page, author it there through + * a fresh shadow line so the guest observes its own QMD release; else + * point release0 at the module semaphore so the host observes engine + * completion. A reduction release is never redirected (its arithmetic + * would change). + */ + if (((qmd[4] >> 10) & 1)) { + u32 red_en = (qmd[24] >> 26) & 1; + u32 one_word = (qmd[24] >> 31) & 1; + + if (req->rel_hpa && one_word && !red_en && + n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + u64 sh_va = 0; + + ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, + req->rel_hpa, + &sh_va); + if (!ret) { + n_shadow++; + qmd_rel_hpa = req->rel_hpa; + qmd_rel_off = req->rel_off; + want_qmd = req->rel_payload; + qmd_rel = true; + qmd[23] = (u32)(sh_va + req->rel_off); + qmd[24] = (qmd[24] & 0xffffff00u) | + ((u32)((sh_va + req->rel_off) >> 32) & + 0xffu); + qmd[25] = req->rel_payload; + } else { + ret = 0; + } + } + if (!qmd_rel) { + qmd[23] = (u32)GSP_PHYS_RMCALL_VA_SEM; + qmd[24] = (qmd[24] & 0xffffff00u) | + ((u32)(GSP_PHYS_RMCALL_VA_SEM >> 32) & + 0xffu); + qmd[25] = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD; + want_qmd = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD; + qmd_rel = true; + } + } + + /* + * Pages: Tier 1 maps the resolved HPA at the guest VA; a page without + * a usable Tier-1 mapping falls to Tier 2 when captured inline bytes + * cover it and a QMD pointer pair names it (the pair is rewritten to + * the shadow VA); anything else refuses the launch. + */ + for (i = 0; i < req->n_pages; i++) { + const struct gsp_phys_gr_page *pg = &req->pages[i]; + u64 pg_va, sh_va = 0; + bool mapped = false; + + if (!pg->guest_va) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + ret = -EINVAL; + goto out; + } + pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + if (pg->hpa) { + ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, + pg->hpa, epoch); + if (!ret) + mapped = true; + else if (ret == -ENOSPC) + gsp_phys_gr_pool_exhausted++; + } + if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE && + pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) { + struct gsp_phys_boot_chunk *sh = + &boot_stage.rmcall_gr_shadow; + u8 *sh_page; + u32 j, copied = 0; + + if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + gsp_phys_gr_pool_exhausted++; + continue; + } + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) { + gsp_phys_gr_pool_exhausted++; + continue; + } + gsp_phys_chunk_zero(sh); + } + sh_page = gsp_phys_chunk_page_va(sh, n_shadow); + if (!sh_page) + continue; + memset(sh_page, 0, GSP_PHYS_PAGE_SIZE); + for (j = 0; j < req->n_inl; j++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[j]; + u32 len; + + if ((inl->dst_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) + != pg_va) + continue; + len = inl->len; + if (len > GSP_PHYS_GR_INLINE_BYTES) + len = GSP_PHYS_GR_INLINE_BYTES; + if ((u64)(inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)) + len > + GSP_PHYS_PAGE_SIZE) + continue; + memcpy(sh_page + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + copied++; + } + if (copied) { + ret = gsp_phys_rmcall_gr_map_shadow_locked( + n_shadow, + (u64)gsp_phys_chunk_page_dma(sh, + n_shadow), + &sh_va); + if (!ret) { + u64 ptr_va = sh_va + + (pg->guest_va & + (GSP_PHYS_PAGE_SIZE - 1)); + + n_shadow++; + qmd[pg->qmd_word] = (u32)ptr_va; + qmd[pg->qmd_word + 1] = + (qmd[pg->qmd_word + 1] & + ~0x1ffffu) | + ((u32)(ptr_va >> 32) & 0x1ffffu); + mapped = true; + } + } + } + if (!mapped) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + ret = -EOPNOTSUPP; + pr_info("vgsp-gr: launch refused: page va=0x%llx hpa=0x%llx qmd_word=%u has no Tier-1 mapping and no captured inline data (pages=%u)\n", + pg->guest_va, pg->hpa, pg->qmd_word, + req->n_pages); + goto out; + } + } + + /* + * Channel release: author it into the guest semaphore page when the + * caller resolved one, else into the module semaphore's second word. + */ + if (req->sem_hpa && req->sem_payload && + n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + u64 sh_va = 0; + + ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, + req->sem_hpa, &sh_va); + if (!ret) { + n_shadow++; + ch_rel_hpa = req->sem_hpa; + ch_rel_off = req->sem_off; + want_ch = req->sem_payload; + ch_rel = true; + ch_sem_va = sh_va + req->sem_off; + } + } + if (!ch_rel) { + want_ch = GSP_PHYS_RMCALL_GR_SEM_PAYLOAD; + ch_rel = true; + } + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)sem_va, 0); + WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + wmb(); + + words = gsp_phys_rmcall_build_gr_launch(methods, qmd, + GSP_PHYS_RMCALL_VA_SRC, + ch_sem_va, want_ch); + memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + + ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, + words, &sub); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + gsp_phys_gr_refused++; + pr_err("vgsp-gr: launch submit refused (%d) pages=%u words=%u\n", + ret, req->n_pages, words); + goto out; + } + + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n", + req->h_object, req->push_va, req->push_len, req->n_pages, + n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token, + qmd_rel, want_ch, GSP_PHYS_RMCALL_GUEST_POLL_MS); + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS); + for (;;) { + if (qmd_rel) { + if (qmd_rel_hpa) + w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + + qmd_rel_off)); + else + w0 = READ_ONCE(*(u32 *)sem_va); + } + if (ch_rel_hpa) + w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + + ch_rel_off)); + else + w1 = READ_ONCE(*(u32 *)(sem_va + 4)); + if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) + break; + if (time_after_eq(jiffies, deadline)) { + gsp_phys_gr_timed_out++; + pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", + GSP_PHYS_RMCALL_GUEST_POLL_MS, w0, want_qmd, w1, + want_ch, req->h_object, req->push_va); + if (gsp_phys_gr_fault_active()) { + struct gsp_phys_fault_info fi; + u32 fep = 0; + + if (!gsp_phys_status_drain_fault(&fi, &fep) && + fi.valid && + fi.function == + GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) { + fault = fi; + gsp_phys_fault_drained++; + } else { + gsp_phys_fault_stale++; + } + } + } + poll = w1; + ret = -ETIMEDOUT; + goto out; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + + gsp_phys_gr_ok++; + poll = w1; + pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n", + req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1, + jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); + ret = 0; + +out: + if (out) { + out->sem_value = poll; + out->elapsed_ms = jiffies_to_msecs(jiffies - start); + out->rc = ret; + out->fault = fault; + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_gr_submit); + /* * P6a trigger variant: identical to the exported forwarding call except that * the gsp_phys_work_forward parameter gate is dropped (every other guard and @@ -11296,6 +12002,9 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx); for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]); + for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++) + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs); @@ -11529,6 +12238,30 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) } { /* + * GR (compute) forwarding: the sema service's capture/offer/ + * fallback counters and the booter backend's submit/refusal + * counters plus the parameter and exec-gate state. All zero + * while gsp_phys_gr_forward=0. + */ + u64 gr_req = 0, gr_ok = 0, gr_fallback = 0, gr_refused = 0; + u64 gr_timeout = 0, gr_fault = 0; + u64 b_sub = 0, b_ok = 0, b_refused = 0, b_timeout = 0; + u64 b_pool = 0; + + gsp_sema_service_fwd_gr_stats(&gr_req, &gr_ok, &gr_fallback, + &gr_refused, &gr_timeout, + &gr_fault); + gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, + &b_timeout, &b_pool); + seq_printf(m, + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n", + gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, + gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, + gsp_phys_gr_forward_enabled(), + boot_stage.rmcall_gr_exec_ok); + } + { + /* * P6c C3 fault observation/relay: the booter's 0x1006 * observation counters (seen/stale/drained/wait_unmatched, * epoch) and the sema service's delivery counters @@ -12835,6 +13568,17 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + (u64)GSP_PHYS_RMCALL_GR_CTX_PT_MAX * (2ULL << 20) <= GSP_PHYS_RMCALL_VAS_SIZE, "the GR ctx window must stay below the VAS vaSize limit"); +static_assert(sizeof(struct gsp_phys_gr_page) == 24, + "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); +static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && + sizeof(struct gsp_phys_gr_inline) == 80, + "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); +static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS, + "the GR launch method block (SET_OBJECT + inline-QMD run + REPORT_SEMAPHORE) must fit the bound"); +static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= + GSP_PHYS_RMCALL_COPY_SIZE, + "the GR launch method block must fit one module pushbuffer page"); /* * GspSystemInfo wire ABI, pinned against the reference header compile diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index f462c4e..73c21ad 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -296,6 +296,92 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, u64 *refused, u64 *pool_exhausted); /* + * GR (compute) guest forwarding (gsp_phys_gr_forward=1 AND + * gsp_phys_guest_mem=1; both default off). The caller has already decoded a + * guest GR launch, resolved the guest VAs of the QMD's program/parameter/ + * pointer pages to guest PAs, and resolved those PAs to host PAs. + * + * gsp_phys_gr_page describes one page the launch references: its guest VA + * (where Tier 1 maps the HPA), the host PA (0 = unresolved) and the QMD word + * index of the 64-bit pointer pair that names it (qmd_word < 48, 32 for the + * parameter bank, 48 for the program, GSP_PHYS_GR_QMD_WORD_NONE when no QMD + * word points at the page). A page with an unresolved HPA can still be + * submitted through Tier 2 when the caller supplies captured inline-write + * bytes for it and qmd_word points at it: the backend allocates a ladder + * shadow page, copies the captured bytes and rewrites that QMD word pair. + * + * gsp_phys_gr_inline is one captured GRCE inline write: the destination guest + * VA and its captured payload bytes. + * + * gsp_phys_gr_submit is one launch: the 64 rewritten-QMD source words, the + * optional guest-visible release page for QMD release0 (rel_hpa/rel_off/ + * rel_payload; zero rel_hpa points release0 at the module semaphore), the + * optional guest-visible channel release page (sem_hpa/sem_off/sem_payload; + * zero sem_hpa points the REPORT_SEMAPHORE block at the module semaphore) and + * push identity for logging. + * + * gsp_phys_backend_gr_submit() consumes only the fresh, never-reused 128 B + * PTE fetch lines (one per shadowed page and one per guest release page; the + * GMMU caches PTE fetches at 128 B with no invalidate), refuses unless the + * GR exec gate is latched and the guest-RAM/identity-domain gates are open, + * builds SET_OBJECT + the inline-QMD loader + REPORT_SEMAPHORE on the compute + * subchannel, publishes the GPFIFO entry/USERD/doorbell and polls the module + * semaphore (and the QMD release word when rewritten to a ladder VA). On + * timeout it drains the epoch-gated fault queue into out->fault, exactly like + * the CE relay path. Every failure returns an errno the sema service falls + * back to; no guest state is lost. Returns 0 on a completed launch. + */ +#define GSP_PHYS_GR_MAX_PAGES 16u +#define GSP_PHYS_GR_MAX_INLINE 8u +#define GSP_PHYS_GR_INLINE_BYTES 64u +#define GSP_PHYS_GR_QMD_WORDS 64u +#define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu + +struct gsp_phys_gr_page { + u64 guest_va; /* guest VA (page aligned) */ + u64 hpa; /* host PA (page aligned; 0 = unresolved) */ + u32 qmd_word; /* QMD word of the pointer pair, or NONE */ + u32 reserved; +}; + +struct gsp_phys_gr_inline { + u64 dst_va; /* captured GRCE inline-write destination */ + u32 len; /* payload bytes (<= GSP_PHYS_GR_INLINE_BYTES) */ + u8 data[GSP_PHYS_GR_INLINE_BYTES]; +}; + +struct gsp_phys_gr_submit { + const u32 *qmd; /* 64 rewritten-QMD source words */ + u32 qmd_words; /* must be GSP_PHYS_GR_QMD_WORDS */ + u64 qmd_va; /* guest VA of the QMD (logging only) */ + const struct gsp_phys_gr_page *pages; + u32 n_pages; + const struct gsp_phys_gr_inline *inl; + u32 n_inl; + u64 rel_hpa; /* guest-visible QMD release0 page, or 0 */ + u32 rel_off; + u32 rel_payload; + u64 sem_hpa; /* guest-visible channel release page, or 0 */ + u32 sem_off; + u32 sem_payload; + u64 push_va; /* guest push identity (logging only) */ + u32 push_len; + u32 h_object; /* guest GR channel handle (logging only) */ +}; + +/* True when gsp_phys_gr_forward is set and the ladder is a live GR channel + * whose boot submission proved out (rmcall_gr_exec_ok). */ +bool gsp_phys_backend_gr_ready(void); + +int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, + struct gsp_phys_backend_copy_result *out); + +/* GR forwarding counters (submitted/verified/cancelled), plain reads. */ +void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, + u64 *timed_out, u64 *pool_exhausted); + + +/* * P6c C3 fault observation/relay (gsp_phys_fault_relay=1; default off). * * gsp_phys_status_drain_fault() walks the unconsumed status-queue records with diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 74e3103..b204961 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -92,6 +92,18 @@ bool gsp_phys_srv_ctxbuf_value(void); bool gsp_phys_work_forward_enabled(void); /* + * True when the gsp_phys_gr_forward module parameter is set (default 0: guest + * GR (compute) launches stay on the existing host CUDA bridge / software + * path). When set, the sema service captures a completed guest inline QMD and + * the booter submits it as real work on the ladder's physical GR channel, + * but only after boot_stage.rmcall_gr_exec_ok proved a real GR submission at + * boot (gsp_phys_backend_gr_ready() enforces the gate). Every forward + * refusal/failure falls back to the CUDA bridge (when enabled) and then the + * unchanged software release path. One-line rollback: gsp_phys_gr_forward=0. + */ +bool gsp_phys_gr_forward_enabled(void); + +/* * True when the gsp_phys_guest_mem module parameter is set (default 0: the * debugfs rm-call ladder stays entirely on module-owned buffers, exactly as * before). When set, ladder triggers 6 (W2G) and 7 (G2H) may point the real diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f897ce0..2b7fa80 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -611,6 +611,23 @@ struct sema_svc { u64 fwd_fault_delivered; u64 fwd_fault_no_event; u64 fwd_fault_late_release; + /* + * GR (compute) forwarding (gsp_phys_gr_forward=1 only): candidate + * launches offered to the request path (fwd_gr_req), launches the + * physical GR channel completed (fwd_gr_ok), offerings that fell back + * to the host CUDA bridge / software release (fwd_gr_fallback), gate + * refusals (fwd_gr_refused), backend timeouts (fwd_gr_timeout), + * timeouts carrying a decoded engine fault (fwd_gr_fault) and CUDA + * bridge launch calls made by a fallback (fwd_gr_bridge). All zero + * while the parameter is off. + */ + u64 fwd_gr_req; + u64 fwd_gr_ok; + u64 fwd_gr_fallback; + u64 fwd_gr_refused; + u64 fwd_gr_timeout; + u64 fwd_gr_fault; + u64 fwd_gr_bridge; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -3154,6 +3171,215 @@ struct svc_fwd_cand { static struct svc_fwd_cand svc_fwd_cand; /* + * GR (compute) forwarding candidate (gsp_phys_gr_forward=1 only). When a GR + * channel decodes a completed 64-word inline QMD, the launch is captured here + * instead of being handed straight to the host CUDA bridge: the QMD words, its + * device VA, the push identity, the newest GRCE inline-write records and the + * QMD fields the bridge decodes (release0, parameter bank, program, pointer + * pairs). svc_fwd_request() consumes the capture on the same service thread: + * it queues a physical GR submission, or falls back to the bridge (when + * enabled) and then the unchanged software release path. `offered` is the + * exactly-once latch: a capture is consumed at most once, and a rewind replay + * never re-captures because the QMD gate skips replay passes. Service-thread + * only, like every other decode/pump state. + */ +#define SVC_GR_QMD_WORDS 64 +#define SVC_GR_INLINE_MAX 8 +#define SVC_GR_INLINE_BYTES 64 +#define SVC_GR_POINTER_MAX 8 +#define SVC_GR_PAGES_MAX 16 +#define SVC_GR_QMD_WORD_NONE 0xffffffffu +/* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ +#define SVC_GR_REL_REDUCTION 0x1u +#define SVC_GR_REL_ONE_WORD 0x2u +#define SVC_GR_REL_ENABLE 0x4u + +struct svc_gr_inline { + u64 dst_va; + u32 len; + u8 data[SVC_GR_INLINE_BYTES]; +}; + +/* One resolved launch page carried in the request/backend hand-off. */ +struct svc_gr_page { + u64 va; /* guest VA, page aligned */ + u64 pa; /* guest PA, page aligned (0 = unresolved) */ + u8 ap; /* SVC_APERTURE_* of pa, 0 when pa = 0 */ + u32 qmd_word; /* QMD pointer pair word index, or NONE */ +}; + +struct svc_gr_cand { + bool valid; + bool offered; /* consumed by a request or a fallback */ + u32 h_object; + u64 push_va; + u32 push_len; + u64 push_hash; + u32 qmd[SVC_GR_QMD_WORDS]; + u64 qmd_dev_va; + u32 n_inline; + struct svc_gr_inline inline_recs[SVC_GR_INLINE_MAX]; + /* Decoded by svc_gr_qmd_decode(); mirrors gsp_cuda_bridge.c:745-759. */ + u64 prog_va; + u64 param_va; + u32 param_size; + u64 rel_va; + u32 rel_payload; + u32 rel_flags; /* bit0 reduction, bit1 one-word, bit2 enable */ + u32 n_pointer; + u64 pointer_va[SVC_GR_POINTER_MAX]; + u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ +}; + +static struct svc_gr_cand svc_gr_cand; + +static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, + "the sema GR QMD word count must match the backend ABI"); +static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, + "the sema GR inline capture must fit the backend ABI"); +static_assert(sizeof(struct svc_gr_inline) == 80 && + offsetof(struct svc_gr_inline, data) == 12, + "the sema GR inline record must stay layout-compatible with the backend"); +static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && + offsetof(struct svc_gr_inline, dst_va) == + offsetof(struct gsp_bridge_inline, dst_va) && + offsetof(struct svc_gr_inline, len) == + offsetof(struct gsp_bridge_inline, len) && + offsetof(struct svc_gr_inline, data) == + offsetof(struct gsp_bridge_inline, data), + "the sema GR inline record must be castable to the CUDA bridge record"); +static_assert(sizeof(struct svc_gr_inline) == + sizeof(struct gsp_phys_gr_inline) && + offsetof(struct svc_gr_inline, dst_va) == + offsetof(struct gsp_phys_gr_inline, dst_va) && + offsetof(struct svc_gr_inline, len) == + offsetof(struct gsp_phys_gr_inline, len) && + offsetof(struct svc_gr_inline, data) == + offsetof(struct gsp_phys_gr_inline, data), + "the sema GR inline record must be castable to the backend record"); +static_assert(SVC_GR_PAGES_MAX == GSP_PHYS_GR_MAX_PAGES, + "the sema GR page list must match the backend ABI"); +static_assert(sizeof(struct svc_gr_page) == 24, + "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes"); + +static u64 svc_push_hash(const u8 *p, u32 len); + +/* QMD field decode: the exact shape the host CUDA bridge uses. */ +static void svc_gr_qmd_decode(struct svc_gr_cand *c) +{ + const u32 *q = c->qmd; + u32 i; + + c->rel_va = q[23] | ((u64)(q[24] & 0xff) << 32); + c->rel_payload = q[25]; + c->rel_flags = 0; + if ((q[24] >> 26) & 1u) + c->rel_flags |= SVC_GR_REL_REDUCTION; + if ((q[24] >> 31) & 1u) + c->rel_flags |= SVC_GR_REL_ONE_WORD; + if ((q[4] >> 10) & 1u) + c->rel_flags |= SVC_GR_REL_ENABLE; + c->param_va = q[32] | ((u64)(q[33] & 0x1ffff) << 32); + c->param_size = (q[33] >> 19) << 4; + c->prog_va = q[48] | ((u64)(q[49] & 0x1ffff) << 32); + c->n_pointer = 0; + /* Pointer-region candidates 34..47; 48/49 is the program pair. */ + for (i = 0; i < SVC_GR_POINTER_MAX - 1; i++) { + u32 w = 34 + 2 * i; + u64 va = q[w] | ((u64)(q[w + 1] & 0x1ffff) << 32); + + if (!va) + continue; + c->pointer_va[c->n_pointer] = va; + c->pointer_word[c->n_pointer] = w; + c->n_pointer++; + } +} + +/* Capture one completed QMD as the current GR forwarding candidate. */ +static void svc_gr_cand_capture(struct svc_channel *ch, + const struct svc_push_state *st, + const struct gsp_bridge_inline *recs, u32 n_recs, + const u8 *push, u32 push_len, u64 push_va) +{ + u32 i, n; + + memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); + svc_gr_cand.valid = true; + svc_gr_cand.h_object = ch->h_object; + svc_gr_cand.push_va = push_va; + svc_gr_cand.push_len = push_len; + if (push && push_len) + svc_gr_cand.push_hash = svc_push_hash(push, push_len); + memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd)); + svc_gr_cand.qmd_dev_va = st->qmd_dev_va; + n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX; + for (i = 0; i < n; i++) { + svc_gr_cand.inline_recs[i].dst_va = recs[i].dst_va; + svc_gr_cand.inline_recs[i].len = + recs[i].len < SVC_GR_INLINE_BYTES ? + recs[i].len : SVC_GR_INLINE_BYTES; + memcpy(svc_gr_cand.inline_recs[i].data, recs[i].data, + svc_gr_cand.inline_recs[i].len); + } + svc_gr_cand.n_inline = n; + svc_gr_qmd_decode(&svc_gr_cand); + pr_info("sema-gr: launch captured h=0x%x push=0x%llx len=%u qmd=0x%llx prog=0x%llx param=0x%llx size=0x%x rel=0x%llx/%u flags=0x%x ptr=%u inline=%u\n", + ch->h_object, push_va, push_len, st->qmd_dev_va, + svc_gr_cand.prog_va, svc_gr_cand.param_va, + svc_gr_cand.param_size, svc_gr_cand.rel_va, + svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, + svc_gr_cand.n_pointer, svc_gr_cand.n_inline); +} + +/* + * Consume the current GR candidate without forwarding it: hand the launch to + * the host CUDA bridge when that path is enabled, then clear the candidate. + * Exactly-once via `offered`. Called on the service thread (capture/refusal) + * or the work item (a failed physical submission uses the request copy). + */ +static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu) +{ + u32 n; + + if (!svc_gr_cand.valid || svc_gr_cand.offered) + return; + svc_gr_cand.offered = true; + if (gsp_cuda_bridge_enabled()) { + n = svc_gr_cand.n_inline; + if (n > GSP_BRIDGE_INLINE_MAX) + n = GSP_BRIDGE_INLINE_MAX; + gsp_cuda_bridge_launch(vgpu, svc_gr_cand.h_object, + svc_gr_cand.qmd, + svc_gr_cand.qmd_dev_va, + svc_gr_cand.push_va, + (const struct gsp_bridge_inline *) + svc_gr_cand.inline_recs, n); + svc.fwd_gr_bridge++; + } + svc_gr_cand.valid = false; +} + +/* + * Should this decode pass's releases be recorded instead of applied inline? + * A captured CE copy defers under the existing CE deferral parameter; a + * captured GR launch defers whenever GR forwarding is armed (its channel + * release must land only on the physical submission's success/failure edge). + * Both still require the guest-memory path that makes any hardware + * submission possible. Service-thread only, like the captures. + */ +static bool svc_fwd_defer_arm(void) +{ + if (!gsp_phys_guest_mem_enabled()) + return false; + if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled()) + return true; + return svc_fwd_cand.valid && + gsp_phys_work_defer_release_enabled(); +} + + +/* * P6b wave-1.5 deferred guest releases: every release group the current * decode pass decoded while a forwarding candidate was captured is recorded * here instead of being applied inline. The record carries exactly what the @@ -4475,15 +4701,23 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) /* * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD - * to the host CUDA bridge. The run may cover the address registers and the - * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1 - * count=66 starting at 0x318), so every data word is mapped to its register - * offset. Each completed QMD resets the accumulator, so a push may carry - * more than one launch. + * to its consumer. With gsp_phys_gr_forward=0 the consumer is the host CUDA + * bridge, exactly as before; with the parameter on a trusted decode captures + * the QMD as the current GR forwarding candidate (svc_gr_cand_capture) and + * an untrusted sweep still goes to the bridge, so the new path never acts on + * a content-scanned page. The bridge is then only reached from a + * request-path fallback, so a forwarded launch is never also handed to the + * bridge and a refused/failed forward still falls back in order forward -> + * bridge -> software release. The run may cover the + * address registers and the data registers as a single INC/NON_INC/ONE_INC + * method (observed: op=1 count=66 starting at 0x318), so every data word is + * mapped to its register offset. Each completed QMD resets the accumulator, + * so a push may carry more than one launch. */ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, - const u8 *data) + const u8 *data, const u8 *push, u32 push_len, + bool trusted) { u32 i; @@ -4542,9 +4776,13 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, memcpy(recs[n].data, rec->data, rec->len); n++; } - gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, - st->qmd_dev_va, st->push_va, - recs, n); + if (gsp_phys_gr_forward_enabled() && trusted) + svc_gr_cand_capture(ch, st, recs, n, push, push_len, + st->push_va); + else + gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, + st->qmd_dev_va, st->push_va, + recs, n); st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -4554,7 +4792,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, - u64 push_va, bool trusted, u64 *ops_done) + u64 push_va, bool trusted, bool replay, + u64 *ops_done) { struct svc_push_state st; /* @@ -4595,6 +4834,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc_fwd_cand.valid = false; svc_fwd_cand.skipped = false; + /* + * A fresh (non-replay) trusted decode pass invalidates any previous GR + * capture: the pump consumes a capture immediately after the pass that + * built it, so anything still here is stale. Replay/rescue decodes + * must not wipe a capture the pump has not consumed yet. + */ + if (!replay && trusted && ch->engine_type == 0) + memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; svc.pages_decoded++; @@ -4739,13 +4986,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * must fall through to the CE decoder. */ if (ch->engine_type == 0 && subch >= 1 && - gsp_cuda_bridge_enabled() && + (!gsp_phys_gr_forward_enabled() || !replay) && + (gsp_cuda_bridge_enabled() || + gsp_phys_gr_forward_enabled()) && addr >= SVC_QMD_ADDR_A && addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && (st.qmd_have || (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) { svc_qmd_accum(vgpu, ch, &st, addr, count, - buf + off + 4); + buf + off + 4, buf, len, trusted); off += 4 + count * 4; continue; } @@ -4871,9 +5120,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * The pre-apply bookkeeping above is unchanged; * only the apply moves. Overflow applies inline. */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid && - gsp_phys_guest_mem_enabled()) { + if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -4988,9 +5235,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * once the forwarded copy completes. * Overflow applies inline. */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid && - gsp_phys_guest_mem_enabled()) { + if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -5386,9 +5631,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * the record path so the pump still sees the * release as decoded. Overflow applies inline. */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid && - gsp_phys_guest_mem_enabled()) { + if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -7730,6 +7973,32 @@ struct svc_fwd_req { * (valid=false) for every non-fault outcome. */ struct gsp_phys_fault_info fault; + /* + * GR (compute) forwarding (gsp_phys_gr_forward=1 only): when gr is + * set the request is a captured guest launch, not a CE copy. The work + * item resolves each page entry's guest PA to an HPA, copies the + * captured inline bytes into the backend's shadow pages and calls + * gsp_phys_backend_guest_gr_submit(); gr_bridge_done is the + * exactly-once latch for the work-item CUDA bridge fallback. + */ + bool gr; + bool gr_bridge_done; + bool gr_page_overflow; + u32 gr_h_object; + u64 gr_push_va; + u32 gr_push_len; + u64 gr_qmd_dev_va; + u32 gr_rel_flags; + u64 gr_rel_va; + u32 gr_rel_payload; + u64 gr_prog_va; + u64 gr_param_va; + u32 gr_param_size; + u32 gr_qmd[SVC_GR_QMD_WORDS]; + u32 gr_n_inline; + struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX]; + u32 gr_n_pages; + struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX]; }; static struct svc_fwd_req svc_fwd_req; @@ -7910,6 +8179,13 @@ struct svc_fwd_hold { */ struct nv_vgpu_device *vgpu; u32 ch_handle; + /* + * The channel's engine type at engage time: the shim channels the + * release-apply paths build must preserve it so a GR (engine_type 0) + * release lands through the exact completion-counter slot, exactly as + * the inline path would have. CE holds stay 1. + */ + u32 engine_type; u64 gpfifo_va; u32 entry_idx; u8 nrel; @@ -8080,6 +8356,7 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.cand = svc_fwd_cand; svc_fwd_hold.vgpu = vgpu; svc_fwd_hold.ch_handle = ch->h_object; + svc_fwd_hold.engine_type = ch->engine_type; svc_fwd_hold.gpfifo_va = ch->gpfifo_va; svc_fwd_hold.entry_idx = entry_idx; svc_fwd_hold.nrel = n; @@ -8277,7 +8554,7 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ + shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } @@ -8441,7 +8718,7 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ + shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } @@ -8493,7 +8770,7 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ + shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } @@ -8508,6 +8785,135 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc_fwd_hold_clear(); } +static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); + +/* + * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The + * request already resolved every launch page's guest VA->PA on the service + * thread; here each PA is turned into an HPA (SYS through the guest-RAM + * backing, VID through the BAR1 shadow), the captured inline bytes are handed + * to the backend for its Tier-2 shadow pages and the whole launch is + * submitted through gsp_phys_backend_guest_gr_submit(). On any failure the + * host CUDA bridge is offered the launch once (the forward -> bridge -> + * software fallback order), then the pump's hold applies the recorded + * releases. Runs on the gsp-phys-work item with the same lock discipline as + * the CE branch. + */ +static void svc_fwd_gr_work(void) +{ + /* + * Static: the ordered gsp-phys-work queue runs one item at a time and + * the GR page array is ~0.4 KB; keeping it off the stack keeps the + * frame under the kernel's warning bound (same reason as the ops array + * in svc_decode_stream). + */ + static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX]; + struct gsp_phys_gr_submit gs; + struct gsp_phys_backend_copy_result cp = { 0 }; + u64 rel_hpa = 0; + u32 off = 0, i; + int hw_rc; + + memset(&gs, 0, sizeof(gs)); + memset(pages, 0, sizeof(pages)); + + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i]; + u8 kind = (pg->ap == SVC_APERTURE_FB) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + u64 hpa = 0; + + if (pg->pa) { + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, pg->pa, + 0, SEMA_PAGE_SIZE, &hpa, + &off) || off) { + hpa = 0; + svc.fwd_guest_resolve_fail++; + } + } + pages[i].guest_va = pg->va; + pages[i].hpa = hpa; + pages[i].qmd_word = pg->qmd_word; + } + + if (svc_fwd_req.gr_rel_va && + (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) && + !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) && + (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) { + u64 page = svc_fwd_req.gr_rel_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 roff = (u32)(svc_fwd_req.gr_rel_va & + (SEMA_PAGE_SIZE - 1)); + + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, SVC_APERTURE_SYS, page, + 0, SEMA_PAGE_SIZE, &rel_hpa, &off) || + off) + rel_hpa = 0; + /* + * Forward-only: a release word already at/ahead of the payload + * means the engine would author nothing this launch; point + * release0 at the module semaphore instead so the host still + * observes completion. + */ + if (rel_hpa && + READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= + svc_fwd_req.gr_rel_payload) + rel_hpa = 0; + } + + gs.qmd = svc_fwd_req.gr_qmd; + gs.qmd_words = SVC_GR_QMD_WORDS; + gs.qmd_va = svc_fwd_req.gr_qmd_dev_va; + gs.pages = pages; + gs.n_pages = svc_fwd_req.gr_n_pages; + gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline; + gs.n_inl = svc_fwd_req.gr_n_inline; + if (gs.n_inl > GSP_PHYS_GR_MAX_INLINE) + gs.n_inl = GSP_PHYS_GR_MAX_INLINE; + gs.rel_hpa = rel_hpa; + gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); + gs.rel_payload = svc_fwd_req.gr_rel_payload; + gs.sem_hpa = 0; + gs.sem_off = 0; + gs.sem_payload = 0; + gs.push_va = svc_fwd_req.gr_push_va; + gs.push_len = svc_fwd_req.gr_push_len; + gs.h_object = svc_fwd_req.gr_h_object; + + hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + if (!hw_rc) { + svc.fwd_gr_ok++; + } else { + svc.fwd_gr_fallback++; + if (hw_rc == -ETIMEDOUT) { + svc.fwd_gr_timeout++; + if (cp.fault.valid) + svc.fwd_gr_fault++; + } + if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", + hw_rc, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, svc.fwd_gr_req, + svc.fwd_gr_ok, svc.fwd_gr_fallback); + svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); + } + + svc_fwd_req.fault = cp.fault; + svc_fwd_req.hw_rc = hw_rc; + /* + * done before INFLIGHT: a reuser that acquires the bit can never + * overwrite the slot before this outcome is visible. + */ + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + { + struct task_struct *task = READ_ONCE(svc.task); + + if (task) + wake_up_process(task); + } +} + static void svc_fwd_work_fn(struct work_struct *work) { bool guest = svc_fwd_req.guest; @@ -8517,6 +8923,12 @@ static void svc_fwd_work_fn(struct work_struct *work) int ret; int hw_rc = 0; + /* GR forwarding requests take their own branch and outcome shape. */ + if (svc_fwd_req.gr) { + svc_fwd_gr_work(); + return; + } + /* * P6b wave 1: try the guest hardware copy first when the pump marked * this request guest. P6b wave 1.5: when the request's releases are @@ -8733,6 +9145,230 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req) } /* + * GR request page collection: append (dedupe) one launch page, resolving its + * guest VA->PA on the service thread now (the work item only does PA->HPA). + * An unresolved page keeps pa=0; whether that is fatal is decided by + * svc_gr_req_fill from its QMD pointer association and the captured inline + * bytes. Overflow is a failure (fail-closed), not a silent drop. + */ +static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + u32 ap = 0; + u32 i; + + if (!page) + return; + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + if (svc_fwd_req.gr_pages[i].va != page) + continue; + if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE && + qmd_word != SVC_GR_QMD_WORD_NONE) + svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + return; + } + if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) { + svc_fwd_req.gr_page_overflow = true; + return; + } + i = svc_fwd_req.gr_n_pages++; + svc_fwd_req.gr_pages[i].va = page; + svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + if (svc_resolve_va(vgpu, page, &pa, &ap)) + return; + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); +} + +/* Does any captured inline write land in this page? */ +static bool svc_gr_page_inline_covers(u64 page) +{ + u32 i; + + for (i = 0; i < svc_gr_cand.n_inline; i++) { + if ((svc_gr_cand.inline_recs[i].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return true; + } + return false; +} + +/* + * Copy the current GR capture into the request slot (INFLIGHT is held). Every + * launch page is collected and resolved; a page with no resolved PA is only + * acceptable when a QMD pointer pair names it and captured inline bytes cover + * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller + * falls back on. + */ +static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 i; + + svc_fwd_req.done = false; + svc_fwd_req.hw_rc = 0; + memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); + svc_fwd_req.guest = false; + svc_fwd_req.gr = true; + svc_fwd_req.gr_bridge_done = false; + svc_fwd_req.defer = true; + svc_fwd_req.relay = false; + svc_fwd_req.relay_index = 0; + svc_fwd_req.vgpu = vgpu; + svc_fwd_req.gr_h_object = ch->h_object; + svc_fwd_req.gr_push_va = svc_gr_cand.push_va; + svc_fwd_req.gr_push_len = svc_gr_cand.push_len; + svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; + svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags; + svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va; + svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload; + svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va; + svc_fwd_req.gr_param_va = svc_gr_cand.param_va; + svc_fwd_req.gr_param_size = svc_gr_cand.param_size; + memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, + sizeof(svc_fwd_req.gr_qmd)); + svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; + if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) + svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; + memcpy(svc_fwd_req.gr_inline, svc_gr_cand.inline_recs, + sizeof(svc_fwd_req.gr_inline)); + svc_fwd_req.gr_n_pages = 0; + svc_fwd_req.gr_page_overflow = false; + + if (svc_gr_cand.param_va) + svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); + if (svc_gr_cand.prog_va) + svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); + for (i = 0; i < svc_gr_cand.n_pointer; i++) + svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], + svc_gr_cand.pointer_word[i]); + for (i = 0; i < svc_gr_cand.n_inline; i++) + svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, + SVC_GR_QMD_WORD_NONE); + if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) + return -E2BIG; + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i]; + + if (pg->pa) + continue; + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) + return -ENOENT; + if (!svc_gr_page_inline_covers(pg->va)) + return -ENOENT; + } + return 0; +} + +/* + * Exactly-once CUDA bridge fallback from the request copy (work item or a + * failed queue): hand the captured launch to the bridge when enabled. The + * latch is per request slot and the slot is single-inflight. + */ +static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) +{ + u32 n; + + if (!svc_fwd_req.gr || svc_fwd_req.gr_bridge_done) + return; + svc_fwd_req.gr_bridge_done = true; + if (!gsp_cuda_bridge_enabled()) + return; + n = svc_fwd_req.gr_n_inline; + if (n > GSP_BRIDGE_INLINE_MAX) + n = GSP_BRIDGE_INLINE_MAX; + gsp_cuda_bridge_launch(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_qmd, svc_fwd_req.gr_qmd_dev_va, + svc_fwd_req.gr_push_va, + (const struct gsp_bridge_inline *) + svc_fwd_req.gr_inline, n); + svc.fwd_gr_bridge++; +} + +/* + * GR request path (gsp_phys_gr_forward=1 only), called by svc_fwd_request() + * for engine_type==0 channels. Queues one physical submission when the GR + * backend is ready; otherwise consumes the capture through the bridge + * fallback. Reuses the CE INFLIGHT/paused/hold exclusion so GR and CE + * forwarding can never overlap; the caller engages the release hold on a + * true return. Service-thread only. + */ +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + int rc; + + if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid) + return false; + svc.fwd_gr_req++; + if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + svc_gr_bridge_fallback(vgpu); + return false; + } + /* + * Never reuse the request slot while a hold is active; the held + * request's done/hw_rc are still being read. The capture falls back. + */ + if (svc_fwd_hold.active) { + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + svc_gr_bridge_fallback(vgpu); + return false; + } + mutex_lock(&svc_fwd_lock); + if (svc_fwd_paused || + test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + svc_gr_bridge_fallback(vgpu); + return false; + } + rc = svc_gr_req_fill(vgpu, ch); + if (rc) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", + rc, ch->h_object, svc_gr_cand.push_va); + svc_gr_bridge_fallback(vgpu); + return false; + } + /* + * The candidate is consumed: the request copy owns it from here, so a + * later capture/replay cannot re-offer it and the bridge is only + * reached through the request's own latch. + */ + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + svc_fwd_requests++; + if (!svc_fwd_wq) { + svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); + if (!svc_fwd_wq) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + svc_gr_req_bridge_fallback(vgpu); + return false; + } + } + if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + svc_gr_req_bridge_fallback(vgpu); + return false; + } + mutex_unlock(&svc_fwd_lock); + return true; +} + +/* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. @@ -8741,7 +9377,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { if (!ch->engine_type) - return false; + return svc_gr_request(vgpu, ch); if (!gsp_phys_work_forward_enabled()) return false; /* Classify this decode pass's candidate for the run summary. */ @@ -8845,7 +9481,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, return false; } mutex_unlock(&svc_fwd_lock); - return svc_fwd_req.guest; + return svc_fwd_req.guest || svc_fwd_req.gr; } /* @@ -8998,6 +9634,27 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, } /* + * GR forwarding counters for the gsp-phys-state work_gr line (see + * gsp_sema_service.h). Same plain-read discipline as the stats above. + */ +void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback, + u64 *refused, u64 *timeout, u64 *fault) +{ + if (req) + *req = svc.fwd_gr_req; + if (ok) + *ok = svc.fwd_gr_ok; + if (fallback) + *fallback = svc.fwd_gr_fallback; + if (refused) + *refused = svc.fwd_gr_refused; + if (timeout) + *timeout = svc.fwd_gr_timeout; + if (fault) + *fault = svc.fwd_gr_fault; +} + +/* * P6c: true while this channel owns an active RELAY hold. The pump's * per-pass notifier is suppressed for such a channel so the hold's verified * success edge (or the next pass after a failure resolution) is the single @@ -9479,6 +10136,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, injected, + entry_replay, entry_replay ? NULL : &entry_done); @@ -9684,10 +10342,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!svc_rescue_push_page(vgpu, ch, va, len) && !svc_read_va_push(vgpu, ch, va, pb, len)) - svc_decode_stream(vgpu, pb, - len, 0, false, - ch, idx, va, true, - NULL); + svc_decode_stream(vgpu, pb, + len, 0, false, + ch, idx, va, true, + true, + NULL); rel_seen = (svc_rel_count != rel_n2); /* @@ -9713,6 +10372,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true, + true, NULL); rel_seen = (svc_rel_count != rel_n2); @@ -9948,6 +10608,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) SVC_ENTRY_NONE, va, false, + false, NULL); if (svc_rel_count != rel_n_before) ch->last_rel_va = diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 22e9edc..4b04892 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -148,6 +148,17 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, u64 *late_release); /* + * GR (compute) forwarding counters for the gsp-phys-state work_gr line: + * captures offered to the request path (req), launches the physical GR + * channel completed (ok), offerings that fell back to the CUDA bridge / + * software release (fallback), gate refusals (refused), backend timeouts + * (timeout) and timeouts carrying a decoded engine fault (fault). All zero + * while gsp_phys_gr_forward=0. Plain reads, safe from the debugfs show path. + */ +void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback, + u64 *refused, u64 *timeout, u64 *fault); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). diff --git a/kernel/main.c b/kernel/main.c index f662b94..258de41 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -174,6 +174,7 @@ static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops; static struct mdev_parent *g_mdev_parent = NULL; static bool gsp_phys_gpu_param; +static bool gsp_phys_gr_forward_param; /* VFIO ops implementation */ static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id); @@ -2596,6 +2597,23 @@ bool gsp_phys_gpu_enabled(void) } /* + * GR (compute) forwarding: when set, a guest GR channel's decoded inline-QMD + * launch is captured by the sema service and submitted as real work on the + * ladder's physical GR channel instead of (or before) the host CUDA bridge. + * Default off: every path is inert and the bridge/software emulation stays + * byte-identical. The backend additionally refuses until a real GR + * submission has proved out at boot (boot_stage.rmcall_gr_exec_ok), so the + * parameter alone can never make the module pretend GR works. + */ +module_param_named(gsp_phys_gr_forward, gsp_phys_gr_forward_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_forward, "Forward guest GR (compute) launches to the physical GR engine on the debugfs ladder channel (default 0; requires gsp_phys_gpu + a proven GR ladder submission)"); + +bool gsp_phys_gr_forward_enabled(void) +{ + return gsp_phys_gr_forward_param; +} + +/* * GR ring discovery: pin the compute channels' rings/USERDs, which follow the * CE rings in the channel video-memory allocation. Default on; set 0 to leave * GR channels untranslated when debugging push-execution side effects. |