From d3f061966bb9e23fdba7f6cc0aa231acdf31f36e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 5 Oct 2026 13:06:39 -0400 Subject: vgsp: guest GR forwarding backend and capture (gsp_phys_gr_forward, default off) Add a default-off guest GR (compute) forwarding path so a guest CUDA launch submitted on its GR channel can be executed by the physical GR engine on the ladder channel, with fail-closed fallback to the host CUDA bridge and the unchanged software release path. - New module parameter gsp_phys_gr_forward (default 0) and accessor. - New GR execution gate: boot_stage.rmcall_gr_exec_ok, latched only after a real GR submission logs REAL GR SEM OK; the backend refuses while it is unset. - Sema capture: a completed 64-word inline QMD on a trusted, non-replay GR decode becomes an svc_gr_cand (push identity, decoded release0, parameter/program/pointer fields, up to 8 captured inline-write records); the CUDA bridge stays authoritative while the parameter is off and is reached in order forward -> bridge -> software release. - Backend: gsp_phys_backend_guest_gr_submit() maps pages at their own guest VAs through fresh Tier-1 small-PT pages/PDE installs or relocates inline-captured pages to ladder shadow pages (Tier 2), builds SET_OBJECT + the inline-QMD method run + REPORT_SEMAPHORE on the compute subchannel, reuses the shared GPFIFO/USERD/doorbell publish tail (extracted from the CE submit path, CE bytes unchanged) and polls the QMD release and/or module semaphore, draining the epoch-gated fault queue on timeout. - Sema wiring: GR requests reuse the INFLIGHT/hold/deferral machinery so the guest channel release applies only on the submit's success/failure edge; hold shims preserve the channel engine type. - Counters and a work_gr state line plus a stats accessor. - Bounds are fixed: 16 pages, 8 inline records, 64 QMD words; every pointer/size checked, pool exhaustion and stale mappings fail closed. - New ABI layouts pinned with static_asserts. Compile-only change: no device test performed. (cherry picked from commit ae507e6f241c7e3498728f9d1ffc2d41ede5cd64) --- kernel/gsp_phys_booter.c | 864 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 804 insertions(+), 60 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 3a65361..bf70cd5 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -613,6 +613,21 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) +/* + * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources, + * allocated lazily under gsp_phys_rm_lock and freed with the staging. The + * Tier-1 window pool holds small-PT pages installed with DMA_UPDATE_PDE_2 at + * the guest's own 2 MiB VA windows (one PDE per window); the mapper's + * conservative no-cached-line rule only allows a new PTE in a fetch line the + * current submission installed, and never rewrites a stale one. The Tier-2 + * shadow chunk backs relocated parameter pages whose content was captured as + * inline writes. Both are deliberately outside rmcall_chunks so the COPY0 + * stage-2 registration walk stays byte-identical. + */ +#define GSP_PHYS_RMCALL_GR_PT_PAGES 8u +#define GSP_PHYS_RMCALL_GR_PT_ENTRIES 32u +#define GSP_PHYS_RMCALL_GR_SHADOW_PAGES 4u + /* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 @@ -1437,6 +1452,29 @@ static struct gsp_phys_boot_stage { */ u64 rmcall_va_guest_pt; /* guest scratch-PT VA window base */ u32 rmcall_guest_pt_next_line; /* next never-used 128 B PTE line */ + /* + * GR forwarding (gsp_phys_gr_forward=1; inert otherwise). The exec + * gate latches only after the stage-4 real GR submission logs + * REAL GR SEM OK, so the forward path can never pretend to work while + * the ladder itself cannot execute GR. The window pool and shadow + * chunk are allocated lazily by the forwarding submit. + */ + bool rmcall_gr_exec_ok; /* a real GR submission proved out */ + u32 rmcall_gr_submit_epoch; /* per-submission mapper epoch */ + u32 rmcall_gr_pt_used; /* Tier-1 window pool slots used */ + struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES]; + struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */ + struct { + u32 pde_index; + u8 pt_slot; + u8 used; + u8 pte_count; + struct { + u32 idx; /* small-PT index in the 2 MiB window */ + u32 epoch; /* submission that installed it */ + u64 hpa; /* the PTE's mapped host PA */ + } pte[GSP_PHYS_RMCALL_GR_PT_ENTRIES]; + } rmcall_gr_win[GSP_PHYS_RMCALL_GR_PT_PAGES]; struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -2040,6 +2078,13 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_guest_selftest_ok = false; boot_stage.rmcall_va_guest_pt = 0; boot_stage.rmcall_guest_pt_next_line = 0; + boot_stage.rmcall_gr_exec_ok = false; + boot_stage.rmcall_gr_submit_epoch = 0; + boot_stage.rmcall_gr_pt_used = 0; + memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt)); + memset(&boot_stage.rmcall_gr_shadow, 0, + sizeof(boot_stage.rmcall_gr_shadow)); + memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win)); for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -9676,84 +9721,39 @@ struct gsp_phys_rmcall_ce_submit { }; /* - * One CE copy block on the ladder channel: clear the module semaphore, build - * the methods for bytes into the module pushbuffer, publish the GPFIFO entry - * and USERD GP_PUT and ring the doorbell. The caller must hold - * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical - * lock-across-poll; the P5 work API polls without the lock). Every guard of - * the original stage-4 submission body is preserved here: server-memory - * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the - * token check. The semaphore is cleared before the doorbell so a stale release - * from a previous run cannot be observed as this submission's completion. - * P6c: sem_payload is the release value the method block carries; legacy - * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is - * still cleared even when the caller polls a guest page: the clear only - * guards the module-sem poll path and is a harmless non-guest write. - * - * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release - * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and - * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call - * site passes false, so the default COPY0 ladder emits the same bytes as - * before. + * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET, + * encode and write the GPFIFO entry for the module pushbuffer VA, advance + * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted + * verbatim from gsp_phys_rmcall_ce_submit_locked so the CE path and the GR + * launch path publish byte-identically; the CE caller's semaphore clear and + * method-block build happen before this call. The caller must hold + * gsp_phys_rm_lock. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, + u32 words, + struct gsp_phys_rmcall_ce_submit *out) { - struct gsp_phys_boot_chunk *pb = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; - resource_size_t bar0_len; - u32 *pb_va, *userd, *ring_va; - u32 words, gpput, gpget, slot, e0, e1; + u32 *ring_va, *userd; + u32 gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; - u64 pb_gpu_va; - - if (boot_stage.rmcall_channel_server_mem) - return -EOPNOTSUPP; - if (!boot_stage.pdev) - return -EINVAL; - bar0_len = pci_resource_len(boot_stage.pdev, 0); - if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) - return -EIO; - if (!bar0 || !boot_stage.rmcall_token || !bytes || - bytes > GSP_PHYS_RMCALL_COPY_SIZE) - return -EINVAL; - pb_gpu_va = boot_stage.rmcall_va_pb; - if (!pb_gpu_va || (pb_gpu_va & 3)) - return -EIO; - pb_va = gsp_phys_chunk_page_va(pb, 0); ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); - if (!pb_va || !ring_va || !userd || - !gsp_phys_chunk_page_va(sem, 0) || - !gsp_phys_chunk_page_va(src, 0) || - !gsp_phys_chunk_page_va(dst, 0)) + if (!ring_va || !userd) return -EIO; - /* A stale release from a previous submission must not be observed. */ - WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); - wmb(); - - if (gr) - words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va, - sem_payload); - else - words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, - dst_va, bytes, - sem_payload); - gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { @@ -9831,6 +9831,87 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, return 0; } +/* + * One CE copy block on the ladder channel: clear the module semaphore, build + * the methods for bytes into the module pushbuffer, publish the GPFIFO entry + * and USERD GP_PUT and ring the doorbell. The caller must hold + * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical + * lock-across-poll; the P5 work API polls without the lock). Every guard of + * the original stage-4 submission body is preserved here: server-memory + * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the + * token check. The semaphore is cleared before the doorbell so a stale release + * from a previous run cannot be observed as this submission's completion. + * P6c: sem_payload is the release value the method block carries; legacy + * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is + * still cleared even when the caller polls a guest page: the clear only + * guards the module-sem poll path and is a harmless non-guest write. + * + * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release + * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and + * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call + * site passes false, so the default COPY0 ladder emits the same bytes as + * before. + */ +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + struct gsp_phys_boot_chunk *ring = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + struct gsp_phys_boot_chunk *userd_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + resource_size_t bar0_len; + u32 *pb_va, *userd, *ring_va; + u32 words; + u64 pb_gpu_va; + + if (boot_stage.rmcall_channel_server_mem) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_gpu_va = boot_stage.rmcall_va_pb; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + + pb_va = gsp_phys_chunk_page_va(pb, 0); + ring_va = gsp_phys_chunk_page_va(ring, 0); + userd = gsp_phys_chunk_page_va(userd_c, 0); + if (!pb_va || !ring_va || !userd || + !gsp_phys_chunk_page_va(sem, 0) || + !gsp_phys_chunk_page_va(src, 0) || + !gsp_phys_chunk_page_va(dst, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + if (gr) + words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va, + sem_payload); + else + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, + dst_va, bytes, + sem_payload); + + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); +} + /* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller @@ -9902,6 +9983,12 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); + /* + * Latch the GR execution gate: this is the only place a real GR + * submission is proven on the ladder, and gsp_phys_backend_gr_ready() + * refuses every forwarded launch until this is observed. + */ + boot_stage.rmcall_gr_exec_ok = true; return 0; } @@ -10011,6 +10098,36 @@ bool gsp_phys_backend_work_ready(void) } EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready); +/* + * GR forwarding readiness: the parameter, a live client-memory GR ladder + * channel and the latched boot proof that a real GR submission executed + * (rmcall_gr_exec_ok, set only by a REAL GR SEM OK). This is deliberately + * separate from gsp_phys_backend_ladder_ready(), which refuses CE work on a + * GR0 ladder; a GR0 ladder is exactly the GR forwarding prerequisite. Every + * submit re-checks the same state under gsp_phys_rm_lock. + */ +bool gsp_phys_backend_gr_ready(void) +{ + if (!gsp_phys_gr_forward_enabled()) + return false; + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return false; + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) + return false; + if (!gsp_phys_rmcall_engine_is_gr() || !boot_stage.rmcall_gr_exec_ok) + return false; + if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || + !boot_stage.rmcall_token) + return false; + if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo || + !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_sem || + !boot_stage.rmcall_pdb_root) + return false; + return true; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_ready); + bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va, u32 *bytes) { @@ -10767,6 +10884,595 @@ int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); +/* ------------------------------------------------------------------ */ +/* GR (compute) forwarding backend (gsp_phys_gr_forward) */ +/* ------------------------------------------------------------------ */ + +/* + * C5C0 inline-QMD method window: SET_INLINE_QMD_ADDRESS_A/B carry the + * shifted QMD device address and LOAD_INLINE_QMD_DATA the 64 QMD words; the + * guest emits the window as one INC run at 0x318 with count 66 (A, B, then 64 + * words), and the launch is replayed in exactly that shape with the QMD words + * the sema service decoded (release0 and, under Tier 2, relocated pointers + * rewritten). The channel release is the same A..D block as the H1 GR sem + * verify, with a payload distinct from the CE/module sentinel. + */ +#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A 0x318u +#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_B 0x31cu +#define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA 0x320u +/* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */ +#define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS 66u +#define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD GSP_PHYS_RMCALL_SEM_PAYLOAD +#define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD 0xc0ffee02u +/* Method block bound: SET_OBJECT(2) + inline run(67) + REPORT_SEM(8). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 96u + +static u64 gsp_phys_gr_submitted; +static u64 gsp_phys_gr_ok; +static u64 gsp_phys_gr_refused; +static u64 gsp_phys_gr_timed_out; +static u64 gsp_phys_gr_pool_exhausted; + +void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, + u64 *timed_out, u64 *pool_exhausted) +{ + if (submitted) + *submitted = gsp_phys_gr_submitted; + if (ok) + *ok = gsp_phys_gr_ok; + if (refused) + *refused = gsp_phys_gr_refused; + if (timed_out) + *timed_out = gsp_phys_gr_timed_out; + if (pool_exhausted) + *pool_exhausted = gsp_phys_gr_pool_exhausted; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_counters); + +/* + * GR launch method block: SET_OBJECT on the compute subchannel, the inline + * QMD run at its original shape and the channel REPORT_SEMAPHORE release. + * Returns the block size in dwords. + */ +static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd, + u64 qmd_va, u64 sem_va, + u32 sem_payload) +{ + u32 *p = pb; + + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT, + GSP_PHYS_RMCALL_CLASS_COMPUTE); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) | + ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) | + (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2); + *p++ = (u32)(qmd_va >> 40); + *p++ = (u32)((qmd_va >> 8) & 0xffffffffu); + memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4); + p += GSP_PHYS_GR_QMD_WORDS; + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + return (u32)(p - pb); +} + +/* + * Tier-1 window pool: find (or install) the small-PT page for one guest + * 2 MiB VA window through DMA_UPDATE_PDE_2. Installation is once per window + * per staging, so a later launch in the same window writes PTEs into the same + * server-installed PT page. Called with gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_win_slot_locked(void __iomem *bar0, + u32 pde_index, u32 *slot_out) +{ + struct gsp_phys_rmcall_update_pde2_params p; + struct gsp_phys_boot_chunk *pt; + u32 rpc_result = 0, status = 0; + u32 i, slot; + int ret; + + for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) { + if (boot_stage.rmcall_gr_win[i].used && + boot_stage.rmcall_gr_win[i].pde_index == pde_index) { + *slot_out = i; + return 0; + } + } + if (boot_stage.rmcall_gr_pt_used >= GSP_PHYS_RMCALL_GR_PT_PAGES) + return -ENOSPC; + slot = boot_stage.rmcall_gr_pt_used; + pt = &boot_stage.rmcall_gr_pt[slot]; + if (!pt->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, pt, 1u); + if (ret) + return ret; + gsp_phys_chunk_zero(pt); + } + memset(&p, 0, sizeof(p)); + p.pde_index = cpu_to_le32(pde_index); + p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); + p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0)); + p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); + p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); + p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + p.pde_buffer = 0; + p.sub_device_id = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2, + &p, sizeof(p), sizeof(p), &p, sizeof(p), + &rpc_result, &status, "GR forward PDE_UPDATE"); + if (ret) { + pr_err("vgsp-gr: forward PDE install refused pdeIndex=%u (window VA 0x%llx) status=0x%x rpc_result=0x%x ret=%d\n", + pde_index, (u64)pde_index << 21, status, rpc_result, ret); + return ret; + } + boot_stage.rmcall_gr_win[slot].pde_index = pde_index; + boot_stage.rmcall_gr_win[slot].pt_slot = (u8)slot; + boot_stage.rmcall_gr_win[slot].used = true; + boot_stage.rmcall_gr_win[slot].pte_count = 0; + boot_stage.rmcall_gr_pt_used = slot + 1; + pr_info("vgsp-gr: forward PDE installed pdeIndex=%u slot=%u ptPa=0x%llx status=0x%x\n", + pde_index, slot, (u64)gsp_phys_chunk_page_dma(pt, 0), status); + *slot_out = slot; + return 0; +} + +/* + * Tier-1 map: write one guest page's HPA into the PT page of its own 2 MiB + * VA window at the guest VA itself, so the QMD and every pointer inside the + * mapped pages resolve without rewriting. The GMMU caches 128 B PTE fetch + * lines with no invalidate, so the mapper never rewrites an installed PTE + * (a different HPA for the same VA is refused rather than made stale) and + * only adds a new PTE to a fetch line this same submission already touched. + * Every refusal is fail-closed: the caller drops to Tier 2 or refuses the + * launch and the sema service falls back. Called with gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, + u64 hpa, u32 epoch) +{ + u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 pde_index = (u32)(page >> 21); + u32 pte_idx = (u32)((page >> 12) & 0x1ffu); + u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; + struct gsp_phys_boot_chunk *pt; + u8 *pt_va; + u32 slot, i; + int ret; + + ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); + if (ret) + return ret; + for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) { + if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) { + if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == hpa) + return 0; /* already mapped, no write */ + return -EALREADY; /* stale: never rewrite */ + } + if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch && + boot_stage.rmcall_gr_win[slot].pte[i].idx / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line) + return -EBUSY; /* fetch line may be cached */ + } + if (boot_stage.rmcall_gr_win[slot].pte_count >= + GSP_PHYS_RMCALL_GR_PT_ENTRIES) + return -ENOSPC; + pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot]; + pt_va = gsp_phys_chunk_page_va(pt, 0); + if (!pt_va) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + pt_va + (size_t)pte_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; + boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch; + boot_stage.rmcall_gr_win[slot].pte[i].hpa = hpa; + boot_stage.rmcall_gr_win[slot].pte_count++; + return 0; +} + +/* + * Tier-2 map: consume one never-reused 128 B fetch line from the guest + * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the + * line's base VA. The shadow chunk is allocated lazily; page slot `slot` + * supplies the backing page. Called with gsp_phys_rm_lock held. + */ +static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, + u64 *va_out) +{ + struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; + struct gsp_phys_boot_chunk *pt; + u32 line = boot_stage.rmcall_guest_pt_next_line; + u32 page, line_in_page; + u64 va; + u8 *pt_va; + int ret; + + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) + return ret; + gsp_phys_chunk_zero(sh); + } + if (slot >= sh->pages) + return -ENOSPC; + if (!boot_stage.rmcall_va_guest_pt) + return -EIO; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + return -ENOSPC; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; + if (!pt->pages) + return -EIO; + pt_va = gsp_phys_chunk_page_va(pt, 0); + if (!pt_va) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page) * 8); + dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1; + va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + ((u64)line_in_page << 16); + *va_out = va; + return 0; +} + +/* GR forward fault observation is armed by the GR parameters, not the CE + * ones (gsp_phys_fault_relay_active() requires gsp_phys_work_forward). */ +static bool gsp_phys_gr_fault_active(void) +{ + return gsp_phys_fault_relay_enabled() && + gsp_phys_gr_forward_enabled() && + gsp_phys_guest_mem_enabled(); +} + +/* + * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The + * caller has resolved the launch's pages; every mapping is fresh or proven + * unchanged, the QMD release0 and the channel release are authored either by + * the engine into ladder-mapped shadow pages or into the module semaphore, + * and the GPFIFO/USERD/doorbell tail is the shared publish helper. The + * release poll runs without gsp_phys_rm_lock; a timeout drains the status + * queue (epoch-gated) into out->fault. Every refusal moves the GR counters + * and returns an errno the sema service falls back on. + */ +int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, + struct gsp_phys_backend_copy_result *out) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + void __iomem *bar0; + struct gsp_phys_rmcall_ce_submit sub; + struct gsp_phys_fault_info fault = { }; + /* + * Static scratch: the only caller is the single-inflight sema work + * item (ordered gsp-phys-work queue), so no concurrent submission can + * touch these. Keeping the 96-dword method block and the 64-word QMD + * copy off the stack keeps the frame under the warning bound. + */ + static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; + static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + unsigned long start = jiffies; + unsigned long deadline; + u8 *sem_va; + u32 epoch, words, i, n_shadow = 0; + u32 want_qmd = 0, want_ch = 0; + bool qmd_rel = false, ch_rel = false; + u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; + u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; + u32 qmd_rel_off = 0, ch_rel_off = 0; + u32 poll = 0, w0 = 0, w1 = 0; + int ret = 0; + + if (out) + memset(out, 0, sizeof(*out)); + if (!req || !req->qmd || + req->qmd_words != GSP_PHYS_GR_QMD_WORDS || + req->n_pages > GSP_PHYS_GR_MAX_PAGES || + req->n_inl > GSP_PHYS_GR_MAX_INLINE) { + gsp_phys_gr_refused++; + return -EINVAL; + } + if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() || + gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { + gsp_phys_gr_refused++; + ret = -EAGAIN; + goto out; + } + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) { + gsp_phys_gr_refused++; + ret = -EAGAIN; + goto out; + } + sem_va = gsp_phys_chunk_page_va(sem, 0); + if (!sem_va || !gsp_phys_chunk_page_va(pb, 0)) { + gsp_phys_gr_refused++; + ret = -EIO; + goto out; + } + + gsp_phys_gr_submitted++; + mutex_lock(&gsp_phys_rm_lock); + if (!gsp_phys_backend_gr_ready()) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + ret = -EAGAIN; + goto out; + } + epoch = ++boot_stage.rmcall_gr_submit_epoch; + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + memcpy(qmd, req->qmd, sizeof(qmd)); + + /* + * QMD release0: when the launch has a one-word absolute release and + * the caller resolved the guest release page, author it there through + * a fresh shadow line so the guest observes its own QMD release; else + * point release0 at the module semaphore so the host observes engine + * completion. A reduction release is never redirected (its arithmetic + * would change). + */ + if (((qmd[4] >> 10) & 1)) { + u32 red_en = (qmd[24] >> 26) & 1; + u32 one_word = (qmd[24] >> 31) & 1; + + if (req->rel_hpa && one_word && !red_en && + n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + u64 sh_va = 0; + + ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, + req->rel_hpa, + &sh_va); + if (!ret) { + n_shadow++; + qmd_rel_hpa = req->rel_hpa; + qmd_rel_off = req->rel_off; + want_qmd = req->rel_payload; + qmd_rel = true; + qmd[23] = (u32)(sh_va + req->rel_off); + qmd[24] = (qmd[24] & 0xffffff00u) | + ((u32)((sh_va + req->rel_off) >> 32) & + 0xffu); + qmd[25] = req->rel_payload; + } else { + ret = 0; + } + } + if (!qmd_rel) { + qmd[23] = (u32)GSP_PHYS_RMCALL_VA_SEM; + qmd[24] = (qmd[24] & 0xffffff00u) | + ((u32)(GSP_PHYS_RMCALL_VA_SEM >> 32) & + 0xffu); + qmd[25] = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD; + want_qmd = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD; + qmd_rel = true; + } + } + + /* + * Pages: Tier 1 maps the resolved HPA at the guest VA; a page without + * a usable Tier-1 mapping falls to Tier 2 when captured inline bytes + * cover it and a QMD pointer pair names it (the pair is rewritten to + * the shadow VA); anything else refuses the launch. + */ + for (i = 0; i < req->n_pages; i++) { + const struct gsp_phys_gr_page *pg = &req->pages[i]; + u64 pg_va, sh_va = 0; + bool mapped = false; + + if (!pg->guest_va) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + ret = -EINVAL; + goto out; + } + pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + if (pg->hpa) { + ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, + pg->hpa, epoch); + if (!ret) + mapped = true; + else if (ret == -ENOSPC) + gsp_phys_gr_pool_exhausted++; + } + if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE && + pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) { + struct gsp_phys_boot_chunk *sh = + &boot_stage.rmcall_gr_shadow; + u8 *sh_page; + u32 j, copied = 0; + + if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + gsp_phys_gr_pool_exhausted++; + continue; + } + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) { + gsp_phys_gr_pool_exhausted++; + continue; + } + gsp_phys_chunk_zero(sh); + } + sh_page = gsp_phys_chunk_page_va(sh, n_shadow); + if (!sh_page) + continue; + memset(sh_page, 0, GSP_PHYS_PAGE_SIZE); + for (j = 0; j < req->n_inl; j++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[j]; + u32 len; + + if ((inl->dst_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) + != pg_va) + continue; + len = inl->len; + if (len > GSP_PHYS_GR_INLINE_BYTES) + len = GSP_PHYS_GR_INLINE_BYTES; + if ((u64)(inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)) + len > + GSP_PHYS_PAGE_SIZE) + continue; + memcpy(sh_page + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + copied++; + } + if (copied) { + ret = gsp_phys_rmcall_gr_map_shadow_locked( + n_shadow, + (u64)gsp_phys_chunk_page_dma(sh, + n_shadow), + &sh_va); + if (!ret) { + u64 ptr_va = sh_va + + (pg->guest_va & + (GSP_PHYS_PAGE_SIZE - 1)); + + n_shadow++; + qmd[pg->qmd_word] = (u32)ptr_va; + qmd[pg->qmd_word + 1] = + (qmd[pg->qmd_word + 1] & + ~0x1ffffu) | + ((u32)(ptr_va >> 32) & 0x1ffffu); + mapped = true; + } + } + } + if (!mapped) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + ret = -EOPNOTSUPP; + pr_info("vgsp-gr: launch refused: page va=0x%llx hpa=0x%llx qmd_word=%u has no Tier-1 mapping and no captured inline data (pages=%u)\n", + pg->guest_va, pg->hpa, pg->qmd_word, + req->n_pages); + goto out; + } + } + + /* + * Channel release: author it into the guest semaphore page when the + * caller resolved one, else into the module semaphore's second word. + */ + if (req->sem_hpa && req->sem_payload && + n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + u64 sh_va = 0; + + ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, + req->sem_hpa, &sh_va); + if (!ret) { + n_shadow++; + ch_rel_hpa = req->sem_hpa; + ch_rel_off = req->sem_off; + want_ch = req->sem_payload; + ch_rel = true; + ch_sem_va = sh_va + req->sem_off; + } + } + if (!ch_rel) { + want_ch = GSP_PHYS_RMCALL_GR_SEM_PAYLOAD; + ch_rel = true; + } + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)sem_va, 0); + WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + wmb(); + + words = gsp_phys_rmcall_build_gr_launch(methods, qmd, + GSP_PHYS_RMCALL_VA_SRC, + ch_sem_va, want_ch); + memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + + ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, + words, &sub); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + gsp_phys_gr_refused++; + pr_err("vgsp-gr: launch submit refused (%d) pages=%u words=%u\n", + ret, req->n_pages, words); + goto out; + } + + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n", + req->h_object, req->push_va, req->push_len, req->n_pages, + n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token, + qmd_rel, want_ch, GSP_PHYS_RMCALL_GUEST_POLL_MS); + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS); + for (;;) { + if (qmd_rel) { + if (qmd_rel_hpa) + w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + + qmd_rel_off)); + else + w0 = READ_ONCE(*(u32 *)sem_va); + } + if (ch_rel_hpa) + w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + + ch_rel_off)); + else + w1 = READ_ONCE(*(u32 *)(sem_va + 4)); + if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) + break; + if (time_after_eq(jiffies, deadline)) { + gsp_phys_gr_timed_out++; + pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", + GSP_PHYS_RMCALL_GUEST_POLL_MS, w0, want_qmd, w1, + want_ch, req->h_object, req->push_va); + if (gsp_phys_gr_fault_active()) { + struct gsp_phys_fault_info fi; + u32 fep = 0; + + if (!gsp_phys_status_drain_fault(&fi, &fep) && + fi.valid && + fi.function == + GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) { + fault = fi; + gsp_phys_fault_drained++; + } else { + gsp_phys_fault_stale++; + } + } + } + poll = w1; + ret = -ETIMEDOUT; + goto out; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + + gsp_phys_gr_ok++; + poll = w1; + pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n", + req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1, + jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); + ret = 0; + +out: + if (out) { + out->sem_value = poll; + out->elapsed_ms = jiffies_to_msecs(jiffies - start); + out->rc = ret; + out->fault = fault; + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_gr_submit); + /* * P6a trigger variant: identical to the exported forwarding call except that * the gsp_phys_work_forward parameter gate is dropped (every other guard and @@ -11538,6 +12244,9 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx); for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]); + for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++) + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs); @@ -11769,6 +12478,30 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) r_resolve_fail, r_timeouts, r_pool, r_refused, gsp_phys_relay_enabled()); } + { + /* + * GR (compute) forwarding: the sema service's capture/offer/ + * fallback counters and the booter backend's submit/refusal + * counters plus the parameter and exec-gate state. All zero + * while gsp_phys_gr_forward=0. + */ + u64 gr_req = 0, gr_ok = 0, gr_fallback = 0, gr_refused = 0; + u64 gr_timeout = 0, gr_fault = 0; + u64 b_sub = 0, b_ok = 0, b_refused = 0, b_timeout = 0; + u64 b_pool = 0; + + gsp_sema_service_fwd_gr_stats(&gr_req, &gr_ok, &gr_fallback, + &gr_refused, &gr_timeout, + &gr_fault); + gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, + &b_timeout, &b_pool); + seq_printf(m, + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n", + gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, + gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, + gsp_phys_gr_forward_enabled(), + boot_stage.rmcall_gr_exec_ok); + } { /* * P6c C3 fault observation/relay: the booter's 0x1006 @@ -13079,6 +13812,17 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + (u64)GSP_PHYS_RMCALL_GR_CTX_PT_MAX * (2ULL << 20) <= GSP_PHYS_RMCALL_VAS_SIZE, "the GR ctx window must stay below the VAS vaSize limit"); +static_assert(sizeof(struct gsp_phys_gr_page) == 24, + "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); +static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && + sizeof(struct gsp_phys_gr_inline) == 80, + "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); +static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS, + "the GR launch method block (SET_OBJECT + inline-QMD run + REPORT_SEMAPHORE) must fit the bound"); +static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= + GSP_PHYS_RMCALL_COPY_SIZE, + "the GR launch method block must fit one module pushbuffer page"); /* * GspSystemInfo wire ABI, pinned against the reference header compile -- cgit v1.2.3