From d3f061966bb9e23fdba7f6cc0aa231acdf31f36e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 5 Oct 2026 13:06:39 -0400 Subject: vgsp: guest GR forwarding backend and capture (gsp_phys_gr_forward, default off) Add a default-off guest GR (compute) forwarding path so a guest CUDA launch submitted on its GR channel can be executed by the physical GR engine on the ladder channel, with fail-closed fallback to the host CUDA bridge and the unchanged software release path. - New module parameter gsp_phys_gr_forward (default 0) and accessor. - New GR execution gate: boot_stage.rmcall_gr_exec_ok, latched only after a real GR submission logs REAL GR SEM OK; the backend refuses while it is unset. - Sema capture: a completed 64-word inline QMD on a trusted, non-replay GR decode becomes an svc_gr_cand (push identity, decoded release0, parameter/program/pointer fields, up to 8 captured inline-write records); the CUDA bridge stays authoritative while the parameter is off and is reached in order forward -> bridge -> software release. - Backend: gsp_phys_backend_guest_gr_submit() maps pages at their own guest VAs through fresh Tier-1 small-PT pages/PDE installs or relocates inline-captured pages to ladder shadow pages (Tier 2), builds SET_OBJECT + the inline-QMD method run + REPORT_SEMAPHORE on the compute subchannel, reuses the shared GPFIFO/USERD/doorbell publish tail (extracted from the CE submit path, CE bytes unchanged) and polls the QMD release and/or module semaphore, draining the epoch-gated fault queue on timeout. - Sema wiring: GR requests reuse the INFLIGHT/hold/deferral machinery so the guest channel release applies only on the submit's success/failure edge; hold shims preserve the channel engine type. - Counters and a work_gr state line plus a stats accessor. - Bounds are fixed: 16 pages, 8 inline records, 64 QMD words; every pointer/size checked, pool exhaustion and stale mappings fail closed. - New ABI layouts pinned with static_asserts. Compile-only change: no device test performed. (cherry picked from commit ae507e6f241c7e3498728f9d1ffc2d41ede5cd64) --- kernel/gsp_sema_service.c | 721 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 691 insertions(+), 30 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f897ce0..2b7fa80 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -611,6 +611,23 @@ struct sema_svc { u64 fwd_fault_delivered; u64 fwd_fault_no_event; u64 fwd_fault_late_release; + /* + * GR (compute) forwarding (gsp_phys_gr_forward=1 only): candidate + * launches offered to the request path (fwd_gr_req), launches the + * physical GR channel completed (fwd_gr_ok), offerings that fell back + * to the host CUDA bridge / software release (fwd_gr_fallback), gate + * refusals (fwd_gr_refused), backend timeouts (fwd_gr_timeout), + * timeouts carrying a decoded engine fault (fwd_gr_fault) and CUDA + * bridge launch calls made by a fallback (fwd_gr_bridge). All zero + * while the parameter is off. + */ + u64 fwd_gr_req; + u64 fwd_gr_ok; + u64 fwd_gr_fallback; + u64 fwd_gr_refused; + u64 fwd_gr_timeout; + u64 fwd_gr_fault; + u64 fwd_gr_bridge; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -3153,6 +3170,215 @@ struct svc_fwd_cand { static struct svc_fwd_cand svc_fwd_cand; +/* + * GR (compute) forwarding candidate (gsp_phys_gr_forward=1 only). When a GR + * channel decodes a completed 64-word inline QMD, the launch is captured here + * instead of being handed straight to the host CUDA bridge: the QMD words, its + * device VA, the push identity, the newest GRCE inline-write records and the + * QMD fields the bridge decodes (release0, parameter bank, program, pointer + * pairs). svc_fwd_request() consumes the capture on the same service thread: + * it queues a physical GR submission, or falls back to the bridge (when + * enabled) and then the unchanged software release path. `offered` is the + * exactly-once latch: a capture is consumed at most once, and a rewind replay + * never re-captures because the QMD gate skips replay passes. Service-thread + * only, like every other decode/pump state. + */ +#define SVC_GR_QMD_WORDS 64 +#define SVC_GR_INLINE_MAX 8 +#define SVC_GR_INLINE_BYTES 64 +#define SVC_GR_POINTER_MAX 8 +#define SVC_GR_PAGES_MAX 16 +#define SVC_GR_QMD_WORD_NONE 0xffffffffu +/* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ +#define SVC_GR_REL_REDUCTION 0x1u +#define SVC_GR_REL_ONE_WORD 0x2u +#define SVC_GR_REL_ENABLE 0x4u + +struct svc_gr_inline { + u64 dst_va; + u32 len; + u8 data[SVC_GR_INLINE_BYTES]; +}; + +/* One resolved launch page carried in the request/backend hand-off. */ +struct svc_gr_page { + u64 va; /* guest VA, page aligned */ + u64 pa; /* guest PA, page aligned (0 = unresolved) */ + u8 ap; /* SVC_APERTURE_* of pa, 0 when pa = 0 */ + u32 qmd_word; /* QMD pointer pair word index, or NONE */ +}; + +struct svc_gr_cand { + bool valid; + bool offered; /* consumed by a request or a fallback */ + u32 h_object; + u64 push_va; + u32 push_len; + u64 push_hash; + u32 qmd[SVC_GR_QMD_WORDS]; + u64 qmd_dev_va; + u32 n_inline; + struct svc_gr_inline inline_recs[SVC_GR_INLINE_MAX]; + /* Decoded by svc_gr_qmd_decode(); mirrors gsp_cuda_bridge.c:745-759. */ + u64 prog_va; + u64 param_va; + u32 param_size; + u64 rel_va; + u32 rel_payload; + u32 rel_flags; /* bit0 reduction, bit1 one-word, bit2 enable */ + u32 n_pointer; + u64 pointer_va[SVC_GR_POINTER_MAX]; + u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ +}; + +static struct svc_gr_cand svc_gr_cand; + +static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, + "the sema GR QMD word count must match the backend ABI"); +static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, + "the sema GR inline capture must fit the backend ABI"); +static_assert(sizeof(struct svc_gr_inline) == 80 && + offsetof(struct svc_gr_inline, data) == 12, + "the sema GR inline record must stay layout-compatible with the backend"); +static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && + offsetof(struct svc_gr_inline, dst_va) == + offsetof(struct gsp_bridge_inline, dst_va) && + offsetof(struct svc_gr_inline, len) == + offsetof(struct gsp_bridge_inline, len) && + offsetof(struct svc_gr_inline, data) == + offsetof(struct gsp_bridge_inline, data), + "the sema GR inline record must be castable to the CUDA bridge record"); +static_assert(sizeof(struct svc_gr_inline) == + sizeof(struct gsp_phys_gr_inline) && + offsetof(struct svc_gr_inline, dst_va) == + offsetof(struct gsp_phys_gr_inline, dst_va) && + offsetof(struct svc_gr_inline, len) == + offsetof(struct gsp_phys_gr_inline, len) && + offsetof(struct svc_gr_inline, data) == + offsetof(struct gsp_phys_gr_inline, data), + "the sema GR inline record must be castable to the backend record"); +static_assert(SVC_GR_PAGES_MAX == GSP_PHYS_GR_MAX_PAGES, + "the sema GR page list must match the backend ABI"); +static_assert(sizeof(struct svc_gr_page) == 24, + "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes"); + +static u64 svc_push_hash(const u8 *p, u32 len); + +/* QMD field decode: the exact shape the host CUDA bridge uses. */ +static void svc_gr_qmd_decode(struct svc_gr_cand *c) +{ + const u32 *q = c->qmd; + u32 i; + + c->rel_va = q[23] | ((u64)(q[24] & 0xff) << 32); + c->rel_payload = q[25]; + c->rel_flags = 0; + if ((q[24] >> 26) & 1u) + c->rel_flags |= SVC_GR_REL_REDUCTION; + if ((q[24] >> 31) & 1u) + c->rel_flags |= SVC_GR_REL_ONE_WORD; + if ((q[4] >> 10) & 1u) + c->rel_flags |= SVC_GR_REL_ENABLE; + c->param_va = q[32] | ((u64)(q[33] & 0x1ffff) << 32); + c->param_size = (q[33] >> 19) << 4; + c->prog_va = q[48] | ((u64)(q[49] & 0x1ffff) << 32); + c->n_pointer = 0; + /* Pointer-region candidates 34..47; 48/49 is the program pair. */ + for (i = 0; i < SVC_GR_POINTER_MAX - 1; i++) { + u32 w = 34 + 2 * i; + u64 va = q[w] | ((u64)(q[w + 1] & 0x1ffff) << 32); + + if (!va) + continue; + c->pointer_va[c->n_pointer] = va; + c->pointer_word[c->n_pointer] = w; + c->n_pointer++; + } +} + +/* Capture one completed QMD as the current GR forwarding candidate. */ +static void svc_gr_cand_capture(struct svc_channel *ch, + const struct svc_push_state *st, + const struct gsp_bridge_inline *recs, u32 n_recs, + const u8 *push, u32 push_len, u64 push_va) +{ + u32 i, n; + + memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); + svc_gr_cand.valid = true; + svc_gr_cand.h_object = ch->h_object; + svc_gr_cand.push_va = push_va; + svc_gr_cand.push_len = push_len; + if (push && push_len) + svc_gr_cand.push_hash = svc_push_hash(push, push_len); + memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd)); + svc_gr_cand.qmd_dev_va = st->qmd_dev_va; + n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX; + for (i = 0; i < n; i++) { + svc_gr_cand.inline_recs[i].dst_va = recs[i].dst_va; + svc_gr_cand.inline_recs[i].len = + recs[i].len < SVC_GR_INLINE_BYTES ? + recs[i].len : SVC_GR_INLINE_BYTES; + memcpy(svc_gr_cand.inline_recs[i].data, recs[i].data, + svc_gr_cand.inline_recs[i].len); + } + svc_gr_cand.n_inline = n; + svc_gr_qmd_decode(&svc_gr_cand); + pr_info("sema-gr: launch captured h=0x%x push=0x%llx len=%u qmd=0x%llx prog=0x%llx param=0x%llx size=0x%x rel=0x%llx/%u flags=0x%x ptr=%u inline=%u\n", + ch->h_object, push_va, push_len, st->qmd_dev_va, + svc_gr_cand.prog_va, svc_gr_cand.param_va, + svc_gr_cand.param_size, svc_gr_cand.rel_va, + svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, + svc_gr_cand.n_pointer, svc_gr_cand.n_inline); +} + +/* + * Consume the current GR candidate without forwarding it: hand the launch to + * the host CUDA bridge when that path is enabled, then clear the candidate. + * Exactly-once via `offered`. Called on the service thread (capture/refusal) + * or the work item (a failed physical submission uses the request copy). + */ +static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu) +{ + u32 n; + + if (!svc_gr_cand.valid || svc_gr_cand.offered) + return; + svc_gr_cand.offered = true; + if (gsp_cuda_bridge_enabled()) { + n = svc_gr_cand.n_inline; + if (n > GSP_BRIDGE_INLINE_MAX) + n = GSP_BRIDGE_INLINE_MAX; + gsp_cuda_bridge_launch(vgpu, svc_gr_cand.h_object, + svc_gr_cand.qmd, + svc_gr_cand.qmd_dev_va, + svc_gr_cand.push_va, + (const struct gsp_bridge_inline *) + svc_gr_cand.inline_recs, n); + svc.fwd_gr_bridge++; + } + svc_gr_cand.valid = false; +} + +/* + * Should this decode pass's releases be recorded instead of applied inline? + * A captured CE copy defers under the existing CE deferral parameter; a + * captured GR launch defers whenever GR forwarding is armed (its channel + * release must land only on the physical submission's success/failure edge). + * Both still require the guest-memory path that makes any hardware + * submission possible. Service-thread only, like the captures. + */ +static bool svc_fwd_defer_arm(void) +{ + if (!gsp_phys_guest_mem_enabled()) + return false; + if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled()) + return true; + return svc_fwd_cand.valid && + gsp_phys_work_defer_release_enabled(); +} + + /* * P6b wave-1.5 deferred guest releases: every release group the current * decode pass decoded while a forwarding candidate was captured is recorded @@ -4475,15 +4701,23 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) /* * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD - * to the host CUDA bridge. The run may cover the address registers and the - * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1 - * count=66 starting at 0x318), so every data word is mapped to its register - * offset. Each completed QMD resets the accumulator, so a push may carry - * more than one launch. + * to its consumer. With gsp_phys_gr_forward=0 the consumer is the host CUDA + * bridge, exactly as before; with the parameter on a trusted decode captures + * the QMD as the current GR forwarding candidate (svc_gr_cand_capture) and + * an untrusted sweep still goes to the bridge, so the new path never acts on + * a content-scanned page. The bridge is then only reached from a + * request-path fallback, so a forwarded launch is never also handed to the + * bridge and a refused/failed forward still falls back in order forward -> + * bridge -> software release. The run may cover the + * address registers and the data registers as a single INC/NON_INC/ONE_INC + * method (observed: op=1 count=66 starting at 0x318), so every data word is + * mapped to its register offset. Each completed QMD resets the accumulator, + * so a push may carry more than one launch. */ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, - const u8 *data) + const u8 *data, const u8 *push, u32 push_len, + bool trusted) { u32 i; @@ -4542,9 +4776,13 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, memcpy(recs[n].data, rec->data, rec->len); n++; } - gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, - st->qmd_dev_va, st->push_va, - recs, n); + if (gsp_phys_gr_forward_enabled() && trusted) + svc_gr_cand_capture(ch, st, recs, n, push, push_len, + st->push_va); + else + gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, + st->qmd_dev_va, st->push_va, + recs, n); st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -4554,7 +4792,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, - u64 push_va, bool trusted, u64 *ops_done) + u64 push_va, bool trusted, bool replay, + u64 *ops_done) { struct svc_push_state st; /* @@ -4595,6 +4834,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc_fwd_cand.valid = false; svc_fwd_cand.skipped = false; + /* + * A fresh (non-replay) trusted decode pass invalidates any previous GR + * capture: the pump consumes a capture immediately after the pass that + * built it, so anything still here is stale. Replay/rescue decodes + * must not wipe a capture the pump has not consumed yet. + */ + if (!replay && trusted && ch->engine_type == 0) + memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; svc.pages_decoded++; @@ -4739,13 +4986,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * must fall through to the CE decoder. */ if (ch->engine_type == 0 && subch >= 1 && - gsp_cuda_bridge_enabled() && + (!gsp_phys_gr_forward_enabled() || !replay) && + (gsp_cuda_bridge_enabled() || + gsp_phys_gr_forward_enabled()) && addr >= SVC_QMD_ADDR_A && addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && (st.qmd_have || (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) { svc_qmd_accum(vgpu, ch, &st, addr, count, - buf + off + 4); + buf + off + 4, buf, len, trusted); off += 4 + count * 4; continue; } @@ -4871,9 +5120,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * The pre-apply bookkeeping above is unchanged; * only the apply moves. Overflow applies inline. */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid && - gsp_phys_guest_mem_enabled()) { + if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -4988,9 +5235,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * once the forwarded copy completes. * Overflow applies inline. */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid && - gsp_phys_guest_mem_enabled()) { + if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -5386,9 +5631,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * the record path so the pump still sees the * release as decoded. Overflow applies inline. */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid && - gsp_phys_guest_mem_enabled()) { + if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -7730,6 +7973,32 @@ struct svc_fwd_req { * (valid=false) for every non-fault outcome. */ struct gsp_phys_fault_info fault; + /* + * GR (compute) forwarding (gsp_phys_gr_forward=1 only): when gr is + * set the request is a captured guest launch, not a CE copy. The work + * item resolves each page entry's guest PA to an HPA, copies the + * captured inline bytes into the backend's shadow pages and calls + * gsp_phys_backend_guest_gr_submit(); gr_bridge_done is the + * exactly-once latch for the work-item CUDA bridge fallback. + */ + bool gr; + bool gr_bridge_done; + bool gr_page_overflow; + u32 gr_h_object; + u64 gr_push_va; + u32 gr_push_len; + u64 gr_qmd_dev_va; + u32 gr_rel_flags; + u64 gr_rel_va; + u32 gr_rel_payload; + u64 gr_prog_va; + u64 gr_param_va; + u32 gr_param_size; + u32 gr_qmd[SVC_GR_QMD_WORDS]; + u32 gr_n_inline; + struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX]; + u32 gr_n_pages; + struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX]; }; static struct svc_fwd_req svc_fwd_req; @@ -7910,6 +8179,13 @@ struct svc_fwd_hold { */ struct nv_vgpu_device *vgpu; u32 ch_handle; + /* + * The channel's engine type at engage time: the shim channels the + * release-apply paths build must preserve it so a GR (engine_type 0) + * release lands through the exact completion-counter slot, exactly as + * the inline path would have. CE holds stay 1. + */ + u32 engine_type; u64 gpfifo_va; u32 entry_idx; u8 nrel; @@ -8080,6 +8356,7 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.cand = svc_fwd_cand; svc_fwd_hold.vgpu = vgpu; svc_fwd_hold.ch_handle = ch->h_object; + svc_fwd_hold.engine_type = ch->engine_type; svc_fwd_hold.gpfifo_va = ch->gpfifo_va; svc_fwd_hold.entry_idx = entry_idx; svc_fwd_hold.nrel = n; @@ -8277,7 +8554,7 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ + shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } @@ -8441,7 +8718,7 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ + shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } @@ -8493,7 +8770,7 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ + shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } @@ -8508,6 +8785,135 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc_fwd_hold_clear(); } +static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); + +/* + * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The + * request already resolved every launch page's guest VA->PA on the service + * thread; here each PA is turned into an HPA (SYS through the guest-RAM + * backing, VID through the BAR1 shadow), the captured inline bytes are handed + * to the backend for its Tier-2 shadow pages and the whole launch is + * submitted through gsp_phys_backend_guest_gr_submit(). On any failure the + * host CUDA bridge is offered the launch once (the forward -> bridge -> + * software fallback order), then the pump's hold applies the recorded + * releases. Runs on the gsp-phys-work item with the same lock discipline as + * the CE branch. + */ +static void svc_fwd_gr_work(void) +{ + /* + * Static: the ordered gsp-phys-work queue runs one item at a time and + * the GR page array is ~0.4 KB; keeping it off the stack keeps the + * frame under the kernel's warning bound (same reason as the ops array + * in svc_decode_stream). + */ + static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX]; + struct gsp_phys_gr_submit gs; + struct gsp_phys_backend_copy_result cp = { 0 }; + u64 rel_hpa = 0; + u32 off = 0, i; + int hw_rc; + + memset(&gs, 0, sizeof(gs)); + memset(pages, 0, sizeof(pages)); + + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i]; + u8 kind = (pg->ap == SVC_APERTURE_FB) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + u64 hpa = 0; + + if (pg->pa) { + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, pg->pa, + 0, SEMA_PAGE_SIZE, &hpa, + &off) || off) { + hpa = 0; + svc.fwd_guest_resolve_fail++; + } + } + pages[i].guest_va = pg->va; + pages[i].hpa = hpa; + pages[i].qmd_word = pg->qmd_word; + } + + if (svc_fwd_req.gr_rel_va && + (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) && + !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) && + (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) { + u64 page = svc_fwd_req.gr_rel_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 roff = (u32)(svc_fwd_req.gr_rel_va & + (SEMA_PAGE_SIZE - 1)); + + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, SVC_APERTURE_SYS, page, + 0, SEMA_PAGE_SIZE, &rel_hpa, &off) || + off) + rel_hpa = 0; + /* + * Forward-only: a release word already at/ahead of the payload + * means the engine would author nothing this launch; point + * release0 at the module semaphore instead so the host still + * observes completion. + */ + if (rel_hpa && + READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= + svc_fwd_req.gr_rel_payload) + rel_hpa = 0; + } + + gs.qmd = svc_fwd_req.gr_qmd; + gs.qmd_words = SVC_GR_QMD_WORDS; + gs.qmd_va = svc_fwd_req.gr_qmd_dev_va; + gs.pages = pages; + gs.n_pages = svc_fwd_req.gr_n_pages; + gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline; + gs.n_inl = svc_fwd_req.gr_n_inline; + if (gs.n_inl > GSP_PHYS_GR_MAX_INLINE) + gs.n_inl = GSP_PHYS_GR_MAX_INLINE; + gs.rel_hpa = rel_hpa; + gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); + gs.rel_payload = svc_fwd_req.gr_rel_payload; + gs.sem_hpa = 0; + gs.sem_off = 0; + gs.sem_payload = 0; + gs.push_va = svc_fwd_req.gr_push_va; + gs.push_len = svc_fwd_req.gr_push_len; + gs.h_object = svc_fwd_req.gr_h_object; + + hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + if (!hw_rc) { + svc.fwd_gr_ok++; + } else { + svc.fwd_gr_fallback++; + if (hw_rc == -ETIMEDOUT) { + svc.fwd_gr_timeout++; + if (cp.fault.valid) + svc.fwd_gr_fault++; + } + if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", + hw_rc, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, svc.fwd_gr_req, + svc.fwd_gr_ok, svc.fwd_gr_fallback); + svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); + } + + svc_fwd_req.fault = cp.fault; + svc_fwd_req.hw_rc = hw_rc; + /* + * done before INFLIGHT: a reuser that acquires the bit can never + * overwrite the slot before this outcome is visible. + */ + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + { + struct task_struct *task = READ_ONCE(svc.task); + + if (task) + wake_up_process(task); + } +} + static void svc_fwd_work_fn(struct work_struct *work) { bool guest = svc_fwd_req.guest; @@ -8517,6 +8923,12 @@ static void svc_fwd_work_fn(struct work_struct *work) int ret; int hw_rc = 0; + /* GR forwarding requests take their own branch and outcome shape. */ + if (svc_fwd_req.gr) { + svc_fwd_gr_work(); + return; + } + /* * P6b wave 1: try the guest hardware copy first when the pump marked * this request guest. P6b wave 1.5: when the request's releases are @@ -8732,6 +9144,230 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req) } } +/* + * GR request page collection: append (dedupe) one launch page, resolving its + * guest VA->PA on the service thread now (the work item only does PA->HPA). + * An unresolved page keeps pa=0; whether that is fatal is decided by + * svc_gr_req_fill from its QMD pointer association and the captured inline + * bytes. Overflow is a failure (fail-closed), not a silent drop. + */ +static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + u32 ap = 0; + u32 i; + + if (!page) + return; + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + if (svc_fwd_req.gr_pages[i].va != page) + continue; + if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE && + qmd_word != SVC_GR_QMD_WORD_NONE) + svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + return; + } + if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) { + svc_fwd_req.gr_page_overflow = true; + return; + } + i = svc_fwd_req.gr_n_pages++; + svc_fwd_req.gr_pages[i].va = page; + svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + if (svc_resolve_va(vgpu, page, &pa, &ap)) + return; + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); +} + +/* Does any captured inline write land in this page? */ +static bool svc_gr_page_inline_covers(u64 page) +{ + u32 i; + + for (i = 0; i < svc_gr_cand.n_inline; i++) { + if ((svc_gr_cand.inline_recs[i].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return true; + } + return false; +} + +/* + * Copy the current GR capture into the request slot (INFLIGHT is held). Every + * launch page is collected and resolved; a page with no resolved PA is only + * acceptable when a QMD pointer pair names it and captured inline bytes cover + * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller + * falls back on. + */ +static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 i; + + svc_fwd_req.done = false; + svc_fwd_req.hw_rc = 0; + memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); + svc_fwd_req.guest = false; + svc_fwd_req.gr = true; + svc_fwd_req.gr_bridge_done = false; + svc_fwd_req.defer = true; + svc_fwd_req.relay = false; + svc_fwd_req.relay_index = 0; + svc_fwd_req.vgpu = vgpu; + svc_fwd_req.gr_h_object = ch->h_object; + svc_fwd_req.gr_push_va = svc_gr_cand.push_va; + svc_fwd_req.gr_push_len = svc_gr_cand.push_len; + svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; + svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags; + svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va; + svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload; + svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va; + svc_fwd_req.gr_param_va = svc_gr_cand.param_va; + svc_fwd_req.gr_param_size = svc_gr_cand.param_size; + memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, + sizeof(svc_fwd_req.gr_qmd)); + svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; + if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) + svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; + memcpy(svc_fwd_req.gr_inline, svc_gr_cand.inline_recs, + sizeof(svc_fwd_req.gr_inline)); + svc_fwd_req.gr_n_pages = 0; + svc_fwd_req.gr_page_overflow = false; + + if (svc_gr_cand.param_va) + svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); + if (svc_gr_cand.prog_va) + svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); + for (i = 0; i < svc_gr_cand.n_pointer; i++) + svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], + svc_gr_cand.pointer_word[i]); + for (i = 0; i < svc_gr_cand.n_inline; i++) + svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, + SVC_GR_QMD_WORD_NONE); + if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) + return -E2BIG; + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i]; + + if (pg->pa) + continue; + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) + return -ENOENT; + if (!svc_gr_page_inline_covers(pg->va)) + return -ENOENT; + } + return 0; +} + +/* + * Exactly-once CUDA bridge fallback from the request copy (work item or a + * failed queue): hand the captured launch to the bridge when enabled. The + * latch is per request slot and the slot is single-inflight. + */ +static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) +{ + u32 n; + + if (!svc_fwd_req.gr || svc_fwd_req.gr_bridge_done) + return; + svc_fwd_req.gr_bridge_done = true; + if (!gsp_cuda_bridge_enabled()) + return; + n = svc_fwd_req.gr_n_inline; + if (n > GSP_BRIDGE_INLINE_MAX) + n = GSP_BRIDGE_INLINE_MAX; + gsp_cuda_bridge_launch(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_qmd, svc_fwd_req.gr_qmd_dev_va, + svc_fwd_req.gr_push_va, + (const struct gsp_bridge_inline *) + svc_fwd_req.gr_inline, n); + svc.fwd_gr_bridge++; +} + +/* + * GR request path (gsp_phys_gr_forward=1 only), called by svc_fwd_request() + * for engine_type==0 channels. Queues one physical submission when the GR + * backend is ready; otherwise consumes the capture through the bridge + * fallback. Reuses the CE INFLIGHT/paused/hold exclusion so GR and CE + * forwarding can never overlap; the caller engages the release hold on a + * true return. Service-thread only. + */ +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + int rc; + + if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid) + return false; + svc.fwd_gr_req++; + if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + svc_gr_bridge_fallback(vgpu); + return false; + } + /* + * Never reuse the request slot while a hold is active; the held + * request's done/hw_rc are still being read. The capture falls back. + */ + if (svc_fwd_hold.active) { + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + svc_gr_bridge_fallback(vgpu); + return false; + } + mutex_lock(&svc_fwd_lock); + if (svc_fwd_paused || + test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + svc_gr_bridge_fallback(vgpu); + return false; + } + rc = svc_gr_req_fill(vgpu, ch); + if (rc) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_fallback++; + if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", + rc, ch->h_object, svc_gr_cand.push_va); + svc_gr_bridge_fallback(vgpu); + return false; + } + /* + * The candidate is consumed: the request copy owns it from here, so a + * later capture/replay cannot re-offer it and the bridge is only + * reached through the request's own latch. + */ + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + svc_fwd_requests++; + if (!svc_fwd_wq) { + svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); + if (!svc_fwd_wq) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + svc_gr_req_bridge_fallback(vgpu); + return false; + } + } + if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + svc_gr_req_bridge_fallback(vgpu); + return false; + } + mutex_unlock(&svc_fwd_lock); + return true; +} + /* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold @@ -8741,7 +9377,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { if (!ch->engine_type) - return false; + return svc_gr_request(vgpu, ch); if (!gsp_phys_work_forward_enabled()) return false; /* Classify this decode pass's candidate for the run summary. */ @@ -8845,7 +9481,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, return false; } mutex_unlock(&svc_fwd_lock); - return svc_fwd_req.guest; + return svc_fwd_req.guest || svc_fwd_req.gr; } /* @@ -8997,6 +9633,27 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, *late_release = svc.fwd_fault_late_release; } +/* + * GR forwarding counters for the gsp-phys-state work_gr line (see + * gsp_sema_service.h). Same plain-read discipline as the stats above. + */ +void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback, + u64 *refused, u64 *timeout, u64 *fault) +{ + if (req) + *req = svc.fwd_gr_req; + if (ok) + *ok = svc.fwd_gr_ok; + if (fallback) + *fallback = svc.fwd_gr_fallback; + if (refused) + *refused = svc.fwd_gr_refused; + if (timeout) + *timeout = svc.fwd_gr_timeout; + if (fault) + *fault = svc.fwd_gr_fault; +} + /* * P6c: true while this channel owns an active RELAY hold. The pump's * per-pass notifier is suppressed for such a channel so the hold's verified @@ -9479,6 +10136,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, injected, + entry_replay, entry_replay ? NULL : &entry_done); @@ -9684,10 +10342,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!svc_rescue_push_page(vgpu, ch, va, len) && !svc_read_va_push(vgpu, ch, va, pb, len)) - svc_decode_stream(vgpu, pb, - len, 0, false, - ch, idx, va, true, - NULL); + svc_decode_stream(vgpu, pb, + len, 0, false, + ch, idx, va, true, + true, + NULL); rel_seen = (svc_rel_count != rel_n2); /* @@ -9713,6 +10372,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true, + true, NULL); rel_seen = (svc_rel_count != rel_n2); @@ -9948,6 +10608,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) SVC_ENTRY_NONE, va, false, + false, NULL); if (svc_rel_count != rel_n_before) ch->last_rel_va = -- cgit v1.2.3