From fe95eadb2bc1cb9af49814528edecbc5e497d6cd Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 07:43:05 -0400 Subject: gsp-phys: P6c C0+C1 - engine-authored guest CE semaphore (default off) New default-off gsp_phys_relay: for a forwarded guest CE copy whose push ends in a one-word release, the release page is resolved at forward time, mapped on a fresh scratch line, and the real engine writes the payload directly into the guest-visible semaphore page; the deferred-release hold then verifies that page and skips only that record's software apply. Any failure (unresolvable target, refused kind, pool exhaustion, poll timeout, data mismatch, channel teardown) falls back to applying every release in software, so a release is never lost or duplicated. - new exported gsp_phys_backend_guest_ce_copy_ex() + relay counters; gsp_phys_backend_guest_ce_copy() behavior unchanged (wrapper) - new work_relay: counters line in gsp-phys-state - relay limited to CE channels and one-word absolute releases; counter and reduction releases stay on the existing paths Live: param-off regression matched the wave-2 baseline exactly (req=101 ok=101 verify_ok=101, copy_skipped=119, smoke matched=64/64, relay zeros). With the parameter on: requested=68 ok=67 skipped_sw=67, verify_fail=0, resolve_fail=0, timeouts=0, smoke 64/64, and the engine-authored guest semaphore was observed directly (sem=0x00000003 / 0x0000007e at the guest page). One copy in 68 released its semaphore but failed the host-side data compare; it fell back to the software apply (fallback accounting applied+fallback==engaged) with the smoke green - residual to triage, which is why the parameter stays default-off. A forced-unresolvable run (requested=103, fallback=105, no relay submissions) kept the smoke green. Every cycle: 5 power-domain locks held, driver rebound, WPR2 down, no rail cycles, pstore empty, same boot. --- kernel/gsp_sema_service.c | 387 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 371 insertions(+), 16 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1f599fc..dea8249 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -470,6 +470,22 @@ struct sema_svc { u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; + /* + * P6c completion relay (gsp_phys_relay=1 only): relay selections armed + * on a deferred guest request (fwd_relay_requested), relays whose + * engine-authored guest semaphore was verified with the software apply + * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to + * the software apply (fwd_relay_fallback), hold re-reads that did not + * see the payload (fwd_relay_verify_fail) and work-item semaphore HPA + * resolution failures (fwd_relay_resolve_fail). All zero while the + * parameter is off. + */ + u64 fwd_relay_requested; + u64 fwd_relay_ok; + u64 fwd_relay_fallback; + u64 fwd_relay_skipped_sw; + u64 fwd_relay_verify_fail; + u64 fwd_relay_resolve_fail; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -2308,6 +2324,18 @@ struct svc_fwd_rel { u8 kind; /* SVC_FWD_REL_* */ u8 op; /* 906f: host sem op / launch: pending flag */ u8 inc_claimed; /* launch: one-shot inc release claimed */ + /* + * P6c completion relay: the release target resolved once at record + * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and + * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or + * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a + * false tgt_ok means the record is software-applied and never + * relayed. + */ + u8 tgt_ok; + u8 tgt_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ + u64 tgt_page; + u32 tgt_off; u64 va; u32 payload; u32 entry_idx; @@ -3133,6 +3161,67 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return false; } +/* + * P6c completion relay: resolve one deferred release record's target once at + * record time so the request can hand the engine a page/offset it may + * program. Fail-open by construction: every miss (parameter off, non-CE + * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an + * unresolvable VA, an unreadable word, or a word already at/ahead of the + * payload) leaves tgt_ok false and the record is software-applied exactly as + * in wave 1.5. The resolution uses the same svc_resolve_va() the software + * apply path uses and pins the page via svc_pin_store(), so the work item's + * later svc_fwd_operand_hpa() resolves the same page. Service-thread only. + */ +static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + struct svc_fwd_rel *r, u32 launch_dma, + bool is_launch) +{ + u64 addr = 0; + u32 ap = 0; + u32 cur = 0; + + r->tgt_ok = false; + r->tgt_kind = 0; + r->tgt_page = 0; + r->tgt_off = 0; + + if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type) + return; + if (svc_counter_va(r->va)) + return; + /* + * The relay always authors an absolute one-word release (the CE block + * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a + * record whose software apply is additive must never be relayed: the + * engine would leave the guest word short of the value it waits on. + * The LAUNCH increment is excluded by the dma check below; the 906F + * REDUCTION op is excluded here. + */ + if (!is_launch && r->kind == SVC_FWD_REL_906F && + r->op == HOST_SEM_OP_REDUCTION) + return; + if (is_launch) { + if (DMA_SEMAPHORE_TYPE(launch_dma) != + SEM_RELEASE_ONE_WORD) + return; + if (DMA_SEM_REDUCTION_EN(launch_dma) && + DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) + return; + } + if (svc_resolve_va(vgpu, r->va, &addr, &ap)) + return; + if (svc_sem_read(vgpu, r->va, &cur)) + return; + /* Forward-only pre-check: only a word behind the payload is relayed. */ + if (cur >= r->payload) + return; + r->tgt_ok = true; + r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); + r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -3399,6 +3488,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = payload; r->entry_idx = entry_idx; r->dma = 0; + svc_fwd_relay_note(vgpu, ch, r, + 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { @@ -3512,6 +3603,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = st.rs_payload; r->entry_idx = entry_idx; r->dma = 0; + svc_fwd_relay_note(vgpu, ch, + r, 0, + false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; break; @@ -3844,6 +3938,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = st.sem_payload; r->entry_idx = entry_idx; r->dma = dma; + svc_fwd_relay_note(vgpu, ch, r, + dma, true); /* * P6b wave 1.5: do NOT bump * svc.pushes_pending here - @@ -5486,6 +5582,25 @@ struct svc_fwd_req { u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ u32 src_off, dst_off; u32 bytes; + /* + * P6c completion relay: when relay is true this request's hardware + * copy additionally authors the guest-visible release of the selected + * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/ + * off identify the resolved semaphore target and relay_payload is the + * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are + * published by the work item before its done release, for the hold's + * verification/logging. Cleared with the rest of the slot whenever the + * releases do not qualify. + */ + bool relay; + u8 relay_index; + u8 relay_kind; + u64 relay_page; + u32 relay_off; + u32 relay_payload; + bool relay_done; + u32 relay_sem; + u32 relay_elapsed_ms; }; static struct svc_fwd_req svc_fwd_req; @@ -5669,6 +5784,15 @@ struct svc_fwd_hold { u64 gpfifo_va; u32 entry_idx; u8 nrel; + /* + * P6c completion relay: the request selected record relay_index and + * the hardware copy was asked to author it into the guest-visible + * semaphore page. On the success edge the hold re-reads the guest + * word and, when it carries the payload, skips that one record's + * software apply. Cleared with the hold. + */ + bool relay_armed; + u8 relay_index; struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; @@ -5733,14 +5857,26 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, inc, r->inc_claimed); } -/* Apply the recorded releases of one deferred push (service thread). */ -static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, - const struct svc_channel *ch, - const struct svc_fwd_rel *rel, u8 n) +/* Skip value that never matches a record index. */ +#define SVC_FWD_REL_SKIP_NONE 0xff + +/* + * Apply the recorded releases of one deferred push (service thread), except + * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip + * on the relay verification success edge: the engine already authored that + * one release into the guest-visible semaphore, so re-applying it in software + * is unnecessary (and would count a second release). + */ +static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n, + u8 skip) { u8 i; for (i = 0; i < n; i++) { + if (i == skip) + continue; switch (rel[i].kind) { case SVC_FWD_REL_906F: svc_rel_apply_906f(vgpu, ch, rel[i].va, @@ -5759,6 +5895,14 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, } } +/* Apply every recorded release (no relay skip). */ +static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n) +{ + svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); +} + static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + @@ -5790,6 +5934,8 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel = n; svc_fwd_hold.last_rel_va = svc_rel_va_seen; svc_fwd_hold.since = jiffies; + svc_fwd_hold.relay_armed = svc_fwd_req.relay; + svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -5813,6 +5959,8 @@ static void svc_fwd_hold_clear(void) memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand)); svc_fwd_hold.vgpu = NULL; svc_fwd_hold.nrel = 0; + svc_fwd_hold.relay_armed = false; + svc_fwd_hold.relay_index = 0; svc.fwd_rel_pending = 0; } @@ -5861,6 +6009,13 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, if (!done) svc.fwd_hold_timeouts++; } + /* + * P6b/P6c: the timeout path never reaches the relay verification + * edge, so a relayed set is software-applied whole (forward-only, so + * the engine's write cannot be duplicated). + */ + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -5883,16 +6038,63 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { - svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, - svc_fwd_hold.nrel); - svc.fwd_rel_applied++; + /* + * P6c: on the relay success edge, re-read the guest-visible + * semaphore through the software view. Only a word that + * actually carries the payload lets the one relayed record be + * skipped; anything else (unreadable, behind) is a relay + * verification failure and the software applies the whole set + * (the forward-only apply makes the relayed write idempotent, + * so the release is never duplicated or lost). + */ + if (svc_fwd_hold.relay_armed && + svc_fwd_hold.relay_index < svc_fwd_hold.nrel) { + struct svc_fwd_rel *rr = + &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; + u32 cur = 0; + + if (!svc_sem_read(hv, rr->va, &cur) && + cur >= rr->payload) { + svc_fwd_rel_apply_all_ex(hv, ch, + svc_fwd_hold.rel, + svc_fwd_hold.nrel, + svc_fwd_hold.relay_index); + svc.fwd_relay_ok++; + svc.fwd_relay_skipped_sw++; + svc.fwd_rel_applied++; + /* + * The hold's post is the single completion + * edge for this channel: the pump's per-pass + * notifier was suppressed while the relay hold + * was active. + */ + svc_notify_complete(hv, ch); + } else { + svc.fwd_relay_verify_fail++; + svc.fwd_relay_fallback++; + svc_fwd_rel_apply_all(hv, ch, + svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + if ((svc.fwd_relay_verify_fail & 15) == 1) + pr_err("gsp-phys-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n", + rr->va, cur, rr->payload); + } + } else { + svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + } } else { /* * P6b wave 2: failure/timeout/abort fallback - author the * skipped data once, then apply the releases exactly as - * wave 1.5 does. + * wave 1.5 does. P6c: a relayed set is applied whole here + * (the engine did not deliver it). */ svc_fwd_hold_soft_copy(hv); + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_fallback++; @@ -5973,6 +6175,13 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) svc_fwd_hold_soft_copy(hv); svc.fwd_hold_canceled++; } + /* + * P6c: a vanished channel cannot run the relay verification edge; a + * relayed set falls back to the software path (which is exactly-once + * against the engine's forward write). + */ + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -6006,6 +6215,8 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc.fwd_rel_applied++; else svc.fwd_rel_fallback++; + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -6029,9 +6240,10 @@ static void svc_fwd_work_fn(struct work_struct *work) * while a hold is active. */ if (guest) { - u64 src_hpa = 0, dst_hpa = 0; - u32 src_off = 0, dst_off = 0; - int src_rc, dst_rc; + u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0; + u32 src_off = 0, dst_off = 0, sem_off = 0; + struct gsp_phys_backend_copy_result cp = { 0 }; + int src_rc, dst_rc, sem_rc = 0; bool resolved; src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, @@ -6047,11 +6259,43 @@ static void svc_fwd_work_fn(struct work_struct *work) svc_fwd_req.bytes, &dst_hpa, &dst_off); resolved = !src_rc && !dst_rc; + /* + * P6c: resolve the selected relay target to an HPA with the + * same operand resolver. A failure is not fatal - hw_rc is + * forced to an error so the hold applies the software + * release exactly as wave 1.5 (never lost, never duplicated). + */ + if (svc_fwd_req.relay) + sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.relay_kind, + svc_fwd_req.relay_page, + svc_fwd_req.relay_off, 4, + &sem_hpa, &sem_off); if (resolved) { - hw_rc = gsp_phys_backend_guest_ce_copy(src_hpa, src_off, - dst_hpa, dst_off, - svc_fwd_req.bytes, - GSP_PHYS_BACKEND_WORK_POLL_MS); + if (svc_fwd_req.relay && !sem_rc) { + hw_rc = gsp_phys_backend_guest_ce_copy_ex( + src_hpa, src_off, dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS, + sem_hpa, sem_off, + svc_fwd_req.relay_payload, &cp); + svc_fwd_req.relay_sem = cp.sem_value; + svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms; + svc_fwd_req.relay_done = (hw_rc == 0); + } else if (svc_fwd_req.relay) { + hw_rc = -EINVAL; + svc.fwd_relay_resolve_fail++; + if (svc.fwd_relay_resolve_fail <= 16) + pr_info("gsp-phys-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n", + svc_fwd_req.relay_kind, + svc_fwd_req.relay_page, + svc_fwd_req.relay_off, sem_rc); + } else { + hw_rc = gsp_phys_backend_guest_ce_copy( + src_hpa, src_off, dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS); + } if (!hw_rc) { svc.fwd_guest_ok++; if (!svc_fwd_req.defer) { @@ -6144,6 +6388,56 @@ static void svc_fwd_work_fn(struct work_struct *work) static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); +/* + * P6c completion relay: pick the ONE deferred release record this request's + * hardware copy may author. Only a deferred guest request whose records were + * armed can relay, and only a record whose record-time resolution succeeded + * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the + * completion edge the guest tracker waits on), else the last eligible record. + * Fail-open: no eligible record leaves req->relay false and the whole release + * set stays software-applied. Service-thread only; called with the request + * slot's INFLIGHT bit held, before queue_work. + */ +static void svc_fwd_relay_select(struct svc_fwd_req *req) +{ + int i; + + req->relay = false; + req->relay_index = 0; + req->relay_kind = 0; + req->relay_page = 0; + req->relay_off = 0; + req->relay_payload = 0; + + if (!req->defer || !gsp_phys_relay_enabled()) + return; + for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { + if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH && + svc_fwd_rel[i].tgt_ok) { + req->relay = true; + req->relay_index = (u8)i; + req->relay_kind = svc_fwd_rel[i].tgt_kind; + req->relay_page = svc_fwd_rel[i].tgt_page; + req->relay_off = svc_fwd_rel[i].tgt_off; + req->relay_payload = svc_fwd_rel[i].payload; + svc.fwd_relay_requested++; + return; + } + } + for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { + if (svc_fwd_rel[i].tgt_ok) { + req->relay = true; + req->relay_index = (u8)i; + req->relay_kind = svc_fwd_rel[i].tgt_kind; + req->relay_page = svc_fwd_rel[i].tgt_page; + req->relay_off = svc_fwd_rel[i].tgt_off; + req->relay_payload = svc_fwd_rel[i].payload; + svc.fwd_relay_requested++; + return; + } + } +} + /* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold @@ -6209,6 +6503,12 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, gsp_phys_guest_mem_enabled(); svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && svc_fwd_rel_n > 0; + /* + * P6c: choose the one relayed record for this request (if any). The + * selection is reset with the rest of the slot every time the releases + * do not qualify, so a later pass can never replay a stale pick. + */ + svc_fwd_relay_select(&svc_fwd_req); if (svc_fwd_req.guest) { svc_fwd_req.vgpu = vgpu; svc_fwd_req.src_kind = svc_fwd_cand.src_kind; @@ -6360,6 +6660,42 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, *canceled = svc.fwd_hold_canceled; } +/* + * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read + * discipline as the wave-1/1.5 stats above. + */ +void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, + u64 *skipped_sw, u64 *verify_fail, + u64 *resolve_fail) +{ + if (requested) + *requested = svc.fwd_relay_requested; + if (ok) + *ok = svc.fwd_relay_ok; + if (fallback) + *fallback = svc.fwd_relay_fallback; + if (skipped_sw) + *skipped_sw = svc.fwd_relay_skipped_sw; + if (verify_fail) + *verify_fail = svc.fwd_relay_verify_fail; + if (resolve_fail) + *resolve_fail = svc.fwd_relay_resolve_fail; +} + +/* + * P6c: true while this channel owns an active RELAY hold. The pump's + * per-pass notifier is suppressed for such a channel so the hold's verified + * success edge (or the next pass after a failure resolution) is the single + * completion post rather than a premature one before the guest release lands. + */ +static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + return svc_fwd_hold.active && svc_fwd_hold.relay_armed && + svc_fwd_hold.vgpu == vgpu && + svc_fwd_hold.ch_handle == ch->h_object; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -6414,7 +6750,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 applied_before; bool held = false; - svc_notify_complete(vgpu, ch); + /* + * P6c: while a relay hold owns this channel, the hold's + * verified success edge is the single completion post; skip + * the per-pass notifier so it cannot fire before the guest + * release is authored. + */ + if (!svc_fwd_relay_hold_active(vgpu, ch)) + svc_notify_complete(vgpu, ch); /* * P6b wave 1.5: enforce the hold cap on EVERY pass for the held @@ -6729,6 +7072,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); + /* + * P6c: a + * selected + * relay whose + * request did + * not queue is + * software-applied + * here. + */ + if (svc_fwd_req.relay) + svc.fwd_relay_fallback++; + svc_fwd_req.relay = false; } } /* -- cgit v1.2.3