diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 387 |
1 files changed, 371 insertions, 16 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1f599fc..dea8249 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -470,6 +470,22 @@ struct sema_svc { u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; + /* + * P6c completion relay (gsp_phys_relay=1 only): relay selections armed + * on a deferred guest request (fwd_relay_requested), relays whose + * engine-authored guest semaphore was verified with the software apply + * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to + * the software apply (fwd_relay_fallback), hold re-reads that did not + * see the payload (fwd_relay_verify_fail) and work-item semaphore HPA + * resolution failures (fwd_relay_resolve_fail). All zero while the + * parameter is off. + */ + u64 fwd_relay_requested; + u64 fwd_relay_ok; + u64 fwd_relay_fallback; + u64 fwd_relay_skipped_sw; + u64 fwd_relay_verify_fail; + u64 fwd_relay_resolve_fail; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -2308,6 +2324,18 @@ struct svc_fwd_rel { u8 kind; /* SVC_FWD_REL_* */ u8 op; /* 906f: host sem op / launch: pending flag */ u8 inc_claimed; /* launch: one-shot inc release claimed */ + /* + * P6c completion relay: the release target resolved once at record + * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and + * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or + * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a + * false tgt_ok means the record is software-applied and never + * relayed. + */ + u8 tgt_ok; + u8 tgt_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ + u64 tgt_page; + u32 tgt_off; u64 va; u32 payload; u32 entry_idx; @@ -3133,6 +3161,67 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return false; } +/* + * P6c completion relay: resolve one deferred release record's target once at + * record time so the request can hand the engine a page/offset it may + * program. Fail-open by construction: every miss (parameter off, non-CE + * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an + * unresolvable VA, an unreadable word, or a word already at/ahead of the + * payload) leaves tgt_ok false and the record is software-applied exactly as + * in wave 1.5. The resolution uses the same svc_resolve_va() the software + * apply path uses and pins the page via svc_pin_store(), so the work item's + * later svc_fwd_operand_hpa() resolves the same page. Service-thread only. + */ +static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + struct svc_fwd_rel *r, u32 launch_dma, + bool is_launch) +{ + u64 addr = 0; + u32 ap = 0; + u32 cur = 0; + + r->tgt_ok = false; + r->tgt_kind = 0; + r->tgt_page = 0; + r->tgt_off = 0; + + if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type) + return; + if (svc_counter_va(r->va)) + return; + /* + * The relay always authors an absolute one-word release (the CE block + * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a + * record whose software apply is additive must never be relayed: the + * engine would leave the guest word short of the value it waits on. + * The LAUNCH increment is excluded by the dma check below; the 906F + * REDUCTION op is excluded here. + */ + if (!is_launch && r->kind == SVC_FWD_REL_906F && + r->op == HOST_SEM_OP_REDUCTION) + return; + if (is_launch) { + if (DMA_SEMAPHORE_TYPE(launch_dma) != + SEM_RELEASE_ONE_WORD) + return; + if (DMA_SEM_REDUCTION_EN(launch_dma) && + DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) + return; + } + if (svc_resolve_va(vgpu, r->va, &addr, &ap)) + return; + if (svc_sem_read(vgpu, r->va, &cur)) + return; + /* Forward-only pre-check: only a word behind the payload is relayed. */ + if (cur >= r->payload) + return; + r->tgt_ok = true; + r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); + r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -3399,6 +3488,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = payload; r->entry_idx = entry_idx; r->dma = 0; + svc_fwd_relay_note(vgpu, ch, r, + 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { @@ -3512,6 +3603,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = st.rs_payload; r->entry_idx = entry_idx; r->dma = 0; + svc_fwd_relay_note(vgpu, ch, + r, 0, + false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; break; @@ -3844,6 +3938,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, r->payload = st.sem_payload; r->entry_idx = entry_idx; r->dma = dma; + svc_fwd_relay_note(vgpu, ch, r, + dma, true); /* * P6b wave 1.5: do NOT bump * svc.pushes_pending here - @@ -5486,6 +5582,25 @@ struct svc_fwd_req { u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ u32 src_off, dst_off; u32 bytes; + /* + * P6c completion relay: when relay is true this request's hardware + * copy additionally authors the guest-visible release of the selected + * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/ + * off identify the resolved semaphore target and relay_payload is the + * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are + * published by the work item before its done release, for the hold's + * verification/logging. Cleared with the rest of the slot whenever the + * releases do not qualify. + */ + bool relay; + u8 relay_index; + u8 relay_kind; + u64 relay_page; + u32 relay_off; + u32 relay_payload; + bool relay_done; + u32 relay_sem; + u32 relay_elapsed_ms; }; static struct svc_fwd_req svc_fwd_req; @@ -5669,6 +5784,15 @@ struct svc_fwd_hold { u64 gpfifo_va; u32 entry_idx; u8 nrel; + /* + * P6c completion relay: the request selected record relay_index and + * the hardware copy was asked to author it into the guest-visible + * semaphore page. On the success edge the hold re-reads the guest + * word and, when it carries the payload, skips that one record's + * software apply. Cleared with the hold. + */ + bool relay_armed; + u8 relay_index; struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; @@ -5733,14 +5857,26 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, inc, r->inc_claimed); } -/* Apply the recorded releases of one deferred push (service thread). */ -static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, - const struct svc_channel *ch, - const struct svc_fwd_rel *rel, u8 n) +/* Skip value that never matches a record index. */ +#define SVC_FWD_REL_SKIP_NONE 0xff + +/* + * Apply the recorded releases of one deferred push (service thread), except + * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip + * on the relay verification success edge: the engine already authored that + * one release into the guest-visible semaphore, so re-applying it in software + * is unnecessary (and would count a second release). + */ +static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n, + u8 skip) { u8 i; for (i = 0; i < n; i++) { + if (i == skip) + continue; switch (rel[i].kind) { case SVC_FWD_REL_906F: svc_rel_apply_906f(vgpu, ch, rel[i].va, @@ -5759,6 +5895,14 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, } } +/* Apply every recorded release (no relay skip). */ +static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n) +{ + svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); +} + static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + @@ -5790,6 +5934,8 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel = n; svc_fwd_hold.last_rel_va = svc_rel_va_seen; svc_fwd_hold.since = jiffies; + svc_fwd_hold.relay_armed = svc_fwd_req.relay; + svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -5813,6 +5959,8 @@ static void svc_fwd_hold_clear(void) memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand)); svc_fwd_hold.vgpu = NULL; svc_fwd_hold.nrel = 0; + svc_fwd_hold.relay_armed = false; + svc_fwd_hold.relay_index = 0; svc.fwd_rel_pending = 0; } @@ -5861,6 +6009,13 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, if (!done) svc.fwd_hold_timeouts++; } + /* + * P6b/P6c: the timeout path never reaches the relay verification + * edge, so a relayed set is software-applied whole (forward-only, so + * the engine's write cannot be duplicated). + */ + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -5883,16 +6038,63 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { - svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, - svc_fwd_hold.nrel); - svc.fwd_rel_applied++; + /* + * P6c: on the relay success edge, re-read the guest-visible + * semaphore through the software view. Only a word that + * actually carries the payload lets the one relayed record be + * skipped; anything else (unreadable, behind) is a relay + * verification failure and the software applies the whole set + * (the forward-only apply makes the relayed write idempotent, + * so the release is never duplicated or lost). + */ + if (svc_fwd_hold.relay_armed && + svc_fwd_hold.relay_index < svc_fwd_hold.nrel) { + struct svc_fwd_rel *rr = + &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; + u32 cur = 0; + + if (!svc_sem_read(hv, rr->va, &cur) && + cur >= rr->payload) { + svc_fwd_rel_apply_all_ex(hv, ch, + svc_fwd_hold.rel, + svc_fwd_hold.nrel, + svc_fwd_hold.relay_index); + svc.fwd_relay_ok++; + svc.fwd_relay_skipped_sw++; + svc.fwd_rel_applied++; + /* + * The hold's post is the single completion + * edge for this channel: the pump's per-pass + * notifier was suppressed while the relay hold + * was active. + */ + svc_notify_complete(hv, ch); + } else { + svc.fwd_relay_verify_fail++; + svc.fwd_relay_fallback++; + svc_fwd_rel_apply_all(hv, ch, + svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + if ((svc.fwd_relay_verify_fail & 15) == 1) + pr_err("gsp-phys-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n", + rr->va, cur, rr->payload); + } + } else { + svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + } } else { /* * P6b wave 2: failure/timeout/abort fallback - author the * skipped data once, then apply the releases exactly as - * wave 1.5 does. + * wave 1.5 does. P6c: a relayed set is applied whole here + * (the engine did not deliver it). */ svc_fwd_hold_soft_copy(hv); + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_fallback++; @@ -5973,6 +6175,13 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) svc_fwd_hold_soft_copy(hv); svc.fwd_hold_canceled++; } + /* + * P6c: a vanished channel cannot run the relay verification edge; a + * relayed set falls back to the software path (which is exactly-once + * against the engine's forward write). + */ + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -6006,6 +6215,8 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc.fwd_rel_applied++; else svc.fwd_rel_fallback++; + if (svc_fwd_hold.relay_armed) + svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } @@ -6029,9 +6240,10 @@ static void svc_fwd_work_fn(struct work_struct *work) * while a hold is active. */ if (guest) { - u64 src_hpa = 0, dst_hpa = 0; - u32 src_off = 0, dst_off = 0; - int src_rc, dst_rc; + u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0; + u32 src_off = 0, dst_off = 0, sem_off = 0; + struct gsp_phys_backend_copy_result cp = { 0 }; + int src_rc, dst_rc, sem_rc = 0; bool resolved; src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, @@ -6047,11 +6259,43 @@ static void svc_fwd_work_fn(struct work_struct *work) svc_fwd_req.bytes, &dst_hpa, &dst_off); resolved = !src_rc && !dst_rc; + /* + * P6c: resolve the selected relay target to an HPA with the + * same operand resolver. A failure is not fatal - hw_rc is + * forced to an error so the hold applies the software + * release exactly as wave 1.5 (never lost, never duplicated). + */ + if (svc_fwd_req.relay) + sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.relay_kind, + svc_fwd_req.relay_page, + svc_fwd_req.relay_off, 4, + &sem_hpa, &sem_off); if (resolved) { - hw_rc = gsp_phys_backend_guest_ce_copy(src_hpa, src_off, - dst_hpa, dst_off, - svc_fwd_req.bytes, - GSP_PHYS_BACKEND_WORK_POLL_MS); + if (svc_fwd_req.relay && !sem_rc) { + hw_rc = gsp_phys_backend_guest_ce_copy_ex( + src_hpa, src_off, dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS, + sem_hpa, sem_off, + svc_fwd_req.relay_payload, &cp); + svc_fwd_req.relay_sem = cp.sem_value; + svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms; + svc_fwd_req.relay_done = (hw_rc == 0); + } else if (svc_fwd_req.relay) { + hw_rc = -EINVAL; + svc.fwd_relay_resolve_fail++; + if (svc.fwd_relay_resolve_fail <= 16) + pr_info("gsp-phys-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n", + svc_fwd_req.relay_kind, + svc_fwd_req.relay_page, + svc_fwd_req.relay_off, sem_rc); + } else { + hw_rc = gsp_phys_backend_guest_ce_copy( + src_hpa, src_off, dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS); + } if (!hw_rc) { svc.fwd_guest_ok++; if (!svc_fwd_req.defer) { @@ -6145,6 +6389,56 @@ static void svc_fwd_work_fn(struct work_struct *work) static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); /* + * P6c completion relay: pick the ONE deferred release record this request's + * hardware copy may author. Only a deferred guest request whose records were + * armed can relay, and only a record whose record-time resolution succeeded + * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the + * completion edge the guest tracker waits on), else the last eligible record. + * Fail-open: no eligible record leaves req->relay false and the whole release + * set stays software-applied. Service-thread only; called with the request + * slot's INFLIGHT bit held, before queue_work. + */ +static void svc_fwd_relay_select(struct svc_fwd_req *req) +{ + int i; + + req->relay = false; + req->relay_index = 0; + req->relay_kind = 0; + req->relay_page = 0; + req->relay_off = 0; + req->relay_payload = 0; + + if (!req->defer || !gsp_phys_relay_enabled()) + return; + for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { + if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH && + svc_fwd_rel[i].tgt_ok) { + req->relay = true; + req->relay_index = (u8)i; + req->relay_kind = svc_fwd_rel[i].tgt_kind; + req->relay_page = svc_fwd_rel[i].tgt_page; + req->relay_off = svc_fwd_rel[i].tgt_off; + req->relay_payload = svc_fwd_rel[i].payload; + svc.fwd_relay_requested++; + return; + } + } + for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { + if (svc_fwd_rel[i].tgt_ok) { + req->relay = true; + req->relay_index = (u8)i; + req->relay_kind = svc_fwd_rel[i].tgt_kind; + req->relay_page = svc_fwd_rel[i].tgt_page; + req->relay_off = svc_fwd_rel[i].tgt_off; + req->relay_payload = svc_fwd_rel[i].payload; + svc.fwd_relay_requested++; + return; + } + } +} + +/* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. @@ -6209,6 +6503,12 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, gsp_phys_guest_mem_enabled(); svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && svc_fwd_rel_n > 0; + /* + * P6c: choose the one relayed record for this request (if any). The + * selection is reset with the rest of the slot every time the releases + * do not qualify, so a later pass can never replay a stale pick. + */ + svc_fwd_relay_select(&svc_fwd_req); if (svc_fwd_req.guest) { svc_fwd_req.vgpu = vgpu; svc_fwd_req.src_kind = svc_fwd_cand.src_kind; @@ -6360,6 +6660,42 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, *canceled = svc.fwd_hold_canceled; } +/* + * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read + * discipline as the wave-1/1.5 stats above. + */ +void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, + u64 *skipped_sw, u64 *verify_fail, + u64 *resolve_fail) +{ + if (requested) + *requested = svc.fwd_relay_requested; + if (ok) + *ok = svc.fwd_relay_ok; + if (fallback) + *fallback = svc.fwd_relay_fallback; + if (skipped_sw) + *skipped_sw = svc.fwd_relay_skipped_sw; + if (verify_fail) + *verify_fail = svc.fwd_relay_verify_fail; + if (resolve_fail) + *resolve_fail = svc.fwd_relay_resolve_fail; +} + +/* + * P6c: true while this channel owns an active RELAY hold. The pump's + * per-pass notifier is suppressed for such a channel so the hold's verified + * success edge (or the next pass after a failure resolution) is the single + * completion post rather than a premature one before the guest release lands. + */ +static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + return svc_fwd_hold.active && svc_fwd_hold.relay_armed && + svc_fwd_hold.vgpu == vgpu && + svc_fwd_hold.ch_handle == ch->h_object; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -6414,7 +6750,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 applied_before; bool held = false; - svc_notify_complete(vgpu, ch); + /* + * P6c: while a relay hold owns this channel, the hold's + * verified success edge is the single completion post; skip + * the per-pass notifier so it cannot fire before the guest + * release is authored. + */ + if (!svc_fwd_relay_hold_active(vgpu, ch)) + svc_notify_complete(vgpu, ch); /* * P6b wave 1.5: enforce the hold cap on EVERY pass for the held @@ -6729,6 +7072,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); + /* + * P6c: a + * selected + * relay whose + * request did + * not queue is + * software-applied + * here. + */ + if (svc_fwd_req.relay) + svc.fwd_relay_fallback++; + svc_fwd_req.relay = false; } } /* |