diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 306 |
1 files changed, 304 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6a4d4ef..c2f3084 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -26071,6 +26071,216 @@ struct svc_fwd_req { static struct svc_fwd_req svc_fwd_req; +/* + * SESSION 153-CONTINUED-61: MISS-ABORT re-arm. The pre-doorbell abort + * (GSP_PHYS_GR_RC_MISSABORT) fires when a 3D-window operand's page is still + * unresolved; the page is demanded through the pin path and resolves within + * the pin cycle (tens of seconds). The historical handling consumed the + * captured candidate and software-completed the client (releases + QMD + * word), so no later forward ever re-ran that chunk and a store-bearing + * window never reached the engine (CONT-60). With the re-arm the candidate + * is copied back into the candidate ring unoffered, the client's releases + * are held back, and a bounded retry tick (svc_gr_missre_tick, called from + * svc_pump_channels) re-offers the same chunk; once the pins land the real + * launch runs and the engine (hold success edge) authors the releases. + * Bounded: after SVC_GR_MISSRE_DEAD_MS / SVC_GR_MISSRE_MAX_TRIES the + * historical consume-with-software-completion path stands so a page that + * never resolves cannot hang the client. Service-thread only. + */ +#define SVC_GR_MISSRE_INT_MS 1500u +#define SVC_GR_MISSRE_DEAD_MS 55000u +#define SVC_GR_MISSRE_MAX_TRIES 64u +static struct { + bool active; + u64 push_va; + u32 h_object; + u32 tries; + unsigned long first_at; + unsigned long last_try; +} svc_gr_missre; +static u32 svc_gr_missre_log; +static u64 svc_gr_missre_arm_n, svc_gr_missre_ok_n, svc_gr_missre_exp_n; + +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch); + +/* + * Episode complete: the re-armed launch ran for real. Clears the retry + * state so the tick stops re-offering. Called from the success edges + * (svc_fwd_outcome_check / svc_fwd_hold_poll). + */ +static void svc_gr_missre_done(void) +{ + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va) + return; + svc_gr_missre.active = false; + svc_gr_missre_ok_n++; + pr_info("vgsp-gr: MISS-RETRY converged push=0x%llx h=0x%x tries=%u ok=%llu (real launch ran after the pins landed)\n", + svc_fwd_req.gr_push_va, svc_fwd_req.gr_h_object, + svc_gr_missre.tries, svc_gr_missre_ok_n); +} + +/* + * Copy the just-aborted request's captured launch back into the candidate + * ring as an unoffered candidate (the re-arm). Returns false when the + * episode bound has expired or the ring has no free slot; the caller then + * falls through to the historical software completion. `rel`/`nrel` are + * the attempt's push-level release records (restored from the park table + * by svc_gr_request and drained by its caller): they are re-parked so the + * retried request restores the same set - the GPFIFO entry is consumed + * with this edge and will not re-decode. + */ +static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel) +{ + struct svc_gr_cand_rec *c = NULL; + unsigned long now = jiffies; + u32 slot; + + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va || + svc_gr_missre.h_object != svc_fwd_req.gr_h_object) { + svc_gr_missre.active = true; + svc_gr_missre.push_va = svc_fwd_req.gr_push_va; + svc_gr_missre.h_object = svc_fwd_req.gr_h_object; + svc_gr_missre.tries = 0; + svc_gr_missre.first_at = now; + } + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(now, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) { + u32 ri; + + svc_gr_missre_exp_n++; + pr_warn("vgsp-gr: MISS-RETRY episode expired push=0x%llx h=0x%x tries=%u waited_ms=%lu (falling back to software completion)\n", + svc_gr_missre.push_va, svc_gr_missre.h_object, + svc_gr_missre.tries, + jiffies_to_msecs(now - svc_gr_missre.first_at)); + /* + * Drop the lingering armed slot(s) for this launch: an + * unoffered candidate surviving the episode could be offered + * much later with a stale window snapshot (the client has + * since reused its buffers - the double-run class). + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ri]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + cc->offered = true; + cc->valid = false; + } + } + svc_gr_missre.active = false; + return false; + } + for (slot = 0; slot < SVC_GR_CAND_RING; slot++) { + if (!svc_gr_cand_ring[slot].valid || + svc_gr_cand_ring[slot].offered) { + c = &svc_gr_cand_ring[slot]; + break; + } + } + if (!c) + return false; /* ring full: fall back historically */ + memset(c, 0, sizeof(*c)); + c->valid = true; + c->offered = false; + c->rel_done = false; + c->captured_jiffies = now; + c->h_object = svc_fwd_req.gr_h_object; + c->push_va = svc_fwd_req.gr_push_va; + c->push_len = svc_fwd_req.gr_push_len; + c->push_n = svc_fwd_req.gr_push_n; + c->push_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + c->push_qmd_end = svc_fwd_req.gr_push_qmd_end; + c->stream_3d = svc_fwd_req.gr_stream_3d; + c->stream_off = svc_fwd_req.gr_stream_off; + c->stream_n = svc_fwd_req.gr_stream_n; + memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw)); + c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va; + memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd)); + c->n_inline = svc_fwd_req.gr_n_inline; + if (c->n_inline > SVC_GR_INLINE_MAX) + c->n_inline = SVC_GR_INLINE_MAX; + memcpy(c->inline_recs, svc_fwd_req.gr_inline, + sizeof(c->inline_recs)); + c->prog_va = svc_fwd_req.gr_prog_va; + c->param_va = svc_fwd_req.gr_param_va; + c->param_size = svc_fwd_req.gr_param_size; + c->rel_va = svc_fwd_req.gr_rel_va; + c->rel_payload = svc_fwd_req.gr_rel_payload; + c->rel_flags = svc_fwd_req.gr_rel_flags; + if (rel && nrel) + svc_grpark_put(svc_fwd_req.vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, rel, nrel); + svc_gr_missre_arm_n++; + if (svc_gr_missre_log < 96) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY re-arm push=0x%llx h=0x%x slot=%u window=%u nrel=%u arm=%llu (candidate kept unoffered; retry tick re-offers after the pins)\n", + c->push_va, c->h_object, slot, c->stream_n, + (unsigned)nrel, svc_gr_missre_arm_n); + } + return true; +} + +/* + * The retry tick (called per pass from svc_pump_channels for the tracked + * channel): re-offer the re-armed candidate on a bounded cadence. Parks + * svc_gr_cand_pos on the tracked slot so svc_gr_request's oldest-unoffered + * scan selects exactly this candidate; svc_gr_request itself defers on + * busy/inflight and no-ops for channels without a matching candidate. + */ +static void svc_gr_missre_tick(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 ti; + + if (ch->engine_type || !svc_gr_missre.active || + svc_gr_missre.h_object != ch->h_object || + !gsp_phys_gr_forward_enabled() || + !time_after_eq(jiffies, svc_gr_missre.last_try + + msecs_to_jiffies(SVC_GR_MISSRE_INT_MS))) + return; + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(jiffies, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) + return; /* the next abort edge expires the episode */ + svc_gr_missre.last_try = jiffies; + svc_gr_missre.tries++; + for (ti = 0; ti < SVC_GR_CAND_RING; ti++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ti]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + svc_gr_cand_pos = ti; + break; + } + } + if (svc_gr_missre_log < 192) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY tick try=%u push=0x%llx h=0x%x (re-offering the armed candidate)\n", + svc_gr_missre.tries, svc_gr_missre.push_va, + svc_gr_missre.h_object); + } + if (svc_gr_request(vgpu, ch)) { + /* + * A tick-queued request never gets an entry hold (the GPFIFO + * entry was consumed with the first abort - there is nothing + * left for the entry walk to hold). The per-push release + * records svc_gr_request just restored are re-parked so they + * survive the work item's lifetime; svc_fwd_outcome_check + * applies them when the retried launch's outcome lands. + */ + if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, ch->h_object, + svc_gr_missre.push_va, + svc_fwd_rel, svc_fwd_rel_n); + } +} + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -27590,8 +27800,57 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) * semaphore and the client's QMD word is still behind (the same edge * the hold's success path covers for held requests). */ - if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) - return; + if (svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT) { + /* + * SESSION 153-CONTINUED-61: MISS-ABORT without a hold - keep + * the candidate armed for a real re-forward and hold the + * client's release back (it completes on the retried + * launch's own edges). Only the expired episode falls + * through to the historical software completion below. + */ + if (svc_gr_missre_arm(NULL, 0)) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } + /* + * SESSION 153-CONTINUED-61: a tick-queued retried launch carries no + * entry hold, so its per-push release records survive in the park + * table (re-parked by svc_gr_missre_tick). Deliver them here on ANY + * final outcome - success, fault or the expired retry episode - so + * the client's tracking words always complete. Exactly-once: the + * park entry is dropped on delivery and the latch below stops any + * later pass from re-running this outcome. + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + static struct svc_channel shim; /* single-threaded service + * path; BSS keeps the kernel + * stack safe */ + u8 pn = 0; + + if (svc_grpark_get(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, parked, &pn)) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_req.gr_h_object; + svc_gr_copy_pend_run(vgpu); + svc_fwd_rel_apply_all(vgpu, &shim, parked, pn); + svc_grpark_drop(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va); + svc.fwd_rel_applied++; + pr_info("vgsp-gr: MISS-RETRY records delivered h=0x%x push=0x%llx nrel=%u (no-entry retry completion)\n", + svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, (unsigned)pn); + } + } + if (svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); + if (!svc_fwd_req.gr_rel_unobserved) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } /* * The outcome is final; latch it first so no later pass, hold edge or * consume tail accounts it again (the release write itself is @@ -27902,6 +28161,20 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* + * SESSION 153-CONTINUED-61: a MISS-ABORT hold expiring with the + * outcome known takes the re-arm edge instead of the software + * completion (the same verdict svc_fwd_hold_poll applies); only an + * episode past its bound falls through to the historical authorship + * below. + */ + if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + svc_fwd_req.gr_swrel_done = true; + svc_fwd_hold_clear(); + return; + } + /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a @@ -27973,6 +28246,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that @@ -28020,6 +28294,23 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } + } else if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + /* + * SESSION 153-CONTINUED-61: the launch aborted pre-doorbell + * because a window operand's page is still unresolved. Do NOT + * software-complete it: the candidate is re-armed for a real + * re-forward once the pin demand resolves the page (the retry + * tick re-offers it) and the client's releases stay unauthored + * (the retried launch's own edges author them on success). + * The attempt's push-level records were re-parked by the arm. + * Bounded by the episode deadline; an expired episode falls + * into the historical failure path below. gr_swrel_done + * latches the request outcome so the pass-top check cannot + * author the release later. + */ + svc_fwd_req.gr_swrel_done = true; } else { /* * P6b wave 2: failure/timeout/abort fallback - author the @@ -34367,6 +34658,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); + /* + * SESSION 153-CONTINUED-61: MISS-ABORT retry tick. A re-armed + * candidate (svc_gr_missre) is re-offered on a bounded cadence + * until the pin demand resolves its pages; the full decode path + * is deliberately NOT involved (the candidate ring slot is the + * queue). svc_gr_request no-ops for other channels and defers on + * busy/inflight. Placed before the early `continue`s so the retry + * still runs when the USERD read happens to fail this pass. + */ + svc_gr_missre_tick(vgpu, ch); + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; |