diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 71 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 10 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 306 |
3 files changed, 380 insertions, 7 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 6f3898f..4c2b460 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17204,6 +17204,19 @@ MODULE_PARM_DESC(sema_gl_mme_prelude, "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); /* + * SESSION 153-CONTINUED-61: generation of the MME prelude already emitted + * into the engine context (0 = none). The r1110 evidence: the prelude was + * emitted once on the first beef launch (gen=1, 512 words), that launch + * priv-errored (grfifo=0xbadf1201) and WEDGE-RECOVER ran TSG preempt + + * resched - which resets the engine context and with it the resident MME + * program - yet the latch kept suppressing re-emission, so every later + * window replay ran its macros against an unloaded MME (mme_pc=513 class + * stall, no retire). The recovery clears the latch so the next launch + * re-preloads the context. + */ +static u32 gsp_phys_gr_mme_prelude_gen; + +/* * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef * window then latched a draw-shader program-header error @@ -17286,7 +17299,6 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, * to also carry the prelude replays without it (logged) - the * emission then happens on the next launch that fits. */ - static u32 mme_prelude_emitted_gen; bool emit_prelude = false; const u32 *pl = NULL; u32 pl_n = 0, pl_gen = 0; @@ -17294,7 +17306,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); if (pl && pl_n && pl_gen && - pl_gen != READ_ONCE(mme_prelude_emitted_gen)) { + pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen)) { if (win3d_n + 2u + 8u + pl_n <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { emit_prelude = true; @@ -17350,7 +17362,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, * emitted (the copy may be a torn prefix). */ gsp_sema_mme_prelude_get(NULL, &g1); if (g1 == g0) { - WRITE_ONCE(mme_prelude_emitted_gen, g0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0); pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u\n", g0, pwords, win3d_n, req->h_object, req->push_va, sdrop); @@ -19921,6 +19933,7 @@ static u64 gsp_phys_gr_surf_dummy_dma_locked(void) * sema_gl_miss_abort=0 restores the historical submit-anyway behaviour. */ static u32 gsp_phys_rmcall_gr_3d_miss_n; +static u64 gsp_phys_rmcall_gr_3d_junk_n; static bool gsp_phys_rmcall_gr_miss_abort = true; static u64 gsp_phys_gr_miss_abort_n; module_param_named(sema_gl_miss_abort, gsp_phys_rmcall_gr_miss_abort, bool, @@ -20005,6 +20018,35 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, } } /* + * SESSION 153-CONTINUED-61: junk-operand filter. The window + * recognizer is byte-shape based, so float/immediate DATA words in + * the method stream also parse as address pairs. The observed junk + * class sits in the 256 MB..4 GB numeric range (0x3d000000..0xc2...... + * float bit patterns: 0x3f800000 = 1.0f, 0xbf800000 = -1.0f, ...) + * and can NEVER resolve - r1109 measured miss=20 constant across + * every attempt, all of it this class, which pinned the abort census + * nonzero forever and blocked every submit. Real client VAs of this + * driver live below 0x5000000 (the 64-80 MB GL/heap range, e.g. + * 0x4250000/0x4350000/0x462fc40) or at/above 0x1_00000000 (the 4G+ + * push-buffer/UVM ranges). A miss outside both is a data-word + * misidentification, not a raw client VA the engine would read as an + * address: it is neither demanded (the demand set was 60 % junk by + * r1109, diverting the pin cycle) nor counted toward the pre-doorbell + * abort. Witnessed per class, bounded. + */ + if (page >= 0x10000000ULL && page < 0x100000000ULL) { + static u32 gr_missjunk_log; + + gsp_phys_rmcall_gr_3d_junk_n++; + if (gr_missjunk_log < 24 || + !(gsp_phys_rmcall_gr_3d_junk_n % 512u)) { + gr_missjunk_log++; + pr_info("vgsp-gr: MISS-JUNK va=0x%llx (data-word class; not demanded, not counted toward the abort) n=%llu\n", + page, (u64)gsp_phys_rmcall_gr_3d_junk_n); + } + return false; + } + /* * Unresolvable: demand the page for the pin path and count the miss * (the submit path aborts pre-doorbell while the count is nonzero). */ @@ -22340,7 +22382,15 @@ gr_page_unmapped: } mutex_unlock(&gsp_phys_rm_lock); gsp_phys_gr_refused++; - ret = -EAGAIN; + /* + * SESSION 153-CONTINUED-61: the dedicated verdict lets + * the sema service keep the captured candidate armed + * and re-forward it (re-arm) instead of consuming it + * with a software completion: the demanded pages + * resolve through the pin path and the retry then runs + * the real launch. + */ + ret = GSP_PHYS_GR_RC_MISSABORT; goto out; } /* @@ -22996,7 +23046,18 @@ gr_page_unmapped: gsp_phys_gr_wedge_hard_rearm_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); g1 = ioread32(bar0 + 0x00400504u); - pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot\n", + /* + * SESSION 153-CONTINUED-61: the TSG preempt + + * resched above resets the engine context, so + * the MME program the emitted prelude loaded is + * gone. Clear the emission latch: the next + * launch that can carry the prelude re-preloads + * it (without this every later replay ran its + * macros against an unloaded MME and stalled + * pre-retire - the r1110 mme_pc=513 class). + */ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", g1, (g1 & 0xffff0000u) == 0xbadf0000u ? "still poisoned" : "cleared", diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 14cf482..39824e3 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -369,6 +369,16 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, #define GSP_PHYS_GR_QMD_SENTINEL_WORD 63u #define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu /* + * SESSION 153-CONTINUED-61: dedicated backend verdict for the 3D-window + * MISS-ABORT (gsp_phys_rmcall_gr_miss_abort): the launch was NOT submitted + * because a window/attr operand's page is still unresolved. The sema + * service uses this code to keep the captured candidate armed (re-arm) + * instead of consuming it with a software completion, so the forward path + * retries the same chunk once the pin demand resolves the page. -ENOMSG + * is returned by no other path on this interface. + */ +#define GSP_PHYS_GR_RC_MISSABORT (-ENOMSG) +/* * r929: grown 256 -> 512 dwords so a whole GL 3D section push (the observed * X server one-shot is 1524 B = 381 dwords) survives in the captured raw * tail; the raw window is the source of the 3D method-stream replay. diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6a4d4ef..c2f3084 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -26071,6 +26071,216 @@ struct svc_fwd_req { static struct svc_fwd_req svc_fwd_req; +/* + * SESSION 153-CONTINUED-61: MISS-ABORT re-arm. The pre-doorbell abort + * (GSP_PHYS_GR_RC_MISSABORT) fires when a 3D-window operand's page is still + * unresolved; the page is demanded through the pin path and resolves within + * the pin cycle (tens of seconds). The historical handling consumed the + * captured candidate and software-completed the client (releases + QMD + * word), so no later forward ever re-ran that chunk and a store-bearing + * window never reached the engine (CONT-60). With the re-arm the candidate + * is copied back into the candidate ring unoffered, the client's releases + * are held back, and a bounded retry tick (svc_gr_missre_tick, called from + * svc_pump_channels) re-offers the same chunk; once the pins land the real + * launch runs and the engine (hold success edge) authors the releases. + * Bounded: after SVC_GR_MISSRE_DEAD_MS / SVC_GR_MISSRE_MAX_TRIES the + * historical consume-with-software-completion path stands so a page that + * never resolves cannot hang the client. Service-thread only. + */ +#define SVC_GR_MISSRE_INT_MS 1500u +#define SVC_GR_MISSRE_DEAD_MS 55000u +#define SVC_GR_MISSRE_MAX_TRIES 64u +static struct { + bool active; + u64 push_va; + u32 h_object; + u32 tries; + unsigned long first_at; + unsigned long last_try; +} svc_gr_missre; +static u32 svc_gr_missre_log; +static u64 svc_gr_missre_arm_n, svc_gr_missre_ok_n, svc_gr_missre_exp_n; + +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch); + +/* + * Episode complete: the re-armed launch ran for real. Clears the retry + * state so the tick stops re-offering. Called from the success edges + * (svc_fwd_outcome_check / svc_fwd_hold_poll). + */ +static void svc_gr_missre_done(void) +{ + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va) + return; + svc_gr_missre.active = false; + svc_gr_missre_ok_n++; + pr_info("vgsp-gr: MISS-RETRY converged push=0x%llx h=0x%x tries=%u ok=%llu (real launch ran after the pins landed)\n", + svc_fwd_req.gr_push_va, svc_fwd_req.gr_h_object, + svc_gr_missre.tries, svc_gr_missre_ok_n); +} + +/* + * Copy the just-aborted request's captured launch back into the candidate + * ring as an unoffered candidate (the re-arm). Returns false when the + * episode bound has expired or the ring has no free slot; the caller then + * falls through to the historical software completion. `rel`/`nrel` are + * the attempt's push-level release records (restored from the park table + * by svc_gr_request and drained by its caller): they are re-parked so the + * retried request restores the same set - the GPFIFO entry is consumed + * with this edge and will not re-decode. + */ +static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel) +{ + struct svc_gr_cand_rec *c = NULL; + unsigned long now = jiffies; + u32 slot; + + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va || + svc_gr_missre.h_object != svc_fwd_req.gr_h_object) { + svc_gr_missre.active = true; + svc_gr_missre.push_va = svc_fwd_req.gr_push_va; + svc_gr_missre.h_object = svc_fwd_req.gr_h_object; + svc_gr_missre.tries = 0; + svc_gr_missre.first_at = now; + } + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(now, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) { + u32 ri; + + svc_gr_missre_exp_n++; + pr_warn("vgsp-gr: MISS-RETRY episode expired push=0x%llx h=0x%x tries=%u waited_ms=%lu (falling back to software completion)\n", + svc_gr_missre.push_va, svc_gr_missre.h_object, + svc_gr_missre.tries, + jiffies_to_msecs(now - svc_gr_missre.first_at)); + /* + * Drop the lingering armed slot(s) for this launch: an + * unoffered candidate surviving the episode could be offered + * much later with a stale window snapshot (the client has + * since reused its buffers - the double-run class). + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ri]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + cc->offered = true; + cc->valid = false; + } + } + svc_gr_missre.active = false; + return false; + } + for (slot = 0; slot < SVC_GR_CAND_RING; slot++) { + if (!svc_gr_cand_ring[slot].valid || + svc_gr_cand_ring[slot].offered) { + c = &svc_gr_cand_ring[slot]; + break; + } + } + if (!c) + return false; /* ring full: fall back historically */ + memset(c, 0, sizeof(*c)); + c->valid = true; + c->offered = false; + c->rel_done = false; + c->captured_jiffies = now; + c->h_object = svc_fwd_req.gr_h_object; + c->push_va = svc_fwd_req.gr_push_va; + c->push_len = svc_fwd_req.gr_push_len; + c->push_n = svc_fwd_req.gr_push_n; + c->push_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + c->push_qmd_end = svc_fwd_req.gr_push_qmd_end; + c->stream_3d = svc_fwd_req.gr_stream_3d; + c->stream_off = svc_fwd_req.gr_stream_off; + c->stream_n = svc_fwd_req.gr_stream_n; + memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw)); + c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va; + memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd)); + c->n_inline = svc_fwd_req.gr_n_inline; + if (c->n_inline > SVC_GR_INLINE_MAX) + c->n_inline = SVC_GR_INLINE_MAX; + memcpy(c->inline_recs, svc_fwd_req.gr_inline, + sizeof(c->inline_recs)); + c->prog_va = svc_fwd_req.gr_prog_va; + c->param_va = svc_fwd_req.gr_param_va; + c->param_size = svc_fwd_req.gr_param_size; + c->rel_va = svc_fwd_req.gr_rel_va; + c->rel_payload = svc_fwd_req.gr_rel_payload; + c->rel_flags = svc_fwd_req.gr_rel_flags; + if (rel && nrel) + svc_grpark_put(svc_fwd_req.vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, rel, nrel); + svc_gr_missre_arm_n++; + if (svc_gr_missre_log < 96) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY re-arm push=0x%llx h=0x%x slot=%u window=%u nrel=%u arm=%llu (candidate kept unoffered; retry tick re-offers after the pins)\n", + c->push_va, c->h_object, slot, c->stream_n, + (unsigned)nrel, svc_gr_missre_arm_n); + } + return true; +} + +/* + * The retry tick (called per pass from svc_pump_channels for the tracked + * channel): re-offer the re-armed candidate on a bounded cadence. Parks + * svc_gr_cand_pos on the tracked slot so svc_gr_request's oldest-unoffered + * scan selects exactly this candidate; svc_gr_request itself defers on + * busy/inflight and no-ops for channels without a matching candidate. + */ +static void svc_gr_missre_tick(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 ti; + + if (ch->engine_type || !svc_gr_missre.active || + svc_gr_missre.h_object != ch->h_object || + !gsp_phys_gr_forward_enabled() || + !time_after_eq(jiffies, svc_gr_missre.last_try + + msecs_to_jiffies(SVC_GR_MISSRE_INT_MS))) + return; + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(jiffies, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) + return; /* the next abort edge expires the episode */ + svc_gr_missre.last_try = jiffies; + svc_gr_missre.tries++; + for (ti = 0; ti < SVC_GR_CAND_RING; ti++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ti]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + svc_gr_cand_pos = ti; + break; + } + } + if (svc_gr_missre_log < 192) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY tick try=%u push=0x%llx h=0x%x (re-offering the armed candidate)\n", + svc_gr_missre.tries, svc_gr_missre.push_va, + svc_gr_missre.h_object); + } + if (svc_gr_request(vgpu, ch)) { + /* + * A tick-queued request never gets an entry hold (the GPFIFO + * entry was consumed with the first abort - there is nothing + * left for the entry walk to hold). The per-push release + * records svc_gr_request just restored are re-parked so they + * survive the work item's lifetime; svc_fwd_outcome_check + * applies them when the retried launch's outcome lands. + */ + if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, ch->h_object, + svc_gr_missre.push_va, + svc_fwd_rel, svc_fwd_rel_n); + } +} + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -27590,8 +27800,57 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) * semaphore and the client's QMD word is still behind (the same edge * the hold's success path covers for held requests). */ - if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) - return; + if (svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT) { + /* + * SESSION 153-CONTINUED-61: MISS-ABORT without a hold - keep + * the candidate armed for a real re-forward and hold the + * client's release back (it completes on the retried + * launch's own edges). Only the expired episode falls + * through to the historical software completion below. + */ + if (svc_gr_missre_arm(NULL, 0)) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } + /* + * SESSION 153-CONTINUED-61: a tick-queued retried launch carries no + * entry hold, so its per-push release records survive in the park + * table (re-parked by svc_gr_missre_tick). Deliver them here on ANY + * final outcome - success, fault or the expired retry episode - so + * the client's tracking words always complete. Exactly-once: the + * park entry is dropped on delivery and the latch below stops any + * later pass from re-running this outcome. + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + static struct svc_channel shim; /* single-threaded service + * path; BSS keeps the kernel + * stack safe */ + u8 pn = 0; + + if (svc_grpark_get(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, parked, &pn)) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_req.gr_h_object; + svc_gr_copy_pend_run(vgpu); + svc_fwd_rel_apply_all(vgpu, &shim, parked, pn); + svc_grpark_drop(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va); + svc.fwd_rel_applied++; + pr_info("vgsp-gr: MISS-RETRY records delivered h=0x%x push=0x%llx nrel=%u (no-entry retry completion)\n", + svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, (unsigned)pn); + } + } + if (svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); + if (!svc_fwd_req.gr_rel_unobserved) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } /* * The outcome is final; latch it first so no later pass, hold edge or * consume tail accounts it again (the release write itself is @@ -27902,6 +28161,20 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* + * SESSION 153-CONTINUED-61: a MISS-ABORT hold expiring with the + * outcome known takes the re-arm edge instead of the software + * completion (the same verdict svc_fwd_hold_poll applies); only an + * episode past its bound falls through to the historical authorship + * below. + */ + if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + svc_fwd_req.gr_swrel_done = true; + svc_fwd_hold_clear(); + return; + } + /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a @@ -27973,6 +28246,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that @@ -28020,6 +28294,23 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } + } else if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + /* + * SESSION 153-CONTINUED-61: the launch aborted pre-doorbell + * because a window operand's page is still unresolved. Do NOT + * software-complete it: the candidate is re-armed for a real + * re-forward once the pin demand resolves the page (the retry + * tick re-offers it) and the client's releases stay unauthored + * (the retried launch's own edges author them on success). + * The attempt's push-level records were re-parked by the arm. + * Bounded by the episode deadline; an expired episode falls + * into the historical failure path below. gr_swrel_done + * latches the request outcome so the pass-top check cannot + * author the release later. + */ + svc_fwd_req.gr_swrel_done = true; } else { /* * P6b wave 2: failure/timeout/abort fallback - author the @@ -34367,6 +34658,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); + /* + * SESSION 153-CONTINUED-61: MISS-ABORT retry tick. A re-armed + * candidate (svc_gr_missre) is re-offered on a bounded cadence + * until the pin demand resolves its pages; the full decode path + * is deliberately NOT involved (the candidate ring slot is the + * queue). svc_gr_request no-ops for other channels and defers on + * busy/inflight. Placed before the early `continue`s so the retry + * still runs when the USERD read happens to fail this pass. + */ + svc_gr_missre_tick(vgpu, ch); + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; |