summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--kernel/gsp_sema_service.c172
1 files changed, 124 insertions, 48 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 4b315f6..1b94505 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -9088,6 +9088,20 @@ static u32 svc_userd_dbg_ch[32];
static u32 svc_ce_firstsub_n[32];
static unsigned long svc_ce_hold_since[32];
/*
+ * SESSION 134c: the FB offset each CE rank's first-submission capture latched
+ * (0 = none). The shadow scan is rank-blind on its own - it takes the first
+ * ring-shaped page in the shadow - and on r779 that latched rank 0's block
+ * (fb 0x80000, userd 0x90000) for rank 1 as well (both channels reported
+ * scan_hits=1 latch=1 off=0x80000). The rank-1 channel's first release was
+ * then decoded from rank 0's ring/page and stale-dropped (cur=1 already from
+ * rank 0's applied release), so the client's rank-1 tracker wait never
+ * completed and the client spun forever (victory_mail stall, three boots).
+ * The capture now prefers the rank's convention offset and refuses any
+ * offset another CE rank already latched. Session state; reset by
+ * svc_session_flush.
+ */
+static u64 svc_ce_firstsub_off[32];
+/*
* ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD
* probe fails at discovery. Session/reset-scoped, scheduled on the first
* failure (+3 s) and re-run once more (+30 s, after the client's follow-up
@@ -13101,6 +13115,7 @@ static void svc_session_flush(void)
memset(svc_userd_dbg_ch, 0, sizeof(svc_userd_dbg_ch));
memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n));
memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since));
+ memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off));
svc_userd_sweep_log = 0;
memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at));
memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n));
@@ -14726,6 +14741,71 @@ static u64 svc_ce_hold_ms;
module_param_named(sema_ce_hold_ms, svc_ce_hold_ms, ullong, 0644);
MODULE_PARM_DESC(sema_ce_hold_ms, "Hold an unlatched CE channel's completion open for up to N ms so the ring stays live for the latch (0 = off)");
+/*
+ * SESSION 134c rank-aware first-submission capture (see the capture below).
+ * The CE block convention: ring at 0x80000 + 0x20000 * rank, USERD at
+ * ring + 0x10000 (observed for every CE channel across boots; the same
+ * convention svc_chan_userd applies in its regular per-pass latch).
+ */
+#define SVC_CE_RING_FB_BASE 0x80000ULL
+#define SVC_CE_RING_FB_STRIDE 0x20000ULL
+
+/*
+ * One candidate offset for the CE first-submission capture: validate the
+ * ring shape (`scan_hits` counts shape hits) and, on a shape hit, run the
+ * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate whose
+ * USERD reads coherent (or idle 0/0) is PINNED and LATCHED exactly like
+ * svc_ring_autopin's CE path. Returns true when latched.
+ */
+static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch, u8 *sbase,
+ u64 lim, u64 off, u32 *scan_hits,
+ u64 *latch_off, u64 *latch_upa,
+ u32 *latch_put, u32 *latch_get)
+{
+ static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 };
+ u32 h0, h1, hlen;
+ u64 hva;
+ unsigned int ui;
+
+ memcpy(&h0, sbase + off, 4);
+ memcpy(&h1, sbase + off + 4, 4);
+ hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc);
+ hlen = ((h1 >> 10) & 0x1fffff) << 2;
+ if ((hva & 0xfffff) || hva < 0x120000000ULL ||
+ hva >= 0x121000000ULL || hlen < 4 || hlen > 0x10000)
+ return false;
+ (*scan_hits)++;
+ for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) {
+ u64 upa = off + udeltas[ui];
+ u32 p3 = 0, g3 = 0;
+ int prc;
+
+ if (upa + 4 > lim)
+ continue;
+ prc = svc_userd_probe(vgpu, upa,
+ SVC_APERTURE_FB,
+ ch->gpfifo_entries,
+ &p3, &g3);
+ if (prc && prc != -EAGAIN)
+ continue;
+ /* latch exactly like the CE autopin */
+ svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB);
+ svc_lat_store(ch->gpfifo_va, off, SVC_APERTURE_FB);
+ ch->userd_pa = upa;
+ ch->userd_as = SVC_APERTURE_FB;
+ ch->userd_ok = true;
+ set_bit(ch->ce_index, &svc_ring_done);
+ svc_ring_n++;
+ *latch_off = off;
+ *latch_upa = upa;
+ *latch_put = p3;
+ *latch_get = g3;
+ return true;
+ }
+ return false;
+}
+
static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
struct svc_channel *ch)
{
@@ -14770,59 +14850,57 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
}
/* 3. shadow scan for a CE-window ring page. */
{
+ /*
+ * SESSION 134c rank-aware order: try the rank's
+ * convention offset first, then the bounded scan,
+ * and never accept an offset another CE rank already
+ * latched. The rank-blind first-hit latched rank 0's
+ * block for rank 1 on r779 (both rows read
+ * off=0x80000 upa=0x90000), so the rank-1 channel's
+ * first release decoded from rank 0's ring/page and
+ * was stale-dropped (cur=1 already) - the client's
+ * rank-1 tracker wait never completed and it spun
+ * forever. A rank whose block is not in the shadow
+ * at this instant now simply stays unlatched for the
+ * regular per-pass CE latch instead of stealing a
+ * foreign block.
+ */
+ u64 exp = SVC_CE_RING_FB_BASE +
+ SVC_CE_RING_FB_STRIDE * ch->ce_index;
u64 off;
- for (off = 0; off + 0x90 <= lim; off += 0x100) {
- static const u64 udeltas[] = { 0x10000, 0x2000,
- 0x1000, 0x0 };
- u32 h0, h1;
- u64 hva;
- u32 hlen;
- unsigned int ui;
+ latched = svc_ce_firstsub_try_off(vgpu, ch, sbase,
+ lim, exp, &scan_hits,
+ &latch_off, &latch_upa,
+ &latch_put, &latch_get);
+ for (off = 0; !latched && off + 0x90 <= lim;
+ off += 0x100) {
+ unsigned int ri;
+ bool foreign = false;
- memcpy(&h0, sbase + off, 4);
- memcpy(&h1, sbase + off + 4, 4);
- hva = ((u64)(h1 & 0xff) << 32) |
- (u64)(h0 & 0xfffffffc);
- hlen = ((h1 >> 10) & 0x1fffff) << 2;
- if ((hva & 0xfffff) || hva < 0x120000000ULL ||
- hva >= 0x121000000ULL || hlen < 4 ||
- hlen > 0x10000)
+ if (off == exp)
continue;
- scan_hits++;
- for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) {
- u64 upa = off + udeltas[ui];
- u32 p3 = 0, g3 = 0;
- int prc;
-
- if (upa + 4 > lim)
+ for (ri = 0; ri < 32; ri++) {
+ if (ri == ch->ce_index ||
+ !svc_ce_firstsub_off[ri])
continue;
- prc = svc_userd_probe(vgpu, upa,
- SVC_APERTURE_FB,
- ch->gpfifo_entries,
- &p3, &g3);
- if (prc && prc != -EAGAIN)
- continue;
- /* latch exactly like the CE autopin */
- svc_pin_store(ch->gpfifo_va, off,
- SVC_APERTURE_FB);
- svc_lat_store(ch->gpfifo_va, off,
- SVC_APERTURE_FB);
- ch->userd_pa = upa;
- ch->userd_as = SVC_APERTURE_FB;
- ch->userd_ok = true;
- set_bit(ch->ce_index, &svc_ring_done);
- svc_ring_n++;
- latched = 1;
- latch_off = off;
- latch_upa = upa;
- latch_put = p3;
- latch_get = g3;
- break;
+ if (svc_ce_firstsub_off[ri] == off) {
+ foreign = true;
+ break;
+ }
}
- if (latched)
- break;
+ if (foreign)
+ continue;
+ latched = svc_ce_firstsub_try_off(vgpu, ch,
+ sbase, lim,
+ off, &scan_hits,
+ &latch_off,
+ &latch_upa,
+ &latch_put,
+ &latch_get);
}
+ if (latched)
+ svc_ce_firstsub_off[ch->ce_index] = latch_off;
}
}
@@ -14970,8 +15048,6 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
* against the ring's first GPFIFO entry, so a layout change costs a failed
* pin, not a wrong one.
*/
-#define SVC_CE_RING_FB_BASE 0x80000ULL
-#define SVC_CE_RING_FB_STRIDE 0x20000ULL
static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */
static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-27 05:08:07 +0000