summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c145
1 files changed, 114 insertions, 31 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 4259314..7a7755f 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -262,6 +262,8 @@ struct svc_channel {
DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS);
u8 slot_pend_tries[SVC_GPFIFO_MAX_BITS]; /* fail4 r308: bounded
* re-holds of a denied device-dst copy slot */
+ u32 counter_gen; /* r311: session generation that authored the
+ * completion word (guards the residue zero) */
/* GPFIFO slots already consumed in the current
* put epoch (the client's put value since it
* last wrapped). A pin-batch rewind replays
@@ -824,6 +826,7 @@ struct sema_svc {
u64 opadd_log; /* fail4: every op appended */
u64 dmatrace_log; /* fail4: denied-transfer trace */
u64 pend_dbg_log; /* fail4 r305: pending/hold witness */
+ u64 snap_gate_log; /* fail4 r311: capture-gate witness */
u64 copyent_log; /* fail4: svc_copy entry trace */
u64 bigrun_log; /* fail4: big method-data run headers */
u64 lid_hdr_log; /* fail4: LOAD_INLINE_DATA headers */
@@ -1310,6 +1313,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
u64 keep_counter_pa = 0;
u64 keep_release_pa = 0;
u8 keep_gr_index = 0, keep_ce_index = 0;
+ bool keep_pub_armed = false;
+ u32 keep_counter_gen = 0;
DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS);
bool same = ch->valid && ch->vgpu == desc->vgpu &&
ch->h_object == desc->h_object;
@@ -1334,6 +1339,18 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
keep_release_pa = ch->release_pa;
keep_gr_index = ch->gr_index;
keep_ce_index = ch->ce_index;
+ /*
+ * r311: a same-session re-registration must not
+ * forget that this session already proved itself
+ * (gr_pub_armed) - losing it re-arms the residual
+ * watchdog, and its recovery zeroes a completion
+ * word this live session authored, which the
+ * client's UVM reads as a downward jump and never
+ * completes on (r311 vector_add: pub-adv/zero
+ * 3 s cycles with the client polling forever).
+ */
+ keep_pub_armed = ch->gr_pub_armed;
+ keep_counter_gen = ch->counter_gen;
bitmap_copy(keep_rel_done, ch->rel_done,
SVC_GPFIFO_MAX_BITS);
}
@@ -1345,21 +1362,12 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
ch->gr_rehold_idx = ~0u;
ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */
ch->session_gen = svc_sess_gen; /* m176 */
- /*
- * STALLFREE: a genuinely new binding - a fresh channel or a
- * same-handle rebinding across a session/context boundary
- * (!same_gen) - must reconcile on its first pump pass: drop
- * opdef slots a dead context left under this handle (only
- * ones whose first-pending stamp predates the binding, so a
- * slot the fresh binding creates can never be touched) and
- * log the transition when the ring already carries content.
- * A same-generation RM re-issue preserves state deliberately
- * (m148/m176) and is not reconciled.
- */
- ch->wd_chanswitch = !same_gen;
- if (same_gen)
+ if (same_gen) {
bitmap_copy(ch->rel_done, keep_rel_done,
SVC_GPFIFO_MAX_BITS);
+ ch->gr_pub_armed = keep_pub_armed;
+ ch->counter_gen = keep_counter_gen;
+ }
/*
* Rank among the live CE channels OF THIS GUEST. The RM hands
* out object handles in creation order and the guest's UVM
@@ -8556,11 +8564,40 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
lm->aperture == CE_APERTURE_VID &&
!lm->src_phys &&
lm->src >= SVC_VAS_FLOOR &&
- lm->src_aperture == CE_APERTURE_SYS &&
- lm->bytes <= SVC_COPY_SNAP_MAX)
- svc_copy_snap_capture(vgpu, lm->dst,
- lm->src,
- lm->bytes);
+ lm->bytes <= SVC_COPY_SNAP_MAX) {
+ /*
+ * r311: the src_aperture equality
+ * gate never matched the sw-copy
+ * client (the driver sends no
+ * SET_SRC_PHYS_MODE, so src_ap
+ * defaults to VID=0 while the
+ * source is host staging; r302-r310
+ * logged zero captures and the
+ * late executions read refilled
+ * staging). The translation is the
+ * truth: capture only when the
+ * first source page resolves
+ * through the SYS aperture.
+ */
+ u64 spa = 0;
+ u32 sap = 0;
+
+ if (!svc_op_translate(vgpu, lm->src,
+ &spa, &sap) &&
+ sap == CE_APERTURE_SYS)
+ svc_copy_snap_capture(vgpu,
+ lm->dst,
+ lm->src,
+ lm->bytes);
+ if (gsp_sema_dbg &&
+ svc.snap_gate_log < 256) {
+ svc.snap_gate_log++;
+ pr_info("sema-snapgate: dst=0x%llx src=0x%llx trc=%d sap=%u\n",
+ lm->dst, lm->src,
+ svc_op_translate(vgpu, lm->src, &spa, &sap),
+ sap);
+ }
+ }
/*
* fail4 CE linear auto-advance, BOTH sides: the
@@ -12915,9 +12952,20 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n",
ch->h_object, pa, val, cur,
ch->counter_last);
- if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &val, sizeof(val)))
- return;
+ {
+ int wrc = svc_write_ap(vgpu, pa, SVC_APERTURE_SYS,
+ &val, sizeof(val));
+ if (wrc) {
+ if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
+ svc.pend_dbg_log++;
+ pr_info("sema-diag: pub-adv-wrc h=0x%x pa=0x%llx val=%u wrc=%d (write did not land)\n",
+ ch->h_object, pa, val, wrc);
+ }
+ return;
+ }
+ }
ch->counter_last = val;
+ ch->counter_gen = svc_sess_gen;
if (gsp_sema_dbg && svc.counter_pub_dbg < 48) {
svc.counter_pub_dbg++;
pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n",
@@ -13787,7 +13835,7 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu,
if (e->n) {
svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n);
svc.opdef_applied++;
- if (gsp_sema_dbg && svc_opdef_log < 32) {
+ if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) {
svc_opdef_log++;
pr_info("sema-opdef: apply h=0x%x entry=%u n=%u (ops complete)\n",
ch->h_object, entry_idx, e->n);
@@ -13864,7 +13912,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu,
if (!svc_opdef_contains(e, r)) {
if (r->payload && e->n < SVC_FWD_REL_MAX) {
e->rel[e->n++] = *r;
- if (gsp_sema_dbg && svc_opdef_log < 32) {
+ if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) {
svc_opdef_log++;
pr_info("sema-opdef: hold h=0x%x entry=%u kind=%u va=0x%llx pay=%u (operand pending)\n",
ch->h_object, entry_idx,
@@ -17444,9 +17492,34 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu,
ch->counter_pa;
u32 zero = 0;
- if (upa)
- (void)svc_write_ap(vgpu, upa, SVC_APERTURE_SYS,
- &zero, sizeof(zero));
+ /*
+ * r311: zero only a word authored by a DEAD
+ * generation. Zeroing a word this live session
+ * authored makes the client's UVM observe a
+ * downward jump (its tracker then never completes)
+ * while the publish re-advances it - the r311
+ * vector_add wedge (pub-adv/zero cycles, client
+ * polling cur=0 forever).
+ */
+ if (ch->counter_gen == svc_sess_gen) {
+ if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
+ svc.pend_dbg_log++;
+ pr_info("sema-diag: stale-recover-zero-skip h=0x%x upa=0x%llx last=%u (authored by live gen)\n",
+ ch->h_object, upa,
+ ch->counter_last);
+ }
+ ch->counter_last = 0;
+ } else if (upa) {
+ int zrc = svc_write_ap(vgpu, upa,
+ SVC_APERTURE_SYS,
+ &zero, sizeof(zero));
+ if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
+ svc.pend_dbg_log++;
+ pr_info("sema-diag: stale-recover-zero h=0x%x upa=0x%llx last=%u zrc=%d\n",
+ ch->h_object, upa,
+ ch->counter_last, zrc);
+ }
+ }
ch->counter_last = 0;
}
}
@@ -19765,12 +19838,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].wd_put = ch->wd_put;
svc.chan[k].wd_sub = ch->wd_sub;
svc.chan[k].wd_since = ch->wd_since;
- svc.chan[k].wd_epoch_done = ch->wd_epoch_done;
- svc.chan[k].wd_nocap_put = ch->wd_nocap_put;
- svc.chan[k].wd_chanswitch = ch->wd_chanswitch;
- svc.chan[k].wd_newbind_until = ch->wd_newbind_until;
- svc.chan[k].wd_newbind_hold_until =
- ch->wd_newbind_hold_until;
+ /*
+ * r312: gr_pub_armed / counter_gen live only on
+ * the snapshot copy if not persisted. Losing the
+ * armed bit re-arms the residual watchdog every
+ * pass; its recovery then keeps zeroing the
+ * completion word and re-publishing - the r311/
+ * r312 vector_add wedge loop (66 stale-wedge
+ * rows with the client polling forever).
+ */
+ svc.chan[k].gr_pub_armed = ch->gr_pub_armed;
+ svc.chan[k].counter_gen = ch->counter_gen;
+ svc.chan[k].wd_rewind_off_until =
+ ch->wd_rewind_off_until;
svc.chan[k].pushpg_va = ch->pushpg_va;
svc.chan[k].pushpg_n = ch->pushpg_n;
svc.chan[k].content_max = ch->content_max;
@@ -19796,6 +19876,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
memcpy(svc.chan[k].slot_ops_ident,
ch->slot_ops_ident,
sizeof(ch->slot_ops_ident));
+ memcpy(svc.chan[k].slot_pend_tries,
+ ch->slot_pend_tries,
+ sizeof(ch->slot_pend_tries));
memcpy(svc.chan[k].slot_ops_done,
ch->slot_ops_done,
sizeof(ch->slot_ops_done));
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-20 23:17:18 +0000