summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c74
1 files changed, 47 insertions, 27 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 77c2315..d3d88e0 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -254,6 +254,11 @@ struct svc_channel {
u64 counter_pa; /* guest PA of the u32 GPU completion counter
* (registered via the sema_counter debug param);
* 0 = not registered */
+ u64 release_pa; /* guest PA the push's REPORT_SEM release
+ * writes (learned from the release VA); the
+ * publish targets counter_pa instead, so both
+ * families of the counter page get written
+ * (SA-COUNTER FORENSICS-grv7-counter F1) */
u32 counter_last; /* last value published to counter_pa */
u64 pushpg_va; /* last push page logged as pin-vs-window
* divergent (rate limit: one line per page) */
@@ -854,6 +859,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
if (ch) {
int k;
u64 keep_counter_pa = 0;
+ u64 keep_release_pa = 0;
u8 keep_gr_index = 0, keep_ce_index = 0;
DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS);
bool same = ch->valid && ch->h_object == desc->h_object;
@@ -867,6 +873,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
*/
if (same) {
keep_counter_pa = ch->counter_pa;
+ keep_release_pa = ch->release_pa;
keep_gr_index = ch->gr_index;
keep_ce_index = ch->ce_index;
bitmap_copy(keep_rel_done, ch->rel_done,
@@ -916,6 +923,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
: SVC_APERTURE_SYS;
if (same) {
ch->counter_pa = keep_counter_pa;
+ ch->release_pa = keep_release_pa;
ch->gr_index = keep_gr_index;
ch->ce_index = keep_ce_index;
}
@@ -4456,7 +4464,16 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
spin_lock_irqsave(&svc.chan_lock, flags);
for (i = 0; i < SVC_MAX_CHANNELS; i++) {
if (svc.chan[i].valid && svc.chan[i].h_object == h) {
- pa = svc.chan[i].counter_pa;
+ /*
+ * The release targets the slot the push itself named
+ * (release_pa learned from the release VA); the
+ * completion counter publish targets counter_pa, a
+ * different word of the same page (SA-COUNTER
+ * FORENSICS-grv7-counter F1). Fall back to counter_pa
+ * when the release slot could not be resolved.
+ */
+ pa = svc.chan[i].release_pa ?
+ svc.chan[i].release_pa : svc.chan[i].counter_pa;
break;
}
}
@@ -4579,10 +4596,11 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
bool inc = ch->pending_rel_inc;
u32 cur = 0;
u32 nv;
+ u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa;
- if (!payload || !ch->counter_pa)
+ if (!payload || !pa)
return;
- if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
+ if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur,
sizeof(cur)))
return;
if (inc) {
@@ -4601,7 +4619,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
return;
}
nv = inc ? cur + payload : payload;
- if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &nv,
+ if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv,
sizeof(nv)))
return;
ch->pending_rel_pay = 0;
@@ -4616,7 +4634,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
if (gsp_sema_dbg && svc.counter_rel_dbg < 96) {
svc.counter_rel_dbg++;
pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n",
- ch->h_object, ch->counter_pa, payload, cur);
+ ch->h_object, pa, payload, cur);
}
}
@@ -4697,15 +4715,14 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
/*
* Auto-register a GR channel's completion-counter slot from the observed
- * layout, without waiting for a readable push: the client zips GR channels
- * in ascending handle order with descending counter slots, so with n_gr
- * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. The
- * slot PA comes from a host-verified pin or from the in-kernel structural
- * derivation (svc_counter_derive), so the write target is a page the client
- * actually polls. This lets the publish path advance every
- * channel's counter even when some GR pushes are unreadable (observed: the
- * lower ranks' 676-byte context-init pushes stayed unresolved while the
- * other 13 channels' counters advanced, and the client timed out waiting).
+ * layout, without waiting for a readable push: the counter records form a
+ * 16-byte-strided array in the client's counter page, allocated in channel
+ * order from the low end (SA-COUNTER FORENSICS-grv7-counter: the ascending
+ * slot fec0 + r*0x10 is the polled completion word; the push's REPORT_SEM
+ * release writes a DIFFERENT (descending) family and is tracked separately
+ * in release_pa). The slot PA comes from a host-verified pin or from the
+ * in-kernel structural derivation (svc_counter_derive), so the write target
+ * is a page the client actually polls.
*/
static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
struct svc_channel *ch, int n_gr)
@@ -4718,7 +4735,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
return;
if (ch->gr_index >= n_gr)
return;
- va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
+ va = 0x20400fec0ULL + (u64)ch->gr_index * 0x10ULL;
if (ch->counter_pa) {
/*
* FORENSICS-v2 anomaly 6: a slot derived in an earlier pass
@@ -4842,13 +4859,14 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
}
/*
- * Learn a GR channel's completion-counter guest PA from a release VA the
- * push carried, so the publish path can advance the exact slot the client
- * polls without any guest-side enumeration. The release VA of a GR push is
- * the counter slot itself (0x20400fXX observed). An injected pin is the
- * authoritative source; on a miss the slot is derived in-kernel
- * (svc_counter_derive) so the release is authored immediately rather than
- * waiting for the external pin watcher. No-op once registered at the same PA.
+ * Learn a GR channel's REPORT_SEM release slot from a release VA the push
+ * carried (0x20400fXX observed). This is the slot the release write
+ * targets (release_pa); it is deliberately NOT the completion-counter slot
+ * (counter_pa), which the publish advances and which the client polls
+ * (SA-COUNTER FORENSICS-grv7-counter F1: green wrote both families of the
+ * counter page). An injected pin is the authoritative source; on a miss the
+ * slot is derived in-kernel (svc_counter_derive) so the release is authored
+ * immediately rather than waiting for the external pin watcher.
*/
static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
{
@@ -4878,18 +4896,18 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
spin_lock_irqsave(&svc.chan_lock, flags);
for (i = 0; i < SVC_MAX_CHANNELS; i++) {
if (svc.chan[i].valid && svc.chan[i].h_object == h) {
- if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) {
- svc.chan[i].counter_pa = pa & ~0x3ULL;
+ if (svc.chan[i].release_pa != (pa & ~0x3ULL)) {
+ svc.chan[i].release_pa = pa & ~0x3ULL;
svc.chan[i].relearn_n++;
if (svc.chan[i].relearn_n == 64)
- pr_warn_ratelimited("sema-chan: counter re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n",
+ pr_warn_ratelimited("sema-chan: release re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n",
h,
svc.chan[i].relearn_n);
if (derived)
- pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n",
+ pr_info("sema-chan: release derived h=0x%x pa=0x%llx (from release va=0x%llx)\n",
h, pa & ~0x3ULL, sem_va);
else
- pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
+ pr_info("sema-chan: release learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
h, pa & ~0x3ULL, sem_va);
}
break;
@@ -12859,6 +12877,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
if (!svc.chan[k].counter_pa)
svc.chan[k].counter_pa = ch->counter_pa;
+ if (!svc.chan[k].release_pa)
+ svc.chan[k].release_pa = ch->release_pa;
break;
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 08:19:51 +0000