summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_rm_emulation.c8
-rw-r--r--kernel/gsp_sema_service.c297
-rw-r--r--kernel/gsp_sema_service.h4
3 files changed, 292 insertions, 17 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 8cd82d1..f6ca2c1 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -3812,6 +3812,14 @@ void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
"vgsp-rm: reset dropped %u stale event record(s)",
stale);
+ /*
+ * m177: RM reload means a new guest boot: the guest's direct-map base
+ * and every content-scanned translation may have changed. Raise the
+ * boot-scoped wipe (executed on the sema svc thread; client session
+ * flushes intentionally keep that state - see svc_session_flush).
+ */
+ gsp_sema_service_guest_reset();
+
if (!rm->parsed)
return;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 8f23178..7b12ecc 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -483,6 +483,7 @@ static u32 svc_ring_n;
* guest's last-channel close is also the global empty transition.
*/
static u8 svc_sess_flush;
+static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */
static u8 svc_sess_active; /* a channel of a session has registered (until flush) */
static u32 svc_sess_gen; /* m176: bumped by every executed session flush;
* stamps channels/scrubbers so per-session learned
@@ -1975,13 +1976,133 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
svc_pd_cands, svc_pd_best, n);
}
+/*
+ * m177c: a SECOND discovered-directory slot for the UVM SYS VA space (the
+ * 0x2xxxxxxx family: GR rings/pushes, QMD windows, counter pages). The
+ * primary slot (above) is validated against rm_mem-content pairs - the
+ * 0x1xxxxxxx pool family - and cannot walk the UVM space, so without pins
+ * the counter page (0x20400fXXX) never resolved: the guest's KMD/UVM
+ * epoch waited on completions that could never be written and re-ran its
+ * ~103 s retry loop forever (r194/r195: 'rel-apply ... slot pending' x8,
+ * then silence). Samples for this slot are content-VERIFIED
+ * (push_va -> push_pa) pairs from the GR content discovery and the CE
+ * push pairing: the push bytes themselves are the proof, and they live in
+ * exactly the space this directory maps.
+ */
+#define SVC_PD_RING_SAMPLES 8
+static u64 svc_pd_r_v[SVC_PD_RING_SAMPLES];
+static u64 svc_pd_r_pa[SVC_PD_RING_SAMPLES];
+static u32 svc_pd_r_n;
+static u64 svc_pd_r_base;
+static int svc_pd_r_ap;
+static u8 svc_pd_r_found;
+static u8 svc_pd_r_attempts;
+static u64 svc_pd_r_ev_hash;
+
+static void svc_pd_ring_note(u64 va, u64 pa)
+{
+ u32 i;
+
+ if (!va || !pa)
+ return;
+ va &= ~0xfffULL;
+ pa &= ~0xfffULL;
+ for (i = 0; i < svc_pd_r_n; i++)
+ if (svc_pd_r_v[i] == va)
+ return;
+ if (svc_pd_r_n < SVC_PD_RING_SAMPLES) {
+ svc_pd_r_v[svc_pd_r_n] = va;
+ svc_pd_r_pa[svc_pd_r_n] = pa;
+ svc_pd_r_n++;
+ }
+}
+
+static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu)
+{
+ u64 chunk = 64 * 1024;
+ u8 *buf;
+ u64 gpa;
+ u64 ev = svc_pd_r_n;
+ u32 i;
+
+ if (svc_pd_r_found || svc_pd_r_attempts >= SVC_PD_ATTEMPTS_MAX)
+ return;
+ if (svc_pd_r_n < SVC_PD_MIN_HITS)
+ return;
+ for (i = 0; i < svc_pd_r_n; i++)
+ ev = (ev + svc_pd_r_v[i] + (svc_pd_r_pa[i] << 12)) * 1000003;
+ if (svc_pd_r_attempts && ev == svc_pd_r_ev_hash)
+ return;
+ svc_pd_r_ev_hash = ev;
+ svc_pd_r_attempts++;
+
+ buf = kmalloc(chunk, GFP_KERNEL);
+ if (!buf)
+ return;
+ for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) {
+ u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa);
+ u64 off;
+
+ if (svc_read(vgpu, gpa, buf, len))
+ continue;
+ for (off = 0; off + 8 <= len && !svc_pd_r_found;
+ off += SEMA_PAGE_SIZE) {
+ u64 e;
+ u64 a = 0;
+ int eap = 0;
+ u32 hits = 1;
+
+ memcpy(&e, buf + off, sizeof(e));
+ if ((e & 1) || !((e >> 1) & 0x3))
+ continue;
+ /* First sample rejects almost every candidate. */
+ if (svc_gmmu_walk(vgpu, gpa + off, 4, svc_pd_r_v[0],
+ SVC_APERTURE_SYS, &a, &eap, false) ||
+ (a & ~0xfffULL) != svc_pd_r_pa[0])
+ continue;
+ for (i = 1; i < svc_pd_r_n; i++) {
+ if (!svc_gmmu_walk(vgpu, gpa + off, 4,
+ svc_pd_r_v[i],
+ SVC_APERTURE_SYS, &a, &eap,
+ false) &&
+ (a & ~0xfffULL) == svc_pd_r_pa[i])
+ hits++;
+ }
+ if (hits >= SVC_PD_MIN_HITS) {
+ svc_pd_r_base = gpa + off;
+ svc_pd_r_ap = SVC_APERTURE_SYS;
+ svc_pd_r_found = 1;
+ pr_info("sema-svc: discovered UVM ring-space page directory 0x%llx (ram), %u/%u content-verified samples\n",
+ svc_pd_r_base, hits, svc_pd_r_n);
+ }
+ }
+ }
+ kfree(buf);
+}
+
static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va,
u64 *addr, int *ap)
{
- if (!svc_pd_found)
- return -ENOENT;
- return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va,
- svc_pd_ap, addr, ap, false);
+ int rc = -ENOENT;
+
+ if (svc_pd_found)
+ rc = svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va,
+ svc_pd_ap, addr, ap, false);
+ if (rc && svc_pd_r_found)
+ rc = svc_gmmu_walk(vgpu, svc_pd_r_base, 4, va,
+ svc_pd_r_ap, addr, ap, false);
+ /*
+ * m177c: the ring-space directory discovery runs on new samples
+ * only (fingerprint + attempt cap inside). A first call with
+ * enough verified ring/push samples performs one bounded scan;
+ * after that this is a flag check.
+ */
+ if (rc && !svc_pd_r_found && svc_pd_r_n >= SVC_PD_MIN_HITS)
+ svc_pd_ring_discover(vgpu);
+ if (rc && svc_pd_r_found)
+ rc = svc_gmmu_walk(vgpu, svc_pd_r_base, 4, va,
+ svc_pd_r_ap, addr, ap, false);
+ return rc;
}
/* Learned translations: a walk costs a handful of guest reads, so cache the
@@ -5836,13 +5957,13 @@ static struct {
u64 pa_page;
} svc_cnt_der[SVC_CNT_DER_MAX];
static u32 svc_cnt_der_n;
+static u32 svc_cnt_der_log;
static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
u64 off = va & (SEMA_PAGE_SIZE - 1);
u64 pa = 0;
- int ap = SVC_APERTURE_SYS;
u32 probe = 0;
u32 i;
bool fresh = true;
@@ -5855,13 +5976,57 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
}
}
if (fresh) {
- if (svc_walk_vaspaces(vgpu, page, &pa, &ap) &&
- svc_walk_discovered(vgpu, page, &pa, &ap) &&
- svc_targeted_resolve(vgpu, page, &pa))
- return -ENOENT;
- if (ap != SVC_APERTURE_SYS)
+ u64 p = 0;
+ int a = SVC_APERTURE_SYS;
+ bool ok = false;
+ const char *src = "none";
+
+ /*
+ * m177b: first READABLE source wins. The previous chain kept
+ * the first source that merely PRODUCED a translation and then
+ * failed -EIO on an unreadable page without trying the rest:
+ * the published directory can yield a stale/incorrect chain
+ * for a counter VA while the rm_mem-content scan would resolve
+ * it (r194: the counter page 0x20400f000 never resolved; its
+ * walk hit probed unreadable and targeted was never reached).
+ */
+ if (!svc_walk_vaspaces(vgpu, page, &p, &a) &&
+ a == SVC_APERTURE_SYS &&
+ !svc_read_ap(vgpu,
+ (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
+ SVC_APERTURE_SYS, &probe, sizeof(probe))) {
+ ok = true;
+ src = "vaspaces";
+ }
+ if (!ok && !svc_walk_discovered(vgpu, page, &p, &a) &&
+ a == SVC_APERTURE_SYS &&
+ !svc_read_ap(vgpu,
+ (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
+ SVC_APERTURE_SYS, &probe, sizeof(probe))) {
+ ok = true;
+ src = "discovered";
+ }
+ if (!ok && !svc_targeted_resolve(vgpu, page, &p) &&
+ !svc_read_ap(vgpu,
+ (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off,
+ SVC_APERTURE_SYS, &probe, sizeof(probe))) {
+ ok = true;
+ src = "targeted";
+ }
+ if (!ok) {
+ if (svc_cnt_der_log < 16) {
+ svc_cnt_der_log++;
+ pr_info("sema-chan: counter page 0x%llx unreadable via vaspaces/discovered/targeted; slot stays pending\n",
+ page);
+ }
return -ENOENT;
- pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off;
+ }
+ pa = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off;
+ if (svc_cnt_der_log < 16) {
+ svc_cnt_der_log++;
+ pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx)\n",
+ page, src, pa);
+ }
}
/* Read the slot before trusting the mapping: an unreadable page is
* not registered, so the caller keeps the "slot pending" fallback. */
@@ -7717,7 +7882,26 @@ static void svc_session_flush(void)
svc_pd_best = 0;
svc_pd_cands = 0;
svc_pd_ev_hash = 0;
- svc.direct_base = 0;
+ /*
+ * m177c: the UVM ring-space directory is per-client guest memory
+ * (and its samples come from this session's rings): reset with the
+ * session like the primary slot.
+ */
+ svc_pd_r_found = 0;
+ svc_pd_r_attempts = 0;
+ svc_pd_r_base = 0;
+ svc_pd_r_ev_hash = 0;
+ svc_pd_r_n = 0;
+ /*
+ * m177: svc.direct_base is NOT cleared here. It is the guest kernel's
+ * direct-map base - a per-BOOT constant, identical for every client
+ * session of the same guest boot - and it is the gate and input of
+ * the pin-free resolution paths (svc_targeted_resolve). Clearing it
+ * per session forced every client to wait for a rebuild scan while
+ * its first cudaMalloc polled, and made the module depend on the
+ * host pin tools entirely. It is wiped on a guest reboot by
+ * svc_guest_reset_wipe() (gsp_sema_service_guest_reset()).
+ */
svc_pending_n = 0;
svc_pending_next = 0;
/*
@@ -7875,6 +8059,52 @@ static void svc_session_flush(void)
}
/*
+ * m177: boot-scoped wipe for a GUEST boot (RM reload). Unlike the
+ * per-client session flush this clears the state that is valid for one
+ * guest boot only: the direct-map base and every content-scanned table.
+ * Runs on the svc thread (svc_guest_reset_wipe is invoked from the thread
+ * loop below), the only writer of this state.
+ */
+static void svc_guest_reset_wipe(void)
+{
+ svc.direct_base = 0;
+ svc.map_n = 0;
+ svc_pd_found = 0;
+ svc_pd_attempts = 0;
+ svc_pd_best = 0;
+ svc_pd_cands = 0;
+ svc_pd_ev_hash = 0;
+ svc_pd_r_found = 0;
+ svc_pd_r_attempts = 0;
+ svc_pd_r_base = 0;
+ svc_pd_r_ev_hash = 0;
+ svc_pd_r_n = 0;
+ svc_pb_n = 0;
+ svc_pool_boot_n = 0;
+ svc_pool_boot_next = 0;
+ svc_pool_pa_n = 0;
+ svc_pool_pa_next = 0;
+ memset(svc_pool_pa, 0, sizeof(svc_pool_pa));
+ memset(tr_failed, 0, sizeof(tr_failed));
+ tr_fail_next = 0;
+ memset(svc.lat, 0, sizeof(svc.lat));
+ pr_info("sema-svc: guest reset: boot-scoped discovery state wiped (direct_base, map, caches)\n");
+}
+
+/*
+ * m177: called by gsp_rm_emu_reset() when the guest's GSP-RM is (re)loaded
+ * - a new guest boot, where every VA->PA relation can have changed. The
+ * request is latched and executed on the svc thread, which owns the
+ * discovery state (a cross-thread wipe here would race svc_rebuild).
+ */
+void gsp_sema_service_guest_reset(void)
+{
+ WRITE_ONCE(svc_guest_reset_req, 1);
+ if (svc.task)
+ wake_up_process(svc.task);
+}
+
+/*
* Loud, bounded telemetry for a starved deferred release (ISSUE-0001).
* Called from the two "no translation yet, keep" paths of
* svc_apply_pending() with an incremented try count; logs at 4/64/1024
@@ -9497,6 +9727,9 @@ static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank,
svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS);
svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS);
svc_mark_hot(push_pa);
+ /* m177c: a content-verified UVM-space pair - feed the ring-space
+ * directory discovery (counter pages resolve through it). */
+ svc_pd_ring_note(push_va, push_pa);
if (ring_ap == SVC_APERTURE_FB)
pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
rank, ring_va, ring_pa, push_va, len, push_pa, prc,
@@ -10367,6 +10600,9 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
}
svc_pb[hit_i].va = va;
set_bit(ch->ce_index, &svc_pb_done);
+ /* m177c: the base page was content-read (svc_chan_entry0) - a
+ * verified UVM-space sample for the ring-space directory. */
+ svc_pd_ring_note(va, hit_pa);
/*
* The pinned pairs are verified samples of UVM's own page directory;
@@ -16214,10 +16450,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
ch->defer_since = 0;
ch->unread_va = 0;
ch->unread_until = 0;
- if (gsp_sema_fake_rel ||
- gsp_sema_rel_recover)
- svc_replay_note(ch, idx,
- put);
+ /*
+ * m177: deliver any
+ * withheld set even
+ * on the GR stale-skip
+ * path. GR channels
+ * have no rescue and
+ * their consume-time
+ * delivery gate is
+ * below; without this
+ * the slot strands
+ * until the sweep.
+ */
+ svc_replay_note(ch, idx,
+ put);
ch->last_get = (idx + 1) %
ch->gpfifo_entries;
svc.gpfifo_skipped++;
@@ -16253,7 +16499,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
}
}
- if (ch->engine_type && svc_rel_count != rel_n_iter)
+ /*
+ * m177: GR channels (engine_type == 0) have no rescue and
+ * no CE-shaped delivery branches above; deliver any
+ * withheld set for this entry whenever the entry is
+ * consumed so a held release can never strand until the
+ * 10 s sweep (evtest3's st[2] sync hang).
+ */
+ if (svc_rel_count != rel_n_iter || !ch->engine_type)
svc_replay_note(ch, idx, put);
ch->last_get = (idx + 1) % ch->gpfifo_entries;
ch->rescue_fail = 0;
@@ -17197,6 +17450,16 @@ static int svc_thread(void *arg)
}
/*
+ * m177: a guest RM (re)load means a new guest boot: wipe the
+ * boot-scoped discovery state here, on the thread that owns
+ * it. Raised by gsp_sema_service_guest_reset().
+ */
+ if (READ_ONCE(svc_guest_reset_req)) {
+ svc_guest_reset_req = 0;
+ svc_guest_reset_wipe();
+ }
+
+ /*
* STALL FIX (r189): expire withheld release sets whose entry
* stopped being re-decoded (the deadline flush is only
* reachable from the entry path). Forward-only delivery to a
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index e0f1587..b8c4dfb 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -81,6 +81,10 @@ struct sema_vaspace_desc {
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
void gsp_sema_service_vaspace_del(u32 h_vaspace);
+/* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state
+ * (direct-map base, content-scanned tables) on the svc thread. Client
+ * session flushes keep that state - see svc_session_flush. */
+void gsp_sema_service_guest_reset(void);
int gsp_sema_service_start(void);
void gsp_sema_service_stop(void);
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-18 06:23:16 +0000