diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 297 |
1 files changed, 280 insertions, 17 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8f23178..7b12ecc 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -483,6 +483,7 @@ static u32 svc_ring_n; * guest's last-channel close is also the global empty transition. */ static u8 svc_sess_flush; +static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */ static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static u32 svc_sess_gen; /* m176: bumped by every executed session flush; * stamps channels/scrubbers so per-session learned @@ -1975,13 +1976,133 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) svc_pd_cands, svc_pd_best, n); } +/* + * m177c: a SECOND discovered-directory slot for the UVM SYS VA space (the + * 0x2xxxxxxx family: GR rings/pushes, QMD windows, counter pages). The + * primary slot (above) is validated against rm_mem-content pairs - the + * 0x1xxxxxxx pool family - and cannot walk the UVM space, so without pins + * the counter page (0x20400fXXX) never resolved: the guest's KMD/UVM + * epoch waited on completions that could never be written and re-ran its + * ~103 s retry loop forever (r194/r195: 'rel-apply ... slot pending' x8, + * then silence). Samples for this slot are content-VERIFIED + * (push_va -> push_pa) pairs from the GR content discovery and the CE + * push pairing: the push bytes themselves are the proof, and they live in + * exactly the space this directory maps. + */ +#define SVC_PD_RING_SAMPLES 8 +static u64 svc_pd_r_v[SVC_PD_RING_SAMPLES]; +static u64 svc_pd_r_pa[SVC_PD_RING_SAMPLES]; +static u32 svc_pd_r_n; +static u64 svc_pd_r_base; +static int svc_pd_r_ap; +static u8 svc_pd_r_found; +static u8 svc_pd_r_attempts; +static u64 svc_pd_r_ev_hash; + +static void svc_pd_ring_note(u64 va, u64 pa) +{ + u32 i; + + if (!va || !pa) + return; + va &= ~0xfffULL; + pa &= ~0xfffULL; + for (i = 0; i < svc_pd_r_n; i++) + if (svc_pd_r_v[i] == va) + return; + if (svc_pd_r_n < SVC_PD_RING_SAMPLES) { + svc_pd_r_v[svc_pd_r_n] = va; + svc_pd_r_pa[svc_pd_r_n] = pa; + svc_pd_r_n++; + } +} + +static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu) +{ + u64 chunk = 64 * 1024; + u8 *buf; + u64 gpa; + u64 ev = svc_pd_r_n; + u32 i; + + if (svc_pd_r_found || svc_pd_r_attempts >= SVC_PD_ATTEMPTS_MAX) + return; + if (svc_pd_r_n < SVC_PD_MIN_HITS) + return; + for (i = 0; i < svc_pd_r_n; i++) + ev = (ev + svc_pd_r_v[i] + (svc_pd_r_pa[i] << 12)) * 1000003; + if (svc_pd_r_attempts && ev == svc_pd_r_ev_hash) + return; + svc_pd_r_ev_hash = ev; + svc_pd_r_attempts++; + + buf = kmalloc(chunk, GFP_KERNEL); + if (!buf) + return; + for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa); + u64 off; + + if (svc_read(vgpu, gpa, buf, len)) + continue; + for (off = 0; off + 8 <= len && !svc_pd_r_found; + off += SEMA_PAGE_SIZE) { + u64 e; + u64 a = 0; + int eap = 0; + u32 hits = 1; + + memcpy(&e, buf + off, sizeof(e)); + if ((e & 1) || !((e >> 1) & 0x3)) + continue; + /* First sample rejects almost every candidate. */ + if (svc_gmmu_walk(vgpu, gpa + off, 4, svc_pd_r_v[0], + SVC_APERTURE_SYS, &a, &eap, false) || + (a & ~0xfffULL) != svc_pd_r_pa[0]) + continue; + for (i = 1; i < svc_pd_r_n; i++) { + if (!svc_gmmu_walk(vgpu, gpa + off, 4, + svc_pd_r_v[i], + SVC_APERTURE_SYS, &a, &eap, + false) && + (a & ~0xfffULL) == svc_pd_r_pa[i]) + hits++; + } + if (hits >= SVC_PD_MIN_HITS) { + svc_pd_r_base = gpa + off; + svc_pd_r_ap = SVC_APERTURE_SYS; + svc_pd_r_found = 1; + pr_info("sema-svc: discovered UVM ring-space page directory 0x%llx (ram), %u/%u content-verified samples\n", + svc_pd_r_base, hits, svc_pd_r_n); + } + } + } + kfree(buf); +} + static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { - if (!svc_pd_found) - return -ENOENT; - return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, - svc_pd_ap, addr, ap, false); + int rc = -ENOENT; + + if (svc_pd_found) + rc = svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, + svc_pd_ap, addr, ap, false); + if (rc && svc_pd_r_found) + rc = svc_gmmu_walk(vgpu, svc_pd_r_base, 4, va, + svc_pd_r_ap, addr, ap, false); + /* + * m177c: the ring-space directory discovery runs on new samples + * only (fingerprint + attempt cap inside). A first call with + * enough verified ring/push samples performs one bounded scan; + * after that this is a flag check. + */ + if (rc && !svc_pd_r_found && svc_pd_r_n >= SVC_PD_MIN_HITS) + svc_pd_ring_discover(vgpu); + if (rc && svc_pd_r_found) + rc = svc_gmmu_walk(vgpu, svc_pd_r_base, 4, va, + svc_pd_r_ap, addr, ap, false); + return rc; } /* Learned translations: a walk costs a handful of guest reads, so cache the @@ -5836,13 +5957,13 @@ static struct { u64 pa_page; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; +static u32 svc_cnt_der_log; static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 off = va & (SEMA_PAGE_SIZE - 1); u64 pa = 0; - int ap = SVC_APERTURE_SYS; u32 probe = 0; u32 i; bool fresh = true; @@ -5855,13 +5976,57 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) } } if (fresh) { - if (svc_walk_vaspaces(vgpu, page, &pa, &ap) && - svc_walk_discovered(vgpu, page, &pa, &ap) && - svc_targeted_resolve(vgpu, page, &pa)) - return -ENOENT; - if (ap != SVC_APERTURE_SYS) + u64 p = 0; + int a = SVC_APERTURE_SYS; + bool ok = false; + const char *src = "none"; + + /* + * m177b: first READABLE source wins. The previous chain kept + * the first source that merely PRODUCED a translation and then + * failed -EIO on an unreadable page without trying the rest: + * the published directory can yield a stale/incorrect chain + * for a counter VA while the rm_mem-content scan would resolve + * it (r194: the counter page 0x20400f000 never resolved; its + * walk hit probed unreadable and targeted was never reached). + */ + if (!svc_walk_vaspaces(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "vaspaces"; + } + if (!ok && !svc_walk_discovered(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "discovered"; + } + if (!ok && !svc_targeted_resolve(vgpu, page, &p) && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "targeted"; + } + if (!ok) { + if (svc_cnt_der_log < 16) { + svc_cnt_der_log++; + pr_info("sema-chan: counter page 0x%llx unreadable via vaspaces/discovered/targeted; slot stays pending\n", + page); + } return -ENOENT; - pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + } + pa = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + if (svc_cnt_der_log < 16) { + svc_cnt_der_log++; + pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx)\n", + page, src, pa); + } } /* Read the slot before trusting the mapping: an unreadable page is * not registered, so the caller keeps the "slot pending" fallback. */ @@ -7717,7 +7882,26 @@ static void svc_session_flush(void) svc_pd_best = 0; svc_pd_cands = 0; svc_pd_ev_hash = 0; - svc.direct_base = 0; + /* + * m177c: the UVM ring-space directory is per-client guest memory + * (and its samples come from this session's rings): reset with the + * session like the primary slot. + */ + svc_pd_r_found = 0; + svc_pd_r_attempts = 0; + svc_pd_r_base = 0; + svc_pd_r_ev_hash = 0; + svc_pd_r_n = 0; + /* + * m177: svc.direct_base is NOT cleared here. It is the guest kernel's + * direct-map base - a per-BOOT constant, identical for every client + * session of the same guest boot - and it is the gate and input of + * the pin-free resolution paths (svc_targeted_resolve). Clearing it + * per session forced every client to wait for a rebuild scan while + * its first cudaMalloc polled, and made the module depend on the + * host pin tools entirely. It is wiped on a guest reboot by + * svc_guest_reset_wipe() (gsp_sema_service_guest_reset()). + */ svc_pending_n = 0; svc_pending_next = 0; /* @@ -7875,6 +8059,52 @@ static void svc_session_flush(void) } /* + * m177: boot-scoped wipe for a GUEST boot (RM reload). Unlike the + * per-client session flush this clears the state that is valid for one + * guest boot only: the direct-map base and every content-scanned table. + * Runs on the svc thread (svc_guest_reset_wipe is invoked from the thread + * loop below), the only writer of this state. + */ +static void svc_guest_reset_wipe(void) +{ + svc.direct_base = 0; + svc.map_n = 0; + svc_pd_found = 0; + svc_pd_attempts = 0; + svc_pd_best = 0; + svc_pd_cands = 0; + svc_pd_ev_hash = 0; + svc_pd_r_found = 0; + svc_pd_r_attempts = 0; + svc_pd_r_base = 0; + svc_pd_r_ev_hash = 0; + svc_pd_r_n = 0; + svc_pb_n = 0; + svc_pool_boot_n = 0; + svc_pool_boot_next = 0; + svc_pool_pa_n = 0; + svc_pool_pa_next = 0; + memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); + memset(tr_failed, 0, sizeof(tr_failed)); + tr_fail_next = 0; + memset(svc.lat, 0, sizeof(svc.lat)); + pr_info("sema-svc: guest reset: boot-scoped discovery state wiped (direct_base, map, caches)\n"); +} + +/* + * m177: called by gsp_rm_emu_reset() when the guest's GSP-RM is (re)loaded + * - a new guest boot, where every VA->PA relation can have changed. The + * request is latched and executed on the svc thread, which owns the + * discovery state (a cross-thread wipe here would race svc_rebuild). + */ +void gsp_sema_service_guest_reset(void) +{ + WRITE_ONCE(svc_guest_reset_req, 1); + if (svc.task) + wake_up_process(svc.task); +} + +/* * Loud, bounded telemetry for a starved deferred release (ISSUE-0001). * Called from the two "no translation yet, keep" paths of * svc_apply_pending() with an incremented try count; logs at 4/64/1024 @@ -9497,6 +9727,9 @@ static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank, svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS); svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS); svc_mark_hot(push_pa); + /* m177c: a content-verified UVM-space pair - feed the ring-space + * directory discovery (counter pages resolve through it). */ + svc_pd_ring_note(push_va, push_pa); if (ring_ap == SVC_APERTURE_FB) pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n", rank, ring_va, ring_pa, push_va, len, push_pa, prc, @@ -10367,6 +10600,9 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) } svc_pb[hit_i].va = va; set_bit(ch->ce_index, &svc_pb_done); + /* m177c: the base page was content-read (svc_chan_entry0) - a + * verified UVM-space sample for the ring-space directory. */ + svc_pd_ring_note(va, hit_pa); /* * The pinned pairs are verified samples of UVM's own page directory; @@ -16214,10 +16450,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; - if (gsp_sema_fake_rel || - gsp_sema_rel_recover) - svc_replay_note(ch, idx, - put); + /* + * m177: deliver any + * withheld set even + * on the GR stale-skip + * path. GR channels + * have no rescue and + * their consume-time + * delivery gate is + * below; without this + * the slot strands + * until the sweep. + */ + svc_replay_note(ch, idx, + put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; @@ -16253,7 +16499,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } - if (ch->engine_type && svc_rel_count != rel_n_iter) + /* + * m177: GR channels (engine_type == 0) have no rescue and + * no CE-shaped delivery branches above; deliver any + * withheld set for this entry whenever the entry is + * consumed so a held release can never strand until the + * 10 s sweep (evtest3's st[2] sync hang). + */ + if (svc_rel_count != rel_n_iter || !ch->engine_type) svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; @@ -17197,6 +17450,16 @@ static int svc_thread(void *arg) } /* + * m177: a guest RM (re)load means a new guest boot: wipe the + * boot-scoped discovery state here, on the thread that owns + * it. Raised by gsp_sema_service_guest_reset(). + */ + if (READ_ONCE(svc_guest_reset_req)) { + svc_guest_reset_req = 0; + svc_guest_reset_wipe(); + } + + /* * STALL FIX (r189): expire withheld release sets whose entry * stopped being re-decoded (the deadline flush is only * reachable from the entry path). Forward-only delivery to a |