diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 392 |
1 files changed, 378 insertions, 14 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9ab609a..f8925df 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -454,6 +454,45 @@ static u8 svc_sess_flush; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ + +/* + * CeUtils scrubber channel (the guest PMA's internal CE channel created at + * driver init). When the PMA must hand out scrubbed pages it waits on the + * channel's finish payload (channel buffer offset finishPayloadOffset = + * channelPbSize + GPFIFO_SIZE + 4; for the regular scrubber = 0x6c004), + * which the physical CE would advance as the scrub work completes. In this + * emulation nothing authors it, so the third+ CUDA session's cudaMalloc + * freezes in scrubCheckAndWaitForSize. + * + * The scrubber channel is identified by its GPFIFO entry count + * (NUM_COPY_BLOCKS = 4096; client channels use 1024); its USERD PA is known + * from the channel ALLOC registration. The channel buffer's PA cannot be + * resolved through the global VA map (the kernel VA space collides + * numerically with the client's CE push-buffer window), so the guest + * scanner (vmallocinfo + kcore page-table walk) registers it through + * sema_scrubbuf_add. The service then authors, from the channel's own + * GP_PUT: USERD GP_GET = put, host semaphore = put (so the guest's + * channelWaitForFreeEntry never blocks), and finish payload = submitted + * block count (so the scrub wait completes). Idempotent, forward-only. + */ +struct svc_scrub { + bool userd_ok; /* a 4096-entry channel was registered */ + u32 h_object; /* handle of the registered scrubber channel */ + u64 userd_pa; + u32 userd_as; + u64 gpfifo_va; + u32 entries; + bool pa_ok; /* scanner registered fin/hsem PAs */ + u64 fin_pa; /* finish payload word (u32) */ + u64 hsem_pa; /* channel host semaphore word (u32) */ + bool seeded; + u32 put_last; + u64 submitted; /* blocks submitted since seeding */ + u64 authored; /* last authored finish value */ + u32 pa_gen; /* scanner registration generation (logged) */ + u32 log_budget; +}; +static struct svc_scrub svc_scrub; static u8 svc_pb_passes; static u64 svc_pb_last; /* @@ -602,6 +641,14 @@ struct sema_svc { u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ + /* + * m151: bounded log for CE pushes that were READABLE (nonzero first + * bytes) yet decoded no release. These are the entries the pump holds + * for the 8 s grace; naming their decode state (have_sem, rel) and + * head bytes is the Class-2 diagnostic (r128: correct content, right + * sem block at +0xC0 with RELEASE_ONE_WORD, yet rel_seen false). + */ + u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -942,6 +989,13 @@ static bool svc_gr_content_rearm; static u32 svc_cepin_log; /* + * m154: injected pins dropped because they collide with a live channel's + * GPFIFO ring range (see svc_handle_map_add). Counted + logged so the + * defensive drop is visible in the boot log. + */ +static u32 svc_ring_pin_skips; + +/* * m148 (second-client wedge): re-arm the per-rank one-shot ring/push-buffer * latches for a (re-)registered channel. The guest driver rebuilds the GSP * session for every client context and RM reuses the same object handles and @@ -987,6 +1041,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; + svc_ring_pin_skips = 0; } /* @@ -1102,6 +1157,37 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ svc_chan_rearm_rank(ch); svc.last_chan_reg = jiffies; + /* + * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO + * entries (client channels use 1024). Stash its USERD and ring + * geometry for the scrub pass; the channel itself is left + * entirely alone (no rank change, no pump touch changes) - the + * pass authors the finish payload from the USERD alone. A + * colliding numeric-handle FREE may still unregister the + * channel; the stashed geometry survives, so the scrub path + * does not depend on the channel-table identity at all. + */ + if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + desc->userd_pa) { + /* Re-seed the counting on a fresh scrubber object. */ + if (!svc_scrub.userd_ok || + svc_scrub.h_object != desc->h_object || + svc_scrub.gpfifo_va != desc->gpfifo_va) { + svc_scrub.seeded = false; + svc_scrub.submitted = 0; + } + svc_scrub.userd_ok = true; + svc_scrub.h_object = desc->h_object; + svc_scrub.userd_pa = desc->userd_pa; + svc_scrub.userd_as = desc->userd_as ? desc->userd_as + : SVC_APERTURE_SYS; + svc_scrub.gpfifo_va = desc->gpfifo_va; + svc_scrub.entries = desc->gpfifo_entries; + pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n", + desc->h_object, desc->gpfifo_va, + desc->gpfifo_entries, desc->userd_pa, + svc_scrub.userd_as); + } pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, @@ -6936,6 +7022,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, tlen ? tail : NULL); } } + + /* + * m151 Class-2 diagnostic: a CE push with real content that decoded no + * release is the one that pays the 8 s grace. Log its decode state + * (bounded, content-gated so all-zero reads do not wash it out). + */ + if (gsp_sema_dbg && ch->engine_type && ch->h_object && !st.have_rel) { + u8 head[64]; + u32 hlen = len < sizeof(head) ? len : (u32)sizeof(head); + bool nz = false; + u32 i; + + for (i = 0; i + 4 <= hlen; i += 4) { + u32 w; + + memcpy(&w, buf + i, 4); + if (w) { + nz = true; + break; + } + } + if (nz && svc.entry_norel_log < 2048) { + svc.entry_norel_log++; + memcpy(head, buf, hlen); + pr_info("sema-norel: h=0x%x eng=%u va=0x%llx len=%u ops=%u have_sem=%u rel=0x%llx pay=%u head=%*phN\n", + ch->h_object, ch->engine_type, push_va, len, + ops_runs, st.have_sem, st.rel_va, + st.rel_payload, hlen, head); + } + } } /* ------------------------------------------------------------------ */ @@ -7942,6 +8058,8 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) /* Fallback-line budget for the whole boot. */ static u32 svc_pushpg_fb_log; +/* Bounded split-read rejection lines (cross-page CE push, r129 class). */ +static u32 svc_pushpg_split_log; /* * Read a push the way the engine does: every 4 KB page of the range is @@ -7961,11 +8079,16 @@ static u32 svc_pushpg_fb_log; * method stream is rejected and the remaining candidates are tried * (observed: a wrong injected pin for the push page made every push * decode no release, starving the client's tracker); - * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * 2. the displaced injected pair (svc_ipin_alt_lookup): what the page + * resolved to before a later injection overwrote the primary slot - + * the fall-back when a foreign/stale injection shadowed a good page + * (observed r129: a continuation page of a cross-page CE push under + * an all-FF pin); + * 3. validated self-pin (svc_pin_lookup_valid): read-back-validated pages * (content rescue, first-push pairing), window guesses excluded; - * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * 4. the channel's linear window mapping (svc_pb_win_lookup): last resort * for a page no validated source knows; - * 4. outside any window: the remaining trusted sources + * 5. outside any window: the remaining trusted sources * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. * * For a contiguous allocation every path resolves to the same PA the old @@ -7993,8 +8116,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); - for (src = 0; src < 4 && !done; src++) { + for (src = 0; src < 5 && !done; src++) { bool via_ipin = false; + bool untrusted = false; pa = 0; ap = SVC_APERTURE_SYS; @@ -8004,22 +8128,39 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_ipin_lookup(va, &pa, &ap)) continue; via_ipin = true; + untrusted = true; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 1: + /* + * Displaced injected pin (svc_ipin_alt): + * the mapping this page had before a later + * injection overwrote it. The push decode + * is the consumer here, so a pin that no + * longer reads like the page's validated + * content (all-FF continuation chunks, + * see below) can fall back to the pair the + * current injection displaced. + */ + if (svc_ipin_alt_lookup(va, &pa, &ap)) + continue; + untrusted = true; + break; + case 2: if (svc_pin_lookup_valid(va, &pa, &ap)) continue; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; - case 2: + case 3: if (!have_win) continue; pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); ap = SVC_APERTURE_SYS; + untrusted = true; break; default: /* learned cache / discovered map / page @@ -8045,18 +8186,45 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, !svc_push_prefix_ok(p, chunk)) continue; /* content says: wrong pin */ /* + * A continuation chunk (not the push start) read from + * an unvalidated source can be a stale/foreign pin: + * observed r129, a 216-byte CE push whose last 24 + * bytes (the SET_SEMAPHORE_A + LAUNCH_DMA) fell in a + * page whose injected pin resolved to an all-0xFF + * page while the first page was correct. A full + * chunk of 0xFF is never a method stream - fresh + * emulation pages read 0xFF - so reject it and try + * the remaining sources (the displaced alt pair + * first). The bounded line names the rejected + * (va, pa, src) for the post-mortem. + */ + if (untrusted && !first && chunk >= 64 && + !memchr_inv(p, 0xff, chunk)) { + if (svc_pushpg_split_log < 8) { + svc_pushpg_split_log++; + pr_info("sema-pushpg: split reject va=0x%llx pa=0x%llx src=%s (all-FF) at page 0x%llx\n", + va, pa, + src == 0 ? "ipin" : + (src == 1 ? "ipin-alt" : + "win"), + page_va); + } + last_rc = -EIO; + continue; + } + /* * ISSUE-0002 (r51): a window guess that reads ALL * ZEROS cannot be right - the paired base page is * real, so a zero first chunk means the linear window * does not describe this page (r49: the second * session's CE push pages read zeros and, because the * read "succeeded", shadowed the per-page resolvers). - * Fall through to src=3 instead. Non-zero window + * Fall through to src=4 instead. Non-zero window * content stays trusted exactly as before (r50: * requiring method-stream plausibility rejected valid * non-method push prefixes and corrupted c1). */ - if (src == 2 && first && !memchr_inv(p, 0, chunk)) + if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; done = true; } @@ -12206,10 +12374,18 @@ static bool svc_gr_page_inline_covers(u64 page) * accesses the tail: with 32 pages the tail read the NEXT candidate's * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap. */ -#define SVC_GR_DATA_PAGES_PER_PTR 33u -/* m114: total pages the pre-commit bounce window can carry (slot 100 is - * the mail message). */ -#define SVC_GR_DATA_BOUNCE_MAX 100u +/* + * m150 (large buffers): the bounce window was grown to 208 pages (mail page + * at slot 206). 68 pages per candidate/pointer covers a 256 KB cudaMalloc'd + * operand whose pointer sits 0x400 into its page (64 full + offset + tail), + * and up to three such operands (3x68 = 204 <= 205) fit below the mail page. + * Larger-than-256 KB operands still hit the cap; they need a chunked data + * path (future work). + */ +#define SVC_GR_DATA_PAGES_PER_PTR 68u +/* Total pages the pre-commit bounce window can carry below the mail page + * (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */ +#define SVC_GR_DATA_BOUNCE_MAX 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ @@ -12456,8 +12632,26 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) u32 before = svc_fwd_req.gr_n_data; u64 pba = 0; u32 pba_ap = 0; + u32 span; const char *pba_src = "?"; + /* + * Window budget discipline (large buffers): only real device + * operands (high, cudaMalloc-shaped VAs) get the full per-pointer + * span. Low-VA candidates (the launch's own QMD/release/reloc + * pointers) need a single page at most, and the launch's QMD VA + * is not a kernel operand at all - relocating 68 pages for it + * starved the operands that follow and blew the 205-page cap + * (r141: a 256 KB operand resolved 0 pages because the QMD and + * param candidates consumed the window). Excluding them lets + * three 256 KB operands (3x68 = 204) fit. + */ + if ((base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_fwd_req.gr_qmd_dev_va & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + span = (base >= 0x40000000000ULL) ? + SVC_GR_DATA_PAGES_PER_PTR : 1u; + if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -12472,7 +12666,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) svc_pd_found, svc.ipin_n); } - for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { + for (p = 0; p < span; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; u64 pa = 0; u32 ap = 0, x; @@ -15218,6 +15412,41 @@ static u32 svc_map_add_head; /* producer */ static u32 svc_map_add_tail; /* consumer (service thread) */ static DEFINE_SPINLOCK(svc_map_add_lock); +/* + * m154: does this injected-pin VA fall inside any live channel's GPFIFO ring + * range? The ring VA translation must come from the channel's own discovery + * path (channel object / push-buffer scan); a scanner run that happens to + * span the ring VA is enumerating a different mapping and its translation is + * always wrong for the ring (r153: an all-zero read consumed a ring entry + * and wedged the client for the boot). Called on the service thread; takes + * chan_lock like the rewind below. + */ +static bool svc_va_hits_chan_ring(u64 va) +{ + unsigned long flags; + u32 i; + bool hit = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + u64 lo, hi; + + if (!ch->valid || !ch->gpfifo_va || !ch->gpfifo_entries) + continue; + lo = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + hi = (ch->gpfifo_va + + (u64)ch->gpfifo_entries * 8 + SEMA_PAGE_SIZE - 1) & + ~(u64)(SEMA_PAGE_SIZE - 1); + if (va >= lo && va < hi) { + hit = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return hit; +} + static void svc_handle_map_add(void) { unsigned long flags; @@ -15242,8 +15471,33 @@ static void svc_handle_map_add(void) ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { - svc_ipin_store(req.va + off, req.pa + off, ap); - svc_lat_store(req.va + off, req.pa + off, ap); + u64 va = req.va + off; + + /* + * m154: never let an injected pin shadow a live + * channel's GPFIFO ring. r153: the scanner's merged + * CE-window run (va=0x121000000 len=0x40000, the + * length varies per session) covered a CE channel's + * ring VA 0x121010000; the injected translation took + * precedence in svc_ipin_lookup/read, the ring then + * read the guest page at 0x102e02000 - an all-zero + * entry was silently consumed, the pool release was + * never authored, and the client spun in userspace + * for the rest of the boot (pushes frozen). The ring + * pages already have their own discovery path (the + * channel object / push-buffer scan); a colliding + * injection is always the wrong translation, so drop + * it and leave the ring to that path. + */ + if (svc_va_hits_chan_ring(va)) { + svc_ring_pin_skips++; + if (svc_ring_pin_skips <= 8) + pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", + va); + continue; + } + svc_ipin_store(va, req.pa + off, ap); + svc_lat_store(va, req.pa + off, ap); } /* * m149: bounded per-session evidence that the CE push-buffer @@ -15465,6 +15719,115 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) return ret; } +/* + * Register the guest PMA scrubber's channel-buffer words. Called from the + * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved + * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish + * payload the guest's scrub wait polls; hsem_pa is the channel's host + * semaphore (channelWaitForFreeEntry polls it). Plain state update. + */ +int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) +{ + if (!svc.task) + return -ENODEV; + /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + return -EINVAL; + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + return -EINVAL; + + svc_scrub.fin_pa = fin_pa; + svc_scrub.hsem_pa = hsem_pa; + svc_scrub.pa_ok = true; + svc_scrub.pa_gen++; + pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", + fin_pa, hsem_pa, svc_scrub.pa_gen); + return 0; +} + +/* + * Author the CeUtils scrubber channel's progress once per service pass. + * + * The physical CE would advance, per completed scrub block, the channel's + * finish payload (channel buffer + 0x6c004) and write the channel's host + * semaphore as it consumes GPFIFO entries. Nothing in this emulation does, + * so the guest PMA's scrub wait (scrubCheckAndWaitForSize) freezes from the + * third CUDA session on. This pass closes that loop from the channel's own + * USERD GPPut (which the guest driver writes before every submit): + * + * submitted = number of scrub blocks submitted (GPPut is end-exclusive + * and counts submissions until the ring wraps; track the wrap ourselves), + * then author USERD GP_GET = put (consumed), host sema = put and finish + * payload = submitted. Forward-only, idempotent, retried every pass so a + * late scanner registration (or a post-flush reset) heals by itself. + * + * The scrub work itself is not executed (no shadow memsets): the guest + * only needs the progress to proceed. Executing the memsets requires + * decoding the channel's pushes, which the scanner can later provide pins + * for; correctness impact is limited to stale data in pages the guest + * believes scrubbed. + */ +static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +{ + struct svc_scrub *s = &svc_scrub; + u32 put = 0; + u64 delta; + + if (!s->userd_ok || !s->pa_ok || !s->entries) + return; + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, + &put, sizeof(put))) { + /* Retry through the alternate aperture (as registration did). */ + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, + s->userd_as == SVC_APERTURE_FB ? + SVC_APERTURE_SYS : SVC_APERTURE_FB, + &put, sizeof(put))) + return; + } + if (put >= s->entries) + return; /* junk translation; retry next pass */ + + if (!s->seeded) { + s->seeded = true; + s->put_last = put; + /* GPPut = (lastSubmittedEntry + 1) % entries: the submission + * count until the ring wraps. Seeding from it adopts any + * pre-host scrub work (the boot session's scrubbers). */ + s->submitted = put; + } else if (put != s->put_last) { + delta = (put + s->entries - s->put_last) % s->entries; + if (!delta) + delta = s->entries; /* exact wrap */ + s->submitted += delta; + s->put_last = put; + } else if (s->submitted == s->authored) { + return; /* nothing new; no guest writes */ + } + + if (s->log_budget < 64) { + s->log_budget++; + pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n", + put, s->submitted, s->fin_pa); + } + + /* + * Author (forward-only). The USERD GP_GET and host-sema writes mark + * the submissions consumed; the finish write releases the scrub + * wait. Written only when GP_PUT advanced (or on the seeding pass), + * so the per-pass cost is zero while the scrubber idles. + */ + svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as, + &put, sizeof(put)); + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put)); + { + u32 fin = (u32)s->submitted; + + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, + sizeof(fin)); + } + s->authored = s->submitted; +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; @@ -15510,6 +15873,7 @@ static int svc_thread(void *arg) svc_handle_walk_probe(vgpu); svc_handle_map_add(); svc_handle_chan_dump(vgpu); + svc_scrub_pass(vgpu); if (svc_sess_flush || !svc.direct_base || time_after(jiffies, |