diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 180 |
1 files changed, 180 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 20dfc54..10992c3 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -454,6 +454,45 @@ static u8 svc_sess_flush; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ + +/* + * CeUtils scrubber channel (the guest PMA's internal CE channel created at + * driver init). When the PMA must hand out scrubbed pages it waits on the + * channel's finish payload (channel buffer offset finishPayloadOffset = + * channelPbSize + GPFIFO_SIZE + 4; for the regular scrubber = 0x6c004), + * which the physical CE would advance as the scrub work completes. In this + * emulation nothing authors it, so the third+ CUDA session's cudaMalloc + * freezes in scrubCheckAndWaitForSize. + * + * The scrubber channel is identified by its GPFIFO entry count + * (NUM_COPY_BLOCKS = 4096; client channels use 1024); its USERD PA is known + * from the channel ALLOC registration. The channel buffer's PA cannot be + * resolved through the global VA map (the kernel VA space collides + * numerically with the client's CE push-buffer window), so the guest + * scanner (vmallocinfo + kcore page-table walk) registers it through + * sema_scrubbuf_add. The service then authors, from the channel's own + * GP_PUT: USERD GP_GET = put, host semaphore = put (so the guest's + * channelWaitForFreeEntry never blocks), and finish payload = submitted + * block count (so the scrub wait completes). Idempotent, forward-only. + */ +struct svc_scrub { + bool userd_ok; /* a 4096-entry channel was registered */ + u32 h_object; /* handle of the registered scrubber channel */ + u64 userd_pa; + u32 userd_as; + u64 gpfifo_va; + u32 entries; + bool pa_ok; /* scanner registered fin/hsem PAs */ + u64 fin_pa; /* finish payload word (u32) */ + u64 hsem_pa; /* channel host semaphore word (u32) */ + bool seeded; + u32 put_last; + u64 submitted; /* blocks submitted since seeding */ + u64 authored; /* last authored finish value */ + u32 pa_gen; /* scanner registration generation (logged) */ + u32 log_budget; +}; +static struct svc_scrub svc_scrub; static u8 svc_pb_passes; static u64 svc_pb_last; /* @@ -1110,6 +1149,37 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ svc_chan_rearm_rank(ch); svc.last_chan_reg = jiffies; + /* + * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO + * entries (client channels use 1024). Stash its USERD and ring + * geometry for the scrub pass; the channel itself is left + * entirely alone (no rank change, no pump touch changes) - the + * pass authors the finish payload from the USERD alone. A + * colliding numeric-handle FREE may still unregister the + * channel; the stashed geometry survives, so the scrub path + * does not depend on the channel-table identity at all. + */ + if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + desc->userd_pa) { + /* Re-seed the counting on a fresh scrubber object. */ + if (!svc_scrub.userd_ok || + svc_scrub.h_object != desc->h_object || + svc_scrub.gpfifo_va != desc->gpfifo_va) { + svc_scrub.seeded = false; + svc_scrub.submitted = 0; + } + svc_scrub.userd_ok = true; + svc_scrub.h_object = desc->h_object; + svc_scrub.userd_pa = desc->userd_pa; + svc_scrub.userd_as = desc->userd_as ? desc->userd_as + : SVC_APERTURE_SYS; + svc_scrub.gpfifo_va = desc->gpfifo_va; + svc_scrub.entries = desc->gpfifo_entries; + pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n", + desc->h_object, desc->gpfifo_va, + desc->gpfifo_entries, desc->userd_pa, + svc_scrub.userd_as); + } pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, @@ -15555,6 +15625,115 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) return ret; } +/* + * Register the guest PMA scrubber's channel-buffer words. Called from the + * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved + * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish + * payload the guest's scrub wait polls; hsem_pa is the channel's host + * semaphore (channelWaitForFreeEntry polls it). Plain state update. + */ +int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) +{ + if (!svc.task) + return -ENODEV; + /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + return -EINVAL; + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + return -EINVAL; + + svc_scrub.fin_pa = fin_pa; + svc_scrub.hsem_pa = hsem_pa; + svc_scrub.pa_ok = true; + svc_scrub.pa_gen++; + pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", + fin_pa, hsem_pa, svc_scrub.pa_gen); + return 0; +} + +/* + * Author the CeUtils scrubber channel's progress once per service pass. + * + * The physical CE would advance, per completed scrub block, the channel's + * finish payload (channel buffer + 0x6c004) and write the channel's host + * semaphore as it consumes GPFIFO entries. Nothing in this emulation does, + * so the guest PMA's scrub wait (scrubCheckAndWaitForSize) freezes from the + * third CUDA session on. This pass closes that loop from the channel's own + * USERD GPPut (which the guest driver writes before every submit): + * + * submitted = number of scrub blocks submitted (GPPut is end-exclusive + * and counts submissions until the ring wraps; track the wrap ourselves), + * then author USERD GP_GET = put (consumed), host sema = put and finish + * payload = submitted. Forward-only, idempotent, retried every pass so a + * late scanner registration (or a post-flush reset) heals by itself. + * + * The scrub work itself is not executed (no shadow memsets): the guest + * only needs the progress to proceed. Executing the memsets requires + * decoding the channel's pushes, which the scanner can later provide pins + * for; correctness impact is limited to stale data in pages the guest + * believes scrubbed. + */ +static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +{ + struct svc_scrub *s = &svc_scrub; + u32 put = 0; + u64 delta; + + if (!s->userd_ok || !s->pa_ok || !s->entries) + return; + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, + &put, sizeof(put))) { + /* Retry through the alternate aperture (as registration did). */ + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, + s->userd_as == SVC_APERTURE_FB ? + SVC_APERTURE_SYS : SVC_APERTURE_FB, + &put, sizeof(put))) + return; + } + if (put >= s->entries) + return; /* junk translation; retry next pass */ + + if (!s->seeded) { + s->seeded = true; + s->put_last = put; + /* GPPut = (lastSubmittedEntry + 1) % entries: the submission + * count until the ring wraps. Seeding from it adopts any + * pre-host scrub work (the boot session's scrubbers). */ + s->submitted = put; + } else if (put != s->put_last) { + delta = (put + s->entries - s->put_last) % s->entries; + if (!delta) + delta = s->entries; /* exact wrap */ + s->submitted += delta; + s->put_last = put; + } else if (s->submitted == s->authored) { + return; /* nothing new; no guest writes */ + } + + if (s->log_budget < 64) { + s->log_budget++; + pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n", + put, s->submitted, s->fin_pa); + } + + /* + * Author (forward-only). The USERD GP_GET and host-sema writes mark + * the submissions consumed; the finish write releases the scrub + * wait. Written only when GP_PUT advanced (or on the seeding pass), + * so the per-pass cost is zero while the scrubber idles. + */ + svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as, + &put, sizeof(put)); + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put)); + { + u32 fin = (u32)s->submitted; + + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, + sizeof(fin)); + } + s->authored = s->submitted; +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; @@ -15600,6 +15779,7 @@ static int svc_thread(void *arg) svc_handle_walk_probe(vgpu); svc_handle_map_add(); svc_handle_chan_dump(vgpu); + svc_scrub_pass(vgpu); if (svc_sess_flush || !svc.direct_base || time_after(jiffies, |