diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 43 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 145 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 8 |
3 files changed, 165 insertions, 31 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 4c848b8..eddfab5 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2509,25 +2509,24 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, } /* - * r496g: this control IS the kernel RM publishing the page-table + * r496j: this control IS the kernel RM publishing the page-table * chain of its internal VAS (the level[0..] instances with their - * phys addresses and page shifts). Register the root with the - * progress service so the walker can translate every VA of this - * client - the X/GL channel's pushbuffer pages and its progress - * semaphore, for which no other translation source exists (the - * fn-54 SET_PAGE_DIRECTORY path is external-VAS/UVM only). The - * level values are sanity-gated; a wrong chain only costs failed - * walks. + * phys addresses and page shifts). The chain can name the X/GL + * channel's pushbuffer pages and progress semaphore, for which no + * other translation source exists (the fn-54 SET_PAGE_DIRECTORY + * path is external-VAS/UVM only). It is NOT registered directly: + * a blast registration of every client's directory let a stale + * kernel directory answer another client's VA in the walker's + * first-success order (the r459 stale-directory class; observed + * as a CUDA client hanging after DEVICE with r496g). Instead the + * root is offered as a CANDIDATE; the progress service registers + * it only after it has been VERIFIED against a content-validated + * pair of that very client (the ring found by the RAM content + * scan). */ if (num_levels >= 1 && rm->pt.levels[0].phys && rm->pt.levels[0].size >= 16 && rm->pt.levels[0].size <= 4096 * 8) { - struct sema_vaspace_desc d; - - memset(&d, 0, sizeof(d)); - d.h_vaspace = h_object; - d.pdb = rm->pt.levels[0].phys; - d.num_entries = (u32)(rm->pt.levels[0].size / 8); /* * The level phys values are guest PAs on this platform * (observed 4+ GiB, above any FB shadow); aperture 3 in the @@ -2536,13 +2535,17 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, * FB, everything else sysmem. The walker also retries the * other aperture, so a misclassification only costs a walk. */ - d.b_vidmem = rm->pt.levels[0].phys < - (u64)ctx->vgpu->bars[1].size; - gsp_sema_service_vaspace_add(&d); + bool vidmem = rm->pt.levels[0].phys < + (u64)ctx->vgpu->bars[1].size; + + gsp_sema_service_vaspace_candidate_add( + h_object, rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), vidmem); nv_vgpu_info(ctx->vgpu, - "vgsp-rm: VAS 0x%x page directory registered from COPY_SERVER_PDES pdb=0x%llx entries=%u (%s)", - h_object, d.pdb, d.num_entries, - d.b_vidmem ? "vidmem" : "sysmem"); + "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)", + h_object, (u64)rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), + vidmem ? "vidmem" : "sysmem"); } } diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b7ce1df..1e9f981 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1640,6 +1640,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + return; + } + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -13056,6 +13140,9 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { u64 off, lim; u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; ch->userd_scans++; lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); @@ -13064,6 +13151,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 hp, hg; bool ok = true; u32 k; + u64 dist; + bool pending; memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); @@ -13106,13 +13195,43 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (!ok) continue; - ch->userd_pa = off; + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; - *put = hp; - *get = hg; - pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n", - ch->h_object, ch->gr_index, off, hp, hg); + *put = best_put; + *get = best_get; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); return 0; } } @@ -14638,13 +14757,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; /* - * r496f: with the ring content-validated, discover the - * client VAS's page directory in vidmem so the - * channel's other VAs (pushbuffer pages, the progress - * semaphore the driver polls) resolve through the - * walker. Runs before the pins below matter and only - * once per channel (the bitmap is set). + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); /* * Persist the latch immediately: a later stalled diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 305a027..ee5c48a 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -81,6 +81,14 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); /* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state * (direct-map base, content-scanned tables) on the svc thread. Client * session flushes keep that state - see svc_session_flush. */ |