diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 181 |
1 files changed, 179 insertions, 2 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9d9f6c8..b2a3803 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -253,6 +253,18 @@ unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); /* + * r1780 (CONT.430b): keep the direct-GL (beef GR) ring content scan alive + * while the ring is unresolved and the channel has no submissions yet, so + * a client that registers an EMPTY ring before its first kick still gets + * found when it finally submits. Default 1 (the validated pre-change path + * is unchanged once a ring is latched or a submission is observed). + */ +static bool gsp_sema_gr_ram_keep = true; +module_param_named(sema_gr_ram_keep, gsp_sema_gr_ram_keep, bool, 0644); +MODULE_PARM_DESC(sema_gr_ram_keep, + "Refresh the beef GR ring content-scan budget while the ring is unresolved and no submission was observed; default 1"); + +/* * r1777 (V12): direct-GL CE-family ring probe gate. Defined here because * the demote/promote paths use it before the probe block; the parameter * description lives with the probe implementation. Default off. @@ -6231,6 +6243,106 @@ static void svc_pool_write_all_vas(struct nv_vgpu_device *vgpu, u32 h, u64 va, } } +/* + * V14 (CONT.430): GLX shared-surface refCount sync. + * + * The GLX client's kickoff-suffix RELEASE_SEMAPHORE advances + * sharedSurface->gpuRefCount[sd].val in the shared surface; the X DDX's + * idle-channel poll checks the CPU mapping of the same page and waits for + * refCount == lastKickoffRefCount, warning after + * 5 s. The single-success VAS walk can name a stale incarnation of the + * client VA and write a page no process maps: r1780 measured the release + * target va=0x420a00800 resolving to pa=0x2493d800 (unmapped) while the + * DDX's page held refCount=8 / lkr=16. Fan the forward-only value out + * through EVERY published VAS translation (the r1648 pool pattern) and, + * when a candidate page carries a lastKickoffRefCount at least the + * decoded payload, author that lkr instead: the DDX stops waiting on + * refCount == lkr and lkr is the client's own last kickoff value (the + * client writes it in CPU memory before the kick; the GPU-side release + * we are emulating would have produced exactly it). + * + * Layout: GLSsharedSurface in the 570.144 DDX has gpuRefCount[] at +0x800 + * of the shared mapping and lastKickoffRefCount at +0x880 (NV_MAX_SUBDEVICES + * = 8, so 8 * 16 B = 0x80; DDX +0x10d237 polls gpuRefCount[j] and + * +0x10d23c reads 0x880(%rcx)). The release VA we decode is + * gpuMapping + 0x800 + 16*sd; lastKickoffRefCount is therefore at + * page_base + 0x880, not at release_va + 0x40. + * + * Forward-only + read-back proven; a stale page absorbing a duplicate is + * harmless. Gated by sema_gl_shsurf_allvas for A/B. + */ +static bool gsp_sema_gl_shsurf_allvas = true; +module_param_named(sema_gl_shsurf_allvas, gsp_sema_gl_shsurf_allvas, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_allvas, + "Sync the GLX shared-surface gpuRefCount through every published VAS translation (and to the client's lastKickoffRefCount when present); default 1"); + +static bool gsp_sema_gl_shsurf_dbg; +module_param_named(sema_gl_shsurf_dbg, gsp_sema_gl_shsurf_dbg, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_dbg, + "Log every candidate VAS translation for the GLX shared-surface refCount sync (bounded)"); + +static bool svc_gl_shsurf_sync(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 pas[SVC_MAX_VASPACES]; + int aps[SVC_MAX_VASPACES]; + u64 off = va & (u64)(SEMA_PAGE_SIZE - 1); + static u32 log, dbg_log; + int n, i, wrote = 0; + + n = svc_walk_vaspaces_collect(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), + pas, aps, SVC_MAX_VASPACES); + for (i = 0; i < n; i++) { + u64 pbase = pas[i] & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 a = pbase | off; + u32 cur = 0, back = 0, want = val; + u64 lkr = 0; + bool have_lkr; + + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &cur, + sizeof(cur))) + continue; + /* + * GLSsharedSurface: gpuRefCount[8] at +0x800, then + * lastKickoffRefCount at +0x880 (64 B aligned). Only trust a + * plausible value: a small non-zero count at least the + * decoded payload and not more than the client could have + * kicked so far. A dense data page is unlikely to satisfy + * all three at once. + */ + have_lkr = !svc_read_ap(vgpu, pbase | 0x880ULL, + svc_ap_of(aps[i]), &lkr, + sizeof(lkr)) && + (lkr >> 32) == 0 && + (lkr & 0xffffffffULL) >= (u64)want && + (lkr & 0xffffffffULL) <= 0x1000000ULL; + if (have_lkr && cur != 0 && cur <= (u32)lkr) + want = (u32)lkr; + if (gsp_sema_gl_shsurf_dbg && dbg_log < 32) { + dbg_log++; + pr_info("sema-gl: shsurf-cand h=0x%x va=0x%llx pa=0x%llx cur=%u lkr=0x%llx ok=%d want=%u\n", + h, va, a, cur, lkr, have_lkr ? 1 : 0, want); + } + if (cur >= want) + continue; + if (svc_write_ap(vgpu, a, svc_ap_of(aps[i]), &want, + sizeof(want))) + continue; + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &back, + sizeof(back)) || back != want) + continue; + wrote++; + if (log < 64) { + log++; + pr_info("sema-gl: shsurf-sync h=0x%x va=0x%llx pa=0x%llx pay=%u lkr=0x%llx (all-VAS)\n", + h, va, a, want, lkr); + } + } + return wrote > 0; +} + static void svc_pool_boot_forget(u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -6669,6 +6781,18 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) static u32 sem_sw_low_log; /* r1656: low GR release-VA write budget */ /* + * V14 (CONT.430): the GLX shared-surface refCount word. The decoded + * release target is <shared gpu mapping>+0x800 in the observed + * boots; the first-success walk may name a stale incarnation, so + * author the value through every published translation (and to the + * client's own lastKickoffRefCount when the page carries one). + * Forward-only, read-back proven, bounded logging. + */ + if (gsp_sema_gl_shsurf_allvas && !is_pool && + va >= 0x200000000ULL && (va & 0xfffULL) == 0x800ULL) + (void)svc_gl_shsurf_sync(vgpu, h, va, val); + + /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem * record before any learned translation; the learned cache and the * discovered map may hold a stale PA for them and a write through it @@ -29569,8 +29693,23 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); u32 o; - if (svc_read(vgpu, gpa, buf, chunk)) + if (svc_read(vgpu, gpa, buf, chunk)) { + if (gsp_sema_dbg && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + static unsigned long rfail_at; + static u64 rfail_gpa; + + if (time_after(jiffies, rfail_at) || + rfail_gpa != gpa) { + rfail_at = jiffies + HZ; + rfail_gpa = gpa; + pr_info("sema-chan: gr-scan READ FAIL h=0x%x gpa=0x%llx chunk=0x%llx cursor=0x%llx\n", + ch->h_object, gpa, chunk, + ch->gr_ram_cursor); + } + } continue; + } for (o = 0; o + eoff + 2 * SVC_GPFIFO_ENTRY_SIZE <= chunk; o += SEMA_PAGE_SIZE) { u32 e0, e1, f0, f1; @@ -31985,8 +32124,21 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->gr_ram_passes < svc_gr_ram_pass_budget() && !test_bit(ch->gr_index, &svc_gr_ring_done)) { u64 hit_pa = 0; + bool hit; + + hit = svc_gr_ram_ring_scan(vgpu, ch, &hit_pa, false, false); + if (gsp_sema_dbg && (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + static unsigned long dbg_at; - if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa, false, false)) { + if (time_after(jiffies, dbg_at)) { + dbg_at = jiffies + HZ; + pr_info("sema-chan: gr-scan h=0x%x rank=%u cursor=0x%llx passes=%u hit=%d pa=0x%llx\n", + ch->h_object, ch->gr_index, + ch->gr_ram_cursor, ch->gr_ram_passes, + hit ? 1 : 0, hit ? hit_pa : 0); + } + } + if (hit) { /* * Pin ONLY the content-validated ring page (validated, * not a window guess). The pushbuffer pages and the @@ -43282,6 +43434,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) msecs_to_jiffies(SVC_GR_RAM_REFRESH_MS); } } + /* + * r1780 (CONT.430b): persistent direct-GL ring scan. The + * beef GR client ring can register BEFORE its first kick + * (empty ring: the content scan legitimately finds nothing) + * while its USERD block is still unresolved, so the block + * above never re-arms the scan and the slice budget retires + * long before the client submits its first push. Measured + * r1780: GZDoom's ring 0xbeef0100 registered at t+0 with an + * empty ring, the 40-slice budget expired during the ~60 s + * GL-init window, and the ring was never found (the X DDX + * then hit its refcount idle timeout). Keep a slow budget + * refresh alive for the beef GR family while the ring is + * unresolved and no submission was observed; the scan stays + * sliced/bounded, the cursor is preserved so a sweep still + * completes, and any content hit latches gr_ram_ring. + */ + if (gsp_sema_gr_ram_keep && !ch->engine_type && + ch->h_object != 0x31415900u && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + ch->gr_index < 32 && !ch->gr_ram_ring && !ch->submissions && + !test_bit(ch->gr_index, &svc_gr_ring_done) && + time_after(jiffies, ch->gr_ram_refresh_at)) { + ch->gr_ram_passes = 0; + ch->gr_ram_refresh_at = jiffies + msecs_to_jiffies(2000); + } svc_chan_vas_candidate_promote(vgpu, ch); svc_ring_autopin(vgpu, ch); /* |