diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 127 |
1 files changed, 116 insertions, 11 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b2a3803..5f8bf07 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1319,7 +1319,15 @@ static unsigned long svc_pb_gr_done; static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ /* SESSION 133: ranks that already consumed their one shadow ring scan. */ static unsigned long svc_gr_ring_scan_done; -static unsigned long svc_gr_ring_scan_last; /* rate limit for the scan */ +/* + * r1780 (CONT.430c): per-rank scan cadence. A single global timestamp let + * the first GR channel in pump order (descending handles; the RC watchdog + * 0x31415900 precedes the direct-GL 0xbeef0100) consume the window every + * 1.5 s, so the direct-GL call always returned at this gate and its ring + * content scan never ran - the ring stayed unresolved and the X DDX hit + * its idle timeout. Per-rank keeps the same bound per channel. + */ +static unsigned long svc_gr_ring_scan_last[32]; /* per-rank scan rate limit */ /* SESSION 138: bounded witness for the GR request builder's page list and * the fill's candidate set (one launch's wiring printed per budget slice). */ static u32 svc_gr_pages_log; @@ -6318,15 +6326,23 @@ static bool svc_gl_shsurf_sync(struct nv_vgpu_device *vgpu, u32 h, u64 va, (lkr >> 32) == 0 && (lkr & 0xffffffffULL) >= (u64)want && (lkr & 0xffffffffULL) <= 0x1000000ULL; - if (have_lkr && cur != 0 && cur <= (u32)lkr) - want = (u32)lkr; + /* + * r1780 (CONT.430e): strict author. Only a page that + * carries the client's own lastKickoffRefCount is the + * shared surface; never fall back to writing the raw + * payload into an unvalidated translation (that branch + * was the one that could hit a data page). The DDX + * asserts refCount <= lkr, so write exactly lkr and only + * forward. + */ + if (!have_lkr || cur >= (u32)lkr) + continue; + want = (u32)lkr; if (gsp_sema_gl_shsurf_dbg && dbg_log < 32) { dbg_log++; pr_info("sema-gl: shsurf-cand h=0x%x va=0x%llx pa=0x%llx cur=%u lkr=0x%llx ok=%d want=%u\n", h, va, a, cur, lkr, have_lkr ? 1 : 0, want); } - if (cur >= want) - continue; if (svc_write_ap(vgpu, a, svc_ap_of(aps[i]), &want, sizeof(want))) continue; @@ -26085,6 +26101,66 @@ static int svc_read_va_push(struct nv_vgpu_device *vgpu, } /* + * r1780 (CONT.430f): dev overrides for the direct-GL USERD experiment. + * `sema_userd_swap` reads GP_PUT at +0x88 and GP_GET at +0x8c (the + * orientation measured on the direct-GL client's control page), and + * `sema_userd_override` ("0x<handle>,0x<pa>") forces a channel's USERD + * candidate to a known sysmem page. Both are diagnostic-only defaults + * off; the normal path is unchanged. + */ +static bool gsp_sema_userd_swap; +module_param_named(sema_userd_swap, gsp_sema_userd_swap, bool, 0644); +MODULE_PARM_DESC(sema_userd_swap, + "Read GP_PUT at +0x88 and GP_GET at +0x8c for the USERD probe (direct-GL layout experiment); default 0"); + +static int svc_userd_override_set(const char *val, const struct kernel_param *kp) +{ + u32 h = 0; + u64 pa = 0; + unsigned long flags; + int i, hits = 0; + char *buf, *comma; + + buf = kstrdup(val, GFP_KERNEL); + if (!buf) + return -ENOMEM; + comma = strchr(buf, ','); + if (!comma) { + kfree(buf); + return -EINVAL; + } + *comma = '\0'; + h = (u32)simple_strtoul(buf, NULL, 0); + pa = simple_strtoull(comma + 1, NULL, 0); + kfree(buf); + if (!h || !pa) + return -EINVAL; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || (u32)c->h_object != h) + continue; + c->userd_pa = pa; + c->userd_as = SVC_APERTURE_SYS; + c->userd_ok = true; + c->userd_reg = false; + hits++; + pr_info("sema-chan: USERD override h=0x%x pa=0x%llx (swap=%d)\n", + h, pa, gsp_sema_userd_swap ? 1 : 0); + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return hits ? 0 : -ENOENT; +} + +static const struct kernel_param_ops svc_userd_override_ops = { + .set = svc_userd_override_set, +}; +module_param_cb(sema_userd_override, &svc_userd_override_ops, NULL, 0200); +MODULE_PARM_DESC(sema_userd_override, + "Force a channel USERD page (handle,pa), diagnostic only"); + +/* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ @@ -26092,12 +26168,14 @@ static int svc_userd_probe_ex(struct nv_vgpu_device *vgpu, u64 pa, u32 as, u32 entries, u32 *put, u32 *get, bool loose_get) { u32 p = 0, g = 0; + u32 put_off = gsp_sema_userd_swap ? SVC_USERD_GP_GET : SVC_USERD_GP_PUT; + u32 get_off = gsp_sema_userd_swap ? SVC_USERD_GP_PUT : SVC_USERD_GP_GET; if (!pa) return -EINVAL; - if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) + if (svc_read_ap(vgpu, pa + put_off, as, &p, sizeof(p))) return -ENOENT; - if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) + if (svc_read_ap(vgpu, pa + get_off, as, &g, sizeof(g))) return -ENOENT; *put = p; *get = g; @@ -29746,6 +29824,19 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, if (va0 < SVC_LOW_REL_VA_LO || va0 >= SVC_LOW_REL_VA_LO + 0x10000000ULL) continue; + /* + * r1780 (CONT.430d): entry0 is the main + * pushbuffer next to the ring. Bound it to + * +/-8 MB of the registered ring VA so a + * dense/transient page elsewhere in the + * 64 MB low arena cannot masquerade as the + * ring (measured false latches at + * PA 0x108587000/0x1100b6000 while the real + * ring sat at PA 0x35331b000). + */ + if (va0 + 0x800000ULL < ch->gpfifo_va || + va0 > ch->gpfifo_va + 0x800000ULL) + continue; } else { if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va || va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) @@ -29814,7 +29905,19 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, * Require an all-zero entry slot within the first 128 * entries of the run. */ - if (sparse) { + /* + * r1780 (CONT.430d): apply the terminus gate to the + * low-aperture family too. The direct-GL ring page + * carries a bounded run of queued entries and then + * zero words (measured: entry0 va=0x4000000 len=2260 + * then zeros at slot 23), while a dense data page in + * the same low arena decoded on every slot and was + * latched first at PA 0x108587000 - the pump then + * read phantom entries, the GET/PUT mismatched and + * the guest corrupted itself (bad pmd/OOM) chasing + * them. A real ring cannot be dense past slot 128. + */ + if (sparse || low) { u32 z; bool term = false; @@ -31848,10 +31951,11 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, * pass can run before the client has written the ring or the * USERD, and a consumed once-bit would never rescan. */ - if (time_before(jiffies, (unsigned long)svc_gr_ring_scan_last + + if (time_before(jiffies, + (unsigned long)svc_gr_ring_scan_last[ch->gr_index] + msecs_to_jiffies(1500))) - return; - svc_gr_ring_scan_last = jiffies; + goto skip_shadow_scan; + svc_gr_ring_scan_last[ch->gr_index] = jiffies; if (sbase && lim > 0x1000) { static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF, 0x1000 }; @@ -32087,6 +32191,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } +skip_shadow_scan: /* * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor * field comment). The driver's kickoff writes GPFIFO entries whose |