diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 293 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1000 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 11 | ||||
| -rw-r--r-- | tools/guest-diag/scrub_pin_qmp.py | 176 |
5 files changed, 1434 insertions, 48 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index f7e7beb..ca14744 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -3290,6 +3290,25 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, { struct sema_channel_desc desc; + /* + * CONT.405: name the registration attempt for the direct-GL client's + * channel families. A "beef"/"bfef" handle that appears more than + * once is the client's alloc -> setup -> free -> retry loop; the + * matching beef step trace in gsp_rm_handle_rm_control names the + * last control it issued before the retry. Bounded. + */ + if (((h_object & 0xffff0000u) == 0xbeef0000u || + (h_object & 0xffffff00u) == 0xbfef0000u)) { + static u32 beef_reg_log; + + if (beef_reg_log < 64) { + beef_reg_log++; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: beef chan register #%u h=0x%x (setup step: alloc_channel params=%u; next client step is BIND/GPFIFO_SCHEDULE, then its first kick)", + beef_reg_log, h_object, params_size); + } + } + if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8) { /* * r828 diag: a channel-class alloc whose params are shorter @@ -3352,26 +3371,109 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, * the direct client's channels never resolved a USERD (CONT-37 * alloc/free retry loop). Index by the subdevice. */ + /* + * CONT.404: read the aperture before the flags log so the logged + * /u value is the real one (it used to print 0 because the memcpy + * sat after the block). + */ + memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM + + GSPRM_CHAN_MEM_APERTURE, 4); { u32 sd = 0; u64 uo = 0; + u32 flags = 0; if (params_size >= 140) memcpy(&sd, params + 136, 4); + /* + * CONT.402: subDeviceId is a one-hot bitmask + * (alloc_channel.h), so the userdOffset[] slot is its first + * set bit, not the raw value. + */ + sd = sd ? (u32)__builtin_ctz(sd) : 0; if (sd >= 8) sd = 0; if (params_size >= 64 + (sd + 1) * 8) { memcpy(&uo, params + 64 + (u64)sd * 8, 8); desc.userd_pa += uo; } + /* + * CONT.402: NVOS04_FLAGS_CHANNEL_USERD_INDEX_VALUE (bits 10:8) + * and USERD_INDEX_PAGE_VALUE (bits 20:12) select the 0x200 + * control block inside the declared userdMem page. Logged so + * the block offset can be validated before it is applied. + * + * CONT.404: apply it. The kernel driver sets idx = ChID % 8 and + * page = ChID / 8 (kernel_channel.c, IS_GSP_CLIENT path), so a + * channel whose ChID is nonzero has its GPGet/GPPut 0x200-aligned + * block somewhere inside the declared userdMem. Reading base+0 + * made the guest's GP_PUT advances invisible - X's channel got + * ChID=3 in r1697 and timed out in "Failed to initialize DMA" + * with the host watching fill at the wrong block. + */ + if (params_size >= 24) { + u64 base = desc.userd_pa; + u32 idx, pg; + + memcpy(&flags, params + 20, 4); + idx = (flags >> 8) & 0x7; + pg = (flags >> 12) & 0x1ff; + if (idx != 0 || pg != 0) + desc.userd_pa += (u64)pg * 0x1000 + + (u64)idx * 0x200; + pr_info("vgsp-rm: chan userd flags=0x%x idx=%u page=%u base=0x%llx block=0x%llx/%u hObject=0x%x\n", + flags, idx, pg, base, desc.userd_pa, + desc.userd_as, h_object); + } } - memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM + - GSPRM_CHAN_MEM_APERTURE, 4); memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8); memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM + GSPRM_CHAN_MEM_APERTURE, 4); /* + * CONT.404 diag: the registered block is not always the block the + * client writes - X's live GP_PUT was adopted by the shape scan at + * fb 0x910000 while the wire declared 0x190000 (+0x600), and the + * direct-GL channel's declared page stayed zero. Name the whole + * wire memory-descriptor set and peek the registered block's + * GP_GET/GP_PUT, so the next cycle separates "wire base is not the + * client's block" from "client never writes FB". Bounded. + */ + { + static u32 md_log; + u64 inst_b = 0, ramfc_b = 0; + u32 inst_sz = 0, ramfc_sz = 0; + u32 inst_as = 0, ramfc_as = 0; + u32 g0 = 0, p0 = 0; + + if (md_log < 32) { + md_log++; + memcpy(&inst_b, params + GSPRM_CHAN_INSTANCE_MEM, 8); + memcpy(&inst_sz, params + GSPRM_CHAN_INSTANCE_MEM + 8, 4); + memcpy(&inst_as, params + GSPRM_CHAN_INSTANCE_MEM + + GSPRM_CHAN_MEM_APERTURE, 4); + memcpy(&ramfc_b, params + 192, 8); + memcpy(&ramfc_sz, params + 200, 4); + memcpy(&ramfc_as, params + 192 + + GSPRM_CHAN_MEM_APERTURE, 4); + if (desc.userd_as == 2 && ctx->vgpu && + ctx->vgpu->bars[1].vaddr && + desc.userd_pa + 0x90 <= + nv_vgpu_fb_emul_size(ctx->vgpu)) { + memcpy(&g0, ctx->vgpu->bars[1].vaddr + + desc.userd_pa + 0x88, 4); + memcpy(&p0, ctx->vgpu->bars[1].vaddr + + desc.userd_pa + 0x8c, 4); + } + pr_info("vgsp-rm: chan mds hObject=0x%x inst=0x%llx/%u/%u userd=0x%llx/%u ramfc=0x%llx/%u/%u notif=0x%llx/%u getput=%08x/%08x\n", + h_object, inst_b, inst_sz, inst_as, + desc.userd_pa, desc.userd_as, + ramfc_b, ramfc_sz, ramfc_as, + desc.notifier_pa, desc.notifier_as, g0, p0); + } + } + + /* * SESSION 153-CONTINUED-15 (draw attribution): every channel-class * allocation is named with its wire shape before registration, so a * channel the service silently refuses (gpfifo VA 0 / entries < 2) @@ -3415,6 +3517,60 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, } /* + * CONT.406: GR class objects (TURING_A 0xc597 and TURING_COMPUTE_A 0xc5c0) + * are allocated on a channel and the client frees them again when the next + * step of its GR context setup fails. Keep the last few allocations so the + * FREE side can name the object class and how long the client kept it: that + * turns "the client freed the 3D object" into a measured keep time, and + * separates "the alloc reply was rejected" (keep time ~0) from "the client + * failed later and cleaned up" (keep time ~80 ms, the observed r1700 case). + * All alloc/free RPC handling is serialized on the emulation thread, so the + * table needs no lock. Bounded. + */ +#define GSPRM_GR3D_LOG_MAX 24 +#define GSPRM_MAX_GROBJS 16 + +static struct { + u32 h; + u32 class_id; + unsigned long at; +} gsp_rm_grobjs[GSPRM_MAX_GROBJS]; +static u32 gsp_rm_grobjs_next; + +static void gsp_rm_grobj_note(u32 h, u32 class_id) +{ + u32 i; + + for (i = 0; i < GSPRM_MAX_GROBJS; i++) { + if (gsp_rm_grobjs[i].h == h) { + gsp_rm_grobjs[i].class_id = class_id; + gsp_rm_grobjs[i].at = jiffies; + return; + } + } + i = gsp_rm_grobjs_next++ % GSPRM_MAX_GROBJS; + gsp_rm_grobjs[i].h = h; + gsp_rm_grobjs[i].class_id = class_id; + gsp_rm_grobjs[i].at = jiffies; +} + +static bool gsp_rm_grobj_retire(u32 h, u32 *class_id, unsigned long *at) +{ + u32 i; + + for (i = 0; i < GSPRM_MAX_GROBJS; i++) { + if (gsp_rm_grobjs[i].h != h || !gsp_rm_grobjs[i].class_id) + continue; + *class_id = gsp_rm_grobjs[i].class_id; + *at = gsp_rm_grobjs[i].at; + gsp_rm_grobjs[i].class_id = 0; + gsp_rm_grobjs[i].h = 0; + return true; + } + return false; +} + +/* * GSP_RM_ALLOC (103) - rpc_gsp_rm_alloc_v03_00 (32 byte header) followed by * the class-specific allocation parameters. * @@ -3454,6 +3610,32 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, "vgsp-rm: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x", class_id, params_size, flags, hc, hp, ho); + /* + * CONT.405: same bounded step trace as gsp_rm_handle_rm_control, for + * the direct-GL client's object allocations. Channel classes get a + * retry counter per handle so the alloc/free loop is visible without + * gsp_sema_dbg. + */ + { + bool beef = (hc >= 0xc1d00010u && hc <= 0xc1d0002fu) || + (ho & 0xffff0000u) == 0xbeef0000u || + (ho & 0xffffff00u) == 0xbfef0000u; + + if (beef) { + static u32 beef_alloc_log; + + if (beef_alloc_log < 128) { + beef_alloc_log++; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: beef alloc step #%u class=0x%x hClient=0x%x hParent=0x%x hObject=0x%x sz=%u%s", + beef_alloc_log, class_id, hc, hp, ho, + params_size, + gsp_rm_is_channel_class(class_id) ? + " (channel)" : ""); + } + } + } + /* Full param dump for memory/channel classes and any class we have not * classified yet (rate-limited). %phN is capped at 64 bytes per print, * so chunk it. */ @@ -3604,6 +3786,39 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, memcpy(reply, p, reply_len); memset(reply + 16, 0, 4); /* status = NV_OK */ + /* + * CONT.406: the direct-GL client allocates a TURING_A (0xc597) 3D + * object on its channel immediately before it would submit its first + * kick, and then frees it again. Record the exact reply the client + * receives (bounded) and remember the object so the FREE side can say + * how long it was kept. The reply must echo the 16-byte + * NV_GR_ALLOCATION_PARAMETERS {version, flags, size, caps} with + * status=NV_OK; caps is a class-dependent *output* that has no + * defined bits for c597 in any 570 header, so the echoed 0 is the + * value the golden/physical path also sends and accepts. + */ + if (class_id == 0xc597u) { + u32 rp[4] = { 0, 0, 0, 0 }; + + if (echo_len >= 4) + memcpy(&rp[0], reply + GSPRM_ALLOC_HDR_SIZE, 4); + if (echo_len >= 8) + memcpy(&rp[1], reply + GSPRM_ALLOC_HDR_SIZE + 4, 4); + if (echo_len >= 12) + memcpy(&rp[2], reply + GSPRM_ALLOC_HDR_SIZE + 8, 4); + if (echo_len >= 16) + memcpy(&rp[3], reply + GSPRM_ALLOC_HDR_SIZE + 12, 4); + + gsp_rm_grobj_note(ho, class_id); + if (rm->gr3d_alloc_log < GSPRM_GR3D_LOG_MAX) { + rm->gr3d_alloc_log++; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: alloc 0xc597 h=0x%x status=0x%x size=0x%x reply=%08x %08x %08x %08x (parent=0x%x client=0x%x)", + ho, 0, params_size, + rp[0], rp[1], rp[2], rp[3], hp, hc); + } + } + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_ALLOC, sequence, 0, reply, reply_len)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_RM_ALLOC reply send failed"); @@ -4309,9 +4524,35 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, u32 out_status = 0; u32 out_params_size = 0; u64 t0 = ktime_get_ns(); + bool beef; memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE); + /* + * CONT.405: bounded step trace for the direct-GL client's RM clients + * (the "beef" device/VAS/BINAPI client and the "bfef" channel + * client). Its init is a fixed control sequence; when it gives up it + * frees the channel and retries with a fresh client, so the LAST + * cmd this trace shows before the next client's device alloc is the + * failing step. Necessary because the default branch answers an + * unhandled control with generic SUCCESS and that is invisible at + * gsp_sema_dbg=0 - which is exactly where the retry loop lived. + */ + beef = (req.hClient >= 0xc1d00010u && req.hClient <= 0xc1d0002fu) || + (req.hObject & 0xffff0000u) == 0xbeef0000u || + (req.hObject & 0xffffff00u) == 0xbfef0000u; + if (beef) { + static u32 beef_step_log; + + if (beef_step_log < 128) { + beef_step_log++; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: beef step #%u cmd=0x%x hClient=0x%x hObject=0x%x in=%u", + beef_step_log, req.cmd, req.hClient, + req.hObject, req.paramsSize); + } + } + if (gsp_sema_dbg) nv_vgpu_info(ctx->vgpu, "vgsp-rm: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x", req.cmd, req.hClient, req.hObject, req.paramsSize, @@ -5090,6 +5331,21 @@ rm_control_reply: req.cmd, req.hClient, req.hObject, req.paramsSize, out_params_size, out_status, dt_us); + /* + * CONT.405: close the beef step row with its outcome so the + * last completed step before the retry is unambiguous. + */ + if (beef) { + static u32 beef_done_log; + + if (beef_done_log < 128) { + beef_done_log++; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: beef done cmd=0x%x -> status=0x%x out=%u", + req.cmd, out_status, + out_params_size); + } + } } if (gsp_rm_send_rm_control(ctx, sequence, reply, @@ -5146,6 +5402,21 @@ static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx, "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo", hDstClient, hDstObject, hSrcClient, hSrcObject, sequence); + /* + * CONT.407: when the duplicated object is a published VA space, the + * destination client's new handle names the SAME page directory (the + * GL client dups X's VAS 0x10005 as 0xbfef0001 and allocates its GR + * channel with that hVASpace). Register the alias so the channel- + * scoped lookup resolves the gpfifo VA through the client's own + * directory instead of the newest-first global walk (r1702/r1703 + * answered 0x1bcfff0/ap=2 because 0xbfef0001 was never known). + * Non-VAS dups simply return false and stay a pure handle echo. + */ + if (gsp_sema_service_vaspace_alias(hDstObject, hSrcObject)) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DUP_OBJECT VAS alias h=0x%x -> src h=0x%x (gpfifo VA now resolves through the source directory)\n", + hDstObject, hSrcObject); + /* Small delay before the reply: same reply-matching race mitigation as * FREE/UNLOADING. */ udelay(2000); @@ -5218,6 +5489,24 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, "vgsp-rm: retired %u event record(s) on FREE hObjOld=0x%x hRoot=0x%x", removed, hObjOld, hRoot); + /* CONT.406: if this is a GR object we noted at alloc time, name the + * class and the keep time. A ~0 ms keep means the alloc reply (or the + * client's params copy-out) was rejected; a longer keep means the + * client proceeded and cleaned up after a later step failed. */ + { + u32 gcls = 0; + unsigned long gat = 0; + + if (gsp_rm_grobj_retire(hObjOld, &gcls, &gat) && + rm->gr3d_free_log < GSPRM_GR3D_LOG_MAX) { + rm->gr3d_free_log++; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: gr-obj free h=0x%x class=0x%x after %ums (client kept it)", + hObjOld, gcls, + jiffies_to_msecs(jiffies - gat)); + } + } + if (gsp_sema_dbg) nv_vgpu_info(ctx->vgpu, "vgsp-rm: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u", hObjOld, sequence, rm->cmd_read, rm->stat_write); diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index db5abdd..1f7b767 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -176,6 +176,8 @@ struct gsp_rm_emu { u64 unknown_rpcs; u32 alloc_dumps; /* raw GSP_RM_ALLOC dumps emitted */ u32 alloc_param_dumps; /* alloc param dumps emitted */ + u32 gr3d_alloc_log; /* CONT.406 c597 alloc-reply witness lines */ + u32 gr3d_free_log; /* CONT.406 c597 free-after witness lines */ u32 ctrl_param_dumps; /* control param dumps emitted */ u64 static_live; /* static-info replies carrying live identity */ u64 static_fallback; /* overlay requested, no snapshot served */ diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 01eddd3..8e6401d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -331,6 +331,15 @@ struct svc_channel { u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ + /* + * CONT.405: the DECLARED FBMEM userd block as it arrived in the + * channel alloc params (userdMem.base + userdOffset + userd + * index/page). userd_pa is overwritten whenever a scan latches the + * live block; this field keeps the wire value so the BAR1 alias can + * be re-derived on every pass and so the alias bias can be learned + * live (wire -> BAR1 page). Zero for non-FBMEM channels. + */ + u64 userd_wire; bool userd_ok; /* userd_pa reads a plausible put/get */ bool userd_wait_fresh; /* ISSUE-0002 r48: the layout USERD was accepted @@ -696,6 +705,19 @@ struct svc_channel { unsigned long gr_ram_userd_latch_at; /* jiffies at the latch */ bool gr_ram_userd_confirmed; /* +1/+2 step observed */ /* + * CONT.407: keep-across-put-reset. r1701-r1703 proved the kick + * family's live block with a +1 step (put 2 -> 3) and then the owner + * zeroed it (`speculative latch REJECTED (put=0 latch=3)`) 7-8 s + * later; dropping the USERD there destroyed a working control block + * and left the frame loop's next kick undecodable. A confirmed kick + * latch now KEEPS the block across a put reset, re-arms at the + * reset baseline (put=0) and lets the next authored step re-confirm. + * Bounded: after 64 keeps a block that keeps zeroing is treated as + * churn by the old path; the log emits at most 8 lines per channel. + */ + u8 gr_ram_userd_keeps; /* keeps applied (bound 64) */ + u8 gr_ram_userd_keep_log; /* keep log lines (bound 8) */ + /* * SESSION 153-CONT-5: rejected-latch memory. A speculative latch the * pump rejects (frozen or jumped PUT) used to exhaust the RAM scan * budget permanently: r936's kick channel latched a FROZEN lookalike @@ -1062,6 +1084,7 @@ static struct svc_guest_sess *svc_guest_get(struct nv_vgpu_device *vgpu, */ struct svc_scrub { struct nv_vgpu_device *vgpu; /* owning mdev (NULL = free slot) */ + bool registered; /* PA came from the guest scanner (vmap) */ bool userd_ok; /* a 4096-entry channel was registered */ u32 h_object; /* handle of the registered scrubber channel */ u64 userd_pa; @@ -1074,6 +1097,7 @@ struct svc_scrub { unsigned long pa_at; /* last successful walk re-resolution */ bool seeded; u32 put_last; + u32 consumed; /* CONT.408: last consumed entry (sema) */ u64 submitted; /* blocks submitted since seeding */ u64 authored; /* last authored finish value */ u32 authored_gen; /* pa_gen at the last authored write */ @@ -1082,32 +1106,93 @@ struct svc_scrub { u32 log_budget; }; /* - * One scrubber record per guest: every guest's PMA has its own 4096-entry - * scrubber channel, and the authored words must be written through the - * owning guest's address space (writing guest A's PA through guest B's VFIO - * device would corrupt B). Keyed by vgpu pointer, freed by - * gsp_sema_service_vgpu_cleanup(). + * One scrubber record per guest-channel buffer: every guest's PMA has its + * own 4096-entry scrubber channel, and the X DDX brings its own CE/PMA + * channel buffer whose host semaphore + finish payload are separate words. + * All authored words must be written through the owning guest's address + * space (writing guest A's PA through guest B's VFIO device would corrupt + * B). Keyed by vgpu pointer, freed by gsp_sema_service_vgpu_cleanup(). + * + * CONT.409: more than one record per guest. The stash path below binds the + * CeUtils scrubber channel (4096 GPFIFO entries) it discovers in the RM + * alloc stream; the guest-side scanner registers additional channel-buffer + * vmap pages through sema_scrubbuf_add. The scanner's PA is the page the + * guest RM actually polls (its memdesc/transfer vmap), while the GPU-VAS + * walk can keep resolving a stale directory after the guest RM re-creates + * its kernel VAS - the r1712 forced-sema boot is the witness (walk target + * 0x126b60000 while the guest polled 0x124fec000). A registered PA is + * therefore authoritative and is never replaced by a walk result. */ -static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS]; +#define SVC_SCRUB_RECS 4 +static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; +/* + * First record of this guest for the CeUtils stash path: prefer the record + * that already owns a stashed 4096-entry channel, else a non-registered + * slot, so a scanner registration is never clobbered by the stash. + */ static struct svc_scrub *svc_scrub_get(struct nv_vgpu_device *vgpu, bool create) { struct svc_scrub *s = NULL; + struct svc_scrub *any = NULL; int i; if (!vgpu) return NULL; - for (i = 0; i < SVC_MAX_GUESTS; i++) { - if (svc_scrubs[i].vgpu == vgpu) + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (svc_scrubs[i].vgpu != vgpu) + continue; + if (svc_scrubs[i].userd_ok) return &svc_scrubs[i]; - if (create && !s && !svc_scrubs[i].vgpu) + if (!svc_scrubs[i].registered && !any) + any = &svc_scrubs[i]; + } + if (any) + return any; + if (!create) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (!svc_scrubs[i].vgpu) { s = &svc_scrubs[i]; + memset(s, 0, sizeof(*s)); + s->vgpu = vgpu; + return s; + } } - if (s) { - memset(s, 0, sizeof(*s)); - s->vgpu = vgpu; + return NULL; +} + +/* + * Register/replace the scanner's channel-buffer record for this guest. + * Matching on hsem_pa keeps a moved buffer as a new record (the old one is + * reused because a registered record without a stashed USERD is disposable + * once the scanner names a different page). + */ +static struct svc_scrub *svc_scrub_reg(struct nv_vgpu_device *vgpu, u64 hsem_pa) +{ + struct svc_scrub *s = NULL; + int i; + + if (!vgpu) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (svc_scrubs[i].vgpu != vgpu) + continue; + if (svc_scrubs[i].hsem_pa == hsem_pa) + return &svc_scrubs[i]; + if (svc_scrubs[i].registered && !svc_scrubs[i].userd_ok && !s) + s = &svc_scrubs[i]; + } + if (s) + return s; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (!svc_scrubs[i].vgpu) { + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + svc_scrubs[i].vgpu = vgpu; + return &svc_scrubs[i]; + } } - return s; + return NULL; } static u8 svc_pb_passes; static u64 svc_pb_last; @@ -2076,6 +2161,50 @@ static bool svc_vaspace_known(u32 h_vaspace) } /* + * CONT.407: install a DUP_OBJECT'd VA-space handle as an alias of the + * source handle's directory. The driver gives a second client the same + * page directory under a new handle (measured: GL dups VAS 0x10005 as + * 0xbfef0001 and allocates its GR channel with hVASpace=0xbfef0001). The + * alias copies the source's pdb/entries/ap/verified state verbatim, so the + * channel-scoped lookup (svc_cur_vas) answers with the client's own + * directory instead of falling through to the newest-first global walk + * (r1702/r1703: gpfifo 0x1203cfff0 re-resolved to the bogus 0x1bcfff0). + * Returns false when the source is not a registered VAS; the caller then + * leaves DUP_OBJECT as a pure handle echo. + */ +bool gsp_sema_service_vaspace_alias(u32 h_dst, u32 h_src) +{ + struct sema_vaspace_desc d; + struct svc_vaspace snap; + unsigned long flags; + bool found = false; + int i; + + if (!h_dst || !h_src || h_dst == h_src) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_src) { + snap = svc.vas[i]; + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + if (!found) + return false; + memset(&d, 0, sizeof(d)); + d.h_vaspace = h_dst; + d.pdb = snap.pdb; + d.num_entries = snap.entries; + d.b_vidmem = snap.ap == SVC_APERTURE_FB; + d.verified = snap.verified; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-svc: VAS alias 0x%x -> 0x%x (pdb=0x%llx entries=%u verified=%d)\n", + h_dst, h_src, snap.pdb, snap.entries, snap.verified ? 1 : 0); + return true; +} + +/* * CONT.400: is this VA space backed by a directory proven against the * client's own content-validated ring (or published by RM)? Only then may * the parked-candidate promote skip its walk. A directory that exists but is @@ -2313,6 +2442,9 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; + /* CONT.405: keep the wire block for BAR1-alias resolution. */ + ch->userd_wire = ch->userd_as == SVC_APERTURE_FB ? + desc->userd_pa : 0; /* * r831: a registered userd value that looks like a real guest * PA (the RC watchdog's channel carries its own userd block) @@ -2322,7 +2454,17 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * pool-relative - observed 0x10000). Tiny registered values * stay guesses. */ - ch->userd_reg = desc->userd_pa >= 0x10000000ULL; + /* + * CONT.402: a declared FBMEM userdMem is the driver's own + * control-page location and must be trusted even though its FB + * offset is small (the direct-GL client declares fb 0x160000). + * The old rule treated every value < 0x10000000 as a guess and + * fell through to the RAM scan, which adopted an unrelated + * channel's page (0x114371000) and read a phantom GP_PUT. + */ + ch->userd_reg = (desc->userd_as == SVC_APERTURE_FB && + desc->userd_pa != 0) || + desc->userd_pa >= 0x10000000ULL; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as @@ -8681,6 +8823,101 @@ static void svc_gl_att_clear(u64 h, u32 subch) } } /* + * CONT.411: THE SECTION BELONGS TO THE ENGINE THE RUN ADDRESSES. + * + * The driver's channel layout fixes one subchannel per engine + * (cla06fsubch.h / NVA06F_SUBCHANNEL_*): + * + * 0 = 3D, 1 = COMPUTE, 2 = I2M, 3 = 2D, 4 = COPY_ENGINE + * + * A SET_OBJECT binds a class object to one of those subchannels and the + * methods that class executes arrive on the SAME subchannel. The section + * recognisers below used to key every check on the subchannel of the + * SET_OBJECT bind, which is wrong for a channel that interleaves engines + * in one push: the observed kick channel (h=0xbfef0002) carries the C597 + * class bind on the 3D subchannel 0 (`op=1 sub=0 addr=0x0 v0=0x0000c597`) + * and its method runs on the 2D subchannel 3 in the same 64 B push, so + * `svc_gl_att_has(h, subch)` rejected the very run that carries the + * window and every draw section stayed open-but-never-captured. + * + * Rule implemented here: attachment stays keyed by (channel, bind + * subchannel) - it is what the SET_OBJECT proved - but a run is accepted + * into the section when its own subchannel is either the bind subchannel + * or the engine subchannel the channel's runs are actually using + * (recorded per channel below). The recorded run subchannel is always a + * valid engine subchannel; nothing outside 0..4 is ever adopted. + */ +#define SVC_SUBCH_3D 0 +#define SVC_SUBCH_COMPUTE 1 +#define SVC_SUBCH_I2M 2 +#define SVC_SUBCH_2D 3 +#define SVC_SUBCH_CE 4 + +static u64 svc_gl_att_run_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_run_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_run_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + if (!h || subch > SVC_SUBCH_CE) + return; + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_run_h[i] == h) { + svc_gl_att_run_sub[i] = (u8)subch; + return; + } + if (!svc_gl_att_run_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) + free = SVC_GL_ATT_MAX - 1; /* evict oldest */ + if (free) { + memmove(&svc_gl_att_run_h[1], &svc_gl_att_run_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_run_sub[1], &svc_gl_att_run_sub[0], + (size_t)free); + } + svc_gl_att_run_h[0] = h; + svc_gl_att_run_sub[0] = (u8)subch; +} + +static u32 svc_gl_att_run_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_run_h[i] == h) + return svc_gl_att_run_sub[i]; + return 0xffu; +} + +static bool svc_gl_att_has_any(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return true; + return false; +} + +/* + * Accept a run's subchannel for section recognition: the bind's own + * subchannel always, or the engine subchannel the channel's runs use + * (only when the channel is attached at all and the run subchannel is a + * valid engine subchannel). + */ +static bool svc_gl_att_accept_run(u64 h, u32 subch) +{ + if (svc_gl_att_has(h, subch)) + return true; + if (!svc_gl_att_has_any(h) || subch > SVC_SUBCH_CE) + return false; + return subch == svc_gl_att_run_subch_of(h); +} + +/* * SESSION 153 (G-B gate: the client's own C597 draws must open sections). * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which * returns true when the push VA sits in the GR candidate ring - even when @@ -9224,6 +9461,67 @@ static void svc_kick_pa_cache_store(u64 h, u64 pa) svc_kick_pa_cache[0].pa = pa; } +/* + * CONT.411 (task 3): the release/counter completion slot the GL kick + * channel polls. The kick executor already authors the NvNotification + * (timestamp + status) into the client-visible page it resolves; record + * the slot it proved (handle, VA, PA, aperture, last value) so a later + * NOTIFY that arrives without its own SET_NOTIFY target can reuse the + * proven slot instead of falling back to "no target" - the client's poll + * then always sees a written completion for this channel. + */ +static struct { + u64 h; /* channel handle */ + u64 va; /* completion VA the client polls */ + u64 pa; /* PA the executor proved writable */ + u32 ap; /* aperture of pa */ + u32 val; /* last value authored (low word of the stamp) */ + u64 n; /* registrations */ +} svc_compl_slot[4]; +static u32 svc_compl_log; + +static void svc_compl_slot_store(struct svc_channel *ch, u64 va, u64 pa, + u32 ap, u32 val) +{ + u32 i, free = ARRAY_SIZE(svc_compl_slot); + + if (!ch || !ch->h_object || !va || ap != SVC_APERTURE_SYS) + return; + for (i = 0; i < ARRAY_SIZE(svc_compl_slot); i++) { + if (svc_compl_slot[i].h == ch->h_object) { + free = i; + break; + } + if (!svc_compl_slot[i].h && free == ARRAY_SIZE(svc_compl_slot)) + free = i; + } + if (free == ARRAY_SIZE(svc_compl_slot)) + free = 0; /* evict oldest */ + svc_compl_slot[free].h = ch->h_object; + svc_compl_slot[free].va = va; + svc_compl_slot[free].pa = pa; + svc_compl_slot[free].ap = ap; + svc_compl_slot[free].val = val; + svc_compl_slot[free].n++; + if (gsp_sema_dbg && (svc_compl_log < 64 || + !(svc_compl_slot[free].n & 127))) { + svc_compl_log++; + pr_info("sema-compl: h=0x%x slot registered va=0x%llx pa=0x%llx ap=%u val=%08x n=%llu (client poll target)\n", + ch->h_object, va, pa, ap, val, + (unsigned long long)svc_compl_slot[free].n); + } +} + +static u64 svc_compl_slot_va_of(u64 h) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_compl_slot); i++) + if (svc_compl_slot[i].h == h) + return svc_compl_slot[i].va; + return 0; +} + static bool gsp_sema_gl_draw_synth; module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); MODULE_PARM_DESC(sema_gl_draw_synth, @@ -9859,6 +10157,21 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, svc_kick_pa_cache_store(ch->h_object, pa); } } + /* + * CONT.412: the channel's registered notifier page is the + * page the guest's submit poll actually watches (the same + * page svc_notify2d_write zeroes). Prefer it over the + * trusted-walk answer, which can resolve the notification + * VA to a different sysmem page (measured: + * notifier_pa+0xb0=0x1288780b0 vs trusted pa=0x124d610b0) + * and leave the poll word unwritten forever. + */ + if (ch && svc_kick_family(ch) && ch->notifier_pa) { + pa = ch->notifier_pa + + (va & (SEMA_PAGE_SIZE - 1)); + ap = ch->notifier_as; + rsrc = "chan-notif"; + } r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); @@ -9876,6 +10189,15 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, } } svc_kick_count++; + /* + * CONT.411 (task 3): register the proven completion slot for this + * channel. Only a successful SYS write is registered - that is the + * page the client's own poll can observe (the FB shadow write was + * the r977 dead end). The slot is reused by the flush's short-kick + * fallback when a later NOTIFY arrives with no target of its own. + */ + if (ok && ch) + svc_compl_slot_store(ch, va, pa, ap, n0); if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) { svc_kick_log++; pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n", @@ -10324,6 +10646,35 @@ static void svc_gl_cover_log(struct nv_vgpu_device *vgpu, const u8 *buf, u32 first, u32 count, u64 iva, u32 stride, u32 voff, const u32 *cand, u32 ncand); +/* + * CONT.411 (task 2): the GL kick channel's flush witness. + * + * The 0xbfef0002 section arrives as a 32/64 B push and its flush was + * invisible: the kick-exec branch exits before the "flush skip" witness, + * so the log could not say whether the flush ran, was rejected by the + * trusted/seen guard, or exited at the operand checks. Witness every exit + * for this channel with the full operand set so the next cycle can name + * the exact branch without a rebuild. Bounded (128 lines) because this + * channel flushes on every pass. + */ +#define SVC_GL_WATCH_H 0xbfef0002ULL +static u32 svc_gl_watch_log; + +static void svc_gl_watch_note(const struct svc_channel *ch, const char *why, + u64 push_va, u32 len, u32 at_off, bool trusted) +{ + if (!gsp_sema_dbg || !ch || ch->h_object != SVC_GL_WATCH_H) + return; + if (svc_gl_watch_log >= 128) + return; + svc_gl_watch_log++; + pr_info("sema-gl-w: %s h=0x%x push=0x%llx len=%u seen=%d trusted=%d prog=0x%llx param=0x%llx rel=0x%llx start=0x%x at=0x%x n_ptr=%u blob=%u subch=%u\n", + why, ch->h_object, push_va, len, svc_gl.seen ? 1 : 0, + trusted ? 1 : 0, svc_gl.prog_va, svc_gl.param_va, + svc_gl.rel_va, svc_gl.start_off, at_off, svc_gl.n_ptr, + svc_gl.blob_len, svc_gl.subch); +} + static void svc_gl_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, const struct svc_push_state *st, const u8 *buf, @@ -10361,9 +10712,15 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, */ if (gsp_sema_dbg) svc_gl_arrival_note(push_va, buf, len); + /* CONT.411: state at every flush entry for the kick channel. */ + svc_gl_watch_note(ch, "entry", push_va, len, at_off, trusted); svc_gl_prog_apply_param(); - if (!svc_gl.seen || !trusted) + if (!svc_gl.seen || !trusted) { + svc_gl_watch_note(ch, !svc_gl.seen ? "early-noseen" : + "early-untrusted", push_va, len, at_off, + trusted); goto out; + } { u64 hp = 0, hm = 0, hr = 0; u32 hms = 0, hrp = 0; @@ -10568,6 +10925,22 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, kick_va = svc_gl.kick_notify_va; if (!kick_va && svc_kick_family(ch)) kick_va = ch->notify2d_va; + /* CONT.411 (task 3): a NOTIFY with no per-push + * target falls back to the completion slot this + * channel proved earlier, so the poll target is + * never left unwritten just because SET_NOTIFY + * was carried by an earlier push. */ + if (!kick_va && svc_gl.kick_notify_seen) + kick_va = svc_compl_slot_va_of( + ch->h_object); + /* CONT.411 (task 3): name the completion slot + * this section resolves to before the exec so + * the witness distinguishes "no slot" from + * "slot write failed". */ + svc_gl_watch_note(ch, kick_va ? + "kick-try" : "kick-noslot", + push_va, len, at_off, + trusted); if (gsp_sema_kick_notify && svc_gl.kick_notify_seen && kick_va && svc_kick_notify_exec(vgpu, ch, kick_va, @@ -10577,6 +10950,9 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * re-author an unchanged member. */ svc_gl.kick_notify_seen = false; svc_gl.kick_notify_va = 0; + svc_gl_watch_note(ch, "kick-done", + push_va, len, at_off, + trusted); goto out; } if (gsp_sema_dbg && svc_gl_skip_log < 2048) { @@ -10588,6 +10964,49 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, at_off > svc_gl.start_off ? (at_off - svc_gl.start_off) / 4u : 0u); } + /* + * CONT.412: an empty/no-op submit must still + * complete. If the 2D NOTIFY path did not + * author the channel's kick completion (no + * NOTIFY seen in this push, or the target was + * dropped earlier), author it from the proven + * completion slot so the client's submit wait + * cannot spin forever on a submit that carried + * no draw operands. + */ + if (trusted && svc_kick_family(ch)) { + u64 cva = svc_compl_slot_va_of( + ch->h_object); + + if (!cva) + cva = ch->notify2d_va; + if (cva && svc_kick_notify_exec( + vgpu, ch, cva, 0, + push_va)) + svc_gl_watch_note(ch, + "empty-submit-complete", + push_va, len, at_off, + trusted); + else if (ch->notifier_pa) { + u8 nw[16]; + u64 ts = ktime_get_real_ns(); + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + if (!svc_write_ap(vgpu, + ch->notifier_pa + + 0x0b0, + ch->notifier_as, + nw, sizeof(nw))) + svc_gl_watch_note(ch, + "empty-submit-complete", + push_va, len, + at_off, + trusted); + } + } + svc_gl_watch_note(ch, "skip", push_va, len, + at_off, trusted); goto out; } else if (gsp_sema_dbg && svc_gl_np_log < 64) { svc_gl_np_log++; @@ -18885,6 +19304,24 @@ static u32 svc_notify2d_log; static bool gsp_sema_2d_notify_immd = true; module_param_named(sema_2d_notify_immd, gsp_sema_2d_notify_immd, bool, 0644); +/* + * CONT.406: the direct-GL kick channel submits a short + * SET_OBJECT(TURING_A)+NOTIFY push before it ever arms a class-2d + * SET_NOTIFY_A/B, so the NOTIFY arrives with no class2d target and used to be + * dropped entirely: the push then had no release edge, the channel was + * consumed without a completion, and the client spun in its kick/fence wait. + * + * A NOTIFY with no programmed address is still this push's completion edge on + * real hardware (the class object's notify address register simply holds its + * creation value and the write goes nowhere). Treat it exactly like a + * completed NOTIFY - mark the release, post the channel event so the client's + * OS event fires, and count it - but do NOT write to any address: there is no + * target, and guessing one would corrupt an unrelated page. The 64-line + * witness is separate from the drop counter so the A/B stays readable. + */ +static u64 svc_kicknoaddr_done; +static u32 svc_kicknoaddr_log; + static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, struct svc_channel *ch, bool completion) { @@ -18905,6 +19342,17 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, pr_info("sema-notify2d: h=0x%x NOTIFY with no class2d target (short kick before SET_NOTIFY; client spin)\n", ch->h_object); } + if (completion) { + /* CONT.406: complete the short kick anyway (no write). */ + svc_kicknoaddr_done++; + if (svc_kicknoaddr_log < 64) { + svc_kicknoaddr_log++; + pr_info("sema-notify2d: h=0x%x short-kick NOTIFY completed with no armed target (n=%llu; release+event)\n", + ch->h_object, svc_kicknoaddr_done); + } + gsp_rm_post_channel_event(vgpu, ch->h_object); + return true; + } return false; } if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { @@ -19292,6 +19740,50 @@ static void svc_twod_run(struct nv_vgpu_device *vgpu, struct svc_channel *ch, */ #define SVC_NOTIFY_SCAN_SLOTS 4 +/* + * CONT.408: 0xFFFF IS THE X DDX'S KICK-IDLE MARKER, NOT AN ARM. + * + * The DDX's own RM channels (the 0x01xxxxxx family, e.g. the X server's + * 0x01000014) drive their push buffer through nvidia_drv.so's + * pb_reserve/pb_flush path: pb_kickIfIdle (+0xb66c0) only calls the kick + * while *(u16 *)(pb->notifier + 0x0e) == 0xFFFF (signature 0xffff8000 is + * the 0x8000 arm pb_armNotifiers +0xb7250 writes at slot 0xb; completion + * is 0x0000). The r864 registration keep-alive below was written for the + * RC-watchdog/beef families where 0xFFFF *is* the armed value, and it + * zeroed every 0xFFFF slot on every pass. On the DDX family that + * destroyed the only value pb_kickIfIdle accepts: r1706 live (Xorg 99 % + * CPU in the kick ioctl, glxgears D on the RM rwlock) had notifier slot 0 + * at 0x0000 and re-zeroed within one pass of any 0xFFFF write (measured + * by writing the live page). Keep 0xFFFF untouched (and restore slot 0 to + * it) for this family; the 0x8000->0 completion still goes through the + * bfef/X-DDX sweep. + */ +static bool svc_ddx_notif_idle(const struct svc_channel *ch) +{ + return (ch->h_object & 0xff000000u) == 0x01000000u; +} + +/* + * CONT.409: the GL client's own channel (e.g. 0xbfef0002) uses the same + * NvPushBuffer kick-idle convention as the DDX: libnvidia-glcore arms its + * class-2D notification at slot 11 (VA ...d0b0) and gates its next kick on + * slot 0's status being 0xFFFF. The r864 keep-alive zeroed that 0xFFFF on + * every pass, so after the first setup kick the client could never kick + * again - the r1716 live probe wrote 0xFFFF to slot 0 and it was re-zeroed + * within one pass (2 s sample), while glxgears kept spinning in glcore + * with only 32/64-byte setup pushes and no draw stream. A channel that has + * captured a class-2D NOTIFY VA therefore owns a kick-idle page for slots + * 0..3; the RC-watchdog/beef kernel channels (which never arm notify2d) + * keep the r864 behaviour. + */ +static bool svc_kick_idle_page(const struct svc_channel *ch) +{ + return svc_ddx_notif_idle(ch) || ch->notify2d_va != 0; +} + +static u32 svc_ddx_idle_log; +static u64 svc_ddx_idle_n; + static void svc_notify2d_keepalive(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -19309,13 +19801,38 @@ static void svc_notify2d_keepalive(struct nv_vgpu_device *vgpu, memcpy(&st, slots + slot * 16 + 14, 2); /* - * r1608: the 2D completion notifier is armed with - * NV9097_NOTIFICATION_STATUS_IN_PROGRESS (0x8000) while the - * GPU works on it (the X DDX's nvFermiSync waits for it to - * become 0); the older watchdog slots use 0xffff. Ack both. + * CONT.408: on the X DDX family 0xFFFF is the kick-idle + * marker pb_kickIfIdle requires, never an arm. Restore it + * on slot 0 (the kick gate) instead of zeroing it, and only + * ack a real 0x8000 arm on the other slots. */ - if (st != 0xffff && !(st & 0x8000)) + if (svc_kick_idle_page(ch)) { + if (slot == 0) { + u16 idle = 0xffff; + + if (st == 0xffff) + continue; + if (svc_write_ap(vgpu, + ch->notifier_pa, + ch->notifier_as, &idle, + sizeof(idle))) + continue; + svc_ddx_idle_n++; + if (svc_ddx_idle_log < 32) { + svc_ddx_idle_log++; + pr_info("sema-chan: kick idle h=0x%x pa=0x%llx restored 0x%04x -> 0xffff (n=%llu)\n", + ch->h_object, ch->notifier_pa, + st, + (unsigned long long) + svc_ddx_idle_n); + } + continue; + } + if (!(st & 0x8000)) + continue; /* 0xffff is idle here */ + } else if (st != 0xffff && !(st & 0x8000)) { continue; /* not armed - nothing to ack */ + } if (svc_write_ap(vgpu, ch->notifier_pa + slot * 16, ch->notifier_as, zero, sizeof(zero))) continue; @@ -19860,8 +20377,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * class), so they must stay outside the sliced window. */ if (gsp_sema_gl_xlate && !ch->engine_type) { - u32 as = svc_gl_att_subch_of(ch->h_object); + u32 as; + /* + * CONT.411: record the engine subchannel this run + * addresses (cla06fsubch.h 0..4) for every attached + * channel, then track the draw window on the section's + * own method subchannel. The previous form keyed the + * tracker on the SET_OBJECT bind's subchannel, so a + * channel that binds C597 on the 3D subchannel and + * emits its runs on another engine subchannel reset + * the window on every one of its own runs. + */ + if (svc_gl_att_has_any(ch->h_object)) + svc_gl_att_run_note(ch->h_object, subch); + as = svc_gl.seen ? svc_gl.subch : + svc_gl_att_subch_of(ch->h_object); if (as != 0xffu) { if (subch != as) gl_win_start = 0; @@ -20140,7 +20671,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ((op == 4 && addr == 0x1618) || (op == 1 && addr == 0x840 && count >= 4)) && !svc_gl.seen && - svc_gl_att_has(ch->h_object, subch)) { + svc_gl_att_accept_run(ch->h_object, subch)) { u64 keep_rel = svc_gl.rel_va; u32 keep_pay = svc_gl.rel_payload; u32 wstart = gl_win_start ? gl_win_start : off; @@ -20355,7 +20886,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl_att_clear(ch->h_object, subch); } } else if (gsp_sema_gl_xlate && svc_gl.seen && - subch == svc_gl.subch) { + (subch == svc_gl.subch || + subch == SVC_SUBCH_3D)) { + /* + * CONT.411: collect operands on the section's own + * method subchannel and on the 3D subchannel (0) - + * the class bind and the state methods it precedes + * live there even when the run that opened the window + * addresses another engine subchannel. + * + * Deliberately NOT every run subchannel: feeding a + * foreign 2D run (subch 3 on a bind-on-0 section) into + * svc_gl_field would interpret 2D method payloads as + * C597 state/pointers and corrupt the section's page + * set. The run-subchannel rule applies to OPENING a + * section (above), not to operand collection. + */ svc_gl_field(addr, count, buf + off + 4, len - off - 4); } @@ -24799,6 +25345,76 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, } /* + * CONT.405 - BAR1-alias resolution for a declared FBMEM USERD. + * + * The channel alloc params name the control block in the FB aperture + * (userdMem.base + userdOffset + index/page), but that is the allocation + * site, not where the client writes. The guest RM maps the USERD memdesc + * into its BAR1 aperture with an offset chosen by its own BAR1 VA + * allocator (kbusMapFbAperture -> dmaAllocMapping): the client then writes + * GP_GET/GP_PUT through that BAR1 window. The emulated BAR1 is an identity + * shadow of FB in this module, so the write lands at the BAR1 offset and + * the declared FB block stays fill (measured: X channel wire 0x190600 is + * 0x01010100 fill while the live pair sits at 0x910000 -> page bias + * 0x780000; both X registrations in r1699 latched the same 0x910000). + * + * The bias lives in the guest, so it cannot be derived here; it is learned + * once from a block the FB shape scan has already validated against a + * channel's ring (svc_chan_userd latch) and then applied to later FBMEM + * registrations as a preferred candidate. Learning is page-granular and + * bounded: a bias that is not page-aligned, zero, or outside the shadow is + * refused. Until it is learned, resolution falls back to the wire block, + * the wire page base, and the existing scans. + */ +static u64 svc_bar1_alias_bias; /* BAR1 page offset - FB page offset */ +static u32 svc_bar1_alias_log; + +static void svc_userd_alias_learn(struct svc_channel *ch, u64 alias_pa) +{ + u64 wire, wp, ap, bias; + + if (!ch || !ch->userd_wire || svc_bar1_alias_bias) + return; + if (ch->userd_as != SVC_APERTURE_FB || !alias_pa) + return; + wire = ch->userd_wire; + wp = wire & ~0xfffULL; + ap = alias_pa & ~0xfffULL; + if (!wp || ap <= wp) + return; + bias = ap - wp; + if ((bias & 0xfff) || bias >= 0x10000000ULL) + return; + WRITE_ONCE(svc_bar1_alias_bias, bias); + pr_info("vgsp-sema: userd alias bias learned 0x%llx (h=0x%x wire=0x%llx alias=0x%llx; applied to later FBMEM userd)\n", + bias, ch->h_object, wire, alias_pa); +} + +/* + * Return the BAR1-alias candidate for a channel's declared FBMEM USERD. + * The client writes the control block at the mapped page's base + 0x88, so + * the page granularity (not the wire's index/page offset) is what the + * alias names; svc_userd_probe_ex still validates the pair. + */ +static bool svc_userd_bar1_alias(struct svc_channel *ch, u64 *alias) +{ + u64 bias = READ_ONCE(svc_bar1_alias_bias); + u64 wire = ch->userd_wire; + + if (!wire || ch->userd_as != SVC_APERTURE_FB || !bias) + return false; + *alias = (wire & ~0xfffULL) + bias; + if (!*alias || *alias > 0x10000000ULL) + return false; + if (svc_bar1_alias_log < 64) { + svc_bar1_alias_log++; + pr_info("vgsp-sema: userd alias h=0x%x wire=0x%llx alias=0x%llx\n", + ch->h_object, wire, *alias); + } + return true; +} + +/* * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the direct GL client's * channel allocation declares its USERD in the FB aperture at a 0x18fc00/ * 0x18fe00-class offset, but the guest's GP_PUT/GP_GET writes for those @@ -25185,8 +25801,8 @@ static bool svc_ce_declared_userd(struct nv_vgpu_device *vgpu, static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc) { - u64 cand[8]; - u32 as[8], nc = 0, i; + u64 cand[16]; + u32 as[16], nc = 0, i; /* * X-DMA-FIX (2026-11-10): "translated" is not "validated". svc_lookup * can answer from a stale pin or a foreign VAS with a page whose @@ -25395,6 +26011,27 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, as[nc++] = ring_alt; } } + /* + * CONT.405: BAR1-alias candidates FIRST for a declared FBMEM USERD. + * The client writes its control block through the guest RM's BAR1 + * mapping, so the alias page can carry the live pair while the + * declared FB block stays fill. The learned bias makes the alias a + * direct translation (X: 0x190000 -> 0x910000); the wire page base + * covers the case where the alloc carried an index offset the client + * did not apply to the block it publishes. Both are still validated + * by svc_userd_probe_ex / the candidate gates below. + */ + if (ch->userd_as == SVC_APERTURE_FB && ch->userd_wire) { + u64 alias = 0; + + if (svc_userd_bar1_alias(ch, &alias)) { + cand[nc] = alias; as[nc++] = SVC_APERTURE_FB; + } + if ((ch->userd_wire & ~0xfffULL) != ch->userd_pa) { + cand[nc] = ch->userd_wire & ~0xfffULL; + as[nc++] = SVC_APERTURE_FB; + } + } if (ch->userd_pa) { cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; cand[nc] = ch->userd_pa; @@ -25811,6 +26448,13 @@ ce_fb_scan: best_pending = pending; } if (best) { + /* + * CONT.405: this latch is ring-validated, so + * it is the ground truth for the FB->BAR1 + * page bias. Learn it once and apply it to + * later FBMEM registrations. + */ + svc_userd_alias_learn(ch, best); ch->userd_pa = best; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; @@ -26579,6 +27223,15 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4)) continue; + /* + * CONT.408: the X DDX family's slot 0 carries the kick-idle + * marker 0xFFFF (pb_kickIfIdle). It is free, not busy - + * zeroing it blocked every later kick (r1706 live). A real + * 0x8000 arm on slot 1 still counts. + */ + if (svc_kick_idle_page(ch) && slot == 0 && + status == 0xffff && !info32) + continue; if (status || info32) busy = 1; } @@ -26623,7 +27276,21 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. - */ + * + * CONT.408: the X DDX family's slot 0 is the 0xFFFF kick-idle marker + * (pb_kickIfIdle), never a completion slot - clear only the + * work-submit-token slot (1) there, or the DDX's push-buffer kick + * gate dies with the clear (r1706 live). + */ + if (svc_kick_idle_page(ch)) { + if (svc_write_ap(vgpu, base + SVC_NOTIFY_SIZE, + ch->notifier_as, zero, SVC_NOTIFY_SIZE)) + return; + gsp_rm_post_channel_event(vgpu, ch->h_object); + svc.post_hint_until = jiffies + + msecs_to_jiffies(SVC_POST_HINT_MS); + return; + } if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { ch->notified++; if (gsp_sema_dbg && @@ -34236,6 +34903,32 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, * mirrors the normal consume tail; returns true when the entry was consumed. * When the copy is still in flight it returns false and the entry stays. */ +static u32 svc_gr_arm_log; /* CONT.408 bounded epoch-arm witness */ + +/* + * CONT.408: arm the GR publish epoch from a proof-of-live-session edge. + * The channel object is the pump's mutable pass-local copy (the callers + * see it const for read-mostly use); the end-of-pass write-back persists + * the flag. A queued/completed launch whose candidate was captured from + * this channel's own guest ring proves the ring belongs to the live + * session, so the residual stale-epoch watchdog (put > 1 && + * last_get == put && !gr_pub_armed) must not declare a wedge on it. + */ +static void svc_gr_epoch_arm(const struct svc_channel *ch, u64 push_va, + const char *why) +{ + struct svc_channel *m = (struct svc_channel *)ch; + + if (m->engine_type || m->gr_pub_armed) + return; + m->gr_pub_armed = true; + if (svc_gr_arm_log < 64) { + svc_gr_arm_log++; + pr_info("sema-chan: GR epoch armed h=0x%x by %s push=0x%llx (residual watchdog defused)\n", + m->h_object, why, push_va); + } +} + static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { @@ -34251,6 +34944,20 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (done && svc_fwd_req.hw_rc == 0) { svc_gr_missre_done(); /* + * CONT.408: a successful forwarded GR launch - including the + * verbatim 3D STREAM replay the X/DDX channel runs - is + * proof this channel's ring belongs to the LIVE session: the + * candidate was captured from the guest's own push and the + * engine executed it. Arm the epoch here so the residual + * stale-epoch watchdog cannot declare a wedge on a channel + * that is demonstrably live (r1704-r1707: the X channel was + * discovered with residual put=2, never dropped, and the + * watchdog fired ~3 s after the last replay while the DDX + * still polled its push buffer). + */ + svc_gr_epoch_arm(ch, svc_fwd_req.gr_push_va, + "forwarded launch"); + /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that * actually carries the payload lets the one relayed record be @@ -39193,6 +39900,13 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } /* + * CONT.408: this launch is being queued from a candidate captured + * out of this channel's own guest ring - proof the session is live. + * Arm the publish epoch (defuses the residual stale-epoch watchdog) + * exactly like the fresh-put-drop rewind does. + */ + svc_gr_epoch_arm(ch, svc_gr_cand.push_va, "queued launch"); + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -41838,6 +42552,30 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", ch->h_object, ch->userd_pa, ch->gr_ram_userd_latch_put, put); + } else if (kfam && ch->gr_ram_userd_confirmed && + put == 0 && + ch->gr_ram_userd_latch_put != 0 && + ch->gr_ram_userd_keeps < 64) { + /* + * CONT.407: a CONFIRMED kick-family block that + * the owner zeroes is a put reset, not a dead + * latch. Keep the block (verified by the +1 + * step above), re-arm at the reset baseline + * and let the next authored step re-confirm; + * r1701-r1703 lost a working USERD exactly + * here 7-8 s before the frame loop's next + * kick. Bounded keeps keep a truly churning + * zeroing block on the old reject path. + */ + ch->gr_ram_userd_keeps++; + if (ch->gr_ram_userd_keep_log < 8) { + ch->gr_ram_userd_keep_log++; + pr_info("userd latch keep h=0x%x blk=0x%llx put=%u\n", + ch->h_object, ch->userd_pa, + put); + } + ch->gr_ram_userd_latch_put = put; + ch->gr_ram_userd_latch_at = jiffies; } else if (put != ch->gr_ram_userd_latch_put || time_after(jiffies, ch->gr_ram_userd_latch_at + @@ -44644,8 +45382,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &pnow, sizeof(pnow)); svc.counter_pub_dbg++; - pr_info("sema-chan: gr-get-back h=0x%x wrote=%u read=%u put=%u\n", - ch->h_object, g, grad, pnow); + pr_info("sema-chan: gr-get-back h=0x%x userd=0x%llx/%u wrote=%u read=%u put=%u\n", + ch->h_object, ch->userd_pa, + ch->userd_as, g, grad, pnow); } } @@ -44738,6 +45477,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_latch_at; svc.chan[k].gr_ram_userd_confirmed = ch->gr_ram_userd_confirmed; + /* CONT.407: keep-across-put-reset state. */ + svc.chan[k].gr_ram_userd_keeps = + ch->gr_ram_userd_keeps; + svc.chan[k].gr_ram_userd_keep_log = + ch->gr_ram_userd_keep_log; /* SESSION 153-CONT-5: rejected-latch memory. */ svc.chan[k].gr_ram_userd_reject_pa = ch->gr_ram_userd_reject_pa; @@ -46034,9 +46778,14 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) /* * Register the guest PMA scrubber's channel-buffer words. Called from the * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved - * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish - * payload the guest's scrub wait polls; hsem_pa is the channel's host - * semaphore (channelWaitForFreeEntry polls it). Plain state update. + * (vmallocinfo vmap area + page-table walk). fin_pa is the finish payload + * the guest's scrub wait polls; hsem_pa is the channel's host semaphore + * (channelWaitForFreeEntry polls it). Plain state update. + * + * CONT.409: one record per channel-buffer page. The CeUtils scrubber and + * the X DDX's own CE/PMA channel buffer can both exist in one guest, and + * the scanner re-emits when a buffer moves. fin_pa=hsem_pa=0 clears every + * scanner-registered record of the open guest (the buffer's vmap is gone). * * Dual-guest: the words are written through a specific guest's address space * (see svc_scrub_pass), so the registration binds to the FIRST open mdev - @@ -46048,9 +46797,36 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; struct svc_scrub *s; + int i; if (!svc.task) return -ENODEV; + if (!fin_pa && !hsem_pa) { + /* Clear: the scanner saw its channel-buffer vmap disappear. */ + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) + return 0; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (svc_scrubs[i].vgpu != vgpu || + !svc_scrubs[i].registered) + continue; + if (svc_scrubs[i].userd_ok) { + /* Keep the stashed CeUtils geometry. */ + svc_scrubs[i].registered = false; + continue; + } + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + } + pr_info("sema-svc: scrubber buffer registration cleared\n"); + return 0; + } /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ if (!fin_pa || (fin_pa & 0x3) || fin_pa >= svc_pa_max()) return -EINVAL; @@ -46069,16 +46845,20 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) pr_warn_ratelimited("sema-svc: scrubber buffer registration ignored: no open mdev to bind it to\n"); return -ENODEV; } - s = svc_scrub_get(vgpu, true); + s = svc_scrub_reg(vgpu, hsem_pa); if (!s) return -ENOSPC; + s->registered = true; s->fin_pa = fin_pa; s->hsem_pa = hsem_pa; s->pa_ok = true; + s->pa_at = jiffies; + s->seeded = false; /* idle-pair authoring runs on the next pass */ s->pa_gen++; - pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", - fin_pa, hsem_pa, s->pa_gen); + pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u slot %d userd_ok=%d)\n", + fin_pa, hsem_pa, s->pa_gen, (int)(s - svc_scrubs), + s->userd_ok); return 0; } @@ -46160,15 +46940,52 @@ static bool svc_scrub_resolve_va(struct nv_vgpu_device *vgpu, u64 va, * for; correctness impact is limited to stale data in pages the guest * believes scrubbed. */ -static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) { - struct svc_scrub *s = svc_scrub_get(vgpu, false); u32 put = 0; u64 delta; if (!s) return; /* + * CONT.409: a scanner-registered channel-buffer page is the vmap the + * guest RM actually polls. Do not walk the GPU VAS for it: after the + * guest RM re-creates its kernel VAS the h=0xc directory can still + * resolve a stale page (r1712: the pass kept authoring 0x126b60000 + * while the guest polled 0x124fec000, so the X scrub wait could never + * complete). Author the proven all-ones pair (CONT.408's forced + * release): channelWaitForFreeEntry passes because 0xffffffff can + * never equal a putIndex below the ring size, and + * channelWaitForFinishPayload passes because the folded payload + * counter is at its maximum. Idempotent: re-assert only when the words + * no longer hold that value, so an idle buffer costs two reads a pass. + */ + if (s->registered && !s->userd_ok) { + u32 sema = 0; + u32 fin = 0; + u32 idle = 0xffffffffu; + bool ok = true; + + if (!s->pa_ok) + return; + ok &= !svc_read_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, + &sema, sizeof(sema)); + ok &= !svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, + &fin, sizeof(fin)); + if (ok && sema == idle && fin == idle) + return; /* already released */ + if (s->log_budget < 16) { + s->log_budget++; + pr_info("sema-svc: scrub reg author hsem_pa=0x%llx fin_pa=0x%llx (was 0x%x/0x%x)\n", + s->hsem_pa, s->fin_pa, sema, fin); + } + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &idle, + sizeof(idle)); + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &idle, + sizeof(idle)); + return; + } + /* * r1665: the guest scanner never registered the scrubber's finish * payload, so the X DDX's vidheap scrub wait (channelFillPbFastScrub * / channelWaitForFinishPayload) never completed. The channel buffer @@ -46203,7 +47020,23 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) * blocked for 200 ms each, starving this very pass). */ if (svc_scrub_resolve_va(vgpu, hva, &pa, &ap, &hv) && pa) { - if (!s->pa_ok || pa != s->hsem_pa) { + /* + * CONT.409: a scanner-registered PA is the page the + * guest RM actually polls; never replace it with a + * walk result. After the guest re-creates its kernel + * VAS the h=0xc directory can keep resolving an old + * page for the reused VA (r1712: walk said + * 0x126b60000 while the guest polled 0x124fec000), + * so adopting the walk would silently re-break the + * wait the registration just fixed. + */ + if (s->registered && pa != s->hsem_pa) { + if (s->log_budget < 24) { + s->log_budget++; + pr_info("sema-svc: scrub walk disagrees hsem_va=0x%llx walk_pa=0x%llx reg_pa=0x%llx (kept registered)\n", + hva, pa, s->hsem_pa); + } + } else if (!s->pa_ok || pa != s->hsem_pa) { if (s->log_budget < 24) { s->log_budget++; pr_info("sema-svc: scrub geometry hsem_va=0x%llx fin_va=0x%llx -> hsem_pa=0x%llx fin_pa=0x%llx vas=0x%x\n", @@ -46212,6 +47045,34 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) s->hsem_pa = pa; s->fin_pa = pa + 4; } + /* + * CONT.409: the scanner may have registered this same + * page before the stash bound (the scanner runs from + * boot; the 4096-entry channel arrives later), which + * leaves a registered-only duplicate record for the same + * words. Two records authoring one page ping-pong between + * the put-based (sema = put-1) and idle (0xffffffff) + * values, and a wait that samples the wrong half can + * still block. The stash record governs; drop the + * duplicate. + */ + { + int j; + + for (j = 0; j < SVC_MAX_GUESTS * SVC_SCRUB_RECS; j++) { + if (&svc_scrubs[j] == s || + svc_scrubs[j].vgpu != vgpu) + continue; + if (svc_scrubs[j].registered && + !svc_scrubs[j].userd_ok && + svc_scrubs[j].hsem_pa == s->hsem_pa) { + pr_info("sema-svc: scrub dedup slot %d (stash owns hsem_pa=0x%llx)\n", + j, s->hsem_pa); + memset(&svc_scrubs[j], 0, + sizeof(svc_scrubs[j])); + } + } + } s->pa_ok = true; s->pa_at = jiffies; } else if (!s->pa_ok) { @@ -46225,7 +47086,15 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) } } } - if (!s->userd_ok || !s->pa_ok || !s->entries) { + /* + * CONT.408: never author through a stale geometry. svc_scrub_pass + * re-resolves the VAs above on every pass; if that walk stops + * succeeding the old PA may already belong to another allocation + * (the r1710 guest panic class). Two seconds is far longer than the + * per-pass cadence, so a healthy boot never trips it. + */ + if (!s->userd_ok || !s->pa_ok || !s->entries || + time_after(jiffies, s->pa_at + msecs_to_jiffies(2000))) { /* m112b: silent-return diagnostic (bounded per boot). */ if (s->log_budget < 8) { s->log_budget++; @@ -46248,6 +47117,14 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n", s->userd_pa, s->userd_as); } + /* + * CONT.408 A/B: authoring the sema/finish from a + * stale geometry when the USERD is unreadable wrote a + * repurposed page and panicked the guest (r1710, + * Fatal exception in interrupt, RIP 0x0). Never write + * without a readable put; the free-run below still + * covers the readable-but-frozen case. + */ return; } } @@ -46268,12 +47145,21 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) * count until the ring wraps. Seeding from it adopts any * pre-host scrub work (the boot session's scrubbers). */ s->submitted = put; + /* + * CONT.408: the host semaphore is the LAST CONSUMED entry, + * not the put position: channelWaitForFreeEntry blocks while + * sema == (lastSubmittedEntry + 1) % entries, so authoring + * the put itself is exactly the value the just-submitted + * entry has and the wait can never pass. Start one behind. + */ + s->consumed = put ? put - 1 : s->entries - 1; } else if (put != s->put_last) { delta = (put + s->entries - s->put_last) % s->entries; if (!delta) delta = s->entries; /* exact wrap */ s->submitted += delta; s->put_last = put; + s->consumed = put ? put - 1 : s->entries - 1; } else if (s->submitted == s->authored && s->authored_gen == s->pa_gen) { return; /* nothing new; no guest writes */ @@ -46281,19 +47167,26 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) if (s->log_budget < 64) { s->log_budget++; - pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n", - put, s->submitted, s->fin_pa); + pr_info("sema-svc: scrub author put=%u consumed=%u submitted=%llu fin_pa=0x%llx\n", + put, s->consumed, s->submitted, s->fin_pa); } /* - * Author (forward-only). The USERD GP_GET and host-sema writes mark - * the submissions consumed; the finish write releases the scrub - * wait. Written only when GP_PUT advanced (or on the seeding pass), - * so the per-pass cost is zero while the scrubber idles. + * Author (forward-only). The USERD GP_GET marks the entries consumed; + * the host sema is the last consumed index the free-entry check reads + * (CONT.408: authoring the put itself made sema == putIndex and the + * wait could never pass), and the finish write releases the scrub + * wait. Written only when GP_PUT advanced (or on the seeding pass) so + * the per-pass cost is zero while the scrubber idles. */ svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as, &put, sizeof(put)); - svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put)); + { + u32 sema = s->consumed; + + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &sema, + sizeof(sema)); + } { u32 fin = (u32)s->submitted; @@ -46305,6 +47198,21 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) } /* + * CONT.409: author every channel-buffer record of this guest. The stashed + * CeUtils scrubber and any scanner-registered buffer (the X DDX's own + * CE/PMA channel) are independent; a singleton record could only serve + * one of them, and the other's wait then never completed. + */ +static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +{ + int i; + + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) + if (svc_scrubs[i].vgpu == vgpu) + svc_scrub_one(vgpu, &svc_scrubs[i]); +} + +/* * r1668: run the scrub step from inside a blocked event post. * * When the guest's status-queue consumer is frozen (typically because the @@ -46977,7 +47885,7 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) if (svc_guests[i].vgpu == vgpu) memset(&svc_guests[i], 0, sizeof(svc_guests[i])); spin_unlock_irqrestore(&svc_guest_lock, flags); - for (i = 0; i < SVC_MAX_GUESTS; i++) + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) if (svc_scrubs[i].vgpu == vgpu) memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 6f591e5..3f2a8dc 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -106,6 +106,17 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * CONT.407: a client's DUP_OBJECT of a VAS gives the destination client a + * NEW handle for the SAME page directory (measured: the GL client dups X's + * VAS 0x10005 as 0xbfef0001, then allocates its GR channel with + * hVASpace=0xbfef0001). Register the destination handle as an alias of the + * source's directory so the channel's gpfifo VA resolves through the + * client's own VAS instead of falling through to another registered + * directory (observed bogus 0x1bcfff0/ap=2 answer). Returns true when the + * source handle was a registered VAS and the alias was installed. + */ +bool gsp_sema_service_vaspace_alias(u32 h_dst, u32 h_src); /* * r820: a driver-declared kernel-buffer translation from an diff --git a/tools/guest-diag/scrub_pin_qmp.py b/tools/guest-diag/scrub_pin_qmp.py new file mode 100644 index 0000000..a69f8eb --- /dev/null +++ b/tools/guest-diag/scrub_pin_qmp.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Open-IOV contributors +# scrub_pin_qmp.py - host-side channel-buffer (PMA scrubber / X DDX CE) +# semaphore+finish page scanner. +# +# The guest RM polls the channel-buffer host semaphore (channelWaitForFreeEntry) +# and finish payload (channelWaitForFinishPayload) through the *vmap* page of +# the buffer memdesc. After the guest RM re-creates its kernel VAS the host's +# GPU-VAS walk can keep resolving a stale page (r1712: pass authored +# 0x126b60000 while the guest polled 0x124fec000), so the waits never +# completed. This tool resolves the guest's own vmap pages and registers them +# through sema_scrubbuf_add; the service then authors the proven all-ones pair +# into the pages the guest actually reads (CONT.409). +# +# Usage (host, root for the QMP socket): +# sudo python3 tools/guest-diag/scrub_pin_qmp.py [--duration 0] [--interval 2] +# +# Every registered page must look like a channel buffer: the ring page +# (base+0x64000) is all-zero or 906F entries whose GET names a push inside +# the channel's own GPU window. A page that fails the shape test is skipped. +import argparse +import importlib.util +import os +import re +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +OIV = os.path.normpath(os.path.join(HERE, "..", "..")) +VMEXEC = os.path.join(OIV, "scripts", "vgpu_vm.py") +PARAM = "/sys/module/vgsp/parameters/sema_scrubbuf_add" + +# nv_vm_map_pages areas seen for channel buffers (pb 0x64000 + gpfifo + sema +# + finish, with per-size variants). The fwimage boots observed 0x6e000 and +# neighbours; keep a small band and rely on the ring shape test. +SCRUB_SIZES = (0x6b000, 0x6c000, 0x6d000, 0x6e000, 0x6f000, 0x70000, + 0x71000, 0x72000) +PAGE = 0x1000 + + +def log(msg): + print("[%s] %s" % (time.strftime("%H:%M:%S"), msg), flush=True) + + +def load_qmp(): + spec = importlib.util.spec_from_file_location( + "semfix_watch", os.path.join(HERE, "semfix_watch.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.Qmp("/tmp/qemu-qmp.sock") + + +def vmexec(cmd, timeout=25): + try: + out = subprocess.run( + ["python3", VMEXEC, "exec", "--timeout", str(timeout), cmd], + capture_output=True, text=True, timeout=timeout + 15).stdout + except Exception: + return "" + return out + + +def guest_vmaps(): + out = vmexec( + "sudo -n grep -E 'nv_vm_map_pages' /proc/vmallocinfo 2>/dev/null | " + "awk '{split($1,a,\"-\"); " + "sz=strtonum(\"0x\" substr(a[2],3))-strtonum(\"0x\" substr(a[1],3)); " + "print a[1], sz}'", timeout=20) + found = [] + for line in out.splitlines(): + m = re.match(r"^(0x[0-9a-fA-F]+)\s+(\d+)$", line.strip()) + if m: + base, size = int(m.group(1), 16), int(m.group(2)) + if size in SCRUB_SIZES: + found.append((base, size)) + return found + + +def phys_read64(q, pa, n=2): + """Read n qwords of guest physical memory through the QEMU monitor. + + HMP prints each output line as "<addr>: <values>"; only the part after + the colon is data (the address tokens are not). + """ + out = q.hmp("xp/%dgx 0x%x" % (n, pa)) + vals = [] + for line in out.splitlines(): + if ":" not in line: + continue + vals += [int(x, 16) for x in re.findall(r"0x([0-9a-fA-F]+)", + line.split(":", 1)[1])] + return vals[:n] + + +def ring_ok(q, ring_pa): + """906F GPFIFO ring page: all-zero or entries whose GET names a push + inside the channel's own GPU window (the guest scanner's shape test, + guest_pin_stream.py _scrub_ring_ok).""" + vals = phys_read64(q, ring_pa, n=16) + if not vals: + return False + for i in range(0, len(vals), 2): + e0 = vals[i] & 0xffffffff + if e0 == 0: + continue + # Low 32 bits of the push GPU VA: the CeUtils scrubber window is + # 0x120000000..0x120080000, the DDX channel buffer pushes reach + # 0x1200a0000+, so accept through 0x120200000. + if not (0x20000000 <= e0 < 0x20200000) or (e0 & 3): + return False + return True + + +def register(fin_pa, hsem_pa): + req = "0x%x,0x%x" % (fin_pa, hsem_pa) + r = subprocess.run(["sudo", "tee", PARAM], input=req + "\n", + capture_output=True, text=True) + return r.returncode == 0 + + +def clear(): + register(0, 0) + + +def scan(q): + pages = [] + for base, size in guest_vmaps(): + try: + hsem = q.gva2gpa(base + 0x6c000) + ring = q.gva2gpa(base + 0x64000) + except Exception: + continue + if not hsem or not ring: + continue + if not ring_ok(q, ring): + log("reject vmap %#x (ring shape)" % base) + continue + pages.append((hsem + 4, hsem)) + return pages + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--duration", type=float, default=0, help="seconds (0 = forever)") + ap.add_argument("--interval", type=float, default=2.0) + args = ap.parse_args() + + q = load_qmp() + cur = None + start = time.time() + while True: + if args.duration and time.time() - start >= args.duration: + break + try: + pages = scan(q) + except Exception as e: + log("scan failed: %s" % e) + pages = [] + key = tuple(sorted(pages)) + if key != cur: + if cur is not None: + clear() + for fin, hsem in pages: + if register(fin, hsem): + log("registered fin=%#x hsem=%#x" % (fin, hsem)) + else: + log("register failed fin=%#x hsem=%#x" % (fin, hsem)) + cur = key + time.sleep(args.interval) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) |