vgsp: USERD index/alias resolution, DDX kick-idle fix, completion-slot authorship (CONT.403-412)

Ten debugging rounds against the direct-GL channel, each fixing the wall that
blocked the previous one:

- USERD index: honour USERD_INDEX_VALUE/PAGE_VALUE from NVOS04 flags
  (X idx=3, GL idx=4) instead of always reading the block at base+0. This
  fixed the X "Failed to initialize DMA" wall.
- USERD alias: learn the BAR1-alias bias from the first ring-validated FB
  latch (wire 0x80600 -> live 0x910000) and prefer alias/wire-page
  candidates before falling back to the RAM scan, which previously adopted
  another channel's control page.
- USERD latch: keep a ring-validated kick-family block across a transient
  put=0 instead of dropping it (bounded, per-channel log).
- VAS alias: DUP_OBJECT copies pdb/entries/ap/verified so hVASpace 0xbfef0001
  resolves to 0x10005 and its gpfifo VA stops re-resolving to a bogus FB PA.
- Short-kick NOTIFY: complete a kick that has no armed class-2D target
  (release + event, no write) so the client proceeds past setup.
- DDX kick-idle: slot 0 of the 0x01xxxxxx family is the pb_kickIfIdle gate and
  0xFFFF is its IDLE value; the keepalive no longer zeroes it. Removes the
  GR-STALL-stale-wedge on the X channel.
- Completion slots: register a per-channel completion on every successful
  kick and reuse it when a NOTIFY has no per-push SET_NOTIFY; prefer the
  channel's notifier page over the VA walk.
- Empty/no-op submit: author the completion even when there is no push to
  replay, so a synchronous GL submit does not wait forever.
- Scrub records: multi-record per guest, stash-first lookup, registered PA is
  authoritative against the GPU-VAS walk, scanner-only records author the
  proven 0xffffffff/0xffffffff pair.
- GL attachment: accept a draw run whose own subchannel is the engine
  subchannel the channel uses (bind is subch 0, draws on subch 3), not only
  the bind subchannel.

Measured: glxgears 427 -> 526 FPS sustained with no GR stall; X binds and
renders its 2D path; 3D draw capture on the GL channel still does not occur
(see CONT.412: the GL thread blocks on the NVRM RM write lock before it
reaches any ioctl).

Adds tools/guest-diag/scrub_pin_qmp.py to register guest channel-buffer
vmap pages as scrub records from the host.

1 files changed, 291 insertions, 2 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index f7e7beb..ca14744 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -3290,6 +3290,25 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
 {
 	struct sema_channel_desc desc;
 
+	/*
+	 * CONT.405: name the registration attempt for the direct-GL client's
+	 * channel families. A "beef"/"bfef" handle that appears more than
+	 * once is the client's alloc -> setup -> free -> retry loop; the
+	 * matching beef step trace in gsp_rm_handle_rm_control names the
+	 * last control it issued before the retry. Bounded.
+	 */
+	if (((h_object & 0xffff0000u) == 0xbeef0000u ||
+	     (h_object & 0xffffff00u) == 0xbfef0000u)) {
+		static u32 beef_reg_log;
+
+		if (beef_reg_log < 64) {
+			beef_reg_log++;
+			nv_vgpu_info(ctx->vgpu,
+				     "vgsp-rm: beef chan register #%u h=0x%x (setup step: alloc_channel params=%u; next client step is BIND/GPFIFO_SCHEDULE, then its first kick)",
+				     beef_reg_log, h_object, params_size);
+		}
+	}
+
 	if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8) {
 		/*
 		 * r828 diag: a channel-class alloc whose params are shorter
@@ -3352,26 +3371,109 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
 	 * the direct client's channels never resolved a USERD (CONT-37
 	 * alloc/free retry loop). Index by the subdevice.
 	 */
+	/*
+	 * CONT.404: read the aperture before the flags log so the logged
+	 * /u value is the real one (it used to print 0 because the memcpy
+	 * sat after the block).
+	 */
+	memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM +
+	       GSPRM_CHAN_MEM_APERTURE, 4);
 	{
 		u32 sd = 0;
 		u64 uo = 0;
+		u32 flags = 0;
 
 		if (params_size >= 140)
 			memcpy(&sd, params + 136, 4);
+		/*
+		 * CONT.402: subDeviceId is a one-hot bitmask
+		 * (alloc_channel.h), so the userdOffset[] slot is its first
+		 * set bit, not the raw value.
+		 */
+		sd = sd ? (u32)__builtin_ctz(sd) : 0;
 		if (sd >= 8)
 			sd = 0;
 		if (params_size >= 64 + (sd + 1) * 8) {
 			memcpy(&uo, params + 64 + (u64)sd * 8, 8);
 			desc.userd_pa += uo;
 		}
+		/*
+		 * CONT.402: NVOS04_FLAGS_CHANNEL_USERD_INDEX_VALUE (bits 10:8)
+		 * and USERD_INDEX_PAGE_VALUE (bits 20:12) select the 0x200
+		 * control block inside the declared userdMem page. Logged so
+		 * the block offset can be validated before it is applied.
+		 *
+		 * CONT.404: apply it. The kernel driver sets idx = ChID % 8 and
+		 * page = ChID / 8 (kernel_channel.c, IS_GSP_CLIENT path), so a
+		 * channel whose ChID is nonzero has its GPGet/GPPut 0x200-aligned
+		 * block somewhere inside the declared userdMem. Reading base+0
+		 * made the guest's GP_PUT advances invisible - X's channel got
+		 * ChID=3 in r1697 and timed out in "Failed to initialize DMA"
+		 * with the host watching fill at the wrong block.
+		 */
+		if (params_size >= 24) {
+			u64 base = desc.userd_pa;
+			u32 idx, pg;
+
+			memcpy(&flags, params + 20, 4);
+			idx = (flags >> 8) & 0x7;
+			pg = (flags >> 12) & 0x1ff;
+			if (idx != 0 || pg != 0)
+				desc.userd_pa += (u64)pg * 0x1000 +
+						 (u64)idx * 0x200;
+			pr_info("vgsp-rm: chan userd flags=0x%x idx=%u page=%u base=0x%llx block=0x%llx/%u hObject=0x%x\n",
+				flags, idx, pg, base, desc.userd_pa,
+				desc.userd_as, h_object);
+		}
 	}
-	memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM +
-	       GSPRM_CHAN_MEM_APERTURE, 4);
 	memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8);
 	memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM +
 	       GSPRM_CHAN_MEM_APERTURE, 4);
 
 	/*
+	 * CONT.404 diag: the registered block is not always the block the
+	 * client writes - X's live GP_PUT was adopted by the shape scan at
+	 * fb 0x910000 while the wire declared 0x190000 (+0x600), and the
+	 * direct-GL channel's declared page stayed zero. Name the whole
+	 * wire memory-descriptor set and peek the registered block's
+	 * GP_GET/GP_PUT, so the next cycle separates "wire base is not the
+	 * client's block" from "client never writes FB". Bounded.
+	 */
+	{
+		static u32 md_log;
+		u64 inst_b = 0, ramfc_b = 0;
+		u32 inst_sz = 0, ramfc_sz = 0;
+		u32 inst_as = 0, ramfc_as = 0;
+		u32 g0 = 0, p0 = 0;
+
+		if (md_log < 32) {
+			md_log++;
+			memcpy(&inst_b, params + GSPRM_CHAN_INSTANCE_MEM, 8);
+			memcpy(&inst_sz, params + GSPRM_CHAN_INSTANCE_MEM + 8, 4);
+			memcpy(&inst_as, params + GSPRM_CHAN_INSTANCE_MEM +
+			       GSPRM_CHAN_MEM_APERTURE, 4);
+			memcpy(&ramfc_b, params + 192, 8);
+			memcpy(&ramfc_sz, params + 200, 4);
+			memcpy(&ramfc_as, params + 192 +
+			       GSPRM_CHAN_MEM_APERTURE, 4);
+			if (desc.userd_as == 2 && ctx->vgpu &&
+			    ctx->vgpu->bars[1].vaddr &&
+			    desc.userd_pa + 0x90 <=
+				    nv_vgpu_fb_emul_size(ctx->vgpu)) {
+				memcpy(&g0, ctx->vgpu->bars[1].vaddr +
+				       desc.userd_pa + 0x88, 4);
+				memcpy(&p0, ctx->vgpu->bars[1].vaddr +
+				       desc.userd_pa + 0x8c, 4);
+			}
+			pr_info("vgsp-rm: chan mds hObject=0x%x inst=0x%llx/%u/%u userd=0x%llx/%u ramfc=0x%llx/%u/%u notif=0x%llx/%u getput=%08x/%08x\n",
+				h_object, inst_b, inst_sz, inst_as,
+				desc.userd_pa, desc.userd_as,
+				ramfc_b, ramfc_sz, ramfc_as,
+				desc.notifier_pa, desc.notifier_as, g0, p0);
+		}
+	}
+
+	/*
 	 * SESSION 153-CONTINUED-15 (draw attribution): every channel-class
 	 * allocation is named with its wire shape before registration, so a
 	 * channel the service silently refuses (gpfifo VA 0 / entries < 2)
@@ -3415,6 +3517,60 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
 }
 
 /*
+ * CONT.406: GR class objects (TURING_A 0xc597 and TURING_COMPUTE_A 0xc5c0)
+ * are allocated on a channel and the client frees them again when the next
+ * step of its GR context setup fails. Keep the last few allocations so the
+ * FREE side can name the object class and how long the client kept it: that
+ * turns "the client freed the 3D object" into a measured keep time, and
+ * separates "the alloc reply was rejected" (keep time ~0) from "the client
+ * failed later and cleaned up" (keep time ~80 ms, the observed r1700 case).
+ * All alloc/free RPC handling is serialized on the emulation thread, so the
+ * table needs no lock. Bounded.
+ */
+#define GSPRM_GR3D_LOG_MAX	24
+#define GSPRM_MAX_GROBJS	16
+
+static struct {
+	u32		h;
+	u32		class_id;
+	unsigned long	at;
+} gsp_rm_grobjs[GSPRM_MAX_GROBJS];
+static u32 gsp_rm_grobjs_next;
+
+static void gsp_rm_grobj_note(u32 h, u32 class_id)
+{
+	u32 i;
+
+	for (i = 0; i < GSPRM_MAX_GROBJS; i++) {
+		if (gsp_rm_grobjs[i].h == h) {
+			gsp_rm_grobjs[i].class_id = class_id;
+			gsp_rm_grobjs[i].at = jiffies;
+			return;
+		}
+	}
+	i = gsp_rm_grobjs_next++ % GSPRM_MAX_GROBJS;
+	gsp_rm_grobjs[i].h = h;
+	gsp_rm_grobjs[i].class_id = class_id;
+	gsp_rm_grobjs[i].at = jiffies;
+}
+
+static bool gsp_rm_grobj_retire(u32 h, u32 *class_id, unsigned long *at)
+{
+	u32 i;
+
+	for (i = 0; i < GSPRM_MAX_GROBJS; i++) {
+		if (gsp_rm_grobjs[i].h != h || !gsp_rm_grobjs[i].class_id)
+			continue;
+		*class_id = gsp_rm_grobjs[i].class_id;
+		*at = gsp_rm_grobjs[i].at;
+		gsp_rm_grobjs[i].class_id = 0;
+		gsp_rm_grobjs[i].h = 0;
+		return true;
+	}
+	return false;
+}
+
+/*
  * GSP_RM_ALLOC (103) - rpc_gsp_rm_alloc_v03_00 (32 byte header) followed by
  * the class-specific allocation parameters.
  *
@@ -3454,6 +3610,32 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
 			     "vgsp-rm: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x",
 			     class_id, params_size, flags, hc, hp, ho);
 
+	/*
+	 * CONT.405: same bounded step trace as gsp_rm_handle_rm_control, for
+	 * the direct-GL client's object allocations. Channel classes get a
+	 * retry counter per handle so the alloc/free loop is visible without
+	 * gsp_sema_dbg.
+	 */
+	{
+		bool beef = (hc >= 0xc1d00010u && hc <= 0xc1d0002fu) ||
+			    (ho & 0xffff0000u) == 0xbeef0000u ||
+			    (ho & 0xffffff00u) == 0xbfef0000u;
+
+		if (beef) {
+			static u32 beef_alloc_log;
+
+			if (beef_alloc_log < 128) {
+				beef_alloc_log++;
+				nv_vgpu_info(ctx->vgpu,
+					     "vgsp-rm: beef alloc step #%u class=0x%x hClient=0x%x hParent=0x%x hObject=0x%x sz=%u%s",
+					     beef_alloc_log, class_id, hc, hp, ho,
+					     params_size,
+					     gsp_rm_is_channel_class(class_id) ?
+					     " (channel)" : "");
+			}
+		}
+	}
+
 	/* Full param dump for memory/channel classes and any class we have not
 	 * classified yet (rate-limited). %phN is capped at 64 bytes per print,
 	 * so chunk it. */
@@ -3604,6 +3786,39 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
 	memcpy(reply, p, reply_len);
 	memset(reply + 16, 0, 4);	/* status = NV_OK */
 
+	/*
+	 * CONT.406: the direct-GL client allocates a TURING_A (0xc597) 3D
+	 * object on its channel immediately before it would submit its first
+	 * kick, and then frees it again. Record the exact reply the client
+	 * receives (bounded) and remember the object so the FREE side can say
+	 * how long it was kept. The reply must echo the 16-byte
+	 * NV_GR_ALLOCATION_PARAMETERS {version, flags, size, caps} with
+	 * status=NV_OK; caps is a class-dependent *output* that has no
+	 * defined bits for c597 in any 570 header, so the echoed 0 is the
+	 * value the golden/physical path also sends and accepts.
+	 */
+	if (class_id == 0xc597u) {
+		u32 rp[4] = { 0, 0, 0, 0 };
+
+		if (echo_len >= 4)
+			memcpy(&rp[0], reply + GSPRM_ALLOC_HDR_SIZE, 4);
+		if (echo_len >= 8)
+			memcpy(&rp[1], reply + GSPRM_ALLOC_HDR_SIZE + 4, 4);
+		if (echo_len >= 12)
+			memcpy(&rp[2], reply + GSPRM_ALLOC_HDR_SIZE + 8, 4);
+		if (echo_len >= 16)
+			memcpy(&rp[3], reply + GSPRM_ALLOC_HDR_SIZE + 12, 4);
+
+		gsp_rm_grobj_note(ho, class_id);
+		if (rm->gr3d_alloc_log < GSPRM_GR3D_LOG_MAX) {
+			rm->gr3d_alloc_log++;
+			nv_vgpu_info(ctx->vgpu,
+				     "vgsp-rm: alloc 0xc597 h=0x%x status=0x%x size=0x%x reply=%08x %08x %08x %08x (parent=0x%x client=0x%x)",
+				     ho, 0, params_size,
+				     rp[0], rp[1], rp[2], rp[3], hp, hc);
+		}
+	}
+
 	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_ALLOC, sequence, 0,
 			reply, reply_len))
 		nv_vgpu_err(ctx->vgpu, "vgsp-rm: GSP_RM_ALLOC reply send failed");
@@ -4309,9 +4524,35 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
 	u32 out_status = 0;
 	u32 out_params_size = 0;
 	u64 t0 = ktime_get_ns();
+	bool beef;
 
 	memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE);
 
+	/*
+	 * CONT.405: bounded step trace for the direct-GL client's RM clients
+	 * (the "beef" device/VAS/BINAPI client and the "bfef" channel
+	 * client). Its init is a fixed control sequence; when it gives up it
+	 * frees the channel and retries with a fresh client, so the LAST
+	 * cmd this trace shows before the next client's device alloc is the
+	 * failing step. Necessary because the default branch answers an
+	 * unhandled control with generic SUCCESS and that is invisible at
+	 * gsp_sema_dbg=0 - which is exactly where the retry loop lived.
+	 */
+	beef = (req.hClient >= 0xc1d00010u && req.hClient <= 0xc1d0002fu) ||
+	       (req.hObject & 0xffff0000u) == 0xbeef0000u ||
+	       (req.hObject & 0xffffff00u) == 0xbfef0000u;
+	if (beef) {
+		static u32 beef_step_log;
+
+		if (beef_step_log < 128) {
+			beef_step_log++;
+			nv_vgpu_info(ctx->vgpu,
+				     "vgsp-rm: beef step #%u cmd=0x%x hClient=0x%x hObject=0x%x in=%u",
+				     beef_step_log, req.cmd, req.hClient,
+				     req.hObject, req.paramsSize);
+		}
+	}
+
 	if (gsp_sema_dbg)
 		nv_vgpu_info(ctx->vgpu, "vgsp-rm: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x",
 			     req.cmd, req.hClient, req.hObject, req.paramsSize,
@@ -5090,6 +5331,21 @@ rm_control_reply:
 				     req.cmd, req.hClient, req.hObject,
 				     req.paramsSize, out_params_size,
 				     out_status, dt_us);
+		/*
+		 * CONT.405: close the beef step row with its outcome so the
+		 * last completed step before the retry is unambiguous.
+		 */
+		if (beef) {
+			static u32 beef_done_log;
+
+			if (beef_done_log < 128) {
+				beef_done_log++;
+				nv_vgpu_info(ctx->vgpu,
+					     "vgsp-rm: beef done cmd=0x%x -> status=0x%x out=%u",
+					     req.cmd, out_status,
+					     out_params_size);
+			}
+		}
 	}
 
 	if (gsp_rm_send_rm_control(ctx, sequence, reply,
@@ -5146,6 +5402,21 @@ static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx,
 		     "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo",
 		     hDstClient, hDstObject, hSrcClient, hSrcObject, sequence);
 
+	/*
+	 * CONT.407: when the duplicated object is a published VA space, the
+	 * destination client's new handle names the SAME page directory (the
+	 * GL client dups X's VAS 0x10005 as 0xbfef0001 and allocates its GR
+	 * channel with that hVASpace). Register the alias so the channel-
+	 * scoped lookup resolves the gpfifo VA through the client's own
+	 * directory instead of the newest-first global walk (r1702/r1703
+	 * answered 0x1bcfff0/ap=2 because 0xbfef0001 was never known).
+	 * Non-VAS dups simply return false and stay a pure handle echo.
+	 */
+	if (gsp_sema_service_vaspace_alias(hDstObject, hSrcObject))
+		nv_vgpu_info(ctx->vgpu,
+			     "vgsp-rm: DUP_OBJECT VAS alias h=0x%x -> src h=0x%x (gpfifo VA now resolves through the source directory)\n",
+			     hDstObject, hSrcObject);
+
 	/* Small delay before the reply: same reply-matching race mitigation as
 	 * FREE/UNLOADING. */
 	udelay(2000);
@@ -5218,6 +5489,24 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
 			     "vgsp-rm: retired %u event record(s) on FREE hObjOld=0x%x hRoot=0x%x",
 			     removed, hObjOld, hRoot);
 
+	/* CONT.406: if this is a GR object we noted at alloc time, name the
+	 * class and the keep time. A ~0 ms keep means the alloc reply (or the
+	 * client's params copy-out) was rejected; a longer keep means the
+	 * client proceeded and cleaned up after a later step failed. */
+	{
+		u32 gcls = 0;
+		unsigned long gat = 0;
+
+		if (gsp_rm_grobj_retire(hObjOld, &gcls, &gat) &&
+		    rm->gr3d_free_log < GSPRM_GR3D_LOG_MAX) {
+			rm->gr3d_free_log++;
+			nv_vgpu_info(ctx->vgpu,
+				     "vgsp-rm: gr-obj free h=0x%x class=0x%x after %ums (client kept it)",
+				     hObjOld, gcls,
+				     jiffies_to_msecs(jiffies - gat));
+		}
+	}
+
 	if (gsp_sema_dbg)
 		nv_vgpu_info(ctx->vgpu, "vgsp-rm: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u",
 			     hObjOld, sequence, rm->cmd_read, rm->stat_write);