vgsp: kick NOTIFY execution (sema_kick_notify; completion word authored + readback verified) + notify2d witness + kick probe (SESSION 153 r971)

1 files changed, 203 insertions, 9 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f15f1d2..e414967 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -7141,6 +7141,17 @@ struct svc_gl_sec {
 	 */
 	u8 blob[SVC_GR_INLINE_BYTES];
 	u32 blob_len;
+	/*
+	 * SESSION 153-CONTINUED-18: the kick notification this push carried
+	 * (class2d SET_NOTIFY_A/B + NOTIFY on subchannel 3, decoded after the
+	 * C597 bind that opened this section). Executed at flush time by
+	 * svc_kick_notify_exec when both the target and the NOTIFY are in
+	 * this same push (the short variant carries the NOTIFY without the
+	 * arm pair; it never executes and stays witness-only).
+	 */
+	u64 kick_notify_va;
+	u32 kick_notify_val;
+	bool kick_notify_seen;
 };
 static struct svc_gl_sec svc_gl;	/* service thread only */
 /*
@@ -7513,6 +7524,28 @@ struct svc_gl_hold {
 static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX];
 static u32 svc_gl_open_log;	/* bounded section-open witness */
 static u32 svc_gl_skip_log;	/* bounded flush-skip witness */
+/*
+ * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor.
+ *
+ * The GL client's per-frame kick push on the kick-family channel
+ * (h=0xbfef0002) is [SET_OBJECT(C597)][SET_NOTIFY_A {1, 0x2000e0b0}]
+ * [NOTIFY 0x10C][NO_OP]: the class2d notification write at VA
+ * 0x12000e0b0 (the shared .nvidia_drv member, page offset 0xB0; its u16
+ * status at +0xE). On real hardware the 2D engine writes a completed
+ * NvNotification there (nonzero timestamp at +0x0, status 0 at +0xE) -
+ * that write is the client's per-frame completion edge; the lane used to
+ * log "flush skip" and drop it, leaving the member all-zero forever. This
+ * executor authors the NvNotification through the same trusted VA walk
+ * the release path uses (svc_lookup_trusted + svc_write_ap) and verifies
+ * by readback. sema_kick_notify=0 disables (A/B); the witness line
+ * "sema-kick: exec ..." is bounded (first 48 + every 512th).
+ */
+static bool gsp_sema_kick_notify = true;
+module_param_named(sema_kick_notify, gsp_sema_kick_notify, bool, 0644);
+MODULE_PARM_DESC(sema_kick_notify,
+	"Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)");
+static u32 svc_kick_log;	/* bounded kick-notify witness */
+static u64 svc_kick_count;	/* executor invocations */
 static bool gsp_sema_gl_draw_synth;
 module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644);
 MODULE_PARM_DESC(sema_gl_draw_synth,
@@ -7814,6 +7847,107 @@ static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len,
 }
 
 /*
+ * SESSION 153-CONTINUED-18 (task 1): execute the GL client's per-frame
+ * kick NOTIFY (see the sema_kick_notify comment above). The GPU's write
+ * is a completed NvNotification: nonzero timestamp at +0x0 (ns since
+ * epoch), info32 at +0x8 (the NOTIFY payload), info16 at +0xC, status 0
+ * at +0xE. Resolution is the release path's trusted walk
+ * (svc_lookup_trusted -> svc_write_ap), then readback verification.
+ */
+static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
+				 struct svc_channel *ch, u64 va, u32 val,
+				 u64 push_va)
+{
+	u64 pa = 0, ts = ktime_get_real_ns();
+	int ap = SVC_APERTURE_SYS;
+	u8 old[16], nw[16], rb[16];
+	bool ok = false, changed = false;
+	u32 o0 = 0, n0 = 0;
+	int rc = -ENOENT;
+
+	memset(nw, 0, sizeof(nw));
+	memcpy(nw, &ts, sizeof(ts));
+	memcpy(nw + 8, &val, sizeof(val));
+	if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) {
+		int r1, r2;
+
+		r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old));
+		r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw));
+		if (!r1 && !r2 &&
+		    !svc_read_ap(vgpu, pa, svc_ap_of(ap), rb, sizeof(rb))) {
+			rc = 0;
+			ok = true;
+			changed = memcmp(old, rb, sizeof(old)) != 0;
+			memcpy(&o0, old, 4);
+			memcpy(&n0, rb, 4);
+		} else {
+			/* CONT.18 probe: name the failing step. */
+			rc = r1 ? (r1 < 0 ? r1 : -100) :
+			     (r2 ? (r2 < 0 ? r2 : -101) : -102);
+		}
+	}
+	svc_kick_count++;
+	if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 511))) {
+		svc_kick_log++;
+		pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n",
+			ch ? ch->h_object : 0, push_va, va, pa, ap, val,
+			(unsigned long long)ts, o0, n0, changed ? 1 : 0,
+			(unsigned long long)svc_kick_count, rc);
+	}
+	return ok;
+}
+
+/*
+ * SESSION 153-CONTINUED-18 (task 1, dev probe): run the kick-notify
+ * executor on an explicit VA against the service's own device. The
+ * execution MUST run in a kernel-thread context: vfio_iommu_type1's
+ * dma_rw only copies when current->mm == dma->mm or the caller is a
+ * kthread (kthread_use_mm); a sysfs write from a shell process has
+ * current->mm != QEMU's mm and gets -EFAULT (measured rc=-14). The
+ * workqueue item below is a kthread - the same context class as the
+ * sema service thread that runs the real kick lane.
+ *   echo 0x12000e0b0 > .../parameters/sema_kick_probe
+ */
+static u64 svc_kick_probe_va;
+static void svc_kick_probe_work_fn(struct work_struct *w);
+static DECLARE_WORK(svc_kick_probe_work, svc_kick_probe_work_fn);
+
+static void svc_kick_probe_work_fn(struct work_struct *w)
+{
+	struct nv_vgpu_device *vgpu = NULL;
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu) {
+			vgpu = svc.chan[i].vgpu;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	if (vgpu)
+		svc_kick_notify_exec(vgpu, NULL, svc_kick_probe_va, 0, 0);
+}
+
+static int nv_vgpu_sema_kick_probe_set(const char *val,
+				       const struct kernel_param *kp)
+{
+	if (!val || kstrtoull(val, 0, &svc_kick_probe_va))
+		return -EINVAL;
+	if (svc_kick_probe_va)
+		schedule_work(&svc_kick_probe_work);
+	return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_kick_probe_ops = {
+	.set = nv_vgpu_sema_kick_probe_set,
+};
+
+module_param_cb(sema_kick_probe, &nv_vgpu_sema_kick_probe_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_kick_probe, "Run the kick-notify executor on a VA (dev probe; session 153 CONT.18)");
+
+/*
  * Close the open section: synthesize the QMD from the collected fields and
  * capture it as the forwarding candidate. The QMD word layout mirrors
  * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33
@@ -7918,6 +8052,29 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu,
 							  svc_gl.param_va,
 							  svc_gl.param_size,
 							  0, 0);
+				/*
+				 * SESSION 153-CONTINUED-18 (task 1): replace
+				 * the old "flush skip" drop with NOTIFY
+				 * EXECUTION when this push carried the kick
+				 * member (SET_NOTIFY_A/B + NOTIFY in the same
+				 * push). The section has no draw operands -
+				 * it is the client's completion kick - so the
+				 * GPU's notification write is the only work
+				 * it asks for.
+				 */
+				if (gsp_sema_kick_notify &&
+				    svc_gl.kick_notify_seen &&
+				    svc_gl.kick_notify_va &&
+				    svc_kick_notify_exec(vgpu, ch,
+							 svc_gl.kick_notify_va,
+							 svc_gl.kick_notify_val,
+							 push_va)) {
+					/* authored once; a replay pass must not
+					 * re-author an unchanged member. */
+					svc_gl.kick_notify_seen = false;
+					svc_gl.kick_notify_va = 0;
+					goto out;
+				}
 				if (gsp_sema_dbg && svc_gl_skip_log < 2048) {
 					svc_gl_skip_log++;
 					pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n",
@@ -12421,16 +12578,34 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu,
 	if (!ch->notify2d_va)
 		return false;
 	if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) {
-		if (gsp_sema_dbg && svc_notify2d_log < 16) {
+		if (gsp_sema_dbg && (svc_notify2d_log < 64 ||
+				     ch->h_object == 0xbfef0002)) {
 			svc_notify2d_log++;
 			pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n",
 				ch->h_object, ch->notify2d_va);
 		}
 		return false;
 	}
-	if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero)))
-		return false;
-	svc_notify2d_done++;
+	/*
+	 * SESSION 153-CONTINUED-18: read the first word for the witness
+	 * (the kick member's timestamp slot: 0 = never GPU-written; the
+	 * completed write makes it nonzero - see svc_kick_notify_exec).
+	 */
+	{
+		u32 o0 = 0;
+
+		if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &o0, sizeof(o0)))
+			o0 = 0xffffffff;
+		if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero)))
+			return false;
+		svc_notify2d_done++;
+		if (gsp_sema_dbg && (svc_notify2d_log < 64 ||
+				     ch->h_object == 0xbfef0002)) {
+			svc_notify2d_log++;
+			pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (w0 old=%08x; ack)\n",
+				ch->h_object, ch->notify2d_va, pa, o0);
+		}
+	}
 	/*
 	 * r834: pin the page that accepted the ack; the same stale-map
 	 * defense as the push-side pin in the decode.
@@ -12447,11 +12622,6 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu,
 	 */
 	if (completion)
 		gsp_rm_post_channel_event(vgpu, ch->h_object);
-	if (gsp_sema_dbg && svc_notify2d_log < 16) {
-		svc_notify2d_log++;
-		pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n",
-			ch->h_object, ch->notify2d_va, pa);
-	}
 	return true;
 }
 
@@ -13511,6 +13681,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				if (count >= 2)
 					ch->notify2d_va |= (u64)v1;
 				/*
+				 * SESSION 153-CONTINUED-18 (task 1): when
+				 * this push also carries the C597 kick
+				 * section, remember the arm so the section
+				 * flush can execute the NOTIFY write (the
+				 * 2D ack below is the watchdog's zero-ack;
+				 * the kick member needs the completed
+				 * NvNotification authored).
+				 */
+				if (gsp_sema_gl_xlate && svc_gl.seen)
+					svc_gl.kick_notify_va =
+						ch->notify2d_va;
+				/*
 				 * r834: this method stream reached us through
 				 * a REAL read (svc_read_last_pa); pin its page
 				 * so the watchdog's push translation cannot go
@@ -13531,6 +13713,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				break;
 			case SVC_2D_NOTIFY:
 				if (count == 1 && v0 == 0) {	/* WRITE_ONLY */
+					/*
+					 * SESSION 153-CONTINUED-18: mark the
+					 * kick section's NOTIFY (payload 0) so
+					 * svc_gl_flush executes the completion
+					 * write at the armed VA - the executor
+					 * authors the NvNotification the 2D
+					 * zero-ack alone does not.
+					 */
+					if (gsp_sema_gl_xlate && svc_gl.seen) {
+						svc_gl.kick_notify_val = v0;
+						svc_gl.kick_notify_seen = true;
+					}
 					if (svc_notify2d_write(vgpu, ch,
 							       true)) {
 						/*